clang 18.0.0git
CGExpr.cpp
Go to the documentation of this file.
1//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCUDARuntime.h"
14#include "CGCXXABI.h"
15#include "CGCall.h"
16#include "CGCleanup.h"
17#include "CGDebugInfo.h"
18#include "CGObjCRuntime.h"
19#include "CGOpenMPRuntime.h"
20#include "CGRecordLayout.h"
21#include "CodeGenFunction.h"
22#include "CodeGenModule.h"
23#include "ConstantEmitter.h"
24#include "TargetInfo.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/DeclObjC.h"
28#include "clang/AST/NSAPI.h"
32#include "llvm/ADT/Hashing.h"
33#include "llvm/ADT/StringExtras.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/Intrinsics.h"
36#include "llvm/IR/IntrinsicsWebAssembly.h"
37#include "llvm/IR/LLVMContext.h"
38#include "llvm/IR/MDBuilder.h"
39#include "llvm/IR/MatrixBuilder.h"
40#include "llvm/Passes/OptimizationLevel.h"
41#include "llvm/Support/ConvertUTF.h"
42#include "llvm/Support/MathExtras.h"
43#include "llvm/Support/Path.h"
44#include "llvm/Support/SaveAndRestore.h"
45#include "llvm/Support/xxhash.h"
46#include "llvm/Transforms/Utils/SanitizerStats.h"
47
48#include <optional>
49#include <string>
50
51using namespace clang;
52using namespace CodeGen;
53
54// Experiment to make sanitizers easier to debug
55static llvm::cl::opt<bool> ClSanitizeDebugDeoptimization(
56 "ubsan-unique-traps", llvm::cl::Optional,
57 llvm::cl::desc("Deoptimize traps for UBSAN so there is 1 trap per check"),
58 llvm::cl::init(false));
59
60//===--------------------------------------------------------------------===//
61// Miscellaneous Helper Methods
62//===--------------------------------------------------------------------===//
63
64/// CreateTempAlloca - This creates a alloca and inserts it into the entry
65/// block.
66Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty,
67 CharUnits Align,
68 const Twine &Name,
69 llvm::Value *ArraySize) {
70 auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);
71 Alloca->setAlignment(Align.getAsAlign());
72 return Address(Alloca, Ty, Align, KnownNonNull);
73}
74
75/// CreateTempAlloca - This creates a alloca and inserts it into the entry
76/// block. The alloca is casted to default address space if necessary.
78 const Twine &Name,
79 llvm::Value *ArraySize,
80 Address *AllocaAddr) {
81 auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
82 if (AllocaAddr)
83 *AllocaAddr = Alloca;
84 llvm::Value *V = Alloca.getPointer();
85 // Alloca always returns a pointer in alloca address space, which may
86 // be different from the type defined by the language. For example,
87 // in C++ the auto variables are in the default address space. Therefore
88 // cast alloca to the default address space when necessary.
90 auto DestAddrSpace = getContext().getTargetAddressSpace(LangAS::Default);
91 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
92 // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,
93 // otherwise alloca is inserted at the current insertion point of the
94 // builder.
95 if (!ArraySize)
96 Builder.SetInsertPoint(getPostAllocaInsertPoint());
99 Ty->getPointerTo(DestAddrSpace), /*non-null*/ true);
100 }
101
102 return Address(V, Ty, Align, KnownNonNull);
103}
104
105/// CreateTempAlloca - This creates an alloca and inserts it into the entry
106/// block if \p ArraySize is nullptr, otherwise inserts it at the current
107/// insertion point of the builder.
108llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
109 const Twine &Name,
110 llvm::Value *ArraySize) {
111 if (ArraySize)
112 return Builder.CreateAlloca(Ty, ArraySize, Name);
113 return new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
114 ArraySize, Name, AllocaInsertPt);
115}
116
117/// CreateDefaultAlignTempAlloca - This creates an alloca with the
118/// default alignment of the corresponding LLVM type, which is *not*
119/// guaranteed to be related in any way to the expected alignment of
120/// an AST type that might have been lowered to Ty.
122 const Twine &Name) {
123 CharUnits Align =
124 CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty));
125 return CreateTempAlloca(Ty, Align, Name);
126}
127
128Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {
130 return CreateTempAlloca(ConvertType(Ty), Align, Name);
131}
132
134 Address *Alloca) {
135 // FIXME: Should we prefer the preferred type alignment here?
136 return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca);
137}
138
140 const Twine &Name, Address *Alloca) {
142 /*ArraySize=*/nullptr, Alloca);
143
144 if (Ty->isConstantMatrixType()) {
145 auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType());
146 auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
147 ArrayTy->getNumElements());
148
149 Result = Address(Result.getPointer(), VectorTy, Result.getAlignment(),
151 }
152 return Result;
153}
154
156 const Twine &Name) {
157 return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name);
158}
159
161 const Twine &Name) {
162 return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty),
163 Name);
164}
165
166/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
167/// expression and compare the result against zero, returning an Int1Ty value.
168llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
169 PGO.setCurrentStmt(E);
170 if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
171 llvm::Value *MemPtr = EmitScalarExpr(E);
172 return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
173 }
174
175 QualType BoolTy = getContext().BoolTy;
176 SourceLocation Loc = E->getExprLoc();
177 CGFPOptionsRAII FPOptsRAII(*this, E);
178 if (!E->getType()->isAnyComplexType())
179 return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy, Loc);
180
182 Loc);
183}
184
185/// EmitIgnoredExpr - Emit code to compute the specified expression,
186/// ignoring the result.
188 if (E->isPRValue())
189 return (void)EmitAnyExpr(E, AggValueSlot::ignored(), true);
190
191 // if this is a bitfield-resulting conditional operator, we can special case
192 // emit this. The normal 'EmitLValue' version of this is particularly
193 // difficult to codegen for, since creating a single "LValue" for two
194 // different sized arguments here is not particularly doable.
195 if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
197 if (CondOp->getObjectKind() == OK_BitField)
198 return EmitIgnoredConditionalOperator(CondOp);
199 }
200
201 // Just emit it as an l-value and drop the result.
202 EmitLValue(E);
203}
204
205/// EmitAnyExpr - Emit code to compute the specified expression which
206/// can have any type. The result is returned as an RValue struct.
207/// If this is an aggregate expression, AggSlot indicates where the
208/// result should be returned.
210 AggValueSlot aggSlot,
211 bool ignoreResult) {
212 switch (getEvaluationKind(E->getType())) {
213 case TEK_Scalar:
214 return RValue::get(EmitScalarExpr(E, ignoreResult));
215 case TEK_Complex:
216 return RValue::getComplex(EmitComplexExpr(E, ignoreResult, ignoreResult));
217 case TEK_Aggregate:
218 if (!ignoreResult && aggSlot.isIgnored())
219 aggSlot = CreateAggTemp(E->getType(), "agg-temp");
220 EmitAggExpr(E, aggSlot);
221 return aggSlot.asRValue();
222 }
223 llvm_unreachable("bad evaluation kind");
224}
225
226/// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will
227/// always be accessible even if no aggregate location is provided.
230
232 AggSlot = CreateAggTemp(E->getType(), "agg.tmp");
233 return EmitAnyExpr(E, AggSlot);
234}
235
236/// EmitAnyExprToMem - Evaluate an expression into a given memory
237/// location.
239 Address Location,
240 Qualifiers Quals,
241 bool IsInit) {
242 // FIXME: This function should take an LValue as an argument.
243 switch (getEvaluationKind(E->getType())) {
244 case TEK_Complex:
246 /*isInit*/ false);
247 return;
248
249 case TEK_Aggregate: {
250 EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals,
255 return;
256 }
257
258 case TEK_Scalar: {
259 RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
260 LValue LV = MakeAddrLValue(Location, E->getType());
262 return;
263 }
264 }
265 llvm_unreachable("bad evaluation kind");
266}
267
268static void
270 const Expr *E, Address ReferenceTemporary) {
271 // Objective-C++ ARC:
272 // If we are binding a reference to a temporary that has ownership, we
273 // need to perform retain/release operations on the temporary.
274 //
275 // FIXME: This should be looking at E, not M.
276 if (auto Lifetime = M->getType().getObjCLifetime()) {
277 switch (Lifetime) {
280 // Carry on to normal cleanup handling.
281 break;
282
284 // Nothing to do; cleaned up by an autorelease pool.
285 return;
286
289 switch (StorageDuration Duration = M->getStorageDuration()) {
290 case SD_Static:
291 // Note: we intentionally do not register a cleanup to release
292 // the object on program termination.
293 return;
294
295 case SD_Thread:
296 // FIXME: We should probably register a cleanup in this case.
297 return;
298
299 case SD_Automatic:
303 if (Lifetime == Qualifiers::OCL_Strong) {
304 const ValueDecl *VD = M->getExtendingDecl();
305 bool Precise =
306 VD && isa<VarDecl>(VD) && VD->hasAttr<ObjCPreciseLifetimeAttr>();
310 } else {
311 // __weak objects always get EH cleanups; otherwise, exceptions
312 // could cause really nasty crashes instead of mere leaks.
315 }
316 if (Duration == SD_FullExpression)
317 CGF.pushDestroy(CleanupKind, ReferenceTemporary,
318 M->getType(), *Destroy,
320 else
321 CGF.pushLifetimeExtendedDestroy(CleanupKind, ReferenceTemporary,
322 M->getType(),
323 *Destroy, CleanupKind & EHCleanup);
324 return;
325
326 case SD_Dynamic:
327 llvm_unreachable("temporary cannot have dynamic storage duration");
328 }
329 llvm_unreachable("unknown storage duration");
330 }
331 }
332
333 CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
334 if (const RecordType *RT =
336 // Get the destructor for the reference temporary.
337 auto *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
338 if (!ClassDecl->hasTrivialDestructor())
339 ReferenceTemporaryDtor = ClassDecl->getDestructor();
340 }
341
342 if (!ReferenceTemporaryDtor)
343 return;
344
345 // Call the destructor for the temporary.
346 switch (M->getStorageDuration()) {
347 case SD_Static:
348 case SD_Thread: {
349 llvm::FunctionCallee CleanupFn;
350 llvm::Constant *CleanupArg;
351 if (E->getType()->isArrayType()) {
353 ReferenceTemporary, E->getType(),
355 dyn_cast_or_null<VarDecl>(M->getExtendingDecl()));
356 CleanupArg = llvm::Constant::getNullValue(CGF.Int8PtrTy);
357 } else {
358 CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
359 GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
360 CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer());
361 }
363 CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg);
364 break;
365 }
366
368 CGF.pushDestroy(NormalAndEHCleanup, ReferenceTemporary, E->getType(),
370 CGF.getLangOpts().Exceptions);
371 break;
372
373 case SD_Automatic:
375 ReferenceTemporary, E->getType(),
377 CGF.getLangOpts().Exceptions);
378 break;
379
380 case SD_Dynamic:
381 llvm_unreachable("temporary cannot have dynamic storage duration");
382 }
383}
384
387 const Expr *Inner,
388 Address *Alloca = nullptr) {
389 auto &TCG = CGF.getTargetHooks();
390 switch (M->getStorageDuration()) {
392 case SD_Automatic: {
393 // If we have a constant temporary array or record try to promote it into a
394 // constant global under the same rules a normal constant would've been
395 // promoted. This is easier on the optimizer and generally emits fewer
396 // instructions.
397 QualType Ty = Inner->getType();
398 if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
399 (Ty->isArrayType() || Ty->isRecordType()) &&
400 Ty.isConstantStorage(CGF.getContext(), true, false))
401 if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(Inner, Ty)) {
402 auto AS = CGF.CGM.GetGlobalConstantAddressSpace();
403 auto *GV = new llvm::GlobalVariable(
404 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
405 llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
406 llvm::GlobalValue::NotThreadLocal,
408 CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty);
409 GV->setAlignment(alignment.getAsAlign());
410 llvm::Constant *C = GV;
411 if (AS != LangAS::Default)
412 C = TCG.performAddrSpaceCast(
413 CGF.CGM, GV, AS, LangAS::Default,
414 GV->getValueType()->getPointerTo(
416 // FIXME: Should we put the new global into a COMDAT?
417 return Address(C, GV->getValueType(), alignment);
418 }
419 return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca);
420 }
421 case SD_Thread:
422 case SD_Static:
423 return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner);
424
425 case SD_Dynamic:
426 llvm_unreachable("temporary can't have dynamic storage duration");
427 }
428 llvm_unreachable("unknown storage duration");
429}
430
431/// Helper method to check if the underlying ABI is AAPCS
432static bool isAAPCS(const TargetInfo &TargetInfo) {
433 return TargetInfo.getABI().startswith("aapcs");
434}
435
438 const Expr *E = M->getSubExpr();
439
440 assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||
441 !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&
442 "Reference should never be pseudo-strong!");
443
444 // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
445 // as that will cause the lifetime adjustment to be lost for ARC
446 auto ownership = M->getType().getObjCLifetime();
447 if (ownership != Qualifiers::OCL_None &&
448 ownership != Qualifiers::OCL_ExplicitNone) {
450 if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {
451 llvm::Type *Ty = ConvertTypeForMem(E->getType());
452 Object = Address(llvm::ConstantExpr::getBitCast(
453 Var, Ty->getPointerTo(Object.getAddressSpace())),
454 Ty, Object.getAlignment());
455
456 // createReferenceTemporary will promote the temporary to a global with a
457 // constant initializer if it can. It can only do this to a value of
458 // ARC-manageable type if the value is global and therefore "immune" to
459 // ref-counting operations. Therefore we have no need to emit either a
460 // dynamic initialization or a cleanup and we can just return the address
461 // of the temporary.
462 if (Var->hasInitializer())
463 return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
464
465 Var->setInitializer(CGM.EmitNullConstant(E->getType()));
466 }
467 LValue RefTempDst = MakeAddrLValue(Object, M->getType(),
469
470 switch (getEvaluationKind(E->getType())) {
471 default: llvm_unreachable("expected scalar or aggregate expression");
472 case TEK_Scalar:
473 EmitScalarInit(E, M->getExtendingDecl(), RefTempDst, false);
474 break;
475 case TEK_Aggregate: {
477 E->getType().getQualifiers(),
482 break;
483 }
484 }
485
486 pushTemporaryCleanup(*this, M, E, Object);
487 return RefTempDst;
488 }
489
492 E = E->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments);
493
494 for (const auto &Ignored : CommaLHSs)
495 EmitIgnoredExpr(Ignored);
496
497 if (const auto *opaque = dyn_cast<OpaqueValueExpr>(E)) {
498 if (opaque->getType()->isRecordType()) {
499 assert(Adjustments.empty());
500 return EmitOpaqueValueLValue(opaque);
501 }
502 }
503
504 // Create and initialize the reference temporary.
505 Address Alloca = Address::invalid();
506 Address Object = createReferenceTemporary(*this, M, E, &Alloca);
507 if (auto *Var = dyn_cast<llvm::GlobalVariable>(
508 Object.getPointer()->stripPointerCasts())) {
509 llvm::Type *TemporaryType = ConvertTypeForMem(E->getType());
510 Object = Address(llvm::ConstantExpr::getBitCast(
511 cast<llvm::Constant>(Object.getPointer()),
512 TemporaryType->getPointerTo()),
513 TemporaryType,
514 Object.getAlignment());
515 // If the temporary is a global and has a constant initializer or is a
516 // constant temporary that we promoted to a global, we may have already
517 // initialized it.
518 if (!Var->hasInitializer()) {
519 Var->setInitializer(CGM.EmitNullConstant(E->getType()));
520 EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
521 }
522 } else {
523 switch (M->getStorageDuration()) {
524 case SD_Automatic:
525 if (auto *Size = EmitLifetimeStart(
526 CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
527 Alloca.getPointer())) {
528 pushCleanupAfterFullExpr<CallLifetimeEnd>(NormalEHLifetimeMarker,
529 Alloca, Size);
530 }
531 break;
532
533 case SD_FullExpression: {
534 if (!ShouldEmitLifetimeMarkers)
535 break;
536
537 // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end
538 // marker. Instead, start the lifetime of a conditional temporary earlier
539 // so that it's unconditional. Don't do this with sanitizers which need
540 // more precise lifetime marks. However when inside an "await.suspend"
541 // block, we should always avoid conditional cleanup because it creates
542 // boolean marker that lives across await_suspend, which can destroy coro
543 // frame.
544 ConditionalEvaluation *OldConditional = nullptr;
545 CGBuilderTy::InsertPoint OldIP;
547 ((!SanOpts.has(SanitizerKind::HWAddress) &&
548 !SanOpts.has(SanitizerKind::Memory) &&
549 !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) ||
550 inSuspendBlock())) {
551 OldConditional = OutermostConditional;
552 OutermostConditional = nullptr;
553
554 OldIP = Builder.saveIP();
555 llvm::BasicBlock *Block = OldConditional->getStartingBlock();
556 Builder.restoreIP(CGBuilderTy::InsertPoint(
557 Block, llvm::BasicBlock::iterator(Block->back())));
558 }
559
560 if (auto *Size = EmitLifetimeStart(
561 CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
562 Alloca.getPointer())) {
563 pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, Alloca,
564 Size);
565 }
566
567 if (OldConditional) {
568 OutermostConditional = OldConditional;
569 Builder.restoreIP(OldIP);
570 }
571 break;
572 }
573
574 default:
575 break;
576 }
577 EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
578 }
579 pushTemporaryCleanup(*this, M, E, Object);
580
581 // Perform derived-to-base casts and/or field accesses, to get from the
582 // temporary object we created (and, potentially, for which we extended
583 // the lifetime) to the subobject we're binding the reference to.
584 for (SubobjectAdjustment &Adjustment : llvm::reverse(Adjustments)) {
585 switch (Adjustment.Kind) {
587 Object =
588 GetAddressOfBaseClass(Object, Adjustment.DerivedToBase.DerivedClass,
589 Adjustment.DerivedToBase.BasePath->path_begin(),
590 Adjustment.DerivedToBase.BasePath->path_end(),
591 /*NullCheckValue=*/ false, E->getExprLoc());
592 break;
593
596 LV = EmitLValueForField(LV, Adjustment.Field);
597 assert(LV.isSimple() &&
598 "materialized temporary field is not a simple lvalue");
599 Object = LV.getAddress(*this);
600 break;
601 }
602
604 llvm::Value *Ptr = EmitScalarExpr(Adjustment.Ptr.RHS);
606 Adjustment.Ptr.MPT);
607 break;
608 }
609 }
610 }
611
612 return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
613}
614
615RValue
617 // Emit the expression as an lvalue.
618 LValue LV = EmitLValue(E);
619 assert(LV.isSimple());
620 llvm::Value *Value = LV.getPointer(*this);
621
623 // C++11 [dcl.ref]p5 (as amended by core issue 453):
624 // If a glvalue to which a reference is directly bound designates neither
625 // an existing object or function of an appropriate type nor a region of
626 // storage of suitable size and alignment to contain an object of the
627 // reference's type, the behavior is undefined.
628 QualType Ty = E->getType();
630 }
631
632 return RValue::get(Value);
633}
634
635
636/// getAccessedFieldNo - Given an encoded value and a result number, return the
637/// input field number being accessed.
638unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
639 const llvm::Constant *Elts) {
640 return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx))
641 ->getZExtValue();
642}
643
644/// Emit the hash_16_bytes function from include/llvm/ADT/Hashing.h.
645static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low,
646 llvm::Value *High) {
647 llvm::Value *KMul = Builder.getInt64(0x9ddfea08eb382d69ULL);
648 llvm::Value *K47 = Builder.getInt64(47);
649 llvm::Value *A0 = Builder.CreateMul(Builder.CreateXor(Low, High), KMul);
650 llvm::Value *A1 = Builder.CreateXor(Builder.CreateLShr(A0, K47), A0);
651 llvm::Value *B0 = Builder.CreateMul(Builder.CreateXor(High, A1), KMul);
652 llvm::Value *B1 = Builder.CreateXor(Builder.CreateLShr(B0, K47), B0);
653 return Builder.CreateMul(B1, KMul);
654}
655
656bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
657 return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
659}
660
661bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
663 return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
664 (TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
667}
668
670 return SanOpts.has(SanitizerKind::Null) ||
671 SanOpts.has(SanitizerKind::Alignment) ||
672 SanOpts.has(SanitizerKind::ObjectSize) ||
673 SanOpts.has(SanitizerKind::Vptr);
674}
675
676void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
677 llvm::Value *Ptr, QualType Ty,
678 CharUnits Alignment,
679 SanitizerSet SkippedChecks,
680 llvm::Value *ArraySize) {
682 return;
683
684 // Don't check pointers outside the default address space. The null check
685 // isn't correct, the object-size check isn't supported by LLVM, and we can't
686 // communicate the addresses to the runtime handler for the vptr check.
687 if (Ptr->getType()->getPointerAddressSpace())
688 return;
689
690 // Don't check pointers to volatile data. The behavior here is implementation-
691 // defined.
692 if (Ty.isVolatileQualified())
693 return;
694
695 SanitizerScope SanScope(this);
696
698 llvm::BasicBlock *Done = nullptr;
699
700 // Quickly determine whether we have a pointer to an alloca. It's possible
701 // to skip null checks, and some alignment checks, for these pointers. This
702 // can reduce compile-time significantly.
703 auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts());
704
705 llvm::Value *True = llvm::ConstantInt::getTrue(getLLVMContext());
706 llvm::Value *IsNonNull = nullptr;
707 bool IsGuaranteedNonNull =
708 SkippedChecks.has(SanitizerKind::Null) || PtrToAlloca;
709 bool AllowNullPointers = isNullPointerAllowed(TCK);
710 if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) &&
711 !IsGuaranteedNonNull) {
712 // The glvalue must not be an empty glvalue.
713 IsNonNull = Builder.CreateIsNotNull(Ptr);
714
715 // The IR builder can constant-fold the null check if the pointer points to
716 // a constant.
717 IsGuaranteedNonNull = IsNonNull == True;
718
719 // Skip the null check if the pointer is known to be non-null.
720 if (!IsGuaranteedNonNull) {
721 if (AllowNullPointers) {
722 // When performing pointer casts, it's OK if the value is null.
723 // Skip the remaining checks in that case.
724 Done = createBasicBlock("null");
725 llvm::BasicBlock *Rest = createBasicBlock("not.null");
726 Builder.CreateCondBr(IsNonNull, Rest, Done);
727 EmitBlock(Rest);
728 } else {
729 Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::Null));
730 }
731 }
732 }
733
734 if (SanOpts.has(SanitizerKind::ObjectSize) &&
735 !SkippedChecks.has(SanitizerKind::ObjectSize) &&
736 !Ty->isIncompleteType()) {
738 llvm::Value *Size = llvm::ConstantInt::get(IntPtrTy, TySize);
739 if (ArraySize)
740 Size = Builder.CreateMul(Size, ArraySize);
741
742 // Degenerate case: new X[0] does not need an objectsize check.
743 llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size);
744 if (!ConstantSize || !ConstantSize->isNullValue()) {
745 // The glvalue must refer to a large enough storage region.
746 // FIXME: If Address Sanitizer is enabled, insert dynamic instrumentation
747 // to check this.
748 // FIXME: Get object address space
749 llvm::Type *Tys[2] = { IntPtrTy, Int8PtrTy };
750 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys);
751 llvm::Value *Min = Builder.getFalse();
752 llvm::Value *NullIsUnknown = Builder.getFalse();
753 llvm::Value *Dynamic = Builder.getFalse();
754 llvm::Value *LargeEnough = Builder.CreateICmpUGE(
755 Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic}), Size);
756 Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize));
757 }
758 }
759
760 llvm::MaybeAlign AlignVal;
761 llvm::Value *PtrAsInt = nullptr;
762
763 if (SanOpts.has(SanitizerKind::Alignment) &&
764 !SkippedChecks.has(SanitizerKind::Alignment)) {
765 AlignVal = Alignment.getAsMaybeAlign();
766 if (!Ty->isIncompleteType() && !AlignVal)
767 AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr,
768 /*ForPointeeType=*/true)
770
771 // The glvalue must be suitably aligned.
772 if (AlignVal && *AlignVal > llvm::Align(1) &&
773 (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
774 PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy);
775 llvm::Value *Align = Builder.CreateAnd(
776 PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal->value() - 1));
777 llvm::Value *Aligned =
778 Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));
779 if (Aligned != True)
780 Checks.push_back(std::make_pair(Aligned, SanitizerKind::Alignment));
781 }
782 }
783
784 if (Checks.size() > 0) {
785 llvm::Constant *StaticData[] = {
787 llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1),
788 llvm::ConstantInt::get(Int8Ty, TCK)};
789 EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData,
790 PtrAsInt ? PtrAsInt : Ptr);
791 }
792
793 // If possible, check that the vptr indicates that there is a subobject of
794 // type Ty at offset zero within this object.
795 //
796 // C++11 [basic.life]p5,6:
797 // [For storage which does not refer to an object within its lifetime]
798 // The program has undefined behavior if:
799 // -- the [pointer or glvalue] is used to access a non-static data member
800 // or call a non-static member function
801 if (SanOpts.has(SanitizerKind::Vptr) &&
802 !SkippedChecks.has(SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
803 // Ensure that the pointer is non-null before loading it. If there is no
804 // compile-time guarantee, reuse the run-time null check or emit a new one.
805 if (!IsGuaranteedNonNull) {
806 if (!IsNonNull)
807 IsNonNull = Builder.CreateIsNotNull(Ptr);
808 if (!Done)
809 Done = createBasicBlock("vptr.null");
810 llvm::BasicBlock *VptrNotNull = createBasicBlock("vptr.not.null");
811 Builder.CreateCondBr(IsNonNull, VptrNotNull, Done);
812 EmitBlock(VptrNotNull);
813 }
814
815 // Compute a hash of the mangled name of the type.
816 //
817 // FIXME: This is not guaranteed to be deterministic! Move to a
818 // fingerprinting mechanism once LLVM provides one. For the time
819 // being the implementation happens to be deterministic.
820 SmallString<64> MangledName;
821 llvm::raw_svector_ostream Out(MangledName);
823 Out);
824
825 // Contained in NoSanitizeList based on the mangled type.
826 if (!CGM.getContext().getNoSanitizeList().containsType(SanitizerKind::Vptr,
827 Out.str())) {
828 llvm::hash_code TypeHash = hash_value(Out.str());
829
830 // Load the vptr, and compute hash_16_bytes(TypeHash, vptr).
831 llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash);
832 Address VPtrAddr(Ptr, IntPtrTy, getPointerAlign());
833 llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr);
834 llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty);
835
836 llvm::Value *Hash = emitHash16Bytes(Builder, Low, High);
837 Hash = Builder.CreateTrunc(Hash, IntPtrTy);
838
839 // Look the hash up in our cache.
840 const int CacheSize = 128;
841 llvm::Type *HashTable = llvm::ArrayType::get(IntPtrTy, CacheSize);
842 llvm::Value *Cache = CGM.CreateRuntimeVariable(HashTable,
843 "__ubsan_vptr_type_cache");
844 llvm::Value *Slot = Builder.CreateAnd(Hash,
845 llvm::ConstantInt::get(IntPtrTy,
846 CacheSize-1));
847 llvm::Value *Indices[] = { Builder.getInt32(0), Slot };
848 llvm::Value *CacheVal = Builder.CreateAlignedLoad(
849 IntPtrTy, Builder.CreateInBoundsGEP(HashTable, Cache, Indices),
851
852 // If the hash isn't in the cache, call a runtime handler to perform the
853 // hard work of checking whether the vptr is for an object of the right
854 // type. This will either fill in the cache and return, or produce a
855 // diagnostic.
856 llvm::Value *EqualHash = Builder.CreateICmpEQ(CacheVal, Hash);
857 llvm::Constant *StaticData[] = {
861 llvm::ConstantInt::get(Int8Ty, TCK)
862 };
863 llvm::Value *DynamicData[] = { Ptr, Hash };
864 EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr),
865 SanitizerHandler::DynamicTypeCacheMiss, StaticData,
866 DynamicData);
867 }
868 }
869
870 if (Done) {
871 Builder.CreateBr(Done);
872 EmitBlock(Done);
873 }
874}
875
876llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,
877 QualType EltTy) {
879 uint64_t EltSize = C.getTypeSizeInChars(EltTy).getQuantity();
880 if (!EltSize)
881 return nullptr;
882
883 auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts());
884 if (!ArrayDeclRef)
885 return nullptr;
886
887 auto *ParamDecl = dyn_cast<ParmVarDecl>(ArrayDeclRef->getDecl());
888 if (!ParamDecl)
889 return nullptr;
890
891 auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
892 if (!POSAttr)
893 return nullptr;
894
895 // Don't load the size if it's a lower bound.
896 int POSType = POSAttr->getType();
897 if (POSType != 0 && POSType != 1)
898 return nullptr;
899
900 // Find the implicit size parameter.
901 auto PassedSizeIt = SizeArguments.find(ParamDecl);
902 if (PassedSizeIt == SizeArguments.end())
903 return nullptr;
904
905 const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;
906 assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");
907 Address AddrOfSize = LocalDeclMap.find(PassedSizeDecl)->second;
908 llvm::Value *SizeInBytes = EmitLoadOfScalar(AddrOfSize, /*Volatile=*/false,
909 C.getSizeType(), E->getExprLoc());
910 llvm::Value *SizeOfElement =
911 llvm::ConstantInt::get(SizeInBytes->getType(), EltSize);
912 return Builder.CreateUDiv(SizeInBytes, SizeOfElement);
913}
914
915/// If Base is known to point to the start of an array, return the length of
916/// that array. Return 0 if the length cannot be determined.
917static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
918 const Expr *Base,
919 QualType &IndexedType,
921 StrictFlexArraysLevel) {
922 // For the vector indexing extension, the bound is the number of elements.
923 if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {
924 IndexedType = Base->getType();
925 return CGF.Builder.getInt32(VT->getNumElements());
926 }
927
928 Base = Base->IgnoreParens();
929
930 if (const auto *CE = dyn_cast<CastExpr>(Base)) {
931 if (CE->getCastKind() == CK_ArrayToPointerDecay &&
932 !CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.getContext(),
933 StrictFlexArraysLevel)) {
934 IndexedType = CE->getSubExpr()->getType();
935 const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
936 if (const auto *CAT = dyn_cast<ConstantArrayType>(AT))
937 return CGF.Builder.getInt(CAT->getSize());
938 else if (const auto *VAT = dyn_cast<VariableArrayType>(AT))
939 return CGF.getVLASize(VAT).NumElts;
940 // Ignore pass_object_size here. It's not applicable on decayed pointers.
941 }
942 }
943
944 QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
945 if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) {
946 IndexedType = Base->getType();
947 return POS;
948 }
949
950 return nullptr;
951}
952
953void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base,
954 llvm::Value *Index, QualType IndexType,
955 bool Accessed) {
956 assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
957 "should not be called unless adding bounds checks");
958 SanitizerScope SanScope(this);
959
960 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
961 getLangOpts().getStrictFlexArraysLevel();
962
963 QualType IndexedType;
964 llvm::Value *Bound =
965 getArrayIndexingBound(*this, Base, IndexedType, StrictFlexArraysLevel);
966 if (!Bound)
967 return;
968
969 bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
970 llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned);
971 llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false);
972
973 llvm::Constant *StaticData[] = {
975 EmitCheckTypeDescriptor(IndexedType),
976 EmitCheckTypeDescriptor(IndexType)
977 };
978 llvm::Value *Check = Accessed ? Builder.CreateICmpULT(IndexVal, BoundVal)
979 : Builder.CreateICmpULE(IndexVal, BoundVal);
980 EmitCheck(std::make_pair(Check, SanitizerKind::ArrayBounds),
981 SanitizerHandler::OutOfBounds, StaticData, Index);
982}
983
984
987 bool isInc, bool isPre) {
989
990 llvm::Value *NextVal;
991 if (isa<llvm::IntegerType>(InVal.first->getType())) {
992 uint64_t AmountVal = isInc ? 1 : -1;
993 NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true);
994
995 // Add the inc/dec to the real part.
996 NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
997 } else {
998 QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
999 llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1);
1000 if (!isInc)
1001 FVal.changeSign();
1002 NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal);
1003
1004 // Add the inc/dec to the real part.
1005 NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
1006 }
1007
1008 ComplexPairTy IncVal(NextVal, InVal.second);
1009
1010 // Store the updated result through the lvalue.
1011 EmitStoreOfComplex(IncVal, LV, /*init*/ false);
1012 if (getLangOpts().OpenMP)
1014 E->getSubExpr());
1015
1016 // If this is a postinc, return the value read from memory, otherwise use the
1017 // updated value.
1018 return isPre ? IncVal : InVal;
1019}
1020
1022 CodeGenFunction *CGF) {
1023 // Bind VLAs in the cast type.
1024 if (CGF && E->getType()->isVariablyModifiedType())
1026
1027 if (CGDebugInfo *DI = getModuleDebugInfo())
1028 DI->EmitExplicitCastType(E->getType());
1029}
1030
1031//===----------------------------------------------------------------------===//
1032// LValue Expression Emission
1033//===----------------------------------------------------------------------===//
1034
1036 TBAAAccessInfo *TBAAInfo,
1037 KnownNonNull_t IsKnownNonNull,
1038 CodeGenFunction &CGF) {
1039 // We allow this with ObjC object pointers because of fragile ABIs.
1040 assert(E->getType()->isPointerType() ||
1042 E = E->IgnoreParens();
1043
1044 // Casts:
1045 if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
1046 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(CE))
1047 CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);
1048
1049 switch (CE->getCastKind()) {
1050 // Non-converting casts (but not C's implicit conversion from void*).
1051 case CK_BitCast:
1052 case CK_NoOp:
1053 case CK_AddressSpaceConversion:
1054 if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
1055 if (PtrTy->getPointeeType()->isVoidType())
1056 break;
1057
1058 LValueBaseInfo InnerBaseInfo;
1059 TBAAAccessInfo InnerTBAAInfo;
1061 CE->getSubExpr(), &InnerBaseInfo, &InnerTBAAInfo, IsKnownNonNull);
1062 if (BaseInfo) *BaseInfo = InnerBaseInfo;
1063 if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1064
1065 if (isa<ExplicitCastExpr>(CE)) {
1066 LValueBaseInfo TargetTypeBaseInfo;
1067 TBAAAccessInfo TargetTypeTBAAInfo;
1069 E->getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo);
1070 if (TBAAInfo)
1071 *TBAAInfo =
1072 CGF.CGM.mergeTBAAInfoForCast(*TBAAInfo, TargetTypeTBAAInfo);
1073 // If the source l-value is opaque, honor the alignment of the
1074 // casted-to type.
1075 if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
1076 if (BaseInfo)
1077 BaseInfo->mergeForCast(TargetTypeBaseInfo);
1078 Addr = Address(Addr.getPointer(), Addr.getElementType(), Align,
1079 IsKnownNonNull);
1080 }
1081 }
1082
1083 if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast) &&
1084 CE->getCastKind() == CK_BitCast) {
1085 if (auto PT = E->getType()->getAs<PointerType>())
1086 CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr,
1087 /*MayBeNull=*/true,
1089 CE->getBeginLoc());
1090 }
1091
1092 llvm::Type *ElemTy =
1094 Addr = Addr.withElementType(ElemTy);
1095 if (CE->getCastKind() == CK_AddressSpaceConversion)
1096 Addr = CGF.Builder.CreateAddrSpaceCast(Addr,
1097 CGF.ConvertType(E->getType()));
1098 return Addr;
1099 }
1100 break;
1101
1102 // Array-to-pointer decay.
1103 case CK_ArrayToPointerDecay:
1104 return CGF.EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo);
1105
1106 // Derived-to-base conversions.
1107 case CK_UncheckedDerivedToBase:
1108 case CK_DerivedToBase: {
1109 // TODO: Support accesses to members of base classes in TBAA. For now, we
1110 // conservatively pretend that the complete object is of the base class
1111 // type.
1112 if (TBAAInfo)
1113 *TBAAInfo = CGF.CGM.getTBAAAccessInfo(E->getType());
1115 CE->getSubExpr(), BaseInfo, nullptr,
1116 (KnownNonNull_t)(IsKnownNonNull ||
1117 CE->getCastKind() == CK_UncheckedDerivedToBase));
1118 auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
1119 return CGF.GetAddressOfBaseClass(
1120 Addr, Derived, CE->path_begin(), CE->path_end(),
1121 CGF.ShouldNullCheckClassCastValue(CE), CE->getExprLoc());
1122 }
1123
1124 // TODO: Is there any reason to treat base-to-derived conversions
1125 // specially?
1126 default:
1127 break;
1128 }
1129 }
1130
1131 // Unary &.
1132 if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) {
1133 if (UO->getOpcode() == UO_AddrOf) {
1134 LValue LV = CGF.EmitLValue(UO->getSubExpr(), IsKnownNonNull);
1135 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1136 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1137 return LV.getAddress(CGF);
1138 }
1139 }
1140
1141 // std::addressof and variants.
1142 if (auto *Call = dyn_cast<CallExpr>(E)) {
1143 switch (Call->getBuiltinCallee()) {
1144 default:
1145 break;
1146 case Builtin::BIaddressof:
1147 case Builtin::BI__addressof:
1148 case Builtin::BI__builtin_addressof: {
1149 LValue LV = CGF.EmitLValue(Call->getArg(0), IsKnownNonNull);
1150 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1151 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1152 return LV.getAddress(CGF);
1153 }
1154 }
1155 }
1156
1157 // TODO: conditional operators, comma.
1158
1159 // Otherwise, use the alignment of the type.
1160 CharUnits Align =
1161 CGF.CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo);
1162 llvm::Type *ElemTy = CGF.ConvertTypeForMem(E->getType()->getPointeeType());
1163 return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull);
1164}
1165
1166/// EmitPointerWithAlignment - Given an expression of pointer type, try to
1167/// derive a more accurate bound on the alignment of the pointer.
1169 const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo,
1170 KnownNonNull_t IsKnownNonNull) {
1171 Address Addr =
1172 ::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, *this);
1173 if (IsKnownNonNull && !Addr.isKnownNonNull())
1174 Addr.setKnownNonNull();
1175 return Addr;
1176}
1177
1179 llvm::Value *V = RV.getScalarVal();
1180 if (auto MPT = T->getAs<MemberPointerType>())
1181 return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, V, MPT);
1182 return Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType()));
1183}
1184
1186 if (Ty->isVoidType())
1187 return RValue::get(nullptr);
1188
1189 switch (getEvaluationKind(Ty)) {
1190 case TEK_Complex: {
1191 llvm::Type *EltTy =
1193 llvm::Value *U = llvm::UndefValue::get(EltTy);
1194 return RValue::getComplex(std::make_pair(U, U));
1195 }
1196
1197 // If this is a use of an undefined aggregate type, the aggregate must have an
1198 // identifiable address. Just because the contents of the value are undefined
1199 // doesn't mean that the address can't be taken and compared.
1200 case TEK_Aggregate: {
1201 Address DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
1202 return RValue::getAggregate(DestPtr);
1203 }
1204
1205 case TEK_Scalar:
1206 return RValue::get(llvm::UndefValue::get(ConvertType(Ty)));
1207 }
1208 llvm_unreachable("bad evaluation kind");
1209}
1210
1212 const char *Name) {
1213 ErrorUnsupported(E, Name);
1214 return GetUndefRValue(E->getType());
1215}
1216
1218 const char *Name) {
1219 ErrorUnsupported(E, Name);
1220 llvm::Type *ElTy = ConvertType(E->getType());
1221 llvm::Type *Ty = UnqualPtrTy;
1222 return MakeAddrLValue(
1223 Address(llvm::UndefValue::get(Ty), ElTy, CharUnits::One()), E->getType());
1224}
1225
1226bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
1227 const Expr *Base = Obj;
1228 while (!isa<CXXThisExpr>(Base)) {
1229 // The result of a dynamic_cast can be null.
1230 if (isa<CXXDynamicCastExpr>(Base))
1231 return false;
1232
1233 if (const auto *CE = dyn_cast<CastExpr>(Base)) {
1234 Base = CE->getSubExpr();
1235 } else if (const auto *PE = dyn_cast<ParenExpr>(Base)) {
1236 Base = PE->getSubExpr();
1237 } else if (const auto *UO = dyn_cast<UnaryOperator>(Base)) {
1238 if (UO->getOpcode() == UO_Extension)
1239 Base = UO->getSubExpr();
1240 else
1241 return false;
1242 } else {
1243 return false;
1244 }
1245 }
1246 return true;
1247}
1248
1249LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
1250 LValue LV;
1251 if (SanOpts.has(SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(E))
1252 LV = EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E), /*Accessed*/true);
1253 else
1254 LV = EmitLValue(E);
1255 if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple()) {
1256 SanitizerSet SkippedChecks;
1257 if (const auto *ME = dyn_cast<MemberExpr>(E)) {
1258 bool IsBaseCXXThis = IsWrappedCXXThis(ME->getBase());
1259 if (IsBaseCXXThis)
1260 SkippedChecks.set(SanitizerKind::Alignment, true);
1261 if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase()))
1262 SkippedChecks.set(SanitizerKind::Null, true);
1263 }
1264 EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(),
1265 LV.getAlignment(), SkippedChecks);
1266 }
1267 return LV;
1268}
1269
1270/// EmitLValue - Emit code to compute a designator that specifies the location
1271/// of the expression.
1272///
1273/// This can return one of two things: a simple address or a bitfield reference.
1274/// In either case, the LLVM Value* in the LValue structure is guaranteed to be
1275/// an LLVM pointer type.
1276///
1277/// If this returns a bitfield reference, nothing about the pointee type of the
1278/// LLVM value is known: For example, it may not be a pointer to an integer.
1279///
1280/// If this returns a normal address, and if the lvalue's C type is fixed size,
1281/// this method guarantees that the returned pointer type will point to an LLVM
1282/// type of the same size of the lvalue's type. If the lvalue has a variable
1283/// length type, this is not possible.
1284///
1286 KnownNonNull_t IsKnownNonNull) {
1287 LValue LV = EmitLValueHelper(E, IsKnownNonNull);
1288 if (IsKnownNonNull && !LV.isKnownNonNull())
1289 LV.setKnownNonNull();
1290 return LV;
1291}
1292
1293LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
1294 KnownNonNull_t IsKnownNonNull) {
1295 ApplyDebugLocation DL(*this, E);
1296 switch (E->getStmtClass()) {
1297 default: return EmitUnsupportedLValue(E, "l-value expression");
1298
1299 case Expr::ObjCPropertyRefExprClass:
1300 llvm_unreachable("cannot emit a property reference directly");
1301
1302 case Expr::ObjCSelectorExprClass:
1303 return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));
1304 case Expr::ObjCIsaExprClass:
1305 return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));
1306 case Expr::BinaryOperatorClass:
1307 return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));
1308 case Expr::CompoundAssignOperatorClass: {
1309 QualType Ty = E->getType();
1310 if (const AtomicType *AT = Ty->getAs<AtomicType>())
1311 Ty = AT->getValueType();
1312 if (!Ty->isAnyComplexType())
1313 return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
1314 return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
1315 }
1316 case Expr::CallExprClass:
1317 case Expr::CXXMemberCallExprClass:
1318 case Expr::CXXOperatorCallExprClass:
1319 case Expr::UserDefinedLiteralClass:
1320 return EmitCallExprLValue(cast<CallExpr>(E));
1321 case Expr::CXXRewrittenBinaryOperatorClass:
1322 return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm(),
1323 IsKnownNonNull);
1324 case Expr::VAArgExprClass:
1325 return EmitVAArgExprLValue(cast<VAArgExpr>(E));
1326 case Expr::DeclRefExprClass:
1327 return EmitDeclRefLValue(cast<DeclRefExpr>(E));
1328 case Expr::ConstantExprClass: {
1329 const ConstantExpr *CE = cast<ConstantExpr>(E);
1330 if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) {
1331 QualType RetType = cast<CallExpr>(CE->getSubExpr()->IgnoreImplicit())
1332 ->getCallReturnType(getContext())
1333 ->getPointeeType();
1334 return MakeNaturalAlignAddrLValue(Result, RetType);
1335 }
1336 return EmitLValue(cast<ConstantExpr>(E)->getSubExpr(), IsKnownNonNull);
1337 }
1338 case Expr::ParenExprClass:
1339 return EmitLValue(cast<ParenExpr>(E)->getSubExpr(), IsKnownNonNull);
1340 case Expr::GenericSelectionExprClass:
1341 return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr(),
1342 IsKnownNonNull);
1343 case Expr::PredefinedExprClass:
1344 return EmitPredefinedLValue(cast<PredefinedExpr>(E));
1345 case Expr::StringLiteralClass:
1346 return EmitStringLiteralLValue(cast<StringLiteral>(E));
1347 case Expr::ObjCEncodeExprClass:
1348 return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E));
1349 case Expr::PseudoObjectExprClass:
1350 return EmitPseudoObjectLValue(cast<PseudoObjectExpr>(E));
1351 case Expr::InitListExprClass:
1352 return EmitInitListLValue(cast<InitListExpr>(E));
1353 case Expr::CXXTemporaryObjectExprClass:
1354 case Expr::CXXConstructExprClass:
1355 return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));
1356 case Expr::CXXBindTemporaryExprClass:
1357 return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
1358 case Expr::CXXUuidofExprClass:
1359 return EmitCXXUuidofLValue(cast<CXXUuidofExpr>(E));
1360 case Expr::LambdaExprClass:
1361 return EmitAggExprToLValue(E);
1362
1363 case Expr::ExprWithCleanupsClass: {
1364 const auto *cleanups = cast<ExprWithCleanups>(E);
1365 RunCleanupsScope Scope(*this);
1366 LValue LV = EmitLValue(cleanups->getSubExpr(), IsKnownNonNull);
1367 if (LV.isSimple()) {
1368 // Defend against branches out of gnu statement expressions surrounded by
1369 // cleanups.
1370 Address Addr = LV.getAddress(*this);
1371 llvm::Value *V = Addr.getPointer();
1372 Scope.ForceCleanup({&V});
1373 return LValue::MakeAddr(Addr.withPointer(V, Addr.isKnownNonNull()),
1374 LV.getType(), getContext(), LV.getBaseInfo(),
1375 LV.getTBAAInfo());
1376 }
1377 // FIXME: Is it possible to create an ExprWithCleanups that produces a
1378 // bitfield lvalue or some other non-simple lvalue?
1379 return LV;
1380 }
1381
1382 case Expr::CXXDefaultArgExprClass: {
1383 auto *DAE = cast<CXXDefaultArgExpr>(E);
1384 CXXDefaultArgExprScope Scope(*this, DAE);
1385 return EmitLValue(DAE->getExpr(), IsKnownNonNull);
1386 }
1387 case Expr::CXXDefaultInitExprClass: {
1388 auto *DIE = cast<CXXDefaultInitExpr>(E);
1389 CXXDefaultInitExprScope Scope(*this, DIE);
1390 return EmitLValue(DIE->getExpr(), IsKnownNonNull);
1391 }
1392 case Expr::CXXTypeidExprClass:
1393 return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E));
1394
1395 case Expr::ObjCMessageExprClass:
1396 return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E));
1397 case Expr::ObjCIvarRefExprClass:
1398 return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E));
1399 case Expr::StmtExprClass:
1400 return EmitStmtExprLValue(cast<StmtExpr>(E));
1401 case Expr::UnaryOperatorClass:
1402 return EmitUnaryOpLValue(cast<UnaryOperator>(E));
1403 case Expr::ArraySubscriptExprClass:
1404 return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E));
1405 case Expr::MatrixSubscriptExprClass:
1406 return EmitMatrixSubscriptExpr(cast<MatrixSubscriptExpr>(E));
1407 case Expr::OMPArraySectionExprClass:
1408 return EmitOMPArraySectionExpr(cast<OMPArraySectionExpr>(E));
1409 case Expr::ExtVectorElementExprClass:
1410 return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E));
1411 case Expr::CXXThisExprClass:
1413 case Expr::MemberExprClass:
1414 return EmitMemberExpr(cast<MemberExpr>(E));
1415 case Expr::CompoundLiteralExprClass:
1416 return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E));
1417 case Expr::ConditionalOperatorClass:
1418 return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E));
1419 case Expr::BinaryConditionalOperatorClass:
1420 return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E));
1421 case Expr::ChooseExprClass:
1422 return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr(), IsKnownNonNull);
1423 case Expr::OpaqueValueExprClass:
1424 return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E));
1425 case Expr::SubstNonTypeTemplateParmExprClass:
1426 return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement(),
1427 IsKnownNonNull);
1428 case Expr::ImplicitCastExprClass:
1429 case Expr::CStyleCastExprClass:
1430 case Expr::CXXFunctionalCastExprClass:
1431 case Expr::CXXStaticCastExprClass:
1432 case Expr::CXXDynamicCastExprClass:
1433 case Expr::CXXReinterpretCastExprClass:
1434 case Expr::CXXConstCastExprClass:
1435 case Expr::CXXAddrspaceCastExprClass:
1436 case Expr::ObjCBridgedCastExprClass:
1437 return EmitCastLValue(cast<CastExpr>(E));
1438
1439 case Expr::MaterializeTemporaryExprClass:
1440 return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E));
1441
1442 case Expr::CoawaitExprClass:
1443 return EmitCoawaitLValue(cast<CoawaitExpr>(E));
1444 case Expr::CoyieldExprClass:
1445 return EmitCoyieldLValue(cast<CoyieldExpr>(E));
1446 }
1447}
1448
1449/// Given an object of the given canonical type, can we safely copy a
1450/// value out of it based on its initializer?
1452 assert(type.isCanonical());
1453 assert(!type->isReferenceType());
1454
1455 // Must be const-qualified but non-volatile.
1456 Qualifiers qs = type.getLocalQualifiers();
1457 if (!qs.hasConst() || qs.hasVolatile()) return false;
1458
1459 // Otherwise, all object types satisfy this except C++ classes with
1460 // mutable subobjects or non-trivial copy/destroy behavior.
1461 if (const auto *RT = dyn_cast<RecordType>(type))
1462 if (const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1463 if (RD->hasMutableFields() || !RD->isTrivial())
1464 return false;
1465
1466 return true;
1467}
1468
1469/// Can we constant-emit a load of a reference to a variable of the
1470/// given type? This is different from predicates like
1471/// Decl::mightBeUsableInConstantExpressions because we do want it to apply
1472/// in situations that don't necessarily satisfy the language's rules
1473/// for this (e.g. C++'s ODR-use rules). For example, we want to able
1474/// to do this with const float variables even if those variables
1475/// aren't marked 'constexpr'.
1483 type = type.getCanonicalType();
1484 if (const auto *ref = dyn_cast<ReferenceType>(type)) {
1485 if (isConstantEmittableObjectType(ref->getPointeeType()))
1487 return CEK_AsReferenceOnly;
1488 }
1490 return CEK_AsValueOnly;
1491 return CEK_None;
1492}
1493
1494/// Try to emit a reference to the given value without producing it as
1495/// an l-value. This is just an optimization, but it avoids us needing
1496/// to emit global copies of variables if they're named without triggering
1497/// a formal use in a context where we can't emit a direct reference to them,
1498/// for instance if a block or lambda or a member of a local class uses a
1499/// const int variable or constexpr variable from an enclosing function.
1500CodeGenFunction::ConstantEmission
1502 ValueDecl *value = refExpr->getDecl();
1503
1504 // The value needs to be an enum constant or a constant variable.
1506 if (isa<ParmVarDecl>(value)) {
1507 CEK = CEK_None;
1508 } else if (auto *var = dyn_cast<VarDecl>(value)) {
1509 CEK = checkVarTypeForConstantEmission(var->getType());
1510 } else if (isa<EnumConstantDecl>(value)) {
1511 CEK = CEK_AsValueOnly;
1512 } else {
1513 CEK = CEK_None;
1514 }
1515 if (CEK == CEK_None) return ConstantEmission();
1516
1517 Expr::EvalResult result;
1518 bool resultIsReference;
1519 QualType resultType;
1520
1521 // It's best to evaluate all the way as an r-value if that's permitted.
1522 if (CEK != CEK_AsReferenceOnly &&
1523 refExpr->EvaluateAsRValue(result, getContext())) {
1524 resultIsReference = false;
1525 resultType = refExpr->getType();
1526
1527 // Otherwise, try to evaluate as an l-value.
1528 } else if (CEK != CEK_AsValueOnly &&
1529 refExpr->EvaluateAsLValue(result, getContext())) {
1530 resultIsReference = true;
1531 resultType = value->getType();
1532
1533 // Failure.
1534 } else {
1535 return ConstantEmission();
1536 }
1537
1538 // In any case, if the initializer has side-effects, abandon ship.
1539 if (result.HasSideEffects)
1540 return ConstantEmission();
1541
1542 // In CUDA/HIP device compilation, a lambda may capture a reference variable
1543 // referencing a global host variable by copy. In this case the lambda should
1544 // make a copy of the value of the global host variable. The DRE of the
1545 // captured reference variable cannot be emitted as load from the host
1546 // global variable as compile time constant, since the host variable is not
1547 // accessible on device. The DRE of the captured reference variable has to be
1548 // loaded from captures.
1549 if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
1551 auto *MD = dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl);
1552 if (MD && MD->getParent()->isLambda() &&
1553 MD->getOverloadedOperator() == OO_Call) {
1554 const APValue::LValueBase &base = result.Val.getLValueBase();
1555 if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
1556 if (const VarDecl *VD = dyn_cast<const VarDecl>(D)) {
1557 if (!VD->hasAttr<CUDADeviceAttr>()) {
1558 return ConstantEmission();
1559 }
1560 }
1561 }
1562 }
1563 }
1564
1565 // Emit as a constant.
1566 auto C = ConstantEmitter(*this).emitAbstract(refExpr->getLocation(),
1567 result.Val, resultType);
1568
1569 // Make sure we emit a debug reference to the global variable.
1570 // This should probably fire even for
1571 if (isa<VarDecl>(value)) {
1572 if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value)))
1573 EmitDeclRefExprDbgValue(refExpr, result.Val);
1574 } else {
1575 assert(isa<EnumConstantDecl>(value));
1576 EmitDeclRefExprDbgValue(refExpr, result.Val);
1577 }
1578
1579 // If we emitted a reference constant, we need to dereference that.
1580 if (resultIsReference)
1582
1584}
1585
1587 const MemberExpr *ME) {
1588 if (auto *VD = dyn_cast<VarDecl>(ME->getMemberDecl())) {
1589 // Try to emit static variable member expressions as DREs.
1590 return DeclRefExpr::Create(
1592 /*RefersToEnclosingVariableOrCapture=*/false, ME->getExprLoc(),
1593 ME->getType(), ME->getValueKind(), nullptr, nullptr, ME->isNonOdrUse());
1594 }
1595 return nullptr;
1596}
1597
1598CodeGenFunction::ConstantEmission
1601 return tryEmitAsConstant(DRE);
1602 return ConstantEmission();
1603}
1604
1606 const CodeGenFunction::ConstantEmission &Constant, Expr *E) {
1607 assert(Constant && "not a constant");
1608 if (Constant.isReference())
1609 return EmitLoadOfLValue(Constant.getReferenceLValue(*this, E),
1610 E->getExprLoc())
1611 .getScalarVal();
1612 return Constant.getValue();
1613}
1614
1615llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
1616 SourceLocation Loc) {
1617 return EmitLoadOfScalar(lvalue.getAddress(*this), lvalue.isVolatile(),
1618 lvalue.getType(), Loc, lvalue.getBaseInfo(),
1619 lvalue.getTBAAInfo(), lvalue.isNontemporal());
1620}
1621
1623 if (Ty->isBooleanType())
1624 return true;
1625
1626 if (const EnumType *ET = Ty->getAs<EnumType>())
1627 return ET->getDecl()->getIntegerType()->isBooleanType();
1628
1629 if (const AtomicType *AT = Ty->getAs<AtomicType>())
1630 return hasBooleanRepresentation(AT->getValueType());
1631
1632 return false;
1633}
1634
1636 llvm::APInt &Min, llvm::APInt &End,
1637 bool StrictEnums, bool IsBool) {
1638 const EnumType *ET = Ty->getAs<EnumType>();
1639 bool IsRegularCPlusPlusEnum = CGF.getLangOpts().CPlusPlus && StrictEnums &&
1640 ET && !ET->getDecl()->isFixed();
1641 if (!IsBool && !IsRegularCPlusPlusEnum)
1642 return false;
1643
1644 if (IsBool) {
1645 Min = llvm::APInt(CGF.getContext().getTypeSize(Ty), 0);
1646 End = llvm::APInt(CGF.getContext().getTypeSize(Ty), 2);
1647 } else {
1648 const EnumDecl *ED = ET->getDecl();
1649 ED->getValueRange(End, Min);
1650 }
1651 return true;
1652}
1653
1654llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
1655 llvm::APInt Min, End;
1656 if (!getRangeForType(*this, Ty, Min, End, CGM.getCodeGenOpts().StrictEnums,
1658 return nullptr;
1659
1660 llvm::MDBuilder MDHelper(getLLVMContext());
1661 return MDHelper.createRange(Min, End);
1662}
1663
1665 SourceLocation Loc) {
1666 bool HasBoolCheck = SanOpts.has(SanitizerKind::Bool);
1667 bool HasEnumCheck = SanOpts.has(SanitizerKind::Enum);
1668 if (!HasBoolCheck && !HasEnumCheck)
1669 return false;
1670
1671 bool IsBool = hasBooleanRepresentation(Ty) ||
1673 bool NeedsBoolCheck = HasBoolCheck && IsBool;
1674 bool NeedsEnumCheck = HasEnumCheck && Ty->getAs<EnumType>();
1675 if (!NeedsBoolCheck && !NeedsEnumCheck)
1676 return false;
1677
1678 // Single-bit booleans don't need to be checked. Special-case this to avoid
1679 // a bit width mismatch when handling bitfield values. This is handled by
1680 // EmitFromMemory for the non-bitfield case.
1681 if (IsBool &&
1682 cast<llvm::IntegerType>(Value->getType())->getBitWidth() == 1)
1683 return false;
1684
1685 llvm::APInt Min, End;
1686 if (!getRangeForType(*this, Ty, Min, End, /*StrictEnums=*/true, IsBool))
1687 return true;
1688
1689 auto &Ctx = getLLVMContext();
1690 SanitizerScope SanScope(this);
1691 llvm::Value *Check;
1692 --End;
1693 if (!Min) {
1694 Check = Builder.CreateICmpULE(Value, llvm::ConstantInt::get(Ctx, End));
1695 } else {
1696 llvm::Value *Upper =
1697 Builder.CreateICmpSLE(Value, llvm::ConstantInt::get(Ctx, End));
1698 llvm::Value *Lower =
1699 Builder.CreateICmpSGE(Value, llvm::ConstantInt::get(Ctx, Min));
1700 Check = Builder.CreateAnd(Upper, Lower);
1701 }
1702 llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
1705 NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool;
1706 EmitCheck(std::make_pair(Check, Kind), SanitizerHandler::LoadInvalidValue,
1707 StaticArgs, EmitCheckValue(Value));
1708 return true;
1709}
1710
1711llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
1712 QualType Ty,
1713 SourceLocation Loc,
1714 LValueBaseInfo BaseInfo,
1715 TBAAAccessInfo TBAAInfo,
1716 bool isNontemporal) {
1717 if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
1718 if (GV->isThreadLocal())
1719 Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),
1721
1722 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
1723 // Boolean vectors use `iN` as storage type.
1724 if (ClangVecTy->isExtVectorBoolType()) {
1725 llvm::Type *ValTy = ConvertType(Ty);
1726 unsigned ValNumElems =
1727 cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1728 // Load the `iP` storage object (P is the padded vector size).
1729 auto *RawIntV = Builder.CreateLoad(Addr, Volatile, "load_bits");
1730 const auto *RawIntTy = RawIntV->getType();
1731 assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
1732 // Bitcast iP --> <P x i1>.
1733 auto *PaddedVecTy = llvm::FixedVectorType::get(
1734 Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1735 llvm::Value *V = Builder.CreateBitCast(RawIntV, PaddedVecTy);
1736 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
1737 V = emitBoolVecConversion(V, ValNumElems, "extractvec");
1738
1739 return EmitFromMemory(V, Ty);
1740 }
1741
1742 // Handle vectors of size 3 like size 4 for better performance.
1743 const llvm::Type *EltTy = Addr.getElementType();
1744 const auto *VTy = cast<llvm::FixedVectorType>(EltTy);
1745
1746 if (!CGM.getCodeGenOpts().PreserveVec3Type && VTy->getNumElements() == 3) {
1747
1748 llvm::VectorType *vec4Ty =
1749 llvm::FixedVectorType::get(VTy->getElementType(), 4);
1750 Address Cast = Addr.withElementType(vec4Ty);
1751 // Now load value.
1752 llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4");
1753
1754 // Shuffle vector to get vec3.
1755 V = Builder.CreateShuffleVector(V, ArrayRef<int>{0, 1, 2}, "extractVec");
1756 return EmitFromMemory(V, Ty);
1757 }
1758 }
1759
1760 // Atomic operations have to be done on integral types.
1761 LValue AtomicLValue =
1762 LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);
1763 if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(AtomicLValue)) {
1764 return EmitAtomicLoad(AtomicLValue, Loc).getScalarVal();
1765 }
1766
1767 llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);
1768 if (isNontemporal) {
1769 llvm::MDNode *Node = llvm::MDNode::get(
1770 Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
1771 Load->setMetadata(llvm::LLVMContext::MD_nontemporal, Node);
1772 }
1773
1774 CGM.DecorateInstructionWithTBAA(Load, TBAAInfo);
1775
1776 if (EmitScalarRangeCheck(Load, Ty, Loc)) {
1777 // In order to prevent the optimizer from throwing away the check, don't
1778 // attach range metadata to the load.
1779 } else if (CGM.getCodeGenOpts().OptimizationLevel > 0)
1780 if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
1781 Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);
1782 Load->setMetadata(llvm::LLVMContext::MD_noundef,
1783 llvm::MDNode::get(getLLVMContext(), std::nullopt));
1784 }
1785
1786 return EmitFromMemory(Load, Ty);
1787}
1788
1789llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
1790 // Bool has a different representation in memory than in registers.
1791 if (hasBooleanRepresentation(Ty)) {
1792 // This should really always be an i1, but sometimes it's already
1793 // an i8, and it's awkward to track those cases down.
1794 if (Value->getType()->isIntegerTy(1))
1795 return Builder.CreateZExt(Value, ConvertTypeForMem(Ty), "frombool");
1796 assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
1797 "wrong value rep of bool");
1798 }
1799
1800 return Value;
1801}
1802
1803llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
1804 // Bool has a different representation in memory than in registers.
1805 if (hasBooleanRepresentation(Ty)) {
1806 assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
1807 "wrong value rep of bool");
1808 return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool");
1809 }
1810 if (Ty->isExtVectorBoolType()) {
1811 const auto *RawIntTy = Value->getType();
1812 // Bitcast iP --> <P x i1>.
1813 auto *PaddedVecTy = llvm::FixedVectorType::get(
1814 Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1815 auto *V = Builder.CreateBitCast(Value, PaddedVecTy);
1816 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
1817 llvm::Type *ValTy = ConvertType(Ty);
1818 unsigned ValNumElems = cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1819 return emitBoolVecConversion(V, ValNumElems, "extractvec");
1820 }
1821
1822 return Value;
1823}
1824
1825// Convert the pointer of \p Addr to a pointer to a vector (the value type of
1826// MatrixType), if it points to a array (the memory type of MatrixType).
1828 bool IsVector = true) {
1829 auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType());
1830 if (ArrayTy && IsVector) {
1831 auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
1832 ArrayTy->getNumElements());
1833
1834 return Addr.withElementType(VectorTy);
1835 }
1836 auto *VectorTy = dyn_cast<llvm::VectorType>(Addr.getElementType());
1837 if (VectorTy && !IsVector) {
1838 auto *ArrayTy = llvm::ArrayType::get(
1839 VectorTy->getElementType(),
1840 cast<llvm::FixedVectorType>(VectorTy)->getNumElements());
1841
1842 return Addr.withElementType(ArrayTy);
1843 }
1844
1845 return Addr;
1846}
1847
1848// Emit a store of a matrix LValue. This may require casting the original
1849// pointer to memory address (ArrayType) to a pointer to the value type
1850// (VectorType).
1851static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,
1852 bool isInit, CodeGenFunction &CGF) {
1853 Address Addr = MaybeConvertMatrixAddress(lvalue.getAddress(CGF), CGF,
1854 value->getType()->isVectorTy());
1855 CGF.EmitStoreOfScalar(value, Addr, lvalue.isVolatile(), lvalue.getType(),
1856 lvalue.getBaseInfo(), lvalue.getTBAAInfo(), isInit,
1857 lvalue.isNontemporal());
1858}
1859
1860void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
1861 bool Volatile, QualType Ty,
1862 LValueBaseInfo BaseInfo,
1863 TBAAAccessInfo TBAAInfo,
1864 bool isInit, bool isNontemporal) {
1865 if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
1866 if (GV->isThreadLocal())
1867 Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),
1869
1870 llvm::Type *SrcTy = Value->getType();
1871 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
1872 auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy);
1873 if (VecTy && ClangVecTy->isExtVectorBoolType()) {
1874 auto *MemIntTy = cast<llvm::IntegerType>(Addr.getElementType());
1875 // Expand to the memory bit width.
1876 unsigned MemNumElems = MemIntTy->getPrimitiveSizeInBits();
1877 // <N x i1> --> <P x i1>.
1878 Value = emitBoolVecConversion(Value, MemNumElems, "insertvec");
1879 // <P x i1> --> iP.
1880 Value = Builder.CreateBitCast(Value, MemIntTy);
1881 } else if (!CGM.getCodeGenOpts().PreserveVec3Type) {
1882 // Handle vec3 special.
1883 if (VecTy && cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) {
1884 // Our source is a vec3, do a shuffle vector to make it a vec4.
1885 Value = Builder.CreateShuffleVector(Value, ArrayRef<int>{0, 1, 2, -1},
1886 "extractVec");
1887 SrcTy = llvm::FixedVectorType::get(VecTy->getElementType(), 4);
1888 }
1889 if (Addr.getElementType() != SrcTy) {
1890 Addr = Addr.withElementType(SrcTy);
1891 }
1892 }
1893 }
1894
1895 Value = EmitToMemory(Value, Ty);
1896
1897 LValue AtomicLValue =
1898 LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);
1899 if (Ty->isAtomicType() ||
1900 (!isInit && LValueIsSuitableForInlineAtomic(AtomicLValue))) {
1901 EmitAtomicStore(RValue::get(Value), AtomicLValue, isInit);
1902 return;
1903 }
1904
1905 llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
1906 if (isNontemporal) {
1907 llvm::MDNode *Node =
1908 llvm::MDNode::get(Store->getContext(),
1909 llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
1910 Store->setMetadata(llvm::LLVMContext::MD_nontemporal, Node);
1911 }
1912
1913 CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
1914}
1915
1916void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
1917 bool isInit) {
1918 if (lvalue.getType()->isConstantMatrixType()) {
1919 EmitStoreOfMatrixScalar(value, lvalue, isInit, *this);
1920 return;
1921 }
1922
1923 EmitStoreOfScalar(value, lvalue.getAddress(*this), lvalue.isVolatile(),
1924 lvalue.getType(), lvalue.getBaseInfo(),
1925 lvalue.getTBAAInfo(), isInit, lvalue.isNontemporal());
1926}
1927
1928// Emit a load of a LValue of matrix type. This may require casting the pointer
1929// to memory address (ArrayType) to a pointer to the value type (VectorType).
1931 CodeGenFunction &CGF) {
1932 assert(LV.getType()->isConstantMatrixType());
1933 Address Addr = MaybeConvertMatrixAddress(LV.getAddress(CGF), CGF);
1934 LV.setAddress(Addr);
1935 return RValue::get(CGF.EmitLoadOfScalar(LV, Loc));
1936}
1937
1938/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
1939/// method emits the address of the lvalue, then loads the result as an rvalue,
1940/// returning the rvalue.
1942 if (LV.isObjCWeak()) {
1943 // load of a __weak object.
1944 Address AddrWeakObj = LV.getAddress(*this);
1946 AddrWeakObj));
1947 }
1949 // In MRC mode, we do a load+autorelease.
1950 if (!getLangOpts().ObjCAutoRefCount) {
1951 return RValue::get(EmitARCLoadWeak(LV.getAddress(*this)));
1952 }
1953
1954 // In ARC mode, we load retained and then consume the value.
1955 llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress(*this));
1956 Object = EmitObjCConsumeObject(LV.getType(), Object);
1957 return RValue::get(Object);
1958 }
1959
1960 if (LV.isSimple()) {
1961 assert(!LV.getType()->isFunctionType());
1962
1963 if (LV.getType()->isConstantMatrixType())
1964 return EmitLoadOfMatrixLValue(LV, Loc, *this);
1965
1966 // Everything needs a load.
1967 return RValue::get(EmitLoadOfScalar(LV, Loc));
1968 }
1969
1970 if (LV.isVectorElt()) {
1971 llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddress(),
1972 LV.isVolatileQualified());
1973 return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(),
1974 "vecext"));
1975 }
1976
1977 // If this is a reference to a subset of the elements of a vector, either
1978 // shuffle the input or extract/insert them as appropriate.
1979 if (LV.isExtVectorElt()) {
1981 }
1982
1983 // Global Register variables always invoke intrinsics
1984 if (LV.isGlobalReg())
1985 return EmitLoadOfGlobalRegLValue(LV);
1986
1987 if (LV.isMatrixElt()) {
1988 llvm::Value *Idx = LV.getMatrixIdx();
1989 if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
1990 const auto *const MatTy = LV.getType()->castAs<ConstantMatrixType>();
1991 llvm::MatrixBuilder MB(Builder);
1992 MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
1993 }
1994 llvm::LoadInst *Load =
1996 return RValue::get(Builder.CreateExtractElement(Load, Idx, "matrixext"));
1997 }
1998
1999 assert(LV.isBitField() && "Unknown LValue type!");
2000 return EmitLoadOfBitfieldLValue(LV, Loc);
2001}
2002
2004 SourceLocation Loc) {
2005 const CGBitFieldInfo &Info = LV.getBitFieldInfo();
2006
2007 // Get the output type.
2008 llvm::Type *ResLTy = ConvertType(LV.getType());
2009
2010 Address Ptr = LV.getBitFieldAddress();
2011 llvm::Value *Val =
2012 Builder.CreateLoad(Ptr, LV.isVolatileQualified(), "bf.load");
2013
2014 bool UseVolatile = LV.isVolatileQualified() &&
2015 Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());
2016 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2017 const unsigned StorageSize =
2018 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2019 if (Info.IsSigned) {
2020 assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);
2021 unsigned HighBits = StorageSize - Offset - Info.Size;
2022 if (HighBits)
2023 Val = Builder.CreateShl(Val, HighBits, "bf.shl");
2024 if (Offset + HighBits)
2025 Val = Builder.CreateAShr(Val, Offset + HighBits, "bf.ashr");
2026 } else {
2027 if (Offset)
2028 Val = Builder.CreateLShr(Val, Offset, "bf.lshr");
2029 if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)
2030 Val = Builder.CreateAnd(
2031 Val, llvm::APInt::getLowBitsSet(StorageSize, Info.Size), "bf.clear");
2032 }
2033 Val = Builder.CreateIntCast(Val, ResLTy, Info.IsSigned, "bf.cast");
2034 EmitScalarRangeCheck(Val, LV.getType(), Loc);
2035 return RValue::get(Val);
2036}
2037
2038// If this is a reference to a subset of the elements of a vector, create an
2039// appropriate shufflevector.
2041 llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddress(),
2042 LV.isVolatileQualified());
2043
2044 const llvm::Constant *Elts = LV.getExtVectorElts();
2045
2046 // If the result of the expression is a non-vector type, we must be extracting
2047 // a single element. Just codegen as an extractelement.
2048 const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
2049 if (!ExprVT) {
2050 unsigned InIdx = getAccessedFieldNo(0, Elts);
2051 llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
2052 return RValue::get(Builder.CreateExtractElement(Vec, Elt));
2053 }
2054
2055 // Always use shuffle vector to try to retain the original program structure
2056 unsigned NumResultElts = ExprVT->getNumElements();
2057
2059 for (unsigned i = 0; i != NumResultElts; ++i)
2060 Mask.push_back(getAccessedFieldNo(i, Elts));
2061
2062 Vec = Builder.CreateShuffleVector(Vec, Mask);
2063 return RValue::get(Vec);
2064}
2065
2066/// Generates lvalue for partial ext_vector access.
2068 Address VectorAddress = LV.getExtVectorAddress();
2069 QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();
2070 llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT);
2071
2072 Address CastToPointerElement = VectorAddress.withElementType(VectorElementTy);
2073
2074 const llvm::Constant *Elts = LV.getExtVectorElts();
2075 unsigned ix = getAccessedFieldNo(0, Elts);
2076
2077 Address VectorBasePtrPlusIx =
2078 Builder.CreateConstInBoundsGEP(CastToPointerElement, ix,
2079 "vector.elt");
2080
2081 return VectorBasePtrPlusIx;
2082}
2083
2084/// Load of global gamed gegisters are always calls to intrinsics.
2086 assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&
2087 "Bad type for register variable");
2088 llvm::MDNode *RegName = cast<llvm::MDNode>(
2089 cast<llvm::MetadataAsValue>(LV.getGlobalReg())->getMetadata());
2090
2091 // We accept integer and pointer types only
2092 llvm::Type *OrigTy = CGM.getTypes().ConvertType(LV.getType());
2093 llvm::Type *Ty = OrigTy;
2094 if (OrigTy->isPointerTy())
2095 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2096 llvm::Type *Types[] = { Ty };
2097
2098 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
2099 llvm::Value *Call = Builder.CreateCall(
2100 F, llvm::MetadataAsValue::get(Ty->getContext(), RegName));
2101 if (OrigTy->isPointerTy())
2102 Call = Builder.CreateIntToPtr(Call, OrigTy);
2103 return RValue::get(Call);
2104}
2105
2106/// EmitStoreThroughLValue - Store the specified rvalue into the specified
2107/// lvalue, where both are guaranteed to the have the same type, and that type
2108/// is 'Ty'.
2110 bool isInit) {
2111 if (!Dst.isSimple()) {
2112 if (Dst.isVectorElt()) {
2113 // Read/modify/write the vector, inserting the new element.
2114 llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(),
2115 Dst.isVolatileQualified());
2116 auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType());
2117 if (IRStoreTy) {
2118 auto *IRVecTy = llvm::FixedVectorType::get(
2119 Builder.getInt1Ty(), IRStoreTy->getPrimitiveSizeInBits());
2120 Vec = Builder.CreateBitCast(Vec, IRVecTy);
2121 // iN --> <N x i1>.
2122 }
2123 Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
2124 Dst.getVectorIdx(), "vecins");
2125 if (IRStoreTy) {
2126 // <N x i1> --> <iN>.
2127 Vec = Builder.CreateBitCast(Vec, IRStoreTy);
2128 }
2130 Dst.isVolatileQualified());
2131 return;
2132 }
2133
2134 // If this is an update of extended vector elements, insert them as
2135 // appropriate.
2136 if (Dst.isExtVectorElt())
2138
2139 if (Dst.isGlobalReg())
2140 return EmitStoreThroughGlobalRegLValue(Src, Dst);
2141
2142 if (Dst.isMatrixElt()) {
2143 llvm::Value *Idx = Dst.getMatrixIdx();
2144 if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2145 const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
2146 llvm::MatrixBuilder MB(Builder);
2147 MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
2148 }
2149 llvm::Instruction *Load = Builder.CreateLoad(Dst.getMatrixAddress());
2150 llvm::Value *Vec =
2151 Builder.CreateInsertElement(Load, Src.getScalarVal(), Idx, "matins");
2153 Dst.isVolatileQualified());
2154 return;
2155 }
2156
2157 assert(Dst.isBitField() && "Unknown LValue type");
2158 return EmitStoreThroughBitfieldLValue(Src, Dst);
2159 }
2160
2161 // There's special magic for assigning into an ARC-qualified l-value.
2162 if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
2163 switch (Lifetime) {
2165 llvm_unreachable("present but none");
2166
2168 // nothing special
2169 break;
2170
2172 if (isInit) {
2173 Src = RValue::get(EmitARCRetain(Dst.getType(), Src.getScalarVal()));
2174 break;
2175 }
2176 EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true);
2177 return;
2178
2180 if (isInit)
2181 // Initialize and then skip the primitive store.
2182 EmitARCInitWeak(Dst.getAddress(*this), Src.getScalarVal());
2183 else
2184 EmitARCStoreWeak(Dst.getAddress(*this), Src.getScalarVal(),
2185 /*ignore*/ true);
2186 return;
2187
2190 Src.getScalarVal()));
2191 // fall into the normal path
2192 break;
2193 }
2194 }
2195
2196 if (Dst.isObjCWeak() && !Dst.isNonGC()) {
2197 // load of a __weak object.
2198 Address LvalueDst = Dst.getAddress(*this);
2199 llvm::Value *src = Src.getScalarVal();
2200 CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst);
2201 return;
2202 }
2203
2204 if (Dst.isObjCStrong() && !Dst.isNonGC()) {
2205 // load of a __strong object.
2206 Address LvalueDst = Dst.getAddress(*this);
2207 llvm::Value *src = Src.getScalarVal();
2208 if (Dst.isObjCIvar()) {
2209 assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
2210 llvm::Type *ResultType = IntPtrTy;
2212 llvm::Value *RHS = dst.getPointer();
2213 RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
2214 llvm::Value *LHS =
2215 Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType,
2216 "sub.ptr.lhs.cast");
2217 llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
2218 CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
2219 BytesBetween);
2220 } else if (Dst.isGlobalObjCRef()) {
2221 CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
2222 Dst.isThreadLocalRef());
2223 }
2224 else
2225 CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst);
2226 return;
2227 }
2228
2229 assert(Src.isScalar() && "Can't emit an agg store with this method");
2230 EmitStoreOfScalar(Src.getScalarVal(), Dst, isInit);
2231}
2232
2234 llvm::Value **Result) {
2235 const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
2236 llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
2237 Address Ptr = Dst.getBitFieldAddress();
2238
2239 // Get the source value, truncated to the width of the bit-field.
2240 llvm::Value *SrcVal = Src.getScalarVal();
2241
2242 // Cast the source to the storage type and shift it into place.
2243 SrcVal = Builder.CreateIntCast(SrcVal, Ptr.getElementType(),
2244 /*isSigned=*/false);
2245 llvm::Value *MaskedVal = SrcVal;
2246
2247 const bool UseVolatile =
2248 CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&
2249 Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());
2250 const unsigned StorageSize =
2251 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2252 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2253 // See if there are other bits in the bitfield's storage we'll need to load
2254 // and mask together with source before storing.
2255 if (StorageSize != Info.Size) {
2256 assert(StorageSize > Info.Size && "Invalid bitfield size.");
2257 llvm::Value *Val =
2258 Builder.CreateLoad(Ptr, Dst.isVolatileQualified(), "bf.load");
2259
2260 // Mask the source value as needed.
2262 SrcVal = Builder.CreateAnd(
2263 SrcVal, llvm::APInt::getLowBitsSet(StorageSize, Info.Size),
2264 "bf.value");
2265 MaskedVal = SrcVal;
2266 if (Offset)
2267 SrcVal = Builder.CreateShl(SrcVal, Offset, "bf.shl");
2268
2269 // Mask out the original value.
2270 Val = Builder.CreateAnd(
2271 Val, ~llvm::APInt::getBitsSet(StorageSize, Offset, Offset + Info.Size),
2272 "bf.clear");
2273
2274 // Or together the unchanged values and the source value.
2275 SrcVal = Builder.CreateOr(Val, SrcVal, "bf.set");
2276 } else {
2277 assert(Offset == 0);
2278 // According to the AACPS:
2279 // When a volatile bit-field is written, and its container does not overlap
2280 // with any non-bit-field member, its container must be read exactly once
2281 // and written exactly once using the access width appropriate to the type
2282 // of the container. The two accesses are not atomic.
2283 if (Dst.isVolatileQualified() && isAAPCS(CGM.getTarget()) &&
2284 CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)
2285 Builder.CreateLoad(Ptr, true, "bf.load");
2286 }
2287
2288 // Write the new value back out.
2289 Builder.CreateStore(SrcVal, Ptr, Dst.isVolatileQualified());
2290
2291 // Return the new value of the bit-field, if requested.
2292 if (Result) {
2293 llvm::Value *ResultVal = MaskedVal;
2294
2295 // Sign extend the value if needed.
2296 if (Info.IsSigned) {
2297 assert(Info.Size <= StorageSize);
2298 unsigned HighBits = StorageSize - Info.Size;
2299 if (HighBits) {
2300 ResultVal = Builder.CreateShl(ResultVal, HighBits, "bf.result.shl");
2301 ResultVal = Builder.CreateAShr(ResultVal, HighBits, "bf.result.ashr");
2302 }
2303 }
2304
2305 ResultVal = Builder.CreateIntCast(ResultVal, ResLTy, Info.IsSigned,
2306 "bf.result.cast");
2307 *Result = EmitFromMemory(ResultVal, Dst.getType());
2308 }
2309}
2310
2312 LValue Dst) {
2313 // This access turns into a read/modify/write of the vector. Load the input
2314 // value now.
2315 llvm::Value *Vec = Builder.CreateLoad(Dst.getExtVectorAddress(),
2316 Dst.isVolatileQualified());
2317 const llvm::Constant *Elts = Dst.getExtVectorElts();
2318
2319 llvm::Value *SrcVal = Src.getScalarVal();
2320
2321 if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
2322 unsigned NumSrcElts = VTy->getNumElements();
2323 unsigned NumDstElts =
2324 cast<llvm::FixedVectorType>(Vec->getType())->getNumElements();
2325 if (NumDstElts == NumSrcElts) {
2326 // Use shuffle vector is the src and destination are the same number of
2327 // elements and restore the vector mask since it is on the side it will be
2328 // stored.
2329 SmallVector<int, 4> Mask(NumDstElts);
2330 for (unsigned i = 0; i != NumSrcElts; ++i)
2331 Mask[getAccessedFieldNo(i, Elts)] = i;
2332
2333 Vec = Builder.CreateShuffleVector(SrcVal, Mask);
2334 } else if (NumDstElts > NumSrcElts) {
2335 // Extended the source vector to the same length and then shuffle it
2336 // into the destination.
2337 // FIXME: since we're shuffling with undef, can we just use the indices
2338 // into that? This could be simpler.
2339 SmallVector<int, 4> ExtMask;
2340 for (unsigned i = 0; i != NumSrcElts; ++i)
2341 ExtMask.push_back(i);
2342 ExtMask.resize(NumDstElts, -1);
2343 llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(SrcVal, ExtMask);
2344 // build identity
2346 for (unsigned i = 0; i != NumDstElts; ++i)
2347 Mask.push_back(i);
2348
2349 // When the vector size is odd and .odd or .hi is used, the last element
2350 // of the Elts constant array will be one past the size of the vector.
2351 // Ignore the last element here, if it is greater than the mask size.
2352 if (getAccessedFieldNo(NumSrcElts - 1, Elts) == Mask.size())
2353 NumSrcElts--;
2354
2355 // modify when what gets shuffled in
2356 for (unsigned i = 0; i != NumSrcElts; ++i)
2357 Mask[getAccessedFieldNo(i, Elts)] = i + NumDstElts;
2358 Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, Mask);
2359 } else {
2360 // We should never shorten the vector
2361 llvm_unreachable("unexpected shorten vector length");
2362 }
2363 } else {
2364 // If the Src is a scalar (not a vector) it must be updating one element.
2365 unsigned InIdx = getAccessedFieldNo(0, Elts);
2366 llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
2367 Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
2368 }
2369
2371 Dst.isVolatileQualified());
2372}
2373
2374/// Store of global named registers are always calls to intrinsics.
2376 assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&
2377 "Bad type for register variable");
2378 llvm::MDNode *RegName = cast<llvm::MDNode>(
2379 cast<llvm::MetadataAsValue>(Dst.getGlobalReg())->getMetadata());
2380 assert(RegName && "Register LValue is not metadata");
2381
2382 // We accept integer and pointer types only
2383 llvm::Type *OrigTy = CGM.getTypes().ConvertType(Dst.getType());
2384 llvm::Type *Ty = OrigTy;
2385 if (OrigTy->isPointerTy())
2386 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2387 llvm::Type *Types[] = { Ty };
2388
2389 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
2390 llvm::Value *Value = Src.getScalarVal();
2391 if (OrigTy->isPointerTy())
2392 Value = Builder.CreatePtrToInt(Value, Ty);
2393 Builder.CreateCall(
2394 F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName), Value});
2395}
2396
2397// setObjCGCLValueClass - sets class of the lvalue for the purpose of
2398// generating write-barries API. It is currently a global, ivar,
2399// or neither.
2400static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
2401 LValue &LV,
2402 bool IsMemberAccess=false) {
2403 if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
2404 return;
2405
2406 if (isa<ObjCIvarRefExpr>(E)) {
2407 QualType ExpTy = E->getType();
2408 if (IsMemberAccess && ExpTy->isPointerType()) {
2409 // If ivar is a structure pointer, assigning to field of
2410 // this struct follows gcc's behavior and makes it a non-ivar
2411 // writer-barrier conservatively.
2412 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2413 if (ExpTy->isRecordType()) {
2414 LV.setObjCIvar(false);
2415 return;
2416 }
2417 }
2418 LV.setObjCIvar(true);
2419 auto *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr *>(E));
2420 LV.setBaseIvarExp(Exp->getBase());
2421 LV.setObjCArray(E->getType()->isArrayType());
2422 return;
2423 }
2424
2425 if (const auto *Exp = dyn_cast<DeclRefExpr>(E)) {
2426 if (const auto *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
2427 if (VD->hasGlobalStorage()) {
2428 LV.setGlobalObjCRef(true);
2429 LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);
2430 }
2431 }
2432 LV.setObjCArray(E->getType()->isArrayType());
2433 return;
2434 }
2435
2436 if (const auto *Exp = dyn_cast<UnaryOperator>(E)) {
2437 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2438 return;
2439 }
2440
2441 if (const auto *Exp = dyn_cast<ParenExpr>(E)) {
2442 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2443 if (LV.isObjCIvar()) {
2444 // If cast is to a structure pointer, follow gcc's behavior and make it
2445 // a non-ivar write-barrier.
2446 QualType ExpTy = E->getType();
2447 if (ExpTy->isPointerType())
2448 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2449 if (ExpTy->isRecordType())
2450 LV.setObjCIvar(false);
2451 }
2452 return;
2453 }
2454
2455 if (const auto *Exp = dyn_cast<GenericSelectionExpr>(E)) {
2456 setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV);
2457 return;
2458 }
2459
2460 if (const auto *Exp = dyn_cast<ImplicitCastExpr>(E)) {
2461 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2462 return;
2463 }
2464
2465 if (const auto *Exp = dyn_cast<CStyleCastExpr>(E)) {
2466 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2467 return;
2468 }
2469
2470 if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {
2471 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2472 return;
2473 }
2474
2475 if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(E)) {
2476 setObjCGCLValueClass(Ctx, Exp->getBase(), LV);
2477 if (LV.isObjCIvar() && !LV.isObjCArray())
2478 // Using array syntax to assigning to what an ivar points to is not
2479 // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
2480 LV.setObjCIvar(false);
2481 else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
2482 // Using array syntax to assigning to what global points to is not
2483 // same as assigning to the global itself. {id *G;} G[i] = 0;
2484 LV.setGlobalObjCRef(false);
2485 return;
2486 }
2487
2488 if (const auto *Exp = dyn_cast<MemberExpr>(E)) {
2489 setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true);
2490 // We don't know if member is an 'ivar', but this flag is looked at
2491 // only in the context of LV.isObjCIvar().
2492 LV.setObjCArray(E->getType()->isArrayType());
2493 return;
2494 }
2495}
2496
2498 CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
2499 llvm::Type *RealVarTy, SourceLocation Loc) {
2500 if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
2502 CGF, VD, Addr, Loc);
2503 else
2504 Addr =
2505 CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc);
2506
2507 Addr = Addr.withElementType(RealVarTy);
2508 return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2509}
2510
2512 const VarDecl *VD, QualType T) {
2513 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2514 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2515 // Return an invalid address if variable is MT_To (or MT_Enter starting with
2516 // OpenMP 5.2) and unified memory is not enabled. For all other cases: MT_Link
2517 // and MT_To (or MT_Enter) with unified memory, return a valid address.
2518 if (!Res || ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2519 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2521 return Address::invalid();
2522 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
2523 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2524 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2526 "Expected link clause OR to clause with unified memory enabled.");
2527 QualType PtrTy = CGF.getContext().getPointerType(VD->getType());
2529 return CGF.EmitLoadOfPointer(Addr, PtrTy->castAs<PointerType>());
2530}
2531
2532Address
2534 LValueBaseInfo *PointeeBaseInfo,
2535 TBAAAccessInfo *PointeeTBAAInfo) {
2536 llvm::LoadInst *Load =
2537 Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile());
2539
2540 QualType PointeeType = RefLVal.getType()->getPointeeType();
2542 PointeeType, PointeeBaseInfo, PointeeTBAAInfo,
2543 /* forPointeeType= */ true);
2544 return Address(Load, ConvertTypeForMem(PointeeType), Align);
2545}
2546
2548 LValueBaseInfo PointeeBaseInfo;
2549 TBAAAccessInfo PointeeTBAAInfo;
2550 Address PointeeAddr = EmitLoadOfReference(RefLVal, &PointeeBaseInfo,
2551 &PointeeTBAAInfo);
2552 return MakeAddrLValue(PointeeAddr, RefLVal.getType()->getPointeeType(),
2553 PointeeBaseInfo, PointeeTBAAInfo);
2554}
2555
2557 const PointerType *PtrTy,
2558 LValueBaseInfo *BaseInfo,
2559 TBAAAccessInfo *TBAAInfo) {
2560 llvm::Value *Addr = Builder.CreateLoad(Ptr);
2561 return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()),
2562 CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo,
2563 TBAAInfo,
2564 /*forPointeeType=*/true));
2565}
2566
2568 const PointerType *PtrTy) {
2569 LValueBaseInfo BaseInfo;
2570 TBAAAccessInfo TBAAInfo;
2571 Address Addr = EmitLoadOfPointer(PtrAddr, PtrTy, &BaseInfo, &TBAAInfo);
2572 return MakeAddrLValue(Addr, PtrTy->getPointeeType(), BaseInfo, TBAAInfo);
2573}
2574
2576 const Expr *E, const VarDecl *VD) {
2577 QualType T = E->getType();
2578
2579 // If it's thread_local, emit a call to its wrapper function instead.
2580 if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
2582 return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T);
2583 // Check if the variable is marked as declare target with link clause in
2584 // device codegen.
2585 if (CGF.getLangOpts().OpenMPIsTargetDevice) {
2586 Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);
2587 if (Addr.isValid())
2588 return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2589 }
2590
2591 llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
2592
2593 if (VD->getTLSKind() != VarDecl::TLS_None)
2594 V = CGF.Builder.CreateThreadLocalAddress(V);
2595
2596 llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
2597 CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
2598 Address Addr(V, RealVarTy, Alignment);
2599 // Emit reference to the private copy of the variable if it is an OpenMP
2600 // threadprivate variable.
2601 if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
2602 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2603 return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
2604 E->getExprLoc());
2605 }
2606 LValue LV = VD->getType()->isReferenceType() ?
2607 CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
2610 setObjCGCLValueClass(CGF.getContext(), E, LV);
2611 return LV;
2612}
2613
2614static llvm::Constant *EmitFunctionDeclPointer(CodeGenModule &CGM,
2615 GlobalDecl GD) {
2616 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
2617 if (FD->hasAttr<WeakRefAttr>()) {
2618 ConstantAddress aliasee = CGM.GetWeakRefReference(FD);
2619 return aliasee.getPointer();
2620 }
2621
2622 llvm::Constant *V = CGM.GetAddrOfFunction(GD);
2623 if (!FD->hasPrototype()) {
2624 if (const FunctionProtoType *Proto =
2625 FD->getType()->getAs<FunctionProtoType>()) {
2626 // Ugly case: for a K&R-style definition, the type of the definition
2627 // isn't the same as the type of a use. Correct for this with a
2628 // bitcast.
2629 QualType NoProtoType =
2630 CGM.getContext().getFunctionNoProtoType(Proto->getReturnType());
2631 NoProtoType = CGM.getContext().getPointerType(NoProtoType);
2632 V = llvm::ConstantExpr::getBitCast(V,
2633 CGM.getTypes().ConvertType(NoProtoType));
2634 }
2635 }
2636 return V;
2637}
2638
2640 GlobalDecl GD) {
2641 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
2642 llvm::Value *V = EmitFunctionDeclPointer(CGF.CGM, GD);
2643 CharUnits Alignment = CGF.getContext().getDeclAlign(FD);
2644 return CGF.MakeAddrLValue(V, E->getType(), Alignment,
2646}
2647
2649 llvm::Value *ThisValue) {
2651 LValue LV = CGF.MakeNaturalAlignAddrLValue(ThisValue, TagType);
2652 return CGF.EmitLValueForField(LV, FD);
2653}
2654
2655/// Named Registers are named metadata pointing to the register name
2656/// which will be read from/written to as an argument to the intrinsic
2657/// @llvm.read/write_register.
2658/// So far, only the name is being passed down, but other options such as
2659/// register type, allocation type or even optimization options could be
2660/// passed down via the metadata node.
2662 SmallString<64> Name("llvm.named.register.");
2663 AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
2664 assert(Asm->getLabel().size() < 64-Name.size() &&
2665 "Register name too big");
2666 Name.append(Asm->getLabel());
2667 llvm::NamedMDNode *M =
2668 CGM.getModule().getOrInsertNamedMetadata(Name);
2669 if (M->getNumOperands() == 0) {
2670 llvm::MDString *Str = llvm::MDString::get(CGM.getLLVMContext(),
2671 Asm->getLabel());
2672 llvm::Metadata *Ops[] = {Str};
2673 M->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops));
2674 }
2675
2676 CharUnits Alignment = CGM.getContext().getDeclAlign(VD);
2677
2678 llvm::Value *Ptr =
2679 llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0));
2680 return LValue::MakeGlobalReg(Ptr, Alignment, VD->getType());
2681}
2682
2683/// Determine whether we can emit a reference to \p VD from the current
2684/// context, despite not necessarily having seen an odr-use of the variable in
2685/// this context.
2687 const DeclRefExpr *E,
2688 const VarDecl *VD) {
2689 // For a variable declared in an enclosing scope, do not emit a spurious
2690 // reference even if we have a capture, as that will emit an unwarranted
2691 // reference to our capture state, and will likely generate worse code than
2692 // emitting a local copy.
2694 return false;
2695
2696 // For a local declaration declared in this function, we can always reference
2697 // it even if we don't have an odr-use.
2698 if (VD->hasLocalStorage()) {
2699 return VD->getDeclContext() ==
2700 dyn_cast_or_null<DeclContext>(CGF.CurCodeDecl);
2701 }
2702
2703 // For a global declaration, we can emit a reference to it if we know
2704 // for sure that we are able to emit a definition of it.
2705 VD = VD->getDefinition(CGF.getContext());
2706 if (!VD)
2707 return false;
2708
2709 // Don't emit a spurious reference if it might be to a variable that only
2710 // exists on a different device / target.
2711 // FIXME: This is unnecessarily broad. Check whether this would actually be a
2712 // cross-target reference.
2713 if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||
2714 CGF.getLangOpts().OpenCL) {
2715 return false;
2716 }
2717
2718 // We can emit a spurious reference only if the linkage implies that we'll
2719 // be emitting a non-interposable symbol that will be retained until link
2720 // time.
2721 switch (CGF.CGM.getLLVMLinkageVarDefinition(VD)) {
2722 case llvm::GlobalValue::ExternalLinkage:
2723 case llvm::GlobalValue::LinkOnceODRLinkage:
2724 case llvm::GlobalValue::WeakODRLinkage:
2725 case llvm::GlobalValue::InternalLinkage:
2726 case llvm::GlobalValue::PrivateLinkage:
2727 return true;
2728 default:
2729 return false;
2730 }
2731}
2732
2734 const NamedDecl *ND = E->getDecl();
2735 QualType T = E->getType();
2736
2737 assert(E->isNonOdrUse() != NOUR_Unevaluated &&
2738 "should not emit an unevaluated operand");
2739
2740 if (const auto *VD = dyn_cast<VarDecl>(ND)) {
2741 // Global Named registers access via intrinsics only
2742 if (VD->getStorageClass() == SC_Register &&
2743 VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
2744 return EmitGlobalNamedRegister(VD, CGM);
2745
2746 // If this DeclRefExpr does not constitute an odr-use of the variable,
2747 // we're not permitted to emit a reference to it in general, and it might
2748 // not be captured if capture would be necessary for a use. Emit the
2749 // constant value directly instead.
2750 if (E->isNonOdrUse() == NOUR_Constant &&
2751 (VD->getType()->isReferenceType() ||
2752 !canEmitSpuriousReferenceToVariable(*this, E, VD))) {
2753 VD->getAnyInitializer(VD);
2754 llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(
2755 E->getLocation(), *VD->evaluateValue(), VD->getType());
2756 assert(Val && "failed to emit constant expression");
2757
2758 Address Addr = Address::invalid();
2759 if (!VD->getType()->isReferenceType()) {
2760 // Spill the constant value to a global.
2761 Addr = CGM.createUnnamedGlobalFrom(*VD, Val,
2762 getContext().getDeclAlign(VD));
2763 llvm::Type *VarTy = getTypes().ConvertTypeForMem(VD->getType());
2764 auto *PTy = llvm::PointerType::get(
2765 VarTy, getTypes().getTargetAddressSpace(VD->getType()));
2766 Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy, VarTy);
2767 } else {
2768 // Should we be using the alignment of the constant pointer we emitted?
2769 CharUnits Alignment =
2771 /* BaseInfo= */ nullptr,
2772 /* TBAAInfo= */ nullptr,
2773 /* forPointeeType= */ true);
2774 Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment);
2775 }
2776 return MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2777 }
2778
2779 // FIXME: Handle other kinds of non-odr-use DeclRefExprs.
2780
2781 // Check for captured variables.
2783 VD = VD->getCanonicalDecl();
2784 if (auto *FD = LambdaCaptureFields.lookup(VD))
2785 return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
2786 if (CapturedStmtInfo) {
2787 auto I = LocalDeclMap.find(VD);
2788 if (I != LocalDeclMap.end()) {
2789 LValue CapLVal;
2790 if (VD->getType()->isReferenceType())
2791 CapLVal = EmitLoadOfReferenceLValue(I->second, VD->getType(),
2793 else
2794 CapLVal = MakeAddrLValue(I->second, T);
2795 // Mark lvalue as nontemporal if the variable is marked as nontemporal
2796 // in simd context.
2797 if (getLangOpts().OpenMP &&
2799 CapLVal.setNontemporal(/*Value=*/true);
2800 return CapLVal;
2801 }
2802 LValue CapLVal =
2805 Address LValueAddress = CapLVal.getAddress(*this);
2806 CapLVal = MakeAddrLValue(
2807 Address(LValueAddress.getPointer(), LValueAddress.getElementType(),
2808 getContext().getDeclAlign(VD)),
2810 CapLVal.getTBAAInfo());
2811 // Mark lvalue as nontemporal if the variable is marked as nontemporal
2812 // in simd context.
2813 if (getLangOpts().OpenMP &&
2815 CapLVal.setNontemporal(/*Value=*/true);
2816 return CapLVal;
2817 }
2818
2819 assert(isa<BlockDecl>(CurCodeDecl));
2820 Address addr = GetAddrOfBlockDecl(VD);
2821 return MakeAddrLValue(addr, T, AlignmentSource::Decl);
2822 }
2823 }
2824
2825 // FIXME: We should be able to assert this for FunctionDecls as well!
2826 // FIXME: We should be able to assert this for all DeclRefExprs, not just
2827 // those with a valid source location.
2828 assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||
2829 !E->getLocation().isValid()) &&
2830 "Should not use decl without marking it used!");
2831
2832 if (ND->hasAttr<WeakRefAttr>()) {
2833 const auto *VD = cast<ValueDecl>(ND);
2835 return MakeAddrLValue(Aliasee, T, AlignmentSource::Decl);
2836 }
2837
2838 if (const auto *VD = dyn_cast<VarDecl>(ND)) {
2839 // Check if this is a global variable.
2840 if (VD->hasLinkage() || VD->isStaticDataMember())
2841 return EmitGlobalVarDeclLValue(*this, E, VD);
2842
2843 Address addr = Address::invalid();
2844
2845 // The variable should generally be present in the local decl map.
2846 auto iter = LocalDeclMap.find(VD);
2847 if (iter != LocalDeclMap.end()) {
2848 addr = iter->second;
2849
2850 // Otherwise, it might be static local we haven't emitted yet for
2851 // some reason; most likely, because it's in an outer function.
2852 } else if (VD->isStaticLocal()) {
2853 llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
2855 addr = Address(
2856 var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD));
2857
2858 // No other cases for now.
2859 } else {
2860 llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
2861 }
2862
2863 // Handle threadlocal function locals.
2864 if (VD->getTLSKind() != VarDecl::TLS_None)
2865 addr = addr.withPointer(
2866 Builder.CreateThreadLocalAddress(addr.getPointer()), NotKnownNonNull);
2867
2868 // Check for OpenMP threadprivate variables.
2869 if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
2870 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2872 *this, VD, T, addr, getTypes().ConvertTypeForMem(VD->getType()),
2873 E->getExprLoc());
2874 }
2875
2876 // Drill into block byref variables.
2877 bool isBlockByref = VD->isEscapingByref();
2878 if (isBlockByref) {
2879 addr = emitBlockByrefAddress(addr, VD);
2880 }
2881
2882 // Drill into reference types.
2883 LValue LV = VD->getType()->isReferenceType() ?
2884 EmitLoadOfReferenceLValue(addr, VD->getType(), AlignmentSource::Decl) :
2886
2887 bool isLocalStorage = VD->hasLocalStorage();
2888
2889 bool NonGCable = isLocalStorage &&
2890 !VD->getType()->isReferenceType() &&
2891 !isBlockByref;
2892 if (NonGCable) {
2894 LV.setNonGC(true);
2895 }
2896
2897 bool isImpreciseLifetime =
2898 (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
2899 if (isImpreciseLifetime)
2902 return LV;
2903 }
2904
2905 if (const auto *FD = dyn_cast<FunctionDecl>(ND)) {
2906 LValue LV = EmitFunctionDeclLValue(*this, E, FD);
2907
2908 // Emit debuginfo for the function declaration if the target wants to.
2909 if (getContext().getTargetInfo().allowDebugInfoForExternalRef()) {
2910 if (CGDebugInfo *DI = CGM.getModuleDebugInfo()) {
2911 auto *Fn =
2912 cast<llvm::Function>(LV.getPointer(*this)->stripPointerCasts());
2913 if (!Fn->getSubprogram())
2914 DI->EmitFunctionDecl(FD, FD->getLocation(), T, Fn);
2915 }
2916 }
2917
2918 return LV;
2919 }
2920
2921 // FIXME: While we're emitting a binding from an enclosing scope, all other
2922 // DeclRefExprs we see should be implicitly treated as if they also refer to
2923 // an enclosing scope.
2924 if (const auto *BD = dyn_cast<BindingDecl>(ND)) {
2926 auto *FD = LambdaCaptureFields.lookup(BD);
2927 return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
2928 }
2929 return EmitLValue(BD->getBinding());
2930 }
2931
2932 // We can form DeclRefExprs naming GUID declarations when reconstituting
2933 // non-type template parameters into expressions.
2934 if (const auto *GD = dyn_cast<MSGuidDecl>(ND))
2937
2938 if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND))
2941
2942 llvm_unreachable("Unhandled DeclRefExpr");
2943}
2944
2946 // __extension__ doesn't affect lvalue-ness.
2947 if (E->getOpcode() == UO_Extension)
2948 return EmitLValue(E->getSubExpr());
2949
2951 switch (E->getOpcode()) {
2952 default: llvm_unreachable("Unknown unary operator lvalue!");
2953 case UO_Deref: {
2955 assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
2956
2957 LValueBaseInfo BaseInfo;
2958 TBAAAccessInfo TBAAInfo;
2959 Address Addr = EmitPointerWithAlignment(E->getSubExpr(), &BaseInfo,
2960 &TBAAInfo);
2961 LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
2963
2964 // We should not generate __weak write barrier on indirect reference
2965 // of a pointer to object; as in void foo (__weak id *param); *param = 0;
2966 // But, we continue to generate __strong write barrier on indirect write
2967 // into a pointer to object.
2968 if (getLangOpts().ObjC &&
2969 getLangOpts().getGC() != LangOptions::NonGC &&
2970 LV.isObjCWeak())
2972 return LV;
2973 }
2974 case UO_Real:
2975 case UO_Imag: {
2976 LValue LV = EmitLValue(E->getSubExpr());
2977 assert(LV.isSimple() && "real/imag on non-ordinary l-value");
2978
2979 // __real is valid on scalars. This is a faster way of testing that.
2980 // __imag can only produce an rvalue on scalars.
2981 if (E->getOpcode() == UO_Real &&
2982 !LV.getAddress(*this).getElementType()->isStructTy()) {
2983 assert(E->getSubExpr()->getType()->isArithmeticType());
2984 return LV;
2985 }
2986
2987 QualType T = ExprTy->castAs<ComplexType>()->getElementType();
2988
2989 Address Component =
2990 (E->getOpcode() == UO_Real
2991 ? emitAddrOfRealComponent(LV.getAddress(*this), LV.getType())
2992 : emitAddrOfImagComponent(LV.getAddress(*this), LV.getType()));
2993 LValue ElemLV = MakeAddrLValue(Component, T, LV.getBaseInfo(),
2995 ElemLV.getQuals().addQualifiers(LV.getQuals());
2996 return ElemLV;
2997 }
2998 case UO_PreInc:
2999 case UO_PreDec: {
3000 LValue LV = EmitLValue(E->getSubExpr());
3001 bool isInc = E->getOpcode() == UO_PreInc;
3002
3003 if (E->getType()->isAnyComplexType())
3004 EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/);
3005 else
3006 EmitScalarPrePostIncDec(E, LV, isInc, true/*isPre*/);
3007 return LV;
3008 }
3009 }
3010}
3011
3015}
3016
3020}
3021
3023 auto SL = E->getFunctionName();
3024 assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
3025 StringRef FnName = CurFn->getName();
3026 if (FnName.startswith("\01"))
3027 FnName = FnName.substr(1);
3028 StringRef NameItems[] = {
3030 std::string GVName = llvm::join(NameItems, NameItems + 2, ".");
3031 if (auto *BD = dyn_cast_or_null<BlockDecl>(CurCodeDecl)) {
3032 std::string Name = std::string(SL->getString());
3033 if (!Name.empty()) {
3034 unsigned Discriminator =
3036 if (Discriminator)
3037 Name += "_" + Twine(Discriminator + 1).str();
3038 auto C = CGM.GetAddrOfConstantCString(Name, GVName.c_str());
3040 } else {
3041 auto C =
3042 CGM.GetAddrOfConstantCString(std::string(FnName), GVName.c_str());
3044 }
3045 }
3046 auto C = CGM.GetAddrOfConstantStringFromLiteral(SL, GVName);
3048}
3049
3050/// Emit a type description suitable for use by a runtime sanitizer library. The
3051/// format of a type descriptor is
3052///
3053/// \code
3054/// { i16 TypeKind, i16 TypeInfo }
3055/// \endcode
3056///
3057/// followed by an array of i8 containing the type name. TypeKind is 0 for an
3058/// integer, 1 for a floating point value, and -1 for anything else.
3060 // Only emit each type's descriptor once.
3061 if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(T))
3062 return C;
3063
3064 uint16_t TypeKind = -1;
3065 uint16_t TypeInfo = 0;
3066
3067 if (T->isIntegerType()) {
3068 TypeKind = 0;
3069 TypeInfo = (llvm::Log2_32(getContext().getTypeSize(T)) << 1) |
3070 (T->isSignedIntegerType() ? 1 : 0);
3071 } else if (T->isFloatingType()) {
3072 TypeKind = 1;
3074 }
3075
3076 // Format the type name as if for a diagnostic, including quotes and
3077 // optionally an 'aka'.
3078 SmallString<32> Buffer;
3081 StringRef(), std::nullopt, Buffer, std::nullopt);
3082
3083 llvm::Constant *Components[] = {
3084 Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo),
3085 llvm::ConstantDataArray::getString(getLLVMContext(), Buffer)
3086 };
3087 llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(Components);
3088
3089 auto *GV = new llvm::GlobalVariable(
3090 CGM.getModule(), Descriptor->getType(),
3091 /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
3092 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3094
3095 // Remember the descriptor for this type.
3097
3098 return GV;
3099}
3100
3101llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
3102 llvm::Type *TargetTy = IntPtrTy;
3103
3104 if (V->getType() == TargetTy)
3105 return V;
3106
3107 // Floating-point types which fit into intptr_t are bitcast to integers
3108 // and then passed directly (after zero-extension, if necessary).
3109 if (V->getType()->isFloatingPointTy()) {
3110 unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue();
3111 if (Bits <= TargetTy->getIntegerBitWidth())
3112 V = Builder.CreateBitCast(V, llvm::Type::getIntNTy(getLLVMContext(),
3113 Bits));
3114 }
3115
3116 // Integers which fit in intptr_t are zero-extended and passed directly.
3117 if (V->getType()->isIntegerTy() &&
3118 V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
3119 return Builder.CreateZExt(V, TargetTy);
3120
3121 // Pointers are passed directly, everything else is passed by address.
3122 if (!V->getType()->isPointerTy()) {
3123 Address Ptr = CreateDefaultAlignTempAlloca(V->getType());
3124 Builder.CreateStore(V, Ptr);
3125 V = Ptr.getPointer();
3126 }
3127 return Builder.CreatePtrToInt(V, TargetTy);
3128}
3129
3130/// Emit a representation of a SourceLocation for passing to a handler
3131/// in a sanitizer runtime library. The format for this data is:
3132/// \code
3133/// struct SourceLocation {
3134/// const char *Filename;
3135/// int32_t Line, Column;
3136/// };
3137/// \endcode
3138/// For an invalid SourceLocation, the Filename pointer is null.
3140 llvm::Constant *Filename;
3141 int Line, Column;
3142
3144 if (PLoc.isValid()) {
3145 StringRef FilenameString = PLoc.getFilename();
3146
3147 int PathComponentsToStrip =
3148 CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;
3149 if (PathComponentsToStrip < 0) {
3150 assert(PathComponentsToStrip != INT_MIN);
3151 int PathComponentsToKeep = -PathComponentsToStrip;
3152 auto I = llvm::sys::path::rbegin(FilenameString);
3153 auto E = llvm::sys::path::rend(FilenameString);
3154 while (I != E && --PathComponentsToKeep)
3155 ++I;
3156
3157 FilenameString = FilenameString.substr(I - E);
3158 } else if (PathComponentsToStrip > 0) {
3159 auto I = llvm::sys::path::begin(FilenameString);
3160 auto E = llvm::sys::path::end(FilenameString);
3161 while (I != E && PathComponentsToStrip--)
3162 ++I;
3163
3164 if (I != E)
3165 FilenameString =
3166 FilenameString.substr(I - llvm::sys::path::begin(FilenameString));
3167 else
3168 FilenameString = llvm::sys::path::filename(FilenameString);
3169 }
3170
3171 auto FilenameGV =
3172 CGM.GetAddrOfConstantCString(std::string(FilenameString), ".src");
3174 cast<llvm::GlobalVariable>(
3175 FilenameGV.getPointer()->stripPointerCasts()));
3176 Filename = FilenameGV.getPointer();
3177 Line = PLoc.getLine();
3178 Column = PLoc.getColumn();
3179 } else {
3180 Filename = llvm::Constant::getNullValue(Int8PtrTy);
3181 Line = Column = 0;
3182 }
3183
3184 llvm::Constant *Data[] = {Filename, Builder.getInt32(Line),
3185 Builder.getInt32(Column)};
3186
3187 return llvm::ConstantStruct::getAnon(Data);
3188}
3189
3190namespace {
3191/// Specify under what conditions this check can be recovered
3192enum class CheckRecoverableKind {
3193 /// Always terminate program execution if this check fails.
3195 /// Check supports recovering, runtime has both fatal (noreturn) and
3196 /// non-fatal handlers for this check.
3197 Recoverable,
3198 /// Runtime conditionally aborts, always need to support recovery.
3200};
3201}
3202
3203static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) {
3204 assert(Kind.countPopulation() == 1);
3205 if (Kind == SanitizerKind::Vptr)
3206 return CheckRecoverableKind::AlwaysRecoverable;
3207 else if (Kind == SanitizerKind::Return || Kind == SanitizerKind::Unreachable)
3208 return CheckRecoverableKind::Unrecoverable;
3209 else
3210 return CheckRecoverableKind::Recoverable;
3211}
3212
3213namespace {
3214struct SanitizerHandlerInfo {
3215 char const *const Name;
3216 unsigned Version;
3217};
3218}
3219
3220const SanitizerHandlerInfo SanitizerHandlers[] = {
3221#define SANITIZER_CHECK(Enum, Name, Version) {#Name, Version},
3223#undef SANITIZER_CHECK
3224};
3225
3227 llvm::FunctionType *FnType,
3229 SanitizerHandler CheckHandler,
3230 CheckRecoverableKind RecoverKind, bool IsFatal,
3231 llvm::BasicBlock *ContBB) {
3232 assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
3233 std::optional<ApplyDebugLocation> DL;
3234 if (!CGF.Builder.getCurrentDebugLocation()) {
3235 // Ensure that the call has at least an artificial debug location.
3236 DL.emplace(CGF, SourceLocation());
3237 }
3238 bool NeedsAbortSuffix =
3239 IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
3240 bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;
3241 const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];
3242 const StringRef CheckName = CheckInfo.Name;
3243 std::string FnName = "__ubsan_handle_" + CheckName.str();
3244 if (CheckInfo.Version && !MinimalRuntime)
3245 FnName += "_v" + llvm::utostr(CheckInfo.Version);
3246 if (MinimalRuntime)
3247 FnName += "_minimal";
3248 if (NeedsAbortSuffix)
3249 FnName += "_abort";
3250 bool MayReturn =
3251 !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
3252
3253 llvm::AttrBuilder B(CGF.getLLVMContext());
3254 if (!MayReturn) {
3255 B.addAttribute(llvm::Attribute::NoReturn)
3256 .addAttribute(llvm::Attribute::NoUnwind);
3257 }
3258 B.addUWTableAttr(llvm::UWTableKind::Default);
3259
3260 llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
3261 FnType, FnName,
3262 llvm::AttributeList::get(CGF.getLLVMContext(),
3263 llvm::AttributeList::FunctionIndex, B),
3264 /*Local=*/true);
3265 llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(Fn, FnArgs);
3266 if (!MayReturn) {
3267 HandlerCall->setDoesNotReturn();
3268 CGF.Builder.CreateUnreachable();
3269 } else {
3270 CGF.Builder.CreateBr(ContBB);
3271 }
3272}
3273
3275 ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
3276 SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,
3277 ArrayRef<llvm::Value *> DynamicArgs) {
3278 assert(IsSanitizerScope);
3279 assert(Checked.size() > 0);
3280 assert(CheckHandler >= 0 &&
3281 size_t(CheckHandler) < std::size(SanitizerHandlers));
3282 const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;
3283
3284 llvm::Value *FatalCond = nullptr;
3285 llvm::Value *RecoverableCond = nullptr;
3286 llvm::Value *TrapCond = nullptr;
3287 for (int i = 0, n = Checked.size(); i < n; ++i) {
3288 llvm::Value *Check = Checked[i].first;
3289 // -fsanitize-trap= overrides -fsanitize-recover=.
3290 llvm::Value *&Cond =
3291 CGM.getCodeGenOpts().SanitizeTrap.has(Checked[i].second)
3292 ? TrapCond
3293 : CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second)
3294 ? RecoverableCond
3295 : FatalCond;
3296 Cond = Cond ? Builder.CreateAnd(Cond, Check) : Check;
3297 }
3298
3299 if (TrapCond)
3300 EmitTrapCheck(TrapCond, CheckHandler);
3301 if (!FatalCond && !RecoverableCond)
3302 return;
3303
3304 llvm::Value *JointCond;
3305 if (FatalCond && RecoverableCond)
3306 JointCond = Builder.CreateAnd(FatalCond, RecoverableCond);
3307 else
3308 JointCond = FatalCond ? FatalCond : RecoverableCond;
3309 assert(JointCond);
3310
3311 CheckRecoverableKind RecoverKind = getRecoverableKind(Checked[0].second);
3312 assert(SanOpts.has(Checked[0].second));
3313#ifndef NDEBUG
3314 for (int i = 1, n = Checked.size(); i < n; ++i) {
3315 assert(RecoverKind == getRecoverableKind(Checked[i].second) &&
3316 "All recoverable kinds in a single check must be same!");
3317 assert(SanOpts.has(Checked[i].second));
3318 }
3319#endif
3320
3321 llvm::BasicBlock *Cont = createBasicBlock("cont");
3322 llvm::BasicBlock *Handlers = createBasicBlock("handler." + CheckName);
3323 llvm::Instruction *Branch = Builder.CreateCondBr(JointCond, Cont, Handlers);
3324 // Give hint that we very much don't expect to execute the handler
3325 // Value chosen to match UR_NONTAKEN_WEIGHT, see BranchProbabilityInfo.cpp
3326 llvm::MDBuilder MDHelper(getLLVMContext());
3327 llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3328 Branch->setMetadata(llvm::LLVMContext::MD_prof, Node);
3329 EmitBlock(Handlers);
3330
3331 // Handler functions take an i8* pointing to the (handler-specific) static
3332 // information block, followed by a sequence of intptr_t arguments
3333 // representing operand values.
3336 if (!CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
3337 Args.reserve(DynamicArgs.size() + 1);
3338 ArgTypes.reserve(DynamicArgs.size() + 1);
3339
3340 // Emit handler arguments and create handler function type.
3341 if (!StaticArgs.empty()) {
3342 llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3343 auto *InfoPtr = new llvm::GlobalVariable(
3344 CGM.getModule(), Info->getType(), false,
3345 llvm::GlobalVariable::PrivateLinkage, Info, "", nullptr,
3346 llvm::GlobalVariable::NotThreadLocal,
3347 CGM.getDataLayout().getDefaultGlobalsAddressSpace());
3348 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3350 Args.push_back(InfoPtr);
3351 ArgTypes.push_back(Args.back()->getType());
3352 }
3353
3354 for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
3355 Args.push_back(EmitCheckValue(DynamicArgs[i]));
3356 ArgTypes.push_back(IntPtrTy);
3357 }
3358 }
3359
3360 llvm::FunctionType *FnType =
3361 llvm::FunctionType::get(CGM.VoidTy, ArgTypes, false);
3362
3363 if (!FatalCond || !RecoverableCond) {
3364 // Simple case: we need to generate a single handler call, either
3365 // fatal, or non-fatal.
3366 emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind,
3367 (FatalCond != nullptr), Cont);
3368 } else {
3369 // Emit two handler calls: first one for set of unrecoverable checks,
3370 // another one for recoverable.
3371 llvm::BasicBlock *NonFatalHandlerBB =
3372 createBasicBlock("non_fatal." + CheckName);
3373 llvm::BasicBlock *FatalHandlerBB = createBasicBlock("fatal." + CheckName);
3374 Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB);
3375 EmitBlock(FatalHandlerBB);
3376 emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, true,
3377 NonFatalHandlerBB);
3378 EmitBlock(NonFatalHandlerBB);
3379 emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, false,
3380 Cont);
3381 }
3382
3383 EmitBlock(Cont);
3384}
3385
3387 SanitizerMask Kind, llvm::Value *Cond, llvm::ConstantInt *TypeId,
3388 llvm::Value *Ptr, ArrayRef<llvm::Constant *> StaticArgs) {
3389 llvm::BasicBlock *Cont = createBasicBlock("cfi.cont");
3390
3391 llvm::BasicBlock *CheckBB = createBasicBlock("cfi.slowpath");
3392 llvm::BranchInst *BI = Builder.CreateCondBr(Cond, Cont, CheckBB);
3393
3394 llvm::MDBuilder MDHelper(getLLVMContext());
3395 llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3396 BI->setMetadata(llvm::LLVMContext::MD_prof, Node);
3397
3398 EmitBlock(CheckBB);
3399
3400 bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(Kind);
3401
3402 llvm::CallInst *CheckCall;
3403 llvm::FunctionCallee SlowPathFn;
3404 if (WithDiag) {
3405 llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3406 auto *InfoPtr =
3407 new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
3408 llvm::GlobalVariable::PrivateLinkage, Info);
3409 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3411
3412 SlowPathFn = CGM.getModule().getOrInsertFunction(
3413 "__cfi_slowpath_diag",
3414 llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy},
3415 false));
3416 CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr, InfoPtr});
3417 } else {
3418 SlowPathFn = CGM.getModule().getOrInsertFunction(
3419 "__cfi_slowpath",
3420 llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy}, false));
3421 CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr});
3422 }
3423
3425 cast<llvm::GlobalValue>(SlowPathFn.getCallee()->stripPointerCasts()));
3426 CheckCall->setDoesNotThrow();
3427
3428 EmitBlock(Cont);
3429}
3430
3431// Emit a stub for __cfi_check function so that the linker knows about this
3432// symbol in LTO mode.
3434 llvm::Module *M = &CGM.getModule();
3435 auto &Ctx = M->getContext();
3436 llvm::Function *F = llvm::Function::Create(
3437 llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy}, false),
3438 llvm::GlobalValue::WeakAnyLinkage, "__cfi_check", M);
3439 F->setAlignment(llvm::Align(4096));
3440 CGM.setDSOLocal(F);
3441 llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", F);
3442 // CrossDSOCFI pass is not executed if there is no executable code.
3443 SmallVector<llvm::Value*> Args{F->getArg(2), F->getArg(1)};
3444 llvm::CallInst::Create(M->getFunction("__cfi_check_fail"), Args, "", BB);
3445 llvm::ReturnInst::Create(Ctx, nullptr, BB);
3446}
3447
3448// This function is basically a switch over the CFI failure kind, which is
3449// extracted from CFICheckFailData (1st function argument). Each case is either
3450// llvm.trap or a call to one of the two runtime handlers, based on
3451// -fsanitize-trap and -fsanitize-recover settings. Default case (invalid
3452// failure kind) traps, but this should really never happen. CFICheckFailData
3453// can be nullptr if the calling module has -fsanitize-trap behavior for this
3454// check kind; in this case __cfi_check_fail traps as well.
3456 SanitizerScope SanScope(this);
3457 FunctionArgList Args;
3462 Args.push_back(&ArgData);
3463 Args.push_back(&ArgAddr);
3464
3465 const CGFunctionInfo &FI =
3467
3468 llvm::Function *F = llvm::Function::Create(
3469 llvm::FunctionType::get(VoidTy, {VoidPtrTy, VoidPtrTy}, false),
3470 llvm::GlobalValue::WeakODRLinkage, "__cfi_check_fail", &CGM.getModule());
3471
3472 CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F, /*IsThunk=*/false);
3474 F->setVisibility(llvm::GlobalValue::HiddenVisibility);
3475
3476 StartFunction(GlobalDecl(), CGM.getContext().VoidTy, F, FI, Args,
3477 SourceLocation());
3478
3479 // This function is not affected by NoSanitizeList. This function does
3480 // not have a source location, but "src:*" would still apply. Revert any
3481 // changes to SanOpts made in StartFunction.
3483
3484 llvm::Value *Data =
3485 EmitLoadOfScalar(GetAddrOfLocalVar(&ArgData), /*Volatile=*/false,
3486 CGM.getContext().VoidPtrTy, ArgData.getLocation());
3487 llvm::Value *Addr =
3488 EmitLoadOfScalar(GetAddrOfLocalVar(&ArgAddr), /*Volatile=*/false,
3489 CGM.getContext().VoidPtrTy, ArgAddr.getLocation());
3490
3491 // Data == nullptr means the calling module has trap behaviour for this check.
3492 llvm::Value *DataIsNotNullPtr =
3493 Builder.CreateICmpNE(Data, llvm::ConstantPointerNull::get(Int8PtrTy));
3494 EmitTrapCheck(DataIsNotNullPtr, SanitizerHandler::CFICheckFail);
3495
3496 llvm::StructType *SourceLocationTy =
3497 llvm::StructType::get(VoidPtrTy, Int32Ty, Int32Ty);
3498 llvm::StructType *CfiCheckFailDataTy =
3499 llvm::StructType::get(Int8Ty, SourceLocationTy, VoidPtrTy);
3500
3501 llvm::Value *V = Builder.CreateConstGEP2_32(
3502 CfiCheckFailDataTy,
3503 Builder.CreatePointerCast(Data, CfiCheckFailDataTy->getPointerTo(0)), 0,
3504 0);
3505
3506 Address CheckKindAddr(V, Int8Ty, getIntAlign());
3507 llvm::Value *CheckKind = Builder.CreateLoad(CheckKindAddr);
3508
3509 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
3511 llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
3512 llvm::Value *ValidVtable = Builder.CreateZExt(
3513 Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
3514 {Addr, AllVtables}),
3515 IntPtrTy);
3516
3517 const std::pair<int, SanitizerMask> CheckKinds[] = {
3518 {CFITCK_VCall, SanitizerKind::CFIVCall},
3519 {CFITCK_NVCall, SanitizerKind::CFINVCall},
3520 {CFITCK_DerivedCast, SanitizerKind::CFIDerivedCast},
3521 {CFITCK_UnrelatedCast, SanitizerKind::CFIUnrelatedCast},
3522 {CFITCK_ICall, SanitizerKind::CFIICall}};
3523
3525 for (auto CheckKindMaskPair : CheckKinds) {
3526 int Kind = CheckKindMaskPair.first;
3527 SanitizerMask Mask = CheckKindMaskPair.second;
3528 llvm::Value *Cond =
3529 Builder.CreateICmpNE(CheckKind, llvm::ConstantInt::get(Int8Ty, Kind));
3530 if (CGM.getLangOpts().Sanitize.has(Mask))
3531 EmitCheck(std::make_pair(Cond, Mask), SanitizerHandler::CFICheckFail, {},
3532 {Data, Addr, ValidVtable});
3533 else
3534 EmitTrapCheck(Cond, SanitizerHandler::CFICheckFail);
3535 }
3536
3538 // The only reference to this function will be created during LTO link.
3539 // Make sure it survives until then.
3540 CGM.addUsedGlobal(F);
3541}
3542
3544 if (SanOpts.has(SanitizerKind::Unreachable)) {
3545 SanitizerScope SanScope(this);
3546 EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()),
3547 SanitizerKind::Unreachable),
3548 SanitizerHandler::BuiltinUnreachable,
3549 EmitCheckSourceLocation(Loc), std::nullopt);
3550 }
3551 Builder.CreateUnreachable();
3552}
3553
3554void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
3555 SanitizerHandler CheckHandlerID) {
3556 llvm::BasicBlock *Cont = createBasicBlock("cont");
3557
3558 // If we're optimizing, collapse all calls to trap down to just one per
3559 // check-type per function to save on code size.
3560 if (TrapBBs.size() <= CheckHandlerID)
3561 TrapBBs.resize(CheckHandlerID + 1);
3562
3563 llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
3564
3566 CGM.getCodeGenOpts().OptimizationLevel && TrapBB &&
3567 (!CurCodeDecl || !CurCodeDecl->hasAttr<OptimizeNoneAttr>())) {
3568 auto Call = TrapBB->begin();
3569 assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
3570
3571 Call->applyMergedLocation(Call->getDebugLoc(),
3572 Builder.getCurrentDebugLocation());
3573 Builder.CreateCondBr(Checked, Cont, TrapBB);
3574 } else {
3575 TrapBB = createBasicBlock("trap");
3576 Builder.CreateCondBr(Checked, Cont, TrapBB);
3577 EmitBlock(TrapBB);
3578
3579 llvm::CallInst *TrapCall = Builder.CreateCall(
3580 CGM.getIntrinsic(llvm::Intrinsic::ubsantrap),
3581 llvm::ConstantInt::get(CGM.Int8Ty, ClSanitizeDebugDeoptimization
3582 ? TrapBB->getParent()->size()
3583 : CheckHandlerID));
3584
3585 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3586 auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
3588 TrapCall->addFnAttr(A);
3589 }
3590 TrapCall->setDoesNotReturn();
3591 TrapCall->setDoesNotThrow();
3592 Builder.CreateUnreachable();
3593 }
3594
3595 EmitBlock(Cont);
3596}
3597
3598llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) {
3599 llvm::CallInst *TrapCall =
3600 Builder.CreateCall(CGM.getIntrinsic(IntrID));
3601
3602 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3603 auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
3605 TrapCall->addFnAttr(A);
3606 }
3607
3608 return TrapCall;
3609}
3610
3612 LValueBaseInfo *BaseInfo,
3613 TBAAAccessInfo *TBAAInfo) {
3614 assert(E->getType()->isArrayType() &&
3615 "Array to pointer decay must have array source type!");
3616
3617 // Expressions of array type can't be bitfields or vector elements.
3618 LValue LV = EmitLValue(E);
3619 Address Addr = LV.getAddress(*this);
3620
3621 // If the array type was an incomplete type, we need to make sure
3622 // the decay ends up being the right type.
3623 llvm::Type *NewTy = ConvertType(E->getType());
3624 Addr = Addr.withElementType(NewTy);
3625
3626 // Note that VLA pointers are always decayed, so we don't need to do
3627 // anything here.
3628 if (!E->getType()->isVariableArrayType()) {
3629 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
3630 "Expected pointer to array");
3631 Addr = Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");
3632 }
3633
3634 // The result of this decay conversion points to an array element within the
3635 // base lvalue. However, since TBAA currently does not support representing
3636 // accesses to elements of member arrays, we conservatively represent accesses
3637 // to the pointee object as if it had no any base lvalue specified.
3638 // TODO: Support TBAA for member arrays.
3640 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
3641 if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(EltType);
3642
3643 return Addr.withElementType(ConvertTypeForMem(EltType));
3644}
3645
3646/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
3647/// array to pointer, return the array subexpression.
3648static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
3649 // If this isn't just an array->pointer decay, bail out.
3650 const auto *CE = dyn_cast<CastExpr>(E);
3651 if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
3652 return nullptr;
3653
3654 // If this is a decay from variable width array, bail out.
3655 const Expr *SubExpr = CE->getSubExpr();
3656 if (SubExpr->getType()->isVariableArrayType())
3657 return nullptr;
3658
3659 return SubExpr;
3660}
3661
3663 llvm::Type *elemType,
3664 llvm::Value *ptr,
3665 ArrayRef<llvm::Value*> indices,
3666 bool inbounds,
3667 bool signedIndices,
3668 SourceLocation loc,
3669 const llvm::Twine &name = "arrayidx") {
3670 if (inbounds) {
3671 return CGF.EmitCheckedInBoundsGEP(elemType, ptr, indices, signedIndices,
3673 name);
3674 } else {
3675 return CGF.Builder.CreateGEP(elemType, ptr, indices, name);
3676 }
3677}
3678
3680 llvm::Value *idx,
3681 CharUnits eltSize) {
3682 // If we have a constant index, we can use the exact offset of the
3683 // element we're accessing.
3684 if (auto constantIdx = dyn_cast<llvm::ConstantInt>(idx)) {
3685 CharUnits offset = constantIdx->getZExtValue() * eltSize;
3686 return arrayAlign.alignmentAtOffset(offset);
3687
3688 // Otherwise, use the worst-case alignment for any element.
3689 } else {
3690 return arrayAlign.alignmentOfArrayElement(eltSize);
3691 }
3692}
3693
3695 const VariableArrayType *vla) {
3696 QualType eltType;
3697 do {
3698 eltType = vla->getElementType();
3699 } while ((vla = ctx.getAsVariableArrayType(eltType)));
3700 return eltType;
3701}
3702
3703/// Given an array base, check whether its member access belongs to a record
3704/// with preserve_access_index attribute or not.
3705static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
3706 if (!ArrayBase || !CGF.getDebugInfo())
3707 return false;
3708
3709 // Only support base as either a MemberExpr or DeclRefExpr.
3710 // DeclRefExpr to cover cases like:
3711 // struct s { int a; int b[10]; };
3712 // struct s *p;
3713 // p[1].a
3714 // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.
3715 // p->b[5] is a MemberExpr example.
3716 const Expr *E = ArrayBase->IgnoreImpCasts();
3717 if (const auto *ME = dyn_cast<MemberExpr>(E))
3718 return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
3719
3720 if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) {
3721 const auto *VarDef = dyn_cast<VarDecl>(DRE->getDecl());
3722 if (!VarDef)
3723 return false;
3724
3725 const auto *PtrT = VarDef->getType()->getAs<PointerType>();
3726 if (!PtrT)
3727 return false;
3728
3729 const auto *PointeeT = PtrT->getPointeeType()
3731 if (const auto *RecT = dyn_cast<RecordType>(PointeeT))
3732 return RecT->getDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
3733 return false;
3734 }
3735
3736 return false;
3737}
3738
3741 QualType eltType, bool inbounds,
3742 bool signedIndices, SourceLocation loc,
3743 QualType *arrayType = nullptr,
3744 const Expr *Base = nullptr,
3745 const llvm::Twine &name = "arrayidx") {
3746 // All the indices except that last must be zero.
3747#ifndef NDEBUG
3748 for (auto *idx : indices.drop_back())
3749 assert(isa<llvm::ConstantInt>(idx) &&
3750 cast<llvm::ConstantInt>(idx)->isZero());
3751#endif
3752
3753 // Determine the element size of the statically-sized base. This is
3754 // the thing that the indices are expressed in terms of.
3755 if (auto vla = CGF.getContext().getAsVariableArrayType(eltType)) {
3756 eltType = getFixedSizeElementType(CGF.getContext(), vla);
3757 }
3758
3759 // We can use that to compute the best alignment of the element.
3760 CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType);
3761 CharUnits eltAlign =
3762 getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize);
3763
3764 llvm::Value *eltPtr;
3765 auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back());
3766 if (!LastIndex ||
3768 eltPtr = emitArraySubscriptGEP(
3769 CGF, addr.getElementType(), addr.getPointer(), indices, inbounds,
3770 signedIndices, loc, name);
3771 } else {
3772 // Remember the original array subscript for bpf target
3773 unsigned idx = LastIndex->getZExtValue();
3774 llvm::DIType *DbgInfo = nullptr;
3775 if (arrayType)
3776 DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc);
3777 eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(),
3778 addr.getPointer(),
3779 indices.size() - 1,
3780 idx, DbgInfo);
3781 }
3782
3783 return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign);
3784}
3785
3787 bool Accessed) {
3788 // The index must always be an integer, which is not an aggregate. Emit it
3789 // in lexical order (this complexity is, sadly, required by C++17).
3790 llvm::Value *IdxPre =
3791 (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E->getIdx()) : nullptr;
3792 bool SignedIndices = false;
3793 auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {
3794 auto *Idx = IdxPre;
3795 if (E->getLHS() != E->getIdx()) {
3796 assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");
3797 Idx = EmitScalarExpr(E->getIdx());
3798 }
3799
3800 QualType IdxTy = E->getIdx()->getType();
3801 bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
3802 SignedIndices |= IdxSigned;
3803
3804 if (SanOpts.has(SanitizerKind::ArrayBounds))
3805 EmitBoundsCheck(E, E->getBase(), Idx, IdxTy, Accessed);
3806
3807 // Extend or truncate the index type to 32 or 64-bits.
3808 if (Promote && Idx->getType() != IntPtrTy)
3809 Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");
3810
3811 return Idx;
3812 };
3813 IdxPre = nullptr;
3814
3815 // If the base is a vector type, then we are forming a vector element lvalue
3816 // with this subscript.
3817 if (E->getBase()->getType()->isVectorType() &&
3818 !isa<ExtVectorElementExpr>(E->getBase())) {
3819 // Emit the vector as an lvalue to get its address.
3820 LValue LHS = EmitLValue(E->getBase());
3821 auto *Idx = EmitIdxAfterBase(/*Promote*/false);
3822 assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
3823 return LValue::MakeVectorElt(LHS.getAddress(*this), Idx,
3824 E->getBase()->getType(), LHS.getBaseInfo(),
3825 TBAAAccessInfo());
3826 }
3827
3828 // All the other cases basically behave like simple offsetting.
3829
3830 // Handle the extvector case we ignored above.
3831 if (isa<ExtVectorElementExpr>(E->getBase())) {
3832 LValue LV = EmitLValue(E->getBase());
3833 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3835
3836 QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
3837 Addr = emitArraySubscriptGEP(*this, Addr, Idx, EltType, /*inbounds*/ true,
3838 SignedIndices, E->getExprLoc());
3839 return MakeAddrLValue(Addr, EltType, LV.getBaseInfo(),
3840 CGM.getTBAAInfoForSubobject(LV, EltType));
3841 }
3842
3843 LValueBaseInfo EltBaseInfo;
3844 TBAAAccessInfo EltTBAAInfo;
3845 Address Addr = Address::invalid();
3846 if (const VariableArrayType *vla =
3847 getContext().getAsVariableArrayType(E->getType())) {
3848 // The base must be a pointer, which is not an aggregate. Emit
3849 // it. It needs to be emitted first in case it's what captures
3850 // the VLA bounds.
3851 Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3852 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3853
3854 // The element count here is the total number of non-VLA elements.
3855 llvm::Value *numElements = getVLASize(vla).NumElts;
3856
3857 // Effectively, the multiply by the VLA size is part of the GEP.
3858 // GEP indexes are signed, and scaling an index isn't permitted to
3859 // signed-overflow, so we use the same semantics for our explicit
3860 // multiply. We suppress this if overflow is not undefined behavior.
3861 if (getLangOpts().isSignedOverflowDefined()) {
3862 Idx = Builder.CreateMul(Idx, numElements);
3863 } else {
3864 Idx = Builder.CreateNSWMul(Idx, numElements);
3865 }
3866
3867 Addr = emitArraySubscriptGEP(*this, Addr, Idx, vla->getElementType(),
3868 !getLangOpts().isSignedOverflowDefined(),
3869 SignedIndices, E->getExprLoc());
3870
3871 } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
3872 // Indexing over an interface, as in "NSString *P; P[4];"
3873
3874 // Emit the base pointer.
3875 Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3876 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3877
3878 CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT);
3879 llvm::Value *InterfaceSizeVal =
3880 llvm::ConstantInt::get(Idx->getType(), InterfaceSize.getQuantity());
3881
3882 llvm::Value *ScaledIdx = Builder.CreateMul(Idx, InterfaceSizeVal);
3883
3884 // We don't necessarily build correct LLVM struct types for ObjC
3885 // interfaces, so we can't rely on GEP to do this scaling
3886 // correctly, so we need to cast to i8*. FIXME: is this actually
3887 // true? A lot of other things in the fragile ABI would break...
3888 llvm::Type *OrigBaseElemTy = Addr.getElementType();
3889
3890 // Do the GEP.
3891 CharUnits EltAlign =
3892 getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize);
3893 llvm::Value *EltPtr =
3894 emitArraySubscriptGEP(*this, Int8Ty, Addr.getPointer(), ScaledIdx,
3895 false, SignedIndices, E->getExprLoc());
3896 Addr = Address(EltPtr, OrigBaseElemTy, EltAlign);
3897 } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
3898 // If this is A[i] where A is an array, the frontend will have decayed the
3899 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
3900 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
3901 // "gep x, i" here. Emit one "gep A, 0, i".
3902 assert(Array->getType()->isArrayType() &&
3903 "Array to pointer decay must have array source type!");
3904 LValue ArrayLV;
3905 // For simple multidimensional array indexing, set the 'accessed' flag for
3906 // better bounds-checking of the base expression.
3907 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
3908 ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
3909 else
3910 ArrayLV = EmitLValue(Array);
3911 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3912
3913 // Propagate the alignment from the array itself to the result.
3914 QualType arrayType = Array->getType();
3915 Addr = emitArraySubscriptGEP(
3916 *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx},
3917 E->getType(), !getLangOpts().isSignedOverflowDefined(), SignedIndices,
3918 E->getExprLoc(), &arrayType, E->getBase());
3919 EltBaseInfo = ArrayLV.getBaseInfo();
3920 EltTBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, E->getType());
3921 } else {
3922 // The base must be a pointer; emit it with an estimate of its alignment.
3923 Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3924 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3925 QualType ptrType = E->getBase()->getType();
3926 Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(),
3927 !getLangOpts().isSignedOverflowDefined(),
3928 SignedIndices, E->getExprLoc(), &ptrType,
3929 E->getBase());
3930 }
3931
3932 LValue LV = MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);
3933
3934 if (getLangOpts().ObjC &&
3935 getLangOpts().getGC() != LangOptions::NonGC) {
3938 }
3939 return LV;
3940}
3941
3943 assert(
3944 !E->isIncomplete() &&
3945 "incomplete matrix subscript expressions should be rejected during Sema");
3946 LValue Base = EmitLValue(E->getBase());
3947 llvm::Value *RowIdx = EmitScalarExpr(E->getRowIdx());
3948 llvm::Value *ColIdx = EmitScalarExpr(E->getColumnIdx());
3949 llvm::Value *NumRows = Builder.getIntN(
3950 RowIdx->getType()->getScalarSizeInBits(),
3952 llvm::Value *FinalIdx =
3953 Builder.CreateAdd(Builder.CreateMul(ColIdx, NumRows), RowIdx);
3954 return LValue::MakeMatrixElt(
3955 MaybeConvertMatrixAddress(Base.getAddress(*this), *this), FinalIdx,
3956 E->getBase()->getType(), Base.getBaseInfo(), TBAAAccessInfo());
3957}
3958
3960 LValueBaseInfo &BaseInfo,
3961 TBAAAccessInfo &TBAAInfo,
3962 QualType BaseTy, QualType ElTy,
3963 bool IsLowerBound) {
3964 LValue BaseLVal;
3965 if (auto *ASE = dyn_cast<OMPArraySectionExpr>(Base->IgnoreParenImpCasts())) {
3966 BaseLVal = CGF.EmitOMPArraySectionExpr(ASE, IsLowerBound);
3967 if (BaseTy->isArrayType()) {
3968 Address Addr = BaseLVal.getAddress(CGF);
3969 BaseInfo = BaseLVal.getBaseInfo();
3970
3971 // If the array type was an incomplete type, we need to make sure
3972 // the decay ends up being the right type.
3973 llvm::Type *NewTy = CGF.ConvertType(BaseTy);
3974 Addr = Addr.withElementType(NewTy);
3975
3976 // Note that VLA pointers are always decayed, so we don't need to do
3977 // anything here.
3978 if (!BaseTy->isVariableArrayType()) {
3979 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
3980 "Expected pointer to array");
3981 Addr = CGF.Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");
3982 }
3983
3984 return Addr.withElementType(CGF.ConvertTypeForMem(ElTy));
3985 }
3986 LValueBaseInfo TypeBaseInfo;
3987 TBAAAccessInfo TypeTBAAInfo;
3988 CharUnits Align =
3989 CGF.CGM.getNaturalTypeAlignment(ElTy, &TypeBaseInfo, &TypeTBAAInfo);
3990 BaseInfo.mergeForCast(TypeBaseInfo);
3991 TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(TBAAInfo, TypeTBAAInfo);
3992 return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)),
3993 CGF.ConvertTypeForMem(ElTy), Align);
3994 }
3995 return CGF.EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo);
3996}
3997
3999 bool IsLowerBound) {
4001 QualType ResultExprTy;
4002 if (auto *AT = getContext().getAsArrayType(BaseTy))
4003 ResultExprTy = AT->getElementType();
4004 else
4005 ResultExprTy = BaseTy->getPointeeType();
4006 llvm::Value *Idx = nullptr;
4007 if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
4008 // Requesting lower bound or upper bound, but without provided length and
4009 // without ':' symbol for the default length -> length = 1.
4010 // Idx = LowerBound ?: 0;
4011 if (auto *LowerBound = E->getLowerBound()) {
4012 Idx = Builder.CreateIntCast(
4013 EmitScalarExpr(LowerBound), IntPtrTy,
4014 LowerBound->getType()->hasSignedIntegerRepresentation());
4015 } else
4016 Idx = llvm::ConstantInt::getNullValue(IntPtrTy);
4017 } else {
4018 // Try to emit length or lower bound as constant. If this is possible, 1
4019 // is subtracted from constant length or lower bound. Otherwise, emit LLVM
4020 // IR (LB + Len) - 1.
4021 auto &C = CGM.getContext();
4022 auto *Length = E->getLength();
4023 llvm::APSInt ConstLength;
4024 if (Length) {
4025 // Idx = LowerBound + Length - 1;
4026 if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(C)) {
4027 ConstLength = CL->zextOrTrunc(PointerWidthInBits);
4028 Length = nullptr;
4029 }
4030 auto *LowerBound = E->getLowerBound();
4031 llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
4032 if (LowerBound) {
4033 if (std::optional<llvm::APSInt> LB =
4034 LowerBound->getIntegerConstantExpr(C)) {
4035 ConstLowerBound = LB->zextOrTrunc(PointerWidthInBits);
4036 LowerBound = nullptr;
4037 }
4038 }
4039 if (!Length)
4040 --ConstLength;
4041 else if (!LowerBound)
4042 --ConstLowerBound;
4043
4044 if (Length || LowerBound) {
4045 auto *LowerBoundVal =
4046 LowerBound
4047 ? Builder.CreateIntCast(
4048 EmitScalarExpr(LowerBound), IntPtrTy,
4049 LowerBound->getType()->hasSignedIntegerRepresentation())
4050 : llvm::ConstantInt::get(IntPtrTy, ConstLowerBound);
4051 auto *LengthVal =
4052 Length
4053 ? Builder.CreateIntCast(
4054 EmitScalarExpr(Length), IntPtrTy,
4055 Length->getType()->hasSignedIntegerRepresentation())
4056 : llvm::ConstantInt::get(IntPtrTy, ConstLength);
4057 Idx = Builder.CreateAdd(LowerBoundVal, LengthVal, "lb_add_len",
4058 /*HasNUW=*/false,
4059 !getLangOpts().isSignedOverflowDefined());
4060 if (Length && LowerBound) {
4061 Idx = Builder.CreateSub(
4062 Idx, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "idx_sub_1",
4063 /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
4064 }
4065 } else
4066 Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength + ConstLowerBound);
4067 } else {
4068 // Idx = ArraySize - 1;
4069 QualType ArrayTy = BaseTy->isPointerType()
4071 : BaseTy;
4072 if (auto *VAT = C.getAsVariableArrayType(ArrayTy)) {
4073 Length = VAT->getSizeExpr();
4074 if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(C)) {
4075 ConstLength = *L;
4076 Length = nullptr;
4077 }
4078 } else {
4079 auto *CAT = C.getAsConstantArrayType(ArrayTy);
4080 assert(CAT && "unexpected type for array initializer");
4081 ConstLength = CAT->getSize();
4082 }
4083 if (Length) {
4084 auto *LengthVal = Builder.CreateIntCast(
4085 EmitScalarExpr(Length), IntPtrTy,
4086 Length->getType()->hasSignedIntegerRepresentation());
4087 Idx = Builder.CreateSub(
4088 LengthVal, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "len_sub_1",
4089 /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
4090 } else {
4091 ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits);
4092 --ConstLength;
4093 Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength);
4094 }
4095 }
4096 }
4097 assert(Idx);
4098
4099 Address EltPtr = Address::invalid();
4100 LValueBaseInfo BaseInfo;
4101 TBAAAccessInfo TBAAInfo;
4102 if (auto *VLA = getContext().getAsVariableArrayType(ResultExprTy)) {
4103 // The base must be a pointer, which is not an aggregate. Emit
4104 // it. It needs to be emitted first in case it's what captures
4105 // the VLA bounds.
4106 Address Base =
4107 emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo,
4108 BaseTy, VLA->getElementType(), IsLowerBound);
4109 // The element count here is the total number of non-VLA elements.
4110 llvm::Value *NumElements = getVLASize(VLA).NumElts;
4111
4112 // Effectively, the multiply by the VLA size is part of the GEP.
4113 // GEP indexes are signed, and scaling an index isn't permitted to
4114 // signed-overflow, so we use the same semantics for our explicit
4115 // multiply. We suppress this if overflow is not undefined behavior.
4116 if (getLangOpts().isSignedOverflowDefined())
4117 Idx = Builder.CreateMul(Idx, NumElements);
4118 else
4119 Idx = Builder.CreateNSWMul(Idx, NumElements);
4120 EltPtr = emitArraySubscriptGEP(*this, Base, Idx, VLA->getElementType(),
4121 !getLangOpts().isSignedOverflowDefined(),
4122 /*signedIndices=*/false, E->getExprLoc());
4123 } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
4124 // If this is A[i] where A is an array, the frontend will have decayed the
4125 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
4126 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
4127 // "gep x, i" here. Emit one "gep A, 0, i".
4128 assert(Array->getType()->isArrayType() &&
4129 "Array to pointer decay must have array source type!");
4130 LValue ArrayLV;
4131 // For simple multidimensional array indexing, set the 'accessed' flag for
4132 // better bounds-checking of the base expression.
4133 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
4134 ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
4135 else
4136 ArrayLV = EmitLValue(Array);
4137
4138 // Propagate the alignment from the array itself to the result.
4139 EltPtr = emitArraySubscriptGEP(
4140 *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx},
4141 ResultExprTy, !getLangOpts().isSignedOverflowDefined(),
4142 /*signedIndices=*/false, E->getExprLoc());
4143 BaseInfo = ArrayLV.getBaseInfo();
4144 TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy);
4145 } else {
4146 Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo,
4147 TBAAInfo, BaseTy, ResultExprTy,
4148 IsLowerBound);
4149 EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy,
4150 !getLangOpts().isSignedOverflowDefined(),
4151 /*signedIndices=*/false, E->getExprLoc());
4152 }
4153
4154 return MakeAddrLValue(EltPtr, ResultExprTy, BaseInfo, TBAAInfo);
4155}
4156
4159 // Emit the base vector as an l-value.
4160 LValue Base;
4161
4162 // ExtVectorElementExpr's base can either be a vector or pointer to vector.
4163 if (E->isArrow()) {
4164 // If it is a pointer to a vector, emit the address and form an lvalue with
4165 // it.
4166 LValueBaseInfo BaseInfo;
4167 TBAAAccessInfo TBAAInfo;
4168 Address Ptr = EmitPointerWithAlignment(E->getBase(), &BaseInfo, &TBAAInfo);
4169 const auto *PT = E->getBase()->getType()->castAs<PointerType>();
4170 Base = MakeAddrLValue(Ptr, PT->getPointeeType(), BaseInfo, TBAAInfo);
4171 Base.getQuals().removeObjCGCAttr();
4172 } else if (E->getBase()->isGLValue()) {
4173 // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
4174 // emit the base as an lvalue.
4175 assert(E->getBase()->getType()->isVectorType());
4176 Base = EmitLValue(E->getBase());
4177 } else {
4178 // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
4179 assert(E->getBase()->getType()->isVectorType() &&
4180 "Result must be a vector");
4181 llvm::Value *Vec = EmitScalarExpr(E->getBase());
4182
4183 // Store the vector to memory (because LValue wants an address).
4184 Address VecMem = CreateMemTemp(E->getBase()->getType());
4185 Builder.CreateStore(Vec, VecMem);
4186 Base = MakeAddrLValue(VecMem, E->getBase()->getType(),
4188 }
4189
4190 QualType type =
4191 E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
4192
4193 // Encode the element access list into a vector of unsigned indices.
4195 E->getEncodedElementAccess(Indices);
4196
4197 if (Base.isSimple()) {
4198 llvm::Constant *CV =
4199 llvm::ConstantDataVector::get(getLLVMContext(), Indices);
4200 return LValue::MakeExtVectorElt(Base.getAddress(*this), CV, type,
4201 Base.getBaseInfo(), TBAAAccessInfo());
4202 }
4203 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
4204
4205 llvm::Constant *BaseElts = Base.getExtVectorElts();
4207
4208 for (unsigned i = 0, e = Indices.size(); i != e; ++i)
4209 CElts.push_back(BaseElts->getAggregateElement(Indices[i]));
4210 llvm::Constant *CV = llvm::ConstantVector::get(CElts);
4211 return LValue::MakeExtVectorElt(Base.getExtVectorAddress(), CV, type,
4212 Base.getBaseInfo(), TBAAAccessInfo());
4213}
4214
4216 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, E)) {
4218 return EmitDeclRefLValue(DRE);
4219 }
4220
4221 Expr *BaseExpr = E->getBase();
4222 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
4223 LValue BaseLV;
4224 if (E->isArrow()) {
4225 LValueBaseInfo BaseInfo;
4226 TBAAAccessInfo TBAAInfo;
4227 Address Addr = EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo);
4228 QualType PtrTy = BaseExpr->getType()->getPointeeType();
4229 SanitizerSet SkippedChecks;
4230 bool IsBaseCXXThis = IsWrappedCXXThis(BaseExpr);
4231 if (IsBaseCXXThis)
4232 SkippedChecks.set(SanitizerKind::Alignment, true);
4233 if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr))
4234 SkippedChecks.set(SanitizerKind::Null, true);
4236 /*Alignment=*/CharUnits::Zero(), SkippedChecks);
4237 BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);
4238 } else
4239 BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess);
4240
4241 NamedDecl *ND = E->getMemberDecl();
4242 if (auto *Field = dyn_cast<FieldDecl>(ND)) {
4243 LValue LV = EmitLValueForField(BaseLV, Field);
4245 if (getLangOpts().OpenMP) {
4246 // If the member was explicitly marked as nontemporal, mark it as
4247 // nontemporal. If the base lvalue is marked as nontemporal, mark access
4248 // to children as nontemporal too.
4249 if ((IsWrappedCXXThis(BaseExpr) &&
4251 BaseLV.isNontemporal())
4252 LV.setNontemporal(/*Value=*/true);
4253 }
4254 return LV;
4255 }
4256
4257 if (const auto *FD = dyn_cast<FunctionDecl>(ND))
4258 return EmitFunctionDeclLValue(*this, E, FD);
4259
4260 llvm_unreachable("Unhandled member declaration!");
4261}
4262
4263/// Given that we are currently emitting a lambda, emit an l-value for
4264/// one of its members.
4266 if (CurCodeDecl) {
4267 assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent()->isLambda());
4268 assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent() == Field->getParent());
4269 }
4270 QualType LambdaTagType =
4271 getContext().getTagDeclType(Field->getParent());
4272 LValue LambdaLV = MakeNaturalAlignAddrLValue(CXXABIThisValue, LambdaTagType);
4273 return EmitLValueForField(LambdaLV, Field);
4274}
4275
4276/// Get the field index in the debug info. The debug info structure/union
4277/// will ignore the unnamed bitfields.
4279 unsigned FieldIndex) {
4280 unsigned I = 0, Skipped = 0;
4281
4282 for (auto *F : Rec->getDefinition()->fields()) {
4283 if (I == FieldIndex)
4284 break;
4285 if (F->isUnnamedBitfield())
4286 Skipped++;
4287 I++;
4288 }
4289
4290 return FieldIndex - Skipped;
4291}
4292
4293/// Get the address of a zero-sized field within a record. The resulting
4294/// address doesn't necessarily have the right type.
4296 const FieldDecl *Field) {
4298 CGF.getContext().getFieldOffset(Field));
4299 if (Offset.isZero())
4300 return Base;
4301 Base = Base.withElementType(CGF.Int8Ty);
4302 return CGF.Builder.CreateConstInBoundsByteGEP(Base, Offset);
4303}
4304
4305/// Drill down to the storage of a field without walking into
4306/// reference types.
4307///
4308/// The resulting address doesn't necessarily have the right type.
4310 const FieldDecl *field) {
4311 if (field->isZeroSize(CGF.getContext()))
4312 return emitAddrOfZeroSizeField(CGF, base, field);
4313
4314 const RecordDecl *rec = field->getParent();
4315
4316 unsigned idx =
4317 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
4318
4319 return CGF.Builder.CreateStructGEP(base, idx, field->getName());
4320}
4321
4323 Address addr, const FieldDecl *field) {
4324 const RecordDecl *rec = field->getParent();
4325 llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(
4326 base.getType(), rec->getLocation());
4327
4328 unsigned idx =
4329 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
4330
4332 addr, idx, CGF.getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo);
4333}
4334
4335static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {
4336 const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();
4337 if (!RD)
4338 return false;
4339
4340 if (RD->isDynamicClass())
4341 return true;
4342
4343 for (const auto &Base : RD->bases())
4344 if (hasAnyVptr(Base.getType(), Context))
4345 return true;
4346
4347 for (const FieldDecl *Field : RD->fields())
4348 if (hasAnyVptr(Field->getType(), Context))
4349 return true;
4350
4351 return false;
4352}
4353
4355 const FieldDecl *field) {
4356 LValueBaseInfo BaseInfo = base.getBaseInfo();
4357
4358 if (field->isBitField()) {
4359 const CGRecordLayout &RL =
4361 const CGBitFieldInfo &Info = RL.getBitFieldInfo(field);
4362 const bool UseVolatile = isAAPCS(CGM.getTarget()) &&
4363 CGM.getCodeGenOpts().AAPCSBitfieldWidth &&
4364 Info.VolatileStorageSize != 0 &&
4365 field->getType()
4368 Address Addr = base.getAddress(*this);
4369 unsigned Idx = RL.getLLVMFieldNo(field);
4370 const RecordDecl *rec = field->getParent();
4371 if (!UseVolatile) {
4372 if (!IsInPreservedAIRegion &&
4373 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4374 if (Idx != 0)
4375 // For structs, we GEP to the field that the record layout suggests.
4376 Addr = Builder.CreateStructGEP(Addr, Idx, field->getName());
4377 } else {
4378 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
4379 getContext().getRecordType(rec), rec->getLocation());
4381 Addr, Idx, getDebugInfoFIndex(rec, field->getFieldIndex()),
4382 DbgInfo);
4383 }
4384 }
4385 const unsigned SS =
4386 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
4387 // Get the access type.
4388 llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), SS);
4389 Addr = Addr.withElementType(FieldIntTy);
4390 if (UseVolatile) {
4391 const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();
4392 if (VolatileOffset)
4393 Addr = Builder.CreateConstInBoundsGEP(Addr, VolatileOffset);
4394 }
4395
4396 QualType fieldType =
4397 field->getType().withCVRQualifiers(base.getVRQualifiers());
4398 // TODO: Support TBAA for bit fields.
4399 LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());
4400 return LValue::MakeBitfield(Addr, Info, fieldType, FieldBaseInfo,
4401 TBAAAccessInfo());
4402 }
4403
4404 // Fields of may-alias structures are may-alias themselves.
4405 // FIXME: this should get propagated down through anonymous structs
4406 // and unions.
4407 QualType FieldType = field->getType();
4408 const RecordDecl *rec = field->getParent();
4409 AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
4410 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(BaseAlignSource));
4411 TBAAAccessInfo FieldTBAAInfo;
4412 if (base.getTBAAInfo().isMayAlias() ||
4413 rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
4414 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4415 } else if (rec->isUnion()) {
4416 // TODO: Support TBAA for unions.
4417 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4418 } else {
4419 // If no base type been assigned for the base access, then try to generate
4420 // one for this base lvalue.
4421 FieldTBAAInfo = base.getTBAAInfo();
4422 if (!FieldTBAAInfo.BaseType) {
4423 FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(base.getType());
4424 assert(!FieldTBAAInfo.Offset &&
4425 "Nonzero offset for an access with no base type!");
4426 }
4427
4428 // Adjust offset to be relative to the base type.
4429 const ASTRecordLayout &Layout =
4431 unsigned CharWidth = getContext().getCharWidth();
4432 if (FieldTBAAInfo.BaseType)
4433 FieldTBAAInfo.Offset +=
4434 Layout.getFieldOffset(field->getFieldIndex()) / CharWidth;
4435
4436 // Update the final access type and size.
4437 FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(FieldType);
4438 FieldTBAAInfo.Size =
4440 }
4441
4442 Address addr = base.getAddress(*this);
4443 if (auto *ClassDef = dyn_cast<CXXRecordDecl>(rec)) {
4444 if (CGM.getCodeGenOpts().StrictVTablePointers &&
4445 ClassDef->isDynamicClass()) {
4446 // Getting to any field of dynamic object requires stripping dynamic
4447 // information provided by invariant.group. This is because accessing
4448 // fields may leak the real address of dynamic object, which could result
4449 // in miscompilation when leaked pointer would be compared.
4450 auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer());
4451 addr = Address(stripped, addr.getElementType(), addr.getAlignment());
4452 }
4453 }
4454
4455 unsigned RecordCVR = base.getVRQualifiers();
4456 if (rec->isUnion()) {
4457 // For unions, there is no pointer adjustment.
4458 if (CGM.getCodeGenOpts().StrictVTablePointers &&
4459 hasAnyVptr(FieldType, getContext()))
4460 // Because unions can easily skip invariant.barriers, we need to add
4461 // a barrier every time CXXRecord field with vptr is referenced.
4463
4465 (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4466 // Remember the original union field index
4467 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(),
4468 rec->getLocation());
4469 addr = Address(
4470 Builder.CreatePreserveUnionAccessIndex(
4471 addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo),
4472 addr.getElementType(), addr.getAlignment());
4473 }
4474
4475 if (FieldType->isReferenceType())
4476 addr = addr.withElementType(CGM.getTypes().ConvertTypeForMem(FieldType));
4477 } else {
4478 if (!IsInPreservedAIRegion &&
4479 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))
4480 // For structs, we GEP to the field that the record layout suggests.
4481 addr = emitAddrOfFieldStorage(*this, addr, field);
4482 else
4483 // Remember the original struct field index
4484 addr = emitPreserveStructAccess(*this, base, addr, field);
4485 }
4486
4487 // If this is a reference field, load the reference right now.
4488 if (FieldType->isReferenceType()) {
4489 LValue RefLVal =
4490 MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
4491 if (RecordCVR & Qualifiers::Volatile)
4492 RefLVal.getQuals().addVolatile();
4493 addr = EmitLoadOfReference(RefLVal, &FieldBaseInfo, &FieldTBAAInfo);
4494
4495 // Qualifiers on the struct don't apply to the referencee.
4496 RecordCVR = 0;
4497 FieldType = FieldType->getPointeeType();
4498 }
4499
4500 // Make sure that the address is pointing to the right type. This is critical
4501 // for both unions and structs.
4502 addr = addr.withElementType(CGM.getTypes().ConvertTypeForMem(FieldType));
4503
4504 if (field->hasAttr<AnnotateAttr>())
4505 addr = EmitFieldAnnotations(field, addr);
4506
4507 LValue LV = MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
4508 LV.getQuals().addCVRQualifiers(RecordCVR);
4509
4510 // __weak attribute on a field is ignored.
4513
4514 return LV;
4515}
4516
4517LValue
4519 const FieldDecl *Field) {
4520 QualType FieldType = Field->getType();
4521
4522 if (!FieldType->isReferenceType())
4523 return EmitLValueForField(Base, Field);
4524
4525 Address V = emitAddrOfFieldStorage(*this, Base.getAddress(*this), Field);
4526
4527 // Make sure that the address is pointing to the right type.
4528 llvm::Type *llvmType = ConvertTypeForMem(FieldType);
4529 V = V.withElementType(llvmType);
4530
4531 // TODO: Generate TBAA information that describes this access as a structure
4532 // member access and not just an access to an object of the field's type. This
4533 // should be similar to what we do in EmitLValueForField().
4534 LValueBaseInfo BaseInfo = Base.getBaseInfo();
4535 AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource();
4536 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(FieldAlignSource));
4537 return MakeAddrLValue(V, FieldType, FieldBaseInfo,
4538 CGM.getTBAAInfoForSubobject(Base, FieldType));
4539}
4540
4542 if (E->isFileScope()) {
4544 return MakeAddrLValue(GlobalPtr, E->getType(), AlignmentSource::Decl);
4545 }