14#include "llvm/IR/BasicBlock.h"
15#include "llvm/IR/CFG.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/InstrTypes.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/Metadata.h"
26 LLVMContext &Ctx = Header->getContext();
28 NewLoopProperties.push_back(
nullptr);
29 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
31 MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
32 LoopID->replaceOperandWith(0, LoopID);
36MDNode *LoopInfo::createPipeliningMetadata(
const LoopAttributes &Attrs,
38 bool &HasUserTransforms) {
39 LLVMContext &Ctx = Header->getContext();
41 std::optional<bool> Enabled;
47 if (Enabled !=
true) {
49 if (Enabled ==
false) {
50 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
51 NewLoopProperties.push_back(
52 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.pipeline.disable"),
53 ConstantAsMetadata::get(ConstantInt::get(
54 llvm::Type::getInt1Ty(Ctx), 1))}));
55 LoopProperties = NewLoopProperties;
57 return createLoopPropertiesMetadata(LoopProperties);
61 Args.push_back(
nullptr);
62 Args.append(LoopProperties.begin(), LoopProperties.end());
66 MDString::get(Ctx,
"llvm.loop.pipeline.initiationinterval"),
67 ConstantAsMetadata::get(ConstantInt::get(
69 Args.push_back(MDNode::get(Ctx, Vals));
74 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
75 LoopID->replaceOperandWith(0, LoopID);
76 HasUserTransforms =
true;
83 bool &HasUserTransforms) {
84 LLVMContext &Ctx = Header->getContext();
86 std::optional<bool> Enabled;
90 Enabled = std::nullopt;
95 if (Enabled !=
true) {
98 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
104 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
107 FollowupLoopProperties.push_back(
108 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.disable")));
110 bool FollowupHasTransforms =
false;
111 MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
112 FollowupHasTransforms);
115 Args.push_back(
nullptr);
116 Args.append(LoopProperties.begin(), LoopProperties.end());
120 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll.count"),
121 ConstantAsMetadata::get(ConstantInt::get(
123 Args.push_back(MDNode::get(Ctx, Vals));
128 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll.enable")};
129 Args.push_back(MDNode::get(Ctx, Vals));
132 if (FollowupHasTransforms)
133 Args.push_back(MDNode::get(
134 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll.followup_all"), Followup}));
136 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137 LoopID->replaceOperandWith(0, LoopID);
138 HasUserTransforms =
true;
145 bool &HasUserTransforms) {
146 LLVMContext &Ctx = Header->getContext();
148 std::optional<bool> Enabled;
155 if (Enabled !=
true) {
157 if (Enabled ==
false) {
158 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159 NewLoopProperties.push_back(MDNode::get(
160 Ctx, MDString::get(Ctx,
"llvm.loop.unroll_and_jam.disable")));
161 LoopProperties = NewLoopProperties;
163 return createPartialUnrollMetadata(Attrs, LoopProperties,
168 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169 FollowupLoopProperties.push_back(
170 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll_and_jam.disable")));
172 bool FollowupHasTransforms =
false;
173 MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174 FollowupHasTransforms);
177 Args.push_back(
nullptr);
178 Args.append(LoopProperties.begin(), LoopProperties.end());
183 MDString::get(Ctx,
"llvm.loop.unroll_and_jam.count"),
184 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
186 Args.push_back(MDNode::get(Ctx, Vals));
190 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.enable")};
191 Args.push_back(MDNode::get(Ctx, Vals));
194 if (FollowupHasTransforms)
195 Args.push_back(MDNode::get(
196 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.followup_outer"),
199 if (UnrollAndJamInnerFollowup)
200 Args.push_back(MDNode::get(
201 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.followup_inner"),
202 UnrollAndJamInnerFollowup}));
204 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
205 LoopID->replaceOperandWith(0, LoopID);
206 HasUserTransforms =
true;
213 bool &HasUserTransforms) {
214 LLVMContext &Ctx = Header->getContext();
216 std::optional<bool> Enabled;
225 if (Enabled !=
true) {
227 if (Enabled ==
false) {
228 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
229 NewLoopProperties.push_back(
230 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.vectorize.enable"),
231 ConstantAsMetadata::get(ConstantInt::get(
232 llvm::Type::getInt1Ty(Ctx), 0))}));
233 LoopProperties = NewLoopProperties;
235 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
240 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
243 FollowupLoopProperties.push_back(
244 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.isvectorized")));
246 bool FollowupHasTransforms =
false;
247 MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
248 FollowupHasTransforms);
251 Args.push_back(
nullptr);
252 Args.append(LoopProperties.begin(), LoopProperties.end());
256 bool IsVectorPredicateEnabled =
false;
258 IsVectorPredicateEnabled =
262 MDString::get(Ctx,
"llvm.loop.vectorize.predicate.enable"),
263 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
264 IsVectorPredicateEnabled))};
265 Args.push_back(MDNode::get(Ctx, Vals));
271 MDString::get(Ctx,
"llvm.loop.vectorize.width"),
272 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
275 Args.push_back(MDNode::get(Ctx, Vals));
281 MDString::get(Ctx,
"llvm.loop.vectorize.scalable.enable"),
282 ConstantAsMetadata::get(
283 ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};
284 Args.push_back(MDNode::get(Ctx, Vals));
290 MDString::get(Ctx,
"llvm.loop.interleave.count"),
291 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
293 Args.push_back(MDNode::get(Ctx, Vals));
312 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.vectorize.enable"),
313 ConstantAsMetadata::get(ConstantInt::get(
314 llvm::Type::getInt1Ty(Ctx), AttrVal))}));
317 if (FollowupHasTransforms)
318 Args.push_back(MDNode::get(
320 {MDString::get(Ctx,
"llvm.loop.vectorize.followup_all"), Followup}));
322 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
323 LoopID->replaceOperandWith(0, LoopID);
324 HasUserTransforms =
true;
329LoopInfo::createLoopDistributeMetadata(
const LoopAttributes &Attrs,
331 bool &HasUserTransforms) {
332 LLVMContext &Ctx = Header->getContext();
334 std::optional<bool> Enabled;
340 if (Enabled !=
true) {
342 if (Enabled ==
false) {
343 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
344 NewLoopProperties.push_back(
345 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.distribute.enable"),
346 ConstantAsMetadata::get(ConstantInt::get(
347 llvm::Type::getInt1Ty(Ctx), 0))}));
348 LoopProperties = NewLoopProperties;
350 return createLoopVectorizeMetadata(Attrs, LoopProperties,
354 bool FollowupHasTransforms =
false;
356 createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
359 Args.push_back(
nullptr);
360 Args.append(LoopProperties.begin(), LoopProperties.end());
362 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.distribute.enable"),
363 ConstantAsMetadata::get(ConstantInt::get(
364 llvm::Type::getInt1Ty(Ctx),
366 Args.push_back(MDNode::get(Ctx, Vals));
368 if (FollowupHasTransforms)
369 Args.push_back(MDNode::get(
371 {MDString::get(Ctx,
"llvm.loop.distribute.followup_all"), Followup}));
373 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
374 LoopID->replaceOperandWith(0, LoopID);
375 HasUserTransforms =
true;
379MDNode *LoopInfo::createFullUnrollMetadata(
const LoopAttributes &Attrs,
381 bool &HasUserTransforms) {
382 LLVMContext &Ctx = Header->getContext();
384 std::optional<bool> Enabled;
390 if (Enabled !=
true) {
392 if (Enabled ==
false) {
393 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
394 NewLoopProperties.push_back(
395 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.disable")));
396 LoopProperties = NewLoopProperties;
398 return createLoopDistributeMetadata(Attrs, LoopProperties,
403 Args.push_back(
nullptr);
404 Args.append(LoopProperties.begin(), LoopProperties.end());
405 Args.push_back(MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.full")));
410 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
411 LoopID->replaceOperandWith(0, LoopID);
412 HasUserTransforms =
true;
416MDNode *LoopInfo::createMetadata(
419 bool &HasUserTransforms) {
424 LoopProperties.push_back(StartLoc.getAsMDNode());
428 LoopProperties.push_back(EndLoc.getAsMDNode());
431 LLVMContext &Ctx = Header->getContext();
433 LoopProperties.push_back(
434 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.mustprogress")));
437 "There must be an access group iff the loop is parallel");
439 LoopProperties.push_back(MDNode::get(
440 Ctx, {MDString::get(Ctx,
"llvm.loop.parallel_accesses"), AccGroup}));
445 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.align"),
446 ConstantAsMetadata::get(ConstantInt::get(
447 llvm::Type::getInt32Ty(Ctx), Attrs.
CodeAlign))};
448 LoopProperties.push_back(MDNode::get(Ctx, Vals));
451 LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
452 AdditionalLoopProperties.end());
453 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
462 UnrollCount(0), UnrollAndJamCount(0),
464 PipelineInitiationInterval(0), CodeAlign(0), MustProgress(
false) {}
485 const llvm::DebugLoc &StartLoc,
const llvm::DebugLoc &EndLoc,
487 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
492 LLVMContext &Ctx = Header->getContext();
493 AccGroup = MDNode::getDistinct(Ctx, {});
509 TempLoopID = MDNode::getTemporary(Header->getContext(), std::nullopt);
520 LLVMContext &Ctx = Header->getContext();
568 if (!Parent->UnrollAndJamInnerFollowup) {
578 BeforeLoopProperties.push_back(
579 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.isvectorized")));
581 bool InnerFollowupHasTransform =
false;
582 MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
583 InnerFollowupHasTransform);
584 if (InnerFollowupHasTransform)
585 Parent->UnrollAndJamInnerFollowup = InnerFollowup;
588 CurLoopAttr = BeforeJam;
591 bool HasUserTransforms =
false;
592 LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
593 TempLoopID->replaceAllUsesWith(LoopID);
597 const llvm::DebugLoc &EndLoc) {
599 new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
600 Active.empty() ?
nullptr : Active.back().get()));
608 const llvm::DebugLoc &StartLoc,
609 const llvm::DebugLoc &EndLoc,
bool MustProgress) {
611 for (
const auto *
Attr : Attrs) {
612 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(
Attr);
613 const OpenCLUnrollHintAttr *OpenCLHint =
614 dyn_cast<OpenCLUnrollHintAttr>(
Attr);
617 if (!LH && !OpenCLHint) {
621 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
622 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
623 unsigned ValueInt = 1;
631 ValueInt = OpenCLHint->getUnrollHint();
633 State = LoopHintAttr::Enable;
634 }
else if (ValueInt != 1) {
635 Option = LoopHintAttr::UnrollCount;
636 State = LoopHintAttr::Numeric;
639 auto *ValueExpr = LH->getValue();
641 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
642 ValueInt = ValueAPS.getSExtValue();
645 Option = LH->getOption();
646 State = LH->getState();
649 case LoopHintAttr::Disable:
651 case LoopHintAttr::Vectorize:
656 case LoopHintAttr::Interleave:
660 case LoopHintAttr::Unroll:
663 case LoopHintAttr::UnrollAndJam:
666 case LoopHintAttr::VectorizePredicate:
669 case LoopHintAttr::Distribute:
672 case LoopHintAttr::PipelineDisabled:
675 case LoopHintAttr::UnrollCount:
676 case LoopHintAttr::UnrollAndJamCount:
677 case LoopHintAttr::VectorizeWidth:
678 case LoopHintAttr::InterleaveCount:
679 case LoopHintAttr::PipelineInitiationInterval:
680 llvm_unreachable(
"Options cannot be disabled.");
684 case LoopHintAttr::Enable:
686 case LoopHintAttr::Vectorize:
687 case LoopHintAttr::Interleave:
690 case LoopHintAttr::Unroll:
693 case LoopHintAttr::UnrollAndJam:
696 case LoopHintAttr::VectorizePredicate:
699 case LoopHintAttr::Distribute:
702 case LoopHintAttr::UnrollCount:
703 case LoopHintAttr::UnrollAndJamCount:
704 case LoopHintAttr::VectorizeWidth:
705 case LoopHintAttr::InterleaveCount:
706 case LoopHintAttr::PipelineDisabled:
707 case LoopHintAttr::PipelineInitiationInterval:
708 llvm_unreachable(
"Options cannot enabled.");
712 case LoopHintAttr::AssumeSafety:
714 case LoopHintAttr::Vectorize:
715 case LoopHintAttr::Interleave:
720 case LoopHintAttr::Unroll:
721 case LoopHintAttr::UnrollAndJam:
722 case LoopHintAttr::VectorizePredicate:
723 case LoopHintAttr::UnrollCount:
724 case LoopHintAttr::UnrollAndJamCount:
725 case LoopHintAttr::VectorizeWidth:
726 case LoopHintAttr::InterleaveCount:
727 case LoopHintAttr::Distribute:
728 case LoopHintAttr::PipelineDisabled:
729 case LoopHintAttr::PipelineInitiationInterval:
730 llvm_unreachable(
"Options cannot be used to assume mem safety.");
734 case LoopHintAttr::Full:
736 case LoopHintAttr::Unroll:
739 case LoopHintAttr::UnrollAndJam:
742 case LoopHintAttr::Vectorize:
743 case LoopHintAttr::Interleave:
744 case LoopHintAttr::UnrollCount:
745 case LoopHintAttr::UnrollAndJamCount:
746 case LoopHintAttr::VectorizeWidth:
747 case LoopHintAttr::InterleaveCount:
748 case LoopHintAttr::Distribute:
749 case LoopHintAttr::PipelineDisabled:
750 case LoopHintAttr::PipelineInitiationInterval:
751 case LoopHintAttr::VectorizePredicate:
752 llvm_unreachable(
"Options cannot be used with 'full' hint.");
756 case LoopHintAttr::FixedWidth:
757 case LoopHintAttr::ScalableWidth:
759 case LoopHintAttr::VectorizeWidth:
767 llvm_unreachable(
"Options cannot be used with 'scalable' hint.");
771 case LoopHintAttr::Numeric:
773 case LoopHintAttr::InterleaveCount:
776 case LoopHintAttr::UnrollCount:
779 case LoopHintAttr::UnrollAndJamCount:
782 case LoopHintAttr::PipelineInitiationInterval:
785 case LoopHintAttr::Unroll:
786 case LoopHintAttr::UnrollAndJam:
787 case LoopHintAttr::VectorizePredicate:
788 case LoopHintAttr::Vectorize:
789 case LoopHintAttr::VectorizeWidth:
790 case LoopHintAttr::Interleave:
791 case LoopHintAttr::Distribute:
792 case LoopHintAttr::PipelineDisabled:
793 llvm_unreachable(
"Options cannot be assigned a value.");
803 if (
const auto *CodeAlign = getSpecificAttr<const CodeAlignAttr>(Attrs)) {
804 const auto *CE = cast<ConstantExpr>(CodeAlign->getAlignment());
805 llvm::APSInt ArgVal = CE->getResultAsAPSInt();
811 if (CGOpts.OptimizationLevel > 0)
814 if (!CGOpts.UnrollLoops &&
820 push(Header, StartLoc, EndLoc);
824 assert(!Active.empty() &&
"No active loops to pop");
825 Active.back()->finish();
830 if (I->mayReadOrWriteMemory()) {
832 for (
const auto &AL : Active) {
834 if (MDNode *Group = AL->getAccessGroup())
835 AccessGroups.push_back(Group);
837 MDNode *UnionMD =
nullptr;
838 if (AccessGroups.size() == 1)
839 UnionMD = cast<MDNode>(AccessGroups[0]);
840 else if (AccessGroups.size() >= 2)
841 UnionMD = MDNode::get(I->getContext(), AccessGroups);
842 I->setMetadata(
"llvm.access.group", UnionMD);
852 if (I->isTerminator()) {
853 for (BasicBlock *Succ : successors(I))
855 I->setMetadata(llvm::LLVMContext::MD_loop, L.
getLoopID());
Defines the clang::ASTContext interface.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Attr - This represents one attribute.
CodeGenOptions - Track various options which control how the code is optimized and passed to the back...
void setPipelineDisabled(bool S)
Set the pipeline disabled state.
void setUnrollCount(unsigned C)
Set the unroll count for the next loop pushed.
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
void InsertHelper(llvm::Instruction *I) const
Function called by the CodeGenFunction when an instruction is created.
void setDistributeState(bool Enable=true)
Set the next pushed loop as a distribution candidate.
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
void setInterleaveCount(unsigned C)
Set the interleave count for the next loop pushed.
void setUnrollState(const LoopAttributes::LVEnableState &State)
Set the next pushed loop unroll state.
void setVectorizeScalable(const LoopAttributes::LVEnableState &State)
void setVectorizePredicateState(const LoopAttributes::LVEnableState &State)
Set the next pushed vectorize predicate state.
void pop()
End the current loop.
void setCodeAlign(unsigned C)
Set value of code align for the next loop pushed.
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
void setMustProgress(bool P)
Set no progress for the next loop pushed.
void setUnrollAndJamState(const LoopAttributes::LVEnableState &State)
Set the next pushed loop unroll_and_jam state.
void setUnrollAndJamCount(unsigned C)
Set the unroll count for the next loop pushed.
void setPipelineInitiationInterval(unsigned C)
Set the pipeline initiation interval.
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
Information used when generating a structured loop.
llvm::BasicBlock * getHeader() const
Get the header block of this loop.
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, LoopInfo *Parent)
Construct a new LoopInfo for the loop with entry Header.
llvm::MDNode * getLoopID() const
Get the loop id metadata for this loop.
void finish()
Create the loop's metadata.
YAML serialization mapping.
Attributes that may be specified on loops.
unsigned UnrollCount
llvm.unroll.
bool MustProgress
Value for whether the loop is required to make progress.
unsigned InterleaveCount
Value for llvm.loop.interleave.count metadata.
LoopAttributes(bool IsParallel=false)
bool IsParallel
Generate llvm.loop.parallel metadata for loads and stores.
LVEnableState VectorizeScalable
LVEnableState UnrollAndJamEnable
Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full).
unsigned UnrollAndJamCount
llvm.unroll.
LVEnableState VectorizePredicateEnable
Value for llvm.loop.vectorize.predicate metadata.
LVEnableState DistributeEnable
Value for llvm.loop.distribute.enable metadata.
bool PipelineDisabled
Value for llvm.loop.pipeline.disable metadata.
unsigned CodeAlign
Value for 'llvm.loop.align' metadata.
LVEnableState UnrollEnable
Value for llvm.loop.unroll.* metadata (enable, disable, or full).
unsigned VectorizeWidth
Value for llvm.loop.vectorize.width metadata.
unsigned PipelineInitiationInterval
Value for llvm.loop.pipeline.iicount metadata.
LVEnableState VectorizeEnable
Value for llvm.loop.vectorize.enable metadata.