clang  14.0.0git
CGLoopInfo.cpp
Go to the documentation of this file.
1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "clang/AST/Expr.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/Constants.h"
17 #include "llvm/IR/InstrTypes.h"
18 #include "llvm/IR/Instructions.h"
19 #include "llvm/IR/Metadata.h"
20 using namespace clang::CodeGen;
21 using namespace llvm;
22 
23 MDNode *
24 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
25  LLVMContext &Ctx = Header->getContext();
26  SmallVector<Metadata *, 4> NewLoopProperties;
27  NewLoopProperties.push_back(nullptr);
28  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
29 
30  MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
31  LoopID->replaceOperandWith(0, LoopID);
32  return LoopID;
33 }
34 
35 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
36  ArrayRef<Metadata *> LoopProperties,
37  bool &HasUserTransforms) {
38  LLVMContext &Ctx = Header->getContext();
39 
40  Optional<bool> Enabled;
41  if (Attrs.PipelineDisabled)
42  Enabled = false;
43  else if (Attrs.PipelineInitiationInterval != 0)
44  Enabled = true;
45 
46  if (Enabled != true) {
47  SmallVector<Metadata *, 4> NewLoopProperties;
48  if (Enabled == false) {
49  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
50  NewLoopProperties.push_back(
51  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
52  ConstantAsMetadata::get(ConstantInt::get(
53  llvm::Type::getInt1Ty(Ctx), 1))}));
54  LoopProperties = NewLoopProperties;
55  }
56  return createLoopPropertiesMetadata(LoopProperties);
57  }
58 
60  Args.push_back(nullptr);
61  Args.append(LoopProperties.begin(), LoopProperties.end());
62 
63  if (Attrs.PipelineInitiationInterval > 0) {
64  Metadata *Vals[] = {
65  MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66  ConstantAsMetadata::get(ConstantInt::get(
67  llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
68  Args.push_back(MDNode::get(Ctx, Vals));
69  }
70 
71  // No follow-up: This is the last transformation.
72 
73  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74  LoopID->replaceOperandWith(0, LoopID);
75  HasUserTransforms = true;
76  return LoopID;
77 }
78 
79 MDNode *
80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81  ArrayRef<Metadata *> LoopProperties,
82  bool &HasUserTransforms) {
83  LLVMContext &Ctx = Header->getContext();
84 
85  Optional<bool> Enabled;
87  Enabled = false;
88  else if (Attrs.UnrollEnable == LoopAttributes::Full)
89  Enabled = None;
90  else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91  Attrs.UnrollCount != 0)
92  Enabled = true;
93 
94  if (Enabled != true) {
95  // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96  // if unrolling is disabled.
97  return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98  }
99 
100  SmallVector<Metadata *, 4> FollowupLoopProperties;
101 
102  // Apply all loop properties to the unrolled loop.
103  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104 
105  // Don't unroll an already unrolled loop.
106  FollowupLoopProperties.push_back(
107  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108 
109  bool FollowupHasTransforms = false;
110  MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111  FollowupHasTransforms);
112 
114  Args.push_back(nullptr);
115  Args.append(LoopProperties.begin(), LoopProperties.end());
116 
117  // Setting unroll.count
118  if (Attrs.UnrollCount > 0) {
119  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
120  ConstantAsMetadata::get(ConstantInt::get(
121  llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
122  Args.push_back(MDNode::get(Ctx, Vals));
123  }
124 
125  // Setting unroll.full or unroll.disable
126  if (Attrs.UnrollEnable == LoopAttributes::Enable) {
127  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
128  Args.push_back(MDNode::get(Ctx, Vals));
129  }
130 
131  if (FollowupHasTransforms)
132  Args.push_back(MDNode::get(
133  Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
134 
135  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
136  LoopID->replaceOperandWith(0, LoopID);
137  HasUserTransforms = true;
138  return LoopID;
139 }
140 
141 MDNode *
142 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
143  ArrayRef<Metadata *> LoopProperties,
144  bool &HasUserTransforms) {
145  LLVMContext &Ctx = Header->getContext();
146 
147  Optional<bool> Enabled;
149  Enabled = false;
150  else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
151  Attrs.UnrollAndJamCount != 0)
152  Enabled = true;
153 
154  if (Enabled != true) {
155  SmallVector<Metadata *, 4> NewLoopProperties;
156  if (Enabled == false) {
157  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
158  NewLoopProperties.push_back(MDNode::get(
159  Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
160  LoopProperties = NewLoopProperties;
161  }
162  return createPartialUnrollMetadata(Attrs, LoopProperties,
163  HasUserTransforms);
164  }
165 
166  SmallVector<Metadata *, 4> FollowupLoopProperties;
167  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
168  FollowupLoopProperties.push_back(
169  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
170 
171  bool FollowupHasTransforms = false;
172  MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
173  FollowupHasTransforms);
174 
176  Args.push_back(nullptr);
177  Args.append(LoopProperties.begin(), LoopProperties.end());
178 
179  // Setting unroll_and_jam.count
180  if (Attrs.UnrollAndJamCount > 0) {
181  Metadata *Vals[] = {
182  MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
183  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
184  Attrs.UnrollAndJamCount))};
185  Args.push_back(MDNode::get(Ctx, Vals));
186  }
187 
189  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
190  Args.push_back(MDNode::get(Ctx, Vals));
191  }
192 
193  if (FollowupHasTransforms)
194  Args.push_back(MDNode::get(
195  Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
196  Followup}));
197 
198  if (UnrollAndJamInnerFollowup)
199  Args.push_back(MDNode::get(
200  Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
201  UnrollAndJamInnerFollowup}));
202 
203  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
204  LoopID->replaceOperandWith(0, LoopID);
205  HasUserTransforms = true;
206  return LoopID;
207 }
208 
209 MDNode *
210 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
211  ArrayRef<Metadata *> LoopProperties,
212  bool &HasUserTransforms) {
213  LLVMContext &Ctx = Header->getContext();
214 
215  Optional<bool> Enabled;
217  Enabled = false;
218  else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
220  Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
222  Enabled = true;
223 
224  if (Enabled != true) {
225  SmallVector<Metadata *, 4> NewLoopProperties;
226  if (Enabled == false) {
227  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
228  NewLoopProperties.push_back(
229  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
230  ConstantAsMetadata::get(ConstantInt::get(
231  llvm::Type::getInt1Ty(Ctx), 0))}));
232  LoopProperties = NewLoopProperties;
233  }
234  return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
235  }
236 
237  // Apply all loop properties to the vectorized loop.
238  SmallVector<Metadata *, 4> FollowupLoopProperties;
239  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
240 
241  // Don't vectorize an already vectorized loop.
242  FollowupLoopProperties.push_back(
243  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
244 
245  bool FollowupHasTransforms = false;
246  MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
247  FollowupHasTransforms);
248 
250  Args.push_back(nullptr);
251  Args.append(LoopProperties.begin(), LoopProperties.end());
252 
253  // Setting vectorize.predicate when it has been specified and vectorization
254  // has not been disabled.
255  bool IsVectorPredicateEnabled = false;
257  IsVectorPredicateEnabled =
259 
260  Metadata *Vals[] = {
261  MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
262  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
263  IsVectorPredicateEnabled))};
264  Args.push_back(MDNode::get(Ctx, Vals));
265  }
266 
267  // Setting vectorize.width
268  if (Attrs.VectorizeWidth > 0) {
269  Metadata *Vals[] = {
270  MDString::get(Ctx, "llvm.loop.vectorize.width"),
271  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
272  Attrs.VectorizeWidth))};
273 
274  Args.push_back(MDNode::get(Ctx, Vals));
275  }
276 
278  bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
279  Metadata *Vals[] = {
280  MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"),
281  ConstantAsMetadata::get(
282  ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};
283  Args.push_back(MDNode::get(Ctx, Vals));
284  }
285 
286  // Setting interleave.count
287  if (Attrs.InterleaveCount > 0) {
288  Metadata *Vals[] = {
289  MDString::get(Ctx, "llvm.loop.interleave.count"),
290  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
291  Attrs.InterleaveCount))};
292  Args.push_back(MDNode::get(Ctx, Vals));
293  }
294 
295  // vectorize.enable is set if:
296  // 1) loop hint vectorize.enable is set, or
297  // 2) it is implied when vectorize.predicate is set, or
298  // 3) it is implied when vectorize.width is set to a value > 1
299  // 4) it is implied when vectorize.scalable.enable is true
300  // 5) it is implied when vectorize.width is unset (0) and the user
301  // explicitly requested fixed-width vectorization, i.e.
302  // vectorize.scalable.enable is false.
304  (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) ||
305  Attrs.VectorizeWidth > 1 ||
308  Attrs.VectorizeWidth != 1)) {
309  bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
310  Args.push_back(
311  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
312  ConstantAsMetadata::get(ConstantInt::get(
313  llvm::Type::getInt1Ty(Ctx), AttrVal))}));
314  }
315 
316  if (FollowupHasTransforms)
317  Args.push_back(MDNode::get(
318  Ctx,
319  {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
320 
321  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
322  LoopID->replaceOperandWith(0, LoopID);
323  HasUserTransforms = true;
324  return LoopID;
325 }
326 
327 MDNode *
328 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
329  ArrayRef<Metadata *> LoopProperties,
330  bool &HasUserTransforms) {
331  LLVMContext &Ctx = Header->getContext();
332 
333  Optional<bool> Enabled;
335  Enabled = false;
337  Enabled = true;
338 
339  if (Enabled != true) {
340  SmallVector<Metadata *, 4> NewLoopProperties;
341  if (Enabled == false) {
342  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
343  NewLoopProperties.push_back(
344  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
345  ConstantAsMetadata::get(ConstantInt::get(
346  llvm::Type::getInt1Ty(Ctx), 0))}));
347  LoopProperties = NewLoopProperties;
348  }
349  return createLoopVectorizeMetadata(Attrs, LoopProperties,
350  HasUserTransforms);
351  }
352 
353  bool FollowupHasTransforms = false;
354  MDNode *Followup =
355  createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
356 
358  Args.push_back(nullptr);
359  Args.append(LoopProperties.begin(), LoopProperties.end());
360 
361  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
362  ConstantAsMetadata::get(ConstantInt::get(
363  llvm::Type::getInt1Ty(Ctx),
365  Args.push_back(MDNode::get(Ctx, Vals));
366 
367  if (FollowupHasTransforms)
368  Args.push_back(MDNode::get(
369  Ctx,
370  {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
371 
372  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
373  LoopID->replaceOperandWith(0, LoopID);
374  HasUserTransforms = true;
375  return LoopID;
376 }
377 
378 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
379  ArrayRef<Metadata *> LoopProperties,
380  bool &HasUserTransforms) {
381  LLVMContext &Ctx = Header->getContext();
382 
383  Optional<bool> Enabled;
385  Enabled = false;
386  else if (Attrs.UnrollEnable == LoopAttributes::Full)
387  Enabled = true;
388 
389  if (Enabled != true) {
390  SmallVector<Metadata *, 4> NewLoopProperties;
391  if (Enabled == false) {
392  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
393  NewLoopProperties.push_back(
394  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
395  LoopProperties = NewLoopProperties;
396  }
397  return createLoopDistributeMetadata(Attrs, LoopProperties,
398  HasUserTransforms);
399  }
400 
402  Args.push_back(nullptr);
403  Args.append(LoopProperties.begin(), LoopProperties.end());
404  Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
405 
406  // No follow-up: there is no loop after full unrolling.
407  // TODO: Warn if there are transformations after full unrolling.
408 
409  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
410  LoopID->replaceOperandWith(0, LoopID);
411  HasUserTransforms = true;
412  return LoopID;
413 }
414 
415 MDNode *LoopInfo::createMetadata(
416  const LoopAttributes &Attrs,
417  llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
418  bool &HasUserTransforms) {
419  SmallVector<Metadata *, 3> LoopProperties;
420 
421  // If we have a valid start debug location for the loop, add it.
422  if (StartLoc) {
423  LoopProperties.push_back(StartLoc.getAsMDNode());
424 
425  // If we also have a valid end debug location for the loop, add it.
426  if (EndLoc)
427  LoopProperties.push_back(EndLoc.getAsMDNode());
428  }
429 
430  LLVMContext &Ctx = Header->getContext();
431  if (Attrs.MustProgress)
432  LoopProperties.push_back(
433  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress")));
434 
435  assert(!!AccGroup == Attrs.IsParallel &&
436  "There must be an access group iff the loop is parallel");
437  if (Attrs.IsParallel) {
438  LoopProperties.push_back(MDNode::get(
439  Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
440  }
441 
442  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
443  AdditionalLoopProperties.end());
444  return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
445 }
446 
448  : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
449  UnrollEnable(LoopAttributes::Unspecified),
450  UnrollAndJamEnable(LoopAttributes::Unspecified),
451  VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
452  VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
453  UnrollCount(0), UnrollAndJamCount(0),
454  DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
455  PipelineInitiationInterval(0), MustProgress(false) {}
456 
458  IsParallel = false;
459  VectorizeWidth = 0;
461  InterleaveCount = 0;
462  UnrollCount = 0;
463  UnrollAndJamCount = 0;
469  PipelineDisabled = false;
471  MustProgress = false;
472 }
473 
474 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
475  const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
476  LoopInfo *Parent)
477  : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
478  Parent(Parent) {
479 
480  if (Attrs.IsParallel) {
481  // Create an access group for this loop.
482  LLVMContext &Ctx = Header->getContext();
483  AccGroup = MDNode::getDistinct(Ctx, {});
484  }
485 
486  if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
488  Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
489  Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
490  Attrs.PipelineInitiationInterval == 0 &&
495  Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
496  !EndLoc && !Attrs.MustProgress)
497  return;
498 
499  TempLoopID = MDNode::getTemporary(Header->getContext(), None);
500 }
501 
503  // We did not annotate the loop body instructions because there are no
504  // attributes for this loop.
505  if (!TempLoopID)
506  return;
507 
508  MDNode *LoopID;
509  LoopAttributes CurLoopAttr = Attrs;
510  LLVMContext &Ctx = Header->getContext();
511 
512  if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
513  Parent->Attrs.UnrollAndJamCount != 0)) {
514  // Parent unroll-and-jams this loop.
515  // Split the transformations in those that happens before the unroll-and-jam
516  // and those after.
517 
518  LoopAttributes BeforeJam, AfterJam;
519 
520  BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
521 
522  BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
523  BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
524  BeforeJam.InterleaveCount = Attrs.InterleaveCount;
525  BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
526  BeforeJam.DistributeEnable = Attrs.DistributeEnable;
528 
529  switch (Attrs.UnrollEnable) {
532  BeforeJam.UnrollEnable = Attrs.UnrollEnable;
533  AfterJam.UnrollEnable = Attrs.UnrollEnable;
534  break;
537  break;
540  break;
541  }
542 
544  AfterJam.UnrollCount = Attrs.UnrollCount;
545  AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
547 
548  // If this loop is subject of an unroll-and-jam by the parent loop, and has
549  // an unroll-and-jam annotation itself, we have to decide whether to first
550  // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
551  // UnrollAndJam pass processes loops from inner to outer, so we apply the
552  // inner first.
553  BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
554  BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
555 
556  // Set the inner followup metadata to process by the outer loop. Only
557  // consider the first inner loop.
558  if (!Parent->UnrollAndJamInnerFollowup) {
559  // Splitting the attributes into a BeforeJam and an AfterJam part will
560  // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
561  // to be forwarded to the AfterJam part. We detect the situation here and
562  // add it manually.
563  SmallVector<Metadata *, 1> BeforeLoopProperties;
564  if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
566  BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
568  BeforeLoopProperties.push_back(
569  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
570 
571  bool InnerFollowupHasTransform = false;
572  MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
573  InnerFollowupHasTransform);
574  if (InnerFollowupHasTransform)
575  Parent->UnrollAndJamInnerFollowup = InnerFollowup;
576  }
577 
578  CurLoopAttr = BeforeJam;
579  }
580 
581  bool HasUserTransforms = false;
582  LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
583  TempLoopID->replaceAllUsesWith(LoopID);
584 }
585 
586 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
587  const llvm::DebugLoc &EndLoc) {
588  Active.emplace_back(
589  new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
590  Active.empty() ? nullptr : Active.back().get()));
591  // Clear the attributes so nested loops do not inherit them.
592  StagedAttrs.clear();
593 }
594 
595 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
596  const clang::CodeGenOptions &CGOpts,
598  const llvm::DebugLoc &StartLoc,
599  const llvm::DebugLoc &EndLoc, bool MustProgress) {
600  // Identify loop hint attributes from Attrs.
601  for (const auto *Attr : Attrs) {
602  const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
603  const OpenCLUnrollHintAttr *OpenCLHint =
604  dyn_cast<OpenCLUnrollHintAttr>(Attr);
605 
606  // Skip non loop hint attributes
607  if (!LH && !OpenCLHint) {
608  continue;
609  }
610 
611  LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
612  LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
613  unsigned ValueInt = 1;
614  // Translate opencl_unroll_hint attribute argument to
615  // equivalent LoopHintAttr enums.
616  // OpenCL v2.0 s6.11.5:
617  // 0 - enable unroll (no argument).
618  // 1 - disable unroll.
619  // other positive integer n - unroll by n.
620  if (OpenCLHint) {
621  ValueInt = OpenCLHint->getUnrollHint();
622  if (ValueInt == 0) {
623  State = LoopHintAttr::Enable;
624  } else if (ValueInt != 1) {
625  Option = LoopHintAttr::UnrollCount;
626  State = LoopHintAttr::Numeric;
627  }
628  } else if (LH) {
629  auto *ValueExpr = LH->getValue();
630  if (ValueExpr) {
631  llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
632  ValueInt = ValueAPS.getSExtValue();
633  }
634 
635  Option = LH->getOption();
636  State = LH->getState();
637  }
638  switch (State) {
639  case LoopHintAttr::Disable:
640  switch (Option) {
641  case LoopHintAttr::Vectorize:
642  // Disable vectorization by specifying a width of 1.
645  break;
646  case LoopHintAttr::Interleave:
647  // Disable interleaving by speciyfing a count of 1.
649  break;
650  case LoopHintAttr::Unroll:
652  break;
653  case LoopHintAttr::UnrollAndJam:
655  break;
656  case LoopHintAttr::VectorizePredicate:
658  break;
659  case LoopHintAttr::Distribute:
660  setDistributeState(false);
661  break;
662  case LoopHintAttr::PipelineDisabled:
663  setPipelineDisabled(true);
664  break;
665  case LoopHintAttr::UnrollCount:
666  case LoopHintAttr::UnrollAndJamCount:
667  case LoopHintAttr::VectorizeWidth:
668  case LoopHintAttr::InterleaveCount:
669  case LoopHintAttr::PipelineInitiationInterval:
670  llvm_unreachable("Options cannot be disabled.");
671  break;
672  }
673  break;
674  case LoopHintAttr::Enable:
675  switch (Option) {
676  case LoopHintAttr::Vectorize:
677  case LoopHintAttr::Interleave:
678  setVectorizeEnable(true);
679  break;
680  case LoopHintAttr::Unroll:
682  break;
683  case LoopHintAttr::UnrollAndJam:
685  break;
686  case LoopHintAttr::VectorizePredicate:
688  break;
689  case LoopHintAttr::Distribute:
690  setDistributeState(true);
691  break;
692  case LoopHintAttr::UnrollCount:
693  case LoopHintAttr::UnrollAndJamCount:
694  case LoopHintAttr::VectorizeWidth:
695  case LoopHintAttr::InterleaveCount:
696  case LoopHintAttr::PipelineDisabled:
697  case LoopHintAttr::PipelineInitiationInterval:
698  llvm_unreachable("Options cannot enabled.");
699  break;
700  }
701  break;
702  case LoopHintAttr::AssumeSafety:
703  switch (Option) {
704  case LoopHintAttr::Vectorize:
705  case LoopHintAttr::Interleave:
706  // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
707  setParallel(true);
708  setVectorizeEnable(true);
709  break;
710  case LoopHintAttr::Unroll:
711  case LoopHintAttr::UnrollAndJam:
712  case LoopHintAttr::VectorizePredicate:
713  case LoopHintAttr::UnrollCount:
714  case LoopHintAttr::UnrollAndJamCount:
715  case LoopHintAttr::VectorizeWidth:
716  case LoopHintAttr::InterleaveCount:
717  case LoopHintAttr::Distribute:
718  case LoopHintAttr::PipelineDisabled:
719  case LoopHintAttr::PipelineInitiationInterval:
720  llvm_unreachable("Options cannot be used to assume mem safety.");
721  break;
722  }
723  break;
724  case LoopHintAttr::Full:
725  switch (Option) {
726  case LoopHintAttr::Unroll:
728  break;
729  case LoopHintAttr::UnrollAndJam:
731  break;
732  case LoopHintAttr::Vectorize:
733  case LoopHintAttr::Interleave:
734  case LoopHintAttr::UnrollCount:
735  case LoopHintAttr::UnrollAndJamCount:
736  case LoopHintAttr::VectorizeWidth:
737  case LoopHintAttr::InterleaveCount:
738  case LoopHintAttr::Distribute:
739  case LoopHintAttr::PipelineDisabled:
740  case LoopHintAttr::PipelineInitiationInterval:
741  case LoopHintAttr::VectorizePredicate:
742  llvm_unreachable("Options cannot be used with 'full' hint.");
743  break;
744  }
745  break;
746  case LoopHintAttr::FixedWidth:
747  case LoopHintAttr::ScalableWidth:
748  switch (Option) {
749  case LoopHintAttr::VectorizeWidth:
750  setVectorizeScalable(State == LoopHintAttr::ScalableWidth
753  if (LH->getValue())
754  setVectorizeWidth(ValueInt);
755  break;
756  default:
757  llvm_unreachable("Options cannot be used with 'scalable' hint.");
758  break;
759  }
760  break;
761  case LoopHintAttr::Numeric:
762  switch (Option) {
763  case LoopHintAttr::InterleaveCount:
764  setInterleaveCount(ValueInt);
765  break;
766  case LoopHintAttr::UnrollCount:
767  setUnrollCount(ValueInt);
768  break;
769  case LoopHintAttr::UnrollAndJamCount:
770  setUnrollAndJamCount(ValueInt);
771  break;
772  case LoopHintAttr::PipelineInitiationInterval:
774  break;
775  case LoopHintAttr::Unroll:
776  case LoopHintAttr::UnrollAndJam:
777  case LoopHintAttr::VectorizePredicate:
778  case LoopHintAttr::Vectorize:
779  case LoopHintAttr::VectorizeWidth:
780  case LoopHintAttr::Interleave:
781  case LoopHintAttr::Distribute:
782  case LoopHintAttr::PipelineDisabled:
783  llvm_unreachable("Options cannot be assigned a value.");
784  break;
785  }
786  break;
787  }
788  }
789 
790  setMustProgress(MustProgress);
791 
792  if (CGOpts.OptimizationLevel > 0)
793  // Disable unrolling for the loop, if unrolling is disabled (via
794  // -fno-unroll-loops) and no pragmas override the decision.
795  if (!CGOpts.UnrollLoops &&
796  (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
797  StagedAttrs.UnrollCount == 0))
799 
800  /// Stage the attributes.
801  push(Header, StartLoc, EndLoc);
802 }
803 
805  assert(!Active.empty() && "No active loops to pop");
806  Active.back()->finish();
807  Active.pop_back();
808 }
809 
810 void LoopInfoStack::InsertHelper(Instruction *I) const {
811  if (I->mayReadOrWriteMemory()) {
812  SmallVector<Metadata *, 4> AccessGroups;
813  for (const auto &AL : Active) {
814  // Here we assume that every loop that has an access group is parallel.
815  if (MDNode *Group = AL->getAccessGroup())
816  AccessGroups.push_back(Group);
817  }
818  MDNode *UnionMD = nullptr;
819  if (AccessGroups.size() == 1)
820  UnionMD = cast<MDNode>(AccessGroups[0]);
821  else if (AccessGroups.size() >= 2)
822  UnionMD = MDNode::get(I->getContext(), AccessGroups);
823  I->setMetadata("llvm.access.group", UnionMD);
824  }
825 
826  if (!hasInfo())
827  return;
828 
829  const LoopInfo &L = getInfo();
830  if (!L.getLoopID())
831  return;
832 
833  if (I->isTerminator()) {
834  for (BasicBlock *Succ : successors(I))
835  if (Succ == L.getHeader()) {
836  I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
837  break;
838  }
839  return;
840  }
841 }
clang::CodeGen::LoopAttributes::MustProgress
bool MustProgress
Value for whether the loop is required to make progress.
Definition: CGLoopInfo.h:83
clang::CodeGen::LoopInfo::getHeader
llvm::BasicBlock * getHeader() const
Get the header block of this loop.
Definition: CGLoopInfo.h:98
llvm
Definition: Dominators.h:30
clang::CodeGen::LoopInfoStack::setVectorizePredicateState
void setVectorizePredicateState(const LoopAttributes::LVEnableState &State)
Set the next pushed vectorize predicate state.
Definition: CGLoopInfo.h:252
clang::CodeGen::LoopAttributes::Unspecified
@ Unspecified
Definition: CGLoopInfo.h:44
clang::CodeGen::LoopAttributes::InterleaveCount
unsigned InterleaveCount
Value for llvm.loop.interleave.count metadata.
Definition: CGLoopInfo.h:65
llvm::SmallVector
Definition: LLVM.h:38
CGLoopInfo.h
clang::CodeGen::LoopInfoStack::setParallel
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
Definition: CGLoopInfo.h:232
clang::CodeGen::LoopInfoStack::setUnrollState
void setUnrollState(const LoopAttributes::LVEnableState &State)
Set the next pushed loop unroll state.
Definition: CGLoopInfo.h:247
clang::CodeGen::LoopAttributes::VectorizeEnable
LVEnableState VectorizeEnable
Value for llvm.loop.vectorize.enable metadata.
Definition: CGLoopInfo.h:47
Attr.h
clang::CodeGen::LoopAttributes::UnrollAndJamCount
unsigned UnrollAndJamCount
llvm.unroll.
Definition: CGLoopInfo.h:71
clang::CodeGen::LoopAttributes::UnrollAndJamEnable
LVEnableState UnrollAndJamEnable
Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full).
Definition: CGLoopInfo.h:53
clang::CodeGen::LoopAttributes::LoopAttributes
LoopAttributes(bool IsParallel=false)
Definition: CGLoopInfo.cpp:447
llvm::Optional< bool >
clang::CodeGen::LoopAttributes::VectorizeScalable
LVEnableState VectorizeScalable
Definition: CGLoopInfo.h:62
clang::CodeGen::LoopInfoStack::InsertHelper
void InsertHelper(llvm::Instruction *I) const
Function called by the CodeGenFunction when an instruction is created.
Definition: CGLoopInfo.cpp:810
clang::CodeGen::LoopAttributes::PipelineInitiationInterval
unsigned PipelineInitiationInterval
Value for llvm.loop.pipeline.iicount metadata.
Definition: CGLoopInfo.h:80
clang::CodeGen::LoopAttributes::VectorizePredicateEnable
LVEnableState VectorizePredicateEnable
Value for llvm.loop.vectorize.predicate metadata.
Definition: CGLoopInfo.h:56
CodeGenOptions.h
APSInt
llvm::APSInt APSInt
Definition: ByteCodeEmitter.cpp:19
clang::CodeGen::LoopInfoStack::setUnrollAndJamCount
void setUnrollAndJamCount(unsigned C)
Set the unroll count for the next loop pushed.
Definition: CGLoopInfo.h:275
clang::CodeGen::LoopInfoStack::setUnrollAndJamState
void setUnrollAndJamState(const LoopAttributes::LVEnableState &State)
Set the next pushed loop unroll_and_jam state.
Definition: CGLoopInfo.h:257
clang::CodeGen::LoopAttributes::IsParallel
bool IsParallel
Generate llvm.loop.parallel metadata for loads and stores.
Definition: CGLoopInfo.h:41
clang::CodeGen::LoopAttributes::UnrollEnable
LVEnableState UnrollEnable
Value for llvm.loop.unroll.* metadata (enable, disable, or full).
Definition: CGLoopInfo.h:50
clang::CodeGen::LoopInfoStack::setVectorizeScalable
void setVectorizeScalable(const LoopAttributes::LVEnableState &State)
Definition: CGLoopInfo.h:264
clang::XRayInstrKind::None
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
clang::CodeGen::LoopAttributes::DistributeEnable
LVEnableState DistributeEnable
Value for llvm.loop.distribute.enable metadata.
Definition: CGLoopInfo.h:74
clang::ASTContext
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:211
clang::CodeGen::LoopInfoStack::setMustProgress
void setMustProgress(bool P)
Set no progress for the next loop pushed.
Definition: CGLoopInfo.h:286
clang::CodeGen::LoopInfoStack::pop
void pop()
End the current loop.
Definition: CGLoopInfo.cpp:804
clang::CodeGen::LoopInfoStack::setVectorizeWidth
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
Definition: CGLoopInfo.h:262
Expr.h
ASTContext.h
clang::CodeGen::LoopInfoStack::push
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
clang::CodeGenOptions
CodeGenOptions - Track various options which control how the code is optimized and passed to the back...
Definition: CodeGenOptions.h:49
clang::CodeGen::LoopInfo::finish
void finish()
Create the loop's metadata.
Definition: CGLoopInfo.cpp:502
clang::CodeGen::LoopInfoStack::setVectorizeEnable
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
Definition: CGLoopInfo.h:235
clang::CodeGen::LoopAttributes::clear
void clear()
Definition: CGLoopInfo.cpp:457
clang::CodeGen::LoopInfoStack::setPipelineInitiationInterval
void setPipelineInitiationInterval(unsigned C)
Set the pipeline initiation interval.
Definition: CGLoopInfo.h:281
false
#define false
Definition: stdbool.h:17
clang::CodeGen::LoopInfoStack::setUnrollCount
void setUnrollCount(unsigned C)
Set the unroll count for the next loop pushed.
Definition: CGLoopInfo.h:272
llvm::ArrayRef
Definition: LLVM.h:34
clang::CodeGen::LoopAttributes::UnrollCount
unsigned UnrollCount
llvm.unroll.
Definition: CGLoopInfo.h:68
clang::CodeGen::LoopInfo::getLoopID
llvm::MDNode * getLoopID() const
Get the loop id metadata for this loop.
Definition: CGLoopInfo.h:95
clang::CodeGen::LoopAttributes
Attributes that may be specified on loops.
Definition: CGLoopInfo.h:36
State
LineState State
Definition: UnwrappedLineFormatter.cpp:986
clang::CodeGen::LoopInfo::LoopInfo
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, LoopInfo *Parent)
Construct a new LoopInfo for the loop with entry Header.
Definition: CGLoopInfo.cpp:474
clang::CodeGen::LoopInfoStack::setInterleaveCount
void setInterleaveCount(unsigned C)
Set the interleave count for the next loop pushed.
Definition: CGLoopInfo.h:269
clang::CodeGen::LoopAttributes::Enable
@ Enable
Definition: CGLoopInfo.h:44
clang::Attr
Attr - This represents one attribute.
Definition: Attr.h:46
clang::CodeGen::LoopInfo
Information used when generating a structured loop.
Definition: CGLoopInfo.h:87
clang::CodeGen::LoopInfoStack::setDistributeState
void setDistributeState(bool Enable=true)
Set the next pushed loop as a distribution candidate.
Definition: CGLoopInfo.h:241
clang::CodeGen::LoopAttributes::PipelineDisabled
bool PipelineDisabled
Value for llvm.loop.pipeline.disable metadata.
Definition: CGLoopInfo.h:77
clang::CodeGen
Definition: CGFunctionInfo.h:28
Parent
NodeId Parent
Definition: ASTDiff.cpp:192
clang::CodeGen::LoopAttributes::Disable
@ Disable
Definition: CGLoopInfo.h:44
clang::CodeGen::LoopAttributes::VectorizeWidth
unsigned VectorizeWidth
Value for llvm.loop.vectorize.width metadata.
Definition: CGLoopInfo.h:59
clang::CodeGen::LoopInfoStack::setPipelineDisabled
void setPipelineDisabled(bool S)
Set the pipeline disabled state.
Definition: CGLoopInfo.h:278
clang::ConstexprSpecKind::Unspecified
@ Unspecified
clang::CodeGen::LoopAttributes::Full
@ Full
Definition: CGLoopInfo.h:44