clang  12.0.0git
CGLoopInfo.cpp
Go to the documentation of this file.
1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "clang/AST/Expr.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/Constants.h"
17 #include "llvm/IR/InstrTypes.h"
18 #include "llvm/IR/Instructions.h"
19 #include "llvm/IR/Metadata.h"
20 using namespace clang::CodeGen;
21 using namespace llvm;
22 
23 MDNode *
24 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
25  LLVMContext &Ctx = Header->getContext();
26  SmallVector<Metadata *, 4> NewLoopProperties;
27  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
28  NewLoopProperties.push_back(TempNode.get());
29  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
30 
31  MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
32  LoopID->replaceOperandWith(0, LoopID);
33  return LoopID;
34 }
35 
36 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
37  ArrayRef<Metadata *> LoopProperties,
38  bool &HasUserTransforms) {
39  LLVMContext &Ctx = Header->getContext();
40 
41  Optional<bool> Enabled;
42  if (Attrs.PipelineDisabled)
43  Enabled = false;
44  else if (Attrs.PipelineInitiationInterval != 0)
45  Enabled = true;
46 
47  if (Enabled != true) {
48  SmallVector<Metadata *, 4> NewLoopProperties;
49  if (Enabled == false) {
50  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
51  NewLoopProperties.push_back(
52  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
53  ConstantAsMetadata::get(ConstantInt::get(
54  llvm::Type::getInt1Ty(Ctx), 1))}));
55  LoopProperties = NewLoopProperties;
56  }
57  return createLoopPropertiesMetadata(LoopProperties);
58  }
59 
61  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
62  Args.push_back(TempNode.get());
63  Args.append(LoopProperties.begin(), LoopProperties.end());
64 
65  if (Attrs.PipelineInitiationInterval > 0) {
66  Metadata *Vals[] = {
67  MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
68  ConstantAsMetadata::get(ConstantInt::get(
69  llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
70  Args.push_back(MDNode::get(Ctx, Vals));
71  }
72 
73  // No follow-up: This is the last transformation.
74 
75  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
76  LoopID->replaceOperandWith(0, LoopID);
77  HasUserTransforms = true;
78  return LoopID;
79 }
80 
81 MDNode *
82 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
83  ArrayRef<Metadata *> LoopProperties,
84  bool &HasUserTransforms) {
85  LLVMContext &Ctx = Header->getContext();
86 
87  Optional<bool> Enabled;
89  Enabled = false;
90  else if (Attrs.UnrollEnable == LoopAttributes::Full)
91  Enabled = None;
92  else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
93  Attrs.UnrollCount != 0)
94  Enabled = true;
95 
96  if (Enabled != true) {
97  // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
98  // if unrolling is disabled.
99  return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
100  }
101 
102  SmallVector<Metadata *, 4> FollowupLoopProperties;
103 
104  // Apply all loop properties to the unrolled loop.
105  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
106 
107  // Don't unroll an already unrolled loop.
108  FollowupLoopProperties.push_back(
109  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
110 
111  bool FollowupHasTransforms = false;
112  MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
113  FollowupHasTransforms);
114 
116  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
117  Args.push_back(TempNode.get());
118  Args.append(LoopProperties.begin(), LoopProperties.end());
119 
120  // Setting unroll.count
121  if (Attrs.UnrollCount > 0) {
122  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
123  ConstantAsMetadata::get(ConstantInt::get(
124  llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
125  Args.push_back(MDNode::get(Ctx, Vals));
126  }
127 
128  // Setting unroll.full or unroll.disable
129  if (Attrs.UnrollEnable == LoopAttributes::Enable) {
130  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
131  Args.push_back(MDNode::get(Ctx, Vals));
132  }
133 
134  if (FollowupHasTransforms)
135  Args.push_back(MDNode::get(
136  Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
137 
138  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
139  LoopID->replaceOperandWith(0, LoopID);
140  HasUserTransforms = true;
141  return LoopID;
142 }
143 
144 MDNode *
145 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
146  ArrayRef<Metadata *> LoopProperties,
147  bool &HasUserTransforms) {
148  LLVMContext &Ctx = Header->getContext();
149 
150  Optional<bool> Enabled;
152  Enabled = false;
153  else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
154  Attrs.UnrollAndJamCount != 0)
155  Enabled = true;
156 
157  if (Enabled != true) {
158  SmallVector<Metadata *, 4> NewLoopProperties;
159  if (Enabled == false) {
160  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
161  NewLoopProperties.push_back(MDNode::get(
162  Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
163  LoopProperties = NewLoopProperties;
164  }
165  return createPartialUnrollMetadata(Attrs, LoopProperties,
166  HasUserTransforms);
167  }
168 
169  SmallVector<Metadata *, 4> FollowupLoopProperties;
170  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
171  FollowupLoopProperties.push_back(
172  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
173 
174  bool FollowupHasTransforms = false;
175  MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
176  FollowupHasTransforms);
177 
179  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
180  Args.push_back(TempNode.get());
181  Args.append(LoopProperties.begin(), LoopProperties.end());
182 
183  // Setting unroll_and_jam.count
184  if (Attrs.UnrollAndJamCount > 0) {
185  Metadata *Vals[] = {
186  MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
187  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
188  Attrs.UnrollAndJamCount))};
189  Args.push_back(MDNode::get(Ctx, Vals));
190  }
191 
193  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
194  Args.push_back(MDNode::get(Ctx, Vals));
195  }
196 
197  if (FollowupHasTransforms)
198  Args.push_back(MDNode::get(
199  Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
200  Followup}));
201 
202  if (UnrollAndJamInnerFollowup)
203  Args.push_back(MDNode::get(
204  Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
205  UnrollAndJamInnerFollowup}));
206 
207  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
208  LoopID->replaceOperandWith(0, LoopID);
209  HasUserTransforms = true;
210  return LoopID;
211 }
212 
213 MDNode *
214 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
215  ArrayRef<Metadata *> LoopProperties,
216  bool &HasUserTransforms) {
217  LLVMContext &Ctx = Header->getContext();
218 
219  Optional<bool> Enabled;
221  Enabled = false;
222  else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
224  Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
225  Enabled = true;
226 
227  if (Enabled != true) {
228  SmallVector<Metadata *, 4> NewLoopProperties;
229  if (Enabled == false) {
230  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
231  NewLoopProperties.push_back(
232  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
233  ConstantAsMetadata::get(ConstantInt::get(
234  llvm::Type::getInt1Ty(Ctx), 0))}));
235  LoopProperties = NewLoopProperties;
236  }
237  return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
238  }
239 
240  // Apply all loop properties to the vectorized loop.
241  SmallVector<Metadata *, 4> FollowupLoopProperties;
242  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
243 
244  // Don't vectorize an already vectorized loop.
245  FollowupLoopProperties.push_back(
246  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
247 
248  bool FollowupHasTransforms = false;
249  MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
250  FollowupHasTransforms);
251 
253  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
254  Args.push_back(TempNode.get());
255  Args.append(LoopProperties.begin(), LoopProperties.end());
256 
257  // Setting vectorize.predicate
258  bool IsVectorPredicateEnabled = false;
261  Attrs.VectorizeWidth < 1) {
262 
263  IsVectorPredicateEnabled =
265 
266  Metadata *Vals[] = {
267  MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
268  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
269  IsVectorPredicateEnabled))};
270  Args.push_back(MDNode::get(Ctx, Vals));
271  }
272 
273  // Setting vectorize.width
274  if (Attrs.VectorizeWidth > 0) {
275  Metadata *Vals[] = {
276  MDString::get(Ctx, "llvm.loop.vectorize.width"),
277  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
278  Attrs.VectorizeWidth))};
279  Args.push_back(MDNode::get(Ctx, Vals));
280  }
281 
282  // Setting interleave.count
283  if (Attrs.InterleaveCount > 0) {
284  Metadata *Vals[] = {
285  MDString::get(Ctx, "llvm.loop.interleave.count"),
286  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
287  Attrs.InterleaveCount))};
288  Args.push_back(MDNode::get(Ctx, Vals));
289  }
290 
291  // vectorize.enable is set if:
292  // 1) loop hint vectorize.enable is set, or
293  // 2) it is implied when vectorize.predicate is set, or
294  // 3) it is implied when vectorize.width is set.
296  IsVectorPredicateEnabled ||
297  Attrs.VectorizeWidth > 1 ) {
298  bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
299  Args.push_back(
300  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
301  ConstantAsMetadata::get(ConstantInt::get(
302  llvm::Type::getInt1Ty(Ctx), AttrVal))}));
303  }
304 
305  if (FollowupHasTransforms)
306  Args.push_back(MDNode::get(
307  Ctx,
308  {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
309 
310  MDNode *LoopID = MDNode::get(Ctx, Args);
311  LoopID->replaceOperandWith(0, LoopID);
312  HasUserTransforms = true;
313  return LoopID;
314 }
315 
316 MDNode *
317 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
318  ArrayRef<Metadata *> LoopProperties,
319  bool &HasUserTransforms) {
320  LLVMContext &Ctx = Header->getContext();
321 
322  Optional<bool> Enabled;
324  Enabled = false;
326  Enabled = true;
327 
328  if (Enabled != true) {
329  SmallVector<Metadata *, 4> NewLoopProperties;
330  if (Enabled == false) {
331  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
332  NewLoopProperties.push_back(
333  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
334  ConstantAsMetadata::get(ConstantInt::get(
335  llvm::Type::getInt1Ty(Ctx), 0))}));
336  LoopProperties = NewLoopProperties;
337  }
338  return createLoopVectorizeMetadata(Attrs, LoopProperties,
339  HasUserTransforms);
340  }
341 
342  bool FollowupHasTransforms = false;
343  MDNode *Followup =
344  createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
345 
347  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
348  Args.push_back(TempNode.get());
349  Args.append(LoopProperties.begin(), LoopProperties.end());
350 
351  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
352  ConstantAsMetadata::get(ConstantInt::get(
353  llvm::Type::getInt1Ty(Ctx),
355  Args.push_back(MDNode::get(Ctx, Vals));
356 
357  if (FollowupHasTransforms)
358  Args.push_back(MDNode::get(
359  Ctx,
360  {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
361 
362  MDNode *LoopID = MDNode::get(Ctx, Args);
363  LoopID->replaceOperandWith(0, LoopID);
364  HasUserTransforms = true;
365  return LoopID;
366 }
367 
368 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
369  ArrayRef<Metadata *> LoopProperties,
370  bool &HasUserTransforms) {
371  LLVMContext &Ctx = Header->getContext();
372 
373  Optional<bool> Enabled;
375  Enabled = false;
376  else if (Attrs.UnrollEnable == LoopAttributes::Full)
377  Enabled = true;
378 
379  if (Enabled != true) {
380  SmallVector<Metadata *, 4> NewLoopProperties;
381  if (Enabled == false) {
382  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
383  NewLoopProperties.push_back(
384  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
385  LoopProperties = NewLoopProperties;
386  }
387  return createLoopDistributeMetadata(Attrs, LoopProperties,
388  HasUserTransforms);
389  }
390 
392  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
393  Args.push_back(TempNode.get());
394  Args.append(LoopProperties.begin(), LoopProperties.end());
395  Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
396 
397  // No follow-up: there is no loop after full unrolling.
398  // TODO: Warn if there are transformations after full unrolling.
399 
400  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
401  LoopID->replaceOperandWith(0, LoopID);
402  HasUserTransforms = true;
403  return LoopID;
404 }
405 
406 MDNode *LoopInfo::createMetadata(
407  const LoopAttributes &Attrs,
408  llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
409  bool &HasUserTransforms) {
410  SmallVector<Metadata *, 3> LoopProperties;
411 
412  // If we have a valid start debug location for the loop, add it.
413  if (StartLoc) {
414  LoopProperties.push_back(StartLoc.getAsMDNode());
415 
416  // If we also have a valid end debug location for the loop, add it.
417  if (EndLoc)
418  LoopProperties.push_back(EndLoc.getAsMDNode());
419  }
420 
421  assert(!!AccGroup == Attrs.IsParallel &&
422  "There must be an access group iff the loop is parallel");
423  if (Attrs.IsParallel) {
424  LLVMContext &Ctx = Header->getContext();
425  LoopProperties.push_back(MDNode::get(
426  Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
427  }
428 
429  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
430  AdditionalLoopProperties.end());
431  return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
432 }
433 
435  : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
436  UnrollEnable(LoopAttributes::Unspecified),
437  UnrollAndJamEnable(LoopAttributes::Unspecified),
438  VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
439  InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
440  DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
441  PipelineInitiationInterval(0) {}
442 
444  IsParallel = false;
445  VectorizeWidth = 0;
446  InterleaveCount = 0;
447  UnrollCount = 0;
448  UnrollAndJamCount = 0;
454  PipelineDisabled = false;
456 }
457 
458 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
459  const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
460  LoopInfo *Parent)
461  : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
462  Parent(Parent) {
463 
464  if (Attrs.IsParallel) {
465  // Create an access group for this loop.
466  LLVMContext &Ctx = Header->getContext();
467  AccGroup = MDNode::getDistinct(Ctx, {});
468  }
469 
470  if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
471  Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
472  Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
473  Attrs.PipelineInitiationInterval == 0 &&
478  Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
479  !EndLoc)
480  return;
481 
482  TempLoopID = MDNode::getTemporary(Header->getContext(), None);
483 }
484 
486  // We did not annotate the loop body instructions because there are no
487  // attributes for this loop.
488  if (!TempLoopID)
489  return;
490 
491  MDNode *LoopID;
492  LoopAttributes CurLoopAttr = Attrs;
493  LLVMContext &Ctx = Header->getContext();
494 
495  if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
496  Parent->Attrs.UnrollAndJamCount != 0)) {
497  // Parent unroll-and-jams this loop.
498  // Split the transformations in those that happens before the unroll-and-jam
499  // and those after.
500 
501  LoopAttributes BeforeJam, AfterJam;
502 
503  BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
504 
505  BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
506  BeforeJam.InterleaveCount = Attrs.InterleaveCount;
507  BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
508  BeforeJam.DistributeEnable = Attrs.DistributeEnable;
510 
511  switch (Attrs.UnrollEnable) {
514  BeforeJam.UnrollEnable = Attrs.UnrollEnable;
515  AfterJam.UnrollEnable = Attrs.UnrollEnable;
516  break;
519  break;
522  break;
523  }
524 
526  AfterJam.UnrollCount = Attrs.UnrollCount;
527  AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
529 
530  // If this loop is subject of an unroll-and-jam by the parent loop, and has
531  // an unroll-and-jam annotation itself, we have to decide whether to first
532  // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
533  // UnrollAndJam pass processes loops from inner to outer, so we apply the
534  // inner first.
535  BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
536  BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
537 
538  // Set the inner followup metadata to process by the outer loop. Only
539  // consider the first inner loop.
540  if (!Parent->UnrollAndJamInnerFollowup) {
541  // Splitting the attributes into a BeforeJam and an AfterJam part will
542  // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
543  // to be forwarded to the AfterJam part. We detect the situation here and
544  // add it manually.
545  SmallVector<Metadata *, 1> BeforeLoopProperties;
546  if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
548  BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
549  BeforeLoopProperties.push_back(
550  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
551 
552  bool InnerFollowupHasTransform = false;
553  MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
554  InnerFollowupHasTransform);
555  if (InnerFollowupHasTransform)
556  Parent->UnrollAndJamInnerFollowup = InnerFollowup;
557  }
558 
559  CurLoopAttr = BeforeJam;
560  }
561 
562  bool HasUserTransforms = false;
563  LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
564  TempLoopID->replaceAllUsesWith(LoopID);
565 }
566 
567 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
568  const llvm::DebugLoc &EndLoc) {
569  Active.emplace_back(
570  new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
571  Active.empty() ? nullptr : Active.back().get()));
572  // Clear the attributes so nested loops do not inherit them.
573  StagedAttrs.clear();
574 }
575 
576 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
577  const clang::CodeGenOptions &CGOpts,
579  const llvm::DebugLoc &StartLoc,
580  const llvm::DebugLoc &EndLoc) {
581 
582  // Identify loop hint attributes from Attrs.
583  for (const auto *Attr : Attrs) {
584  const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
585  const OpenCLUnrollHintAttr *OpenCLHint =
586  dyn_cast<OpenCLUnrollHintAttr>(Attr);
587 
588  // Skip non loop hint attributes
589  if (!LH && !OpenCLHint) {
590  continue;
591  }
592 
593  LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
594  LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
595  unsigned ValueInt = 1;
596  // Translate opencl_unroll_hint attribute argument to
597  // equivalent LoopHintAttr enums.
598  // OpenCL v2.0 s6.11.5:
599  // 0 - enable unroll (no argument).
600  // 1 - disable unroll.
601  // other positive integer n - unroll by n.
602  if (OpenCLHint) {
603  ValueInt = OpenCLHint->getUnrollHint();
604  if (ValueInt == 0) {
605  State = LoopHintAttr::Enable;
606  } else if (ValueInt != 1) {
607  Option = LoopHintAttr::UnrollCount;
608  State = LoopHintAttr::Numeric;
609  }
610  } else if (LH) {
611  auto *ValueExpr = LH->getValue();
612  if (ValueExpr) {
613  llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
614  ValueInt = ValueAPS.getSExtValue();
615  }
616 
617  Option = LH->getOption();
618  State = LH->getState();
619  }
620  switch (State) {
621  case LoopHintAttr::Disable:
622  switch (Option) {
623  case LoopHintAttr::Vectorize:
624  // Disable vectorization by specifying a width of 1.
625  setVectorizeWidth(1);
626  break;
627  case LoopHintAttr::Interleave:
628  // Disable interleaving by speciyfing a count of 1.
629  setInterleaveCount(1);
630  break;
631  case LoopHintAttr::Unroll:
632  setUnrollState(LoopAttributes::Disable);
633  break;
634  case LoopHintAttr::UnrollAndJam:
635  setUnrollAndJamState(LoopAttributes::Disable);
636  break;
637  case LoopHintAttr::VectorizePredicate:
638  setVectorizePredicateState(LoopAttributes::Disable);
639  break;
640  case LoopHintAttr::Distribute:
641  setDistributeState(false);
642  break;
643  case LoopHintAttr::PipelineDisabled:
644  setPipelineDisabled(true);
645  break;
646  case LoopHintAttr::UnrollCount:
647  case LoopHintAttr::UnrollAndJamCount:
648  case LoopHintAttr::VectorizeWidth:
649  case LoopHintAttr::InterleaveCount:
650  case LoopHintAttr::PipelineInitiationInterval:
651  llvm_unreachable("Options cannot be disabled.");
652  break;
653  }
654  break;
655  case LoopHintAttr::Enable:
656  switch (Option) {
657  case LoopHintAttr::Vectorize:
658  case LoopHintAttr::Interleave:
659  setVectorizeEnable(true);
660  break;
661  case LoopHintAttr::Unroll:
662  setUnrollState(LoopAttributes::Enable);
663  break;
664  case LoopHintAttr::UnrollAndJam:
665  setUnrollAndJamState(LoopAttributes::Enable);
666  break;
667  case LoopHintAttr::VectorizePredicate:
668  setVectorizePredicateState(LoopAttributes::Enable);
669  break;
670  case LoopHintAttr::Distribute:
671  setDistributeState(true);
672  break;
673  case LoopHintAttr::UnrollCount:
674  case LoopHintAttr::UnrollAndJamCount:
675  case LoopHintAttr::VectorizeWidth:
676  case LoopHintAttr::InterleaveCount:
677  case LoopHintAttr::PipelineDisabled:
678  case LoopHintAttr::PipelineInitiationInterval:
679  llvm_unreachable("Options cannot enabled.");
680  break;
681  }
682  break;
683  case LoopHintAttr::AssumeSafety:
684  switch (Option) {
685  case LoopHintAttr::Vectorize:
686  case LoopHintAttr::Interleave:
687  // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
688  setParallel(true);
689  setVectorizeEnable(true);
690  break;
691  case LoopHintAttr::Unroll:
692  case LoopHintAttr::UnrollAndJam:
693  case LoopHintAttr::VectorizePredicate:
694  case LoopHintAttr::UnrollCount:
695  case LoopHintAttr::UnrollAndJamCount:
696  case LoopHintAttr::VectorizeWidth:
697  case LoopHintAttr::InterleaveCount:
698  case LoopHintAttr::Distribute:
699  case LoopHintAttr::PipelineDisabled:
700  case LoopHintAttr::PipelineInitiationInterval:
701  llvm_unreachable("Options cannot be used to assume mem safety.");
702  break;
703  }
704  break;
705  case LoopHintAttr::Full:
706  switch (Option) {
707  case LoopHintAttr::Unroll:
708  setUnrollState(LoopAttributes::Full);
709  break;
710  case LoopHintAttr::UnrollAndJam:
711  setUnrollAndJamState(LoopAttributes::Full);
712  break;
713  case LoopHintAttr::Vectorize:
714  case LoopHintAttr::Interleave:
715  case LoopHintAttr::UnrollCount:
716  case LoopHintAttr::UnrollAndJamCount:
717  case LoopHintAttr::VectorizeWidth:
718  case LoopHintAttr::InterleaveCount:
719  case LoopHintAttr::Distribute:
720  case LoopHintAttr::PipelineDisabled:
721  case LoopHintAttr::PipelineInitiationInterval:
722  case LoopHintAttr::VectorizePredicate:
723  llvm_unreachable("Options cannot be used with 'full' hint.");
724  break;
725  }
726  break;
727  case LoopHintAttr::Numeric:
728  switch (Option) {
729  case LoopHintAttr::VectorizeWidth:
730  setVectorizeWidth(ValueInt);
731  break;
732  case LoopHintAttr::InterleaveCount:
733  setInterleaveCount(ValueInt);
734  break;
735  case LoopHintAttr::UnrollCount:
736  setUnrollCount(ValueInt);
737  break;
738  case LoopHintAttr::UnrollAndJamCount:
739  setUnrollAndJamCount(ValueInt);
740  break;
741  case LoopHintAttr::PipelineInitiationInterval:
742  setPipelineInitiationInterval(ValueInt);
743  break;
744  case LoopHintAttr::Unroll:
745  case LoopHintAttr::UnrollAndJam:
746  case LoopHintAttr::VectorizePredicate:
747  case LoopHintAttr::Vectorize:
748  case LoopHintAttr::Interleave:
749  case LoopHintAttr::Distribute:
750  case LoopHintAttr::PipelineDisabled:
751  llvm_unreachable("Options cannot be assigned a value.");
752  break;
753  }
754  break;
755  }
756  }
757 
758  if (CGOpts.OptimizationLevel > 0)
759  // Disable unrolling for the loop, if unrolling is disabled (via
760  // -fno-unroll-loops) and no pragmas override the decision.
761  if (!CGOpts.UnrollLoops &&
762  (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
763  StagedAttrs.UnrollCount == 0))
764  setUnrollState(LoopAttributes::Disable);
765 
766  /// Stage the attributes.
767  push(Header, StartLoc, EndLoc);
768 }
769 
771  assert(!Active.empty() && "No active loops to pop");
772  Active.back()->finish();
773  Active.pop_back();
774 }
775 
776 void LoopInfoStack::InsertHelper(Instruction *I) const {
777  if (I->mayReadOrWriteMemory()) {
778  SmallVector<Metadata *, 4> AccessGroups;
779  for (const auto &AL : Active) {
780  // Here we assume that every loop that has an access group is parallel.
781  if (MDNode *Group = AL->getAccessGroup())
782  AccessGroups.push_back(Group);
783  }
784  MDNode *UnionMD = nullptr;
785  if (AccessGroups.size() == 1)
786  UnionMD = cast<MDNode>(AccessGroups[0]);
787  else if (AccessGroups.size() >= 2)
788  UnionMD = MDNode::get(I->getContext(), AccessGroups);
789  I->setMetadata("llvm.access.group", UnionMD);
790  }
791 
792  if (!hasInfo())
793  return;
794 
795  const LoopInfo &L = getInfo();
796  if (!L.getLoopID())
797  return;
798 
799  if (I->isTerminator()) {
800  for (BasicBlock *Succ : successors(I))
801  if (Succ == L.getHeader()) {
802  I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
803  break;
804  }
805  return;
806  }
807 }
Defines the clang::ASTContext interface.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
Definition: Dominators.h:30
Attributes that may be specified on loops.
Definition: CGLoopInfo.h:36
unsigned UnrollAndJamCount
llvm.unroll.
Definition: CGLoopInfo.h:68
Information used when generating a structured loop.
Definition: CGLoopInfo.h:81
LVEnableState VectorizePredicateEnable
Value for llvm.loop.vectorize.predicate metadata.
Definition: CGLoopInfo.h:56
LoopAttributes(bool IsParallel=false)
Definition: CGLoopInfo.cpp:434
LVEnableState UnrollEnable
Value for llvm.loop.unroll.* metadata (enable, disable, or full).
Definition: CGLoopInfo.h:50
unsigned PipelineInitiationInterval
Value for llvm.loop.pipeline.iicount metadata.
Definition: CGLoopInfo.h:77
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:174
LineState State
unsigned InterleaveCount
Value for llvm.loop.interleave.count metadata.
Definition: CGLoopInfo.h:62
LVEnableState VectorizeEnable
Value for llvm.loop.vectorize.enable metadata.
Definition: CGLoopInfo.h:47
void pop()
End the current loop.
Definition: CGLoopInfo.cpp:770
Whether values of this type can be null is (explicitly) unspecified.
LVEnableState UnrollAndJamEnable
Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full).
Definition: CGLoopInfo.h:53
void finish()
Create the loop&#39;s metadata.
Definition: CGLoopInfo.cpp:485
NodeId Parent
Definition: ASTDiff.cpp:192
void InsertHelper(llvm::Instruction *I) const
Function called by the CodeGenFunction when an instruction is created.
Definition: CGLoopInfo.cpp:776
bool IsParallel
Generate llvm.loop.parallel metadata for loads and stores.
Definition: CGLoopInfo.h:41
unsigned UnrollCount
llvm.unroll.
Definition: CGLoopInfo.h:65
LVEnableState DistributeEnable
Value for llvm.loop.distribute.enable metadata.
Definition: CGLoopInfo.h:71
#define false
Definition: stdbool.h:17
llvm::MDNode * getLoopID() const
Get the loop id metadata for this loop.
Definition: CGLoopInfo.h:89
llvm::APSInt APSInt
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
bool PipelineDisabled
Value for llvm.loop.pipeline.disable metadata.
Definition: CGLoopInfo.h:74
static const TypeInfo & getInfo(unsigned id)
Definition: Types.cpp:47
CodeGenOptions - Track various options which control how the code is optimized and passed to the back...
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, LoopInfo *Parent)
Construct a new LoopInfo for the loop with entry Header.
Definition: CGLoopInfo.cpp:458
unsigned VectorizeWidth
Value for llvm.loop.vectorize.width metadata.
Definition: CGLoopInfo.h:59
llvm::BasicBlock * getHeader() const
Get the header block of this loop.
Definition: CGLoopInfo.h:92
Attr - This represents one attribute.
Definition: Attr.h:46