clang  10.0.0svn
CGLoopInfo.cpp
Go to the documentation of this file.
1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "llvm/IR/BasicBlock.h"
13 #include "llvm/IR/CFG.h"
14 #include "llvm/IR/Constants.h"
15 #include "llvm/IR/InstrTypes.h"
16 #include "llvm/IR/Instructions.h"
17 #include "llvm/IR/Metadata.h"
18 using namespace clang::CodeGen;
19 using namespace llvm;
20 
21 MDNode *
22 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
23  LLVMContext &Ctx = Header->getContext();
24  SmallVector<Metadata *, 4> NewLoopProperties;
25  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
26  NewLoopProperties.push_back(TempNode.get());
27  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
28 
29  MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
30  LoopID->replaceOperandWith(0, LoopID);
31  return LoopID;
32 }
33 
34 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
35  ArrayRef<Metadata *> LoopProperties,
36  bool &HasUserTransforms) {
37  LLVMContext &Ctx = Header->getContext();
38 
39  Optional<bool> Enabled;
40  if (Attrs.PipelineDisabled)
41  Enabled = false;
42  else if (Attrs.PipelineInitiationInterval != 0)
43  Enabled = true;
44 
45  if (Enabled != true) {
46  SmallVector<Metadata *, 4> NewLoopProperties;
47  if (Enabled == false) {
48  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
49  NewLoopProperties.push_back(
50  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
51  ConstantAsMetadata::get(ConstantInt::get(
52  llvm::Type::getInt1Ty(Ctx), 1))}));
53  LoopProperties = NewLoopProperties;
54  }
55  return createLoopPropertiesMetadata(LoopProperties);
56  }
57 
59  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
60  Args.push_back(TempNode.get());
61  Args.append(LoopProperties.begin(), LoopProperties.end());
62 
63  if (Attrs.PipelineInitiationInterval > 0) {
64  Metadata *Vals[] = {
65  MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66  ConstantAsMetadata::get(ConstantInt::get(
67  llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
68  Args.push_back(MDNode::get(Ctx, Vals));
69  }
70 
71  // No follow-up: This is the last transformation.
72 
73  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74  LoopID->replaceOperandWith(0, LoopID);
75  HasUserTransforms = true;
76  return LoopID;
77 }
78 
79 MDNode *
80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81  ArrayRef<Metadata *> LoopProperties,
82  bool &HasUserTransforms) {
83  LLVMContext &Ctx = Header->getContext();
84 
85  Optional<bool> Enabled;
87  Enabled = false;
88  else if (Attrs.UnrollEnable == LoopAttributes::Full)
89  Enabled = None;
90  else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91  Attrs.UnrollCount != 0)
92  Enabled = true;
93 
94  if (Enabled != true) {
95  // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96  // if unrolling is disabled.
97  return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98  }
99 
100  SmallVector<Metadata *, 4> FollowupLoopProperties;
101 
102  // Apply all loop properties to the unrolled loop.
103  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104 
105  // Don't unroll an already unrolled loop.
106  FollowupLoopProperties.push_back(
107  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108 
109  bool FollowupHasTransforms = false;
110  MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111  FollowupHasTransforms);
112 
114  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
115  Args.push_back(TempNode.get());
116  Args.append(LoopProperties.begin(), LoopProperties.end());
117 
118  // Setting unroll.count
119  if (Attrs.UnrollCount > 0) {
120  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
121  ConstantAsMetadata::get(ConstantInt::get(
122  llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
123  Args.push_back(MDNode::get(Ctx, Vals));
124  }
125 
126  // Setting unroll.full or unroll.disable
127  if (Attrs.UnrollEnable == LoopAttributes::Enable) {
128  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
129  Args.push_back(MDNode::get(Ctx, Vals));
130  }
131 
132  if (FollowupHasTransforms)
133  Args.push_back(MDNode::get(
134  Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
135 
136  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137  LoopID->replaceOperandWith(0, LoopID);
138  HasUserTransforms = true;
139  return LoopID;
140 }
141 
142 MDNode *
143 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
144  ArrayRef<Metadata *> LoopProperties,
145  bool &HasUserTransforms) {
146  LLVMContext &Ctx = Header->getContext();
147 
148  Optional<bool> Enabled;
150  Enabled = false;
151  else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
152  Attrs.UnrollAndJamCount != 0)
153  Enabled = true;
154 
155  if (Enabled != true) {
156  SmallVector<Metadata *, 4> NewLoopProperties;
157  if (Enabled == false) {
158  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159  NewLoopProperties.push_back(MDNode::get(
160  Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
161  LoopProperties = NewLoopProperties;
162  }
163  return createPartialUnrollMetadata(Attrs, LoopProperties,
164  HasUserTransforms);
165  }
166 
167  SmallVector<Metadata *, 4> FollowupLoopProperties;
168  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169  FollowupLoopProperties.push_back(
170  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
171 
172  bool FollowupHasTransforms = false;
173  MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174  FollowupHasTransforms);
175 
177  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
178  Args.push_back(TempNode.get());
179  Args.append(LoopProperties.begin(), LoopProperties.end());
180 
181  // Setting unroll_and_jam.count
182  if (Attrs.UnrollAndJamCount > 0) {
183  Metadata *Vals[] = {
184  MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
185  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
186  Attrs.UnrollAndJamCount))};
187  Args.push_back(MDNode::get(Ctx, Vals));
188  }
189 
191  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
192  Args.push_back(MDNode::get(Ctx, Vals));
193  }
194 
195  if (FollowupHasTransforms)
196  Args.push_back(MDNode::get(
197  Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
198  Followup}));
199 
200  if (UnrollAndJamInnerFollowup)
201  Args.push_back(MDNode::get(
202  Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
203  UnrollAndJamInnerFollowup}));
204 
205  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
206  LoopID->replaceOperandWith(0, LoopID);
207  HasUserTransforms = true;
208  return LoopID;
209 }
210 
211 MDNode *
212 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
213  ArrayRef<Metadata *> LoopProperties,
214  bool &HasUserTransforms) {
215  LLVMContext &Ctx = Header->getContext();
216 
217  Optional<bool> Enabled;
219  Enabled = false;
220  else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
222  Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
223  Enabled = true;
224 
225  if (Enabled != true) {
226  SmallVector<Metadata *, 4> NewLoopProperties;
227  if (Enabled == false) {
228  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
229  NewLoopProperties.push_back(
230  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
231  ConstantAsMetadata::get(ConstantInt::get(
232  llvm::Type::getInt1Ty(Ctx), 0))}));
233  LoopProperties = NewLoopProperties;
234  }
235  return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
236  }
237 
238  // Apply all loop properties to the vectorized loop.
239  SmallVector<Metadata *, 4> FollowupLoopProperties;
240  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
241 
242  // Don't vectorize an already vectorized loop.
243  FollowupLoopProperties.push_back(
244  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
245 
246  bool FollowupHasTransforms = false;
247  MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
248  FollowupHasTransforms);
249 
251  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
252  Args.push_back(TempNode.get());
253  Args.append(LoopProperties.begin(), LoopProperties.end());
254 
255  // Setting vectorize.predicate
256  bool IsVectorPredicateEnabled = false;
259  Attrs.VectorizeWidth < 1) {
260 
261  IsVectorPredicateEnabled =
263 
264  Metadata *Vals[] = {
265  MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
266  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
267  IsVectorPredicateEnabled))};
268  Args.push_back(MDNode::get(Ctx, Vals));
269  }
270 
271  // Setting vectorize.width
272  if (Attrs.VectorizeWidth > 0) {
273  // This implies vectorize.enable = true, but only add it when it is not
274  // already enabled.
276  Args.push_back(
277  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
278  ConstantAsMetadata::get(ConstantInt::get(
279  llvm::Type::getInt1Ty(Ctx), 1))}));
280 
281  Metadata *Vals[] = {
282  MDString::get(Ctx, "llvm.loop.vectorize.width"),
283  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
284  Attrs.VectorizeWidth))};
285  Args.push_back(MDNode::get(Ctx, Vals));
286  }
287 
288  // Setting interleave.count
289  if (Attrs.InterleaveCount > 0) {
290  Metadata *Vals[] = {
291  MDString::get(Ctx, "llvm.loop.interleave.count"),
292  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
293  Attrs.InterleaveCount))};
294  Args.push_back(MDNode::get(Ctx, Vals));
295  }
296 
297  // Setting vectorize.enable
299  IsVectorPredicateEnabled) {
300  Metadata *Vals[] = {
301  MDString::get(Ctx, "llvm.loop.vectorize.enable"),
302  ConstantAsMetadata::get(ConstantInt::get(
303  llvm::Type::getInt1Ty(Ctx),
304  IsVectorPredicateEnabled
305  ? true
306  : (Attrs.VectorizeEnable == LoopAttributes::Enable)))};
307  Args.push_back(MDNode::get(Ctx, Vals));
308  }
309 
310  if (FollowupHasTransforms)
311  Args.push_back(MDNode::get(
312  Ctx,
313  {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
314 
315  MDNode *LoopID = MDNode::get(Ctx, Args);
316  LoopID->replaceOperandWith(0, LoopID);
317  HasUserTransforms = true;
318  return LoopID;
319 }
320 
321 MDNode *
322 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
323  ArrayRef<Metadata *> LoopProperties,
324  bool &HasUserTransforms) {
325  LLVMContext &Ctx = Header->getContext();
326 
327  Optional<bool> Enabled;
329  Enabled = false;
331  Enabled = true;
332 
333  if (Enabled != true) {
334  SmallVector<Metadata *, 4> NewLoopProperties;
335  if (Enabled == false) {
336  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
337  NewLoopProperties.push_back(
338  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
339  ConstantAsMetadata::get(ConstantInt::get(
340  llvm::Type::getInt1Ty(Ctx), 0))}));
341  LoopProperties = NewLoopProperties;
342  }
343  return createLoopVectorizeMetadata(Attrs, LoopProperties,
344  HasUserTransforms);
345  }
346 
347  bool FollowupHasTransforms = false;
348  MDNode *Followup =
349  createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
350 
352  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
353  Args.push_back(TempNode.get());
354  Args.append(LoopProperties.begin(), LoopProperties.end());
355 
356  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
357  ConstantAsMetadata::get(ConstantInt::get(
358  llvm::Type::getInt1Ty(Ctx),
360  Args.push_back(MDNode::get(Ctx, Vals));
361 
362  if (FollowupHasTransforms)
363  Args.push_back(MDNode::get(
364  Ctx,
365  {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
366 
367  MDNode *LoopID = MDNode::get(Ctx, Args);
368  LoopID->replaceOperandWith(0, LoopID);
369  HasUserTransforms = true;
370  return LoopID;
371 }
372 
373 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
374  ArrayRef<Metadata *> LoopProperties,
375  bool &HasUserTransforms) {
376  LLVMContext &Ctx = Header->getContext();
377 
378  Optional<bool> Enabled;
380  Enabled = false;
381  else if (Attrs.UnrollEnable == LoopAttributes::Full)
382  Enabled = true;
383 
384  if (Enabled != true) {
385  SmallVector<Metadata *, 4> NewLoopProperties;
386  if (Enabled == false) {
387  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
388  NewLoopProperties.push_back(
389  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
390  LoopProperties = NewLoopProperties;
391  }
392  return createLoopDistributeMetadata(Attrs, LoopProperties,
393  HasUserTransforms);
394  }
395 
397  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
398  Args.push_back(TempNode.get());
399  Args.append(LoopProperties.begin(), LoopProperties.end());
400  Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
401 
402  // No follow-up: there is no loop after full unrolling.
403  // TODO: Warn if there are transformations after full unrolling.
404 
405  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
406  LoopID->replaceOperandWith(0, LoopID);
407  HasUserTransforms = true;
408  return LoopID;
409 }
410 
411 MDNode *LoopInfo::createMetadata(
412  const LoopAttributes &Attrs,
413  llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
414  bool &HasUserTransforms) {
415  SmallVector<Metadata *, 3> LoopProperties;
416 
417  // If we have a valid start debug location for the loop, add it.
418  if (StartLoc) {
419  LoopProperties.push_back(StartLoc.getAsMDNode());
420 
421  // If we also have a valid end debug location for the loop, add it.
422  if (EndLoc)
423  LoopProperties.push_back(EndLoc.getAsMDNode());
424  }
425 
426  assert(!!AccGroup == Attrs.IsParallel &&
427  "There must be an access group iff the loop is parallel");
428  if (Attrs.IsParallel) {
429  LLVMContext &Ctx = Header->getContext();
430  LoopProperties.push_back(MDNode::get(
431  Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
432  }
433 
434  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
435  AdditionalLoopProperties.end());
436  return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
437 }
438 
440  : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
441  UnrollEnable(LoopAttributes::Unspecified),
442  UnrollAndJamEnable(LoopAttributes::Unspecified),
443  VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
444  InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
445  DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
446  PipelineInitiationInterval(0) {}
447 
449  IsParallel = false;
450  VectorizeWidth = 0;
451  InterleaveCount = 0;
452  UnrollCount = 0;
453  UnrollAndJamCount = 0;
459  PipelineDisabled = false;
461 }
462 
463 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
464  const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
465  LoopInfo *Parent)
466  : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
467  Parent(Parent) {
468 
469  if (Attrs.IsParallel) {
470  // Create an access group for this loop.
471  LLVMContext &Ctx = Header->getContext();
472  AccGroup = MDNode::getDistinct(Ctx, {});
473  }
474 
475  if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
476  Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
477  Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
478  Attrs.PipelineInitiationInterval == 0 &&
483  Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
484  !EndLoc)
485  return;
486 
487  TempLoopID = MDNode::getTemporary(Header->getContext(), None);
488 }
489 
491  // We did not annotate the loop body instructions because there are no
492  // attributes for this loop.
493  if (!TempLoopID)
494  return;
495 
496  MDNode *LoopID;
497  LoopAttributes CurLoopAttr = Attrs;
498  LLVMContext &Ctx = Header->getContext();
499 
500  if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
501  Parent->Attrs.UnrollAndJamCount != 0)) {
502  // Parent unroll-and-jams this loop.
503  // Split the transformations in those that happens before the unroll-and-jam
504  // and those after.
505 
506  LoopAttributes BeforeJam, AfterJam;
507 
508  BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
509 
510  BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
511  BeforeJam.InterleaveCount = Attrs.InterleaveCount;
512  BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
513  BeforeJam.DistributeEnable = Attrs.DistributeEnable;
515 
516  switch (Attrs.UnrollEnable) {
519  BeforeJam.UnrollEnable = Attrs.UnrollEnable;
520  AfterJam.UnrollEnable = Attrs.UnrollEnable;
521  break;
524  break;
527  break;
528  }
529 
531  AfterJam.UnrollCount = Attrs.UnrollCount;
532  AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
534 
535  // If this loop is subject of an unroll-and-jam by the parent loop, and has
536  // an unroll-and-jam annotation itself, we have to decide whether to first
537  // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
538  // UnrollAndJam pass processes loops from inner to outer, so we apply the
539  // inner first.
540  BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
541  BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
542 
543  // Set the inner followup metadata to process by the outer loop. Only
544  // consider the first inner loop.
545  if (!Parent->UnrollAndJamInnerFollowup) {
546  // Splitting the attributes into a BeforeJam and an AfterJam part will
547  // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
548  // to be forwarded to the AfterJam part. We detect the situation here and
549  // add it manually.
550  SmallVector<Metadata *, 1> BeforeLoopProperties;
551  if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
553  BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
554  BeforeLoopProperties.push_back(
555  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
556 
557  bool InnerFollowupHasTransform = false;
558  MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
559  InnerFollowupHasTransform);
560  if (InnerFollowupHasTransform)
561  Parent->UnrollAndJamInnerFollowup = InnerFollowup;
562  }
563 
564  CurLoopAttr = BeforeJam;
565  }
566 
567  bool HasUserTransforms = false;
568  LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
569  TempLoopID->replaceAllUsesWith(LoopID);
570 }
571 
572 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
573  const llvm::DebugLoc &EndLoc) {
574  Active.emplace_back(
575  new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
576  Active.empty() ? nullptr : Active.back().get()));
577  // Clear the attributes so nested loops do not inherit them.
578  StagedAttrs.clear();
579 }
580 
581 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
583  const llvm::DebugLoc &StartLoc,
584  const llvm::DebugLoc &EndLoc) {
585 
586  // Identify loop hint attributes from Attrs.
587  for (const auto *Attr : Attrs) {
588  const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
589  const OpenCLUnrollHintAttr *OpenCLHint =
590  dyn_cast<OpenCLUnrollHintAttr>(Attr);
591 
592  // Skip non loop hint attributes
593  if (!LH && !OpenCLHint) {
594  continue;
595  }
596 
597  LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
598  LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
599  unsigned ValueInt = 1;
600  // Translate opencl_unroll_hint attribute argument to
601  // equivalent LoopHintAttr enums.
602  // OpenCL v2.0 s6.11.5:
603  // 0 - enable unroll (no argument).
604  // 1 - disable unroll.
605  // other positive integer n - unroll by n.
606  if (OpenCLHint) {
607  ValueInt = OpenCLHint->getUnrollHint();
608  if (ValueInt == 0) {
609  State = LoopHintAttr::Enable;
610  } else if (ValueInt != 1) {
611  Option = LoopHintAttr::UnrollCount;
612  State = LoopHintAttr::Numeric;
613  }
614  } else if (LH) {
615  auto *ValueExpr = LH->getValue();
616  if (ValueExpr) {
617  llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
618  ValueInt = ValueAPS.getSExtValue();
619  }
620 
621  Option = LH->getOption();
622  State = LH->getState();
623  }
624  switch (State) {
625  case LoopHintAttr::Disable:
626  switch (Option) {
627  case LoopHintAttr::Vectorize:
628  // Disable vectorization by specifying a width of 1.
629  setVectorizeWidth(1);
630  break;
631  case LoopHintAttr::Interleave:
632  // Disable interleaving by speciyfing a count of 1.
633  setInterleaveCount(1);
634  break;
635  case LoopHintAttr::Unroll:
636  setUnrollState(LoopAttributes::Disable);
637  break;
638  case LoopHintAttr::UnrollAndJam:
639  setUnrollAndJamState(LoopAttributes::Disable);
640  break;
641  case LoopHintAttr::VectorizePredicate:
642  setVectorizePredicateState(LoopAttributes::Disable);
643  break;
644  case LoopHintAttr::Distribute:
645  setDistributeState(false);
646  break;
647  case LoopHintAttr::PipelineDisabled:
648  setPipelineDisabled(true);
649  break;
650  case LoopHintAttr::UnrollCount:
651  case LoopHintAttr::UnrollAndJamCount:
652  case LoopHintAttr::VectorizeWidth:
653  case LoopHintAttr::InterleaveCount:
654  case LoopHintAttr::PipelineInitiationInterval:
655  llvm_unreachable("Options cannot be disabled.");
656  break;
657  }
658  break;
659  case LoopHintAttr::Enable:
660  switch (Option) {
661  case LoopHintAttr::Vectorize:
662  case LoopHintAttr::Interleave:
663  setVectorizeEnable(true);
664  break;
665  case LoopHintAttr::Unroll:
666  setUnrollState(LoopAttributes::Enable);
667  break;
668  case LoopHintAttr::UnrollAndJam:
669  setUnrollAndJamState(LoopAttributes::Enable);
670  break;
671  case LoopHintAttr::VectorizePredicate:
672  setVectorizePredicateState(LoopAttributes::Enable);
673  break;
674  case LoopHintAttr::Distribute:
675  setDistributeState(true);
676  break;
677  case LoopHintAttr::UnrollCount:
678  case LoopHintAttr::UnrollAndJamCount:
679  case LoopHintAttr::VectorizeWidth:
680  case LoopHintAttr::InterleaveCount:
681  case LoopHintAttr::PipelineDisabled:
682  case LoopHintAttr::PipelineInitiationInterval:
683  llvm_unreachable("Options cannot enabled.");
684  break;
685  }
686  break;
687  case LoopHintAttr::AssumeSafety:
688  switch (Option) {
689  case LoopHintAttr::Vectorize:
690  case LoopHintAttr::Interleave:
691  // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
692  setParallel(true);
693  setVectorizeEnable(true);
694  break;
695  case LoopHintAttr::Unroll:
696  case LoopHintAttr::UnrollAndJam:
697  case LoopHintAttr::VectorizePredicate:
698  case LoopHintAttr::UnrollCount:
699  case LoopHintAttr::UnrollAndJamCount:
700  case LoopHintAttr::VectorizeWidth:
701  case LoopHintAttr::InterleaveCount:
702  case LoopHintAttr::Distribute:
703  case LoopHintAttr::PipelineDisabled:
704  case LoopHintAttr::PipelineInitiationInterval:
705  llvm_unreachable("Options cannot be used to assume mem safety.");
706  break;
707  }
708  break;
709  case LoopHintAttr::Full:
710  switch (Option) {
711  case LoopHintAttr::Unroll:
712  setUnrollState(LoopAttributes::Full);
713  break;
714  case LoopHintAttr::UnrollAndJam:
715  setUnrollAndJamState(LoopAttributes::Full);
716  break;
717  case LoopHintAttr::Vectorize:
718  case LoopHintAttr::Interleave:
719  case LoopHintAttr::UnrollCount:
720  case LoopHintAttr::UnrollAndJamCount:
721  case LoopHintAttr::VectorizeWidth:
722  case LoopHintAttr::InterleaveCount:
723  case LoopHintAttr::Distribute:
724  case LoopHintAttr::PipelineDisabled:
725  case LoopHintAttr::PipelineInitiationInterval:
726  case LoopHintAttr::VectorizePredicate:
727  llvm_unreachable("Options cannot be used with 'full' hint.");
728  break;
729  }
730  break;
731  case LoopHintAttr::Numeric:
732  switch (Option) {
733  case LoopHintAttr::VectorizeWidth:
734  setVectorizeWidth(ValueInt);
735  break;
736  case LoopHintAttr::InterleaveCount:
737  setInterleaveCount(ValueInt);
738  break;
739  case LoopHintAttr::UnrollCount:
740  setUnrollCount(ValueInt);
741  break;
742  case LoopHintAttr::UnrollAndJamCount:
743  setUnrollAndJamCount(ValueInt);
744  break;
745  case LoopHintAttr::PipelineInitiationInterval:
746  setPipelineInitiationInterval(ValueInt);
747  break;
748  case LoopHintAttr::Unroll:
749  case LoopHintAttr::UnrollAndJam:
750  case LoopHintAttr::VectorizePredicate:
751  case LoopHintAttr::Vectorize:
752  case LoopHintAttr::Interleave:
753  case LoopHintAttr::Distribute:
754  case LoopHintAttr::PipelineDisabled:
755  llvm_unreachable("Options cannot be assigned a value.");
756  break;
757  }
758  break;
759  }
760  }
761 
762  /// Stage the attributes.
763  push(Header, StartLoc, EndLoc);
764 }
765 
767  assert(!Active.empty() && "No active loops to pop");
768  Active.back()->finish();
769  Active.pop_back();
770 }
771 
772 void LoopInfoStack::InsertHelper(Instruction *I) const {
773  if (I->mayReadOrWriteMemory()) {
774  SmallVector<Metadata *, 4> AccessGroups;
775  for (const auto &AL : Active) {
776  // Here we assume that every loop that has an access group is parallel.
777  if (MDNode *Group = AL->getAccessGroup())
778  AccessGroups.push_back(Group);
779  }
780  MDNode *UnionMD = nullptr;
781  if (AccessGroups.size() == 1)
782  UnionMD = cast<MDNode>(AccessGroups[0]);
783  else if (AccessGroups.size() >= 2)
784  UnionMD = MDNode::get(I->getContext(), AccessGroups);
785  I->setMetadata("llvm.access.group", UnionMD);
786  }
787 
788  if (!hasInfo())
789  return;
790 
791  const LoopInfo &L = getInfo();
792  if (!L.getLoopID())
793  return;
794 
795  if (I->isTerminator()) {
796  for (BasicBlock *Succ : successors(I))
797  if (Succ == L.getHeader()) {
798  I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
799  break;
800  }
801  return;
802  }
803 }
Defines the clang::ASTContext interface.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
Definition: Dominators.h:30
Attributes that may be specified on loops.
Definition: CGLoopInfo.h:35
unsigned UnrollAndJamCount
llvm.unroll.
Definition: CGLoopInfo.h:67
Information used when generating a structured loop.
Definition: CGLoopInfo.h:80
LVEnableState VectorizePredicateEnable
Value for llvm.loop.vectorize.predicate metadata.
Definition: CGLoopInfo.h:55
LoopAttributes(bool IsParallel=false)
Definition: CGLoopInfo.cpp:439
LVEnableState UnrollEnable
Value for llvm.loop.unroll.* metadata (enable, disable, or full).
Definition: CGLoopInfo.h:49
unsigned PipelineInitiationInterval
Value for llvm.loop.pipeline.iicount metadata.
Definition: CGLoopInfo.h:76
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:160
LineState State
unsigned InterleaveCount
Value for llvm.loop.interleave.count metadata.
Definition: CGLoopInfo.h:61
LVEnableState VectorizeEnable
Value for llvm.loop.vectorize.enable metadata.
Definition: CGLoopInfo.h:46
void pop()
End the current loop.
Definition: CGLoopInfo.cpp:766
Whether values of this type can be null is (explicitly) unspecified.
LVEnableState UnrollAndJamEnable
Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full).
Definition: CGLoopInfo.h:52
void finish()
Create the loop&#39;s metadata.
Definition: CGLoopInfo.cpp:490
NodeId Parent
Definition: ASTDiff.cpp:191
void InsertHelper(llvm::Instruction *I) const
Function called by the CodeGenFunction when an instruction is created.
Definition: CGLoopInfo.cpp:772
bool IsParallel
Generate llvm.loop.parallel metadata for loads and stores.
Definition: CGLoopInfo.h:40
unsigned UnrollCount
llvm.unroll.
Definition: CGLoopInfo.h:64
LVEnableState DistributeEnable
Value for llvm.loop.distribute.enable metadata.
Definition: CGLoopInfo.h:70
#define false
Definition: stdbool.h:17
llvm::MDNode * getLoopID() const
Get the loop id metadata for this loop.
Definition: CGLoopInfo.h:88
llvm::APSInt APSInt
constexpr XRayInstrMask None
Definition: XRayInstr.h:37
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
bool PipelineDisabled
Value for llvm.loop.pipeline.disable metadata.
Definition: CGLoopInfo.h:73
static const TypeInfo & getInfo(unsigned id)
Definition: Types.cpp:38
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, LoopInfo *Parent)
Construct a new LoopInfo for the loop with entry Header.
Definition: CGLoopInfo.cpp:463
unsigned VectorizeWidth
Value for llvm.loop.vectorize.width metadata.
Definition: CGLoopInfo.h:58
llvm::BasicBlock * getHeader() const
Get the header block of this loop.
Definition: CGLoopInfo.h:91
Attr - This represents one attribute.
Definition: Attr.h:45