clang  9.0.0svn
CGLoopInfo.cpp
Go to the documentation of this file.
1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "llvm/IR/BasicBlock.h"
13 #include "llvm/IR/CFG.h"
14 #include "llvm/IR/Constants.h"
15 #include "llvm/IR/InstrTypes.h"
16 #include "llvm/IR/Instructions.h"
17 #include "llvm/IR/Metadata.h"
18 using namespace clang::CodeGen;
19 using namespace llvm;
20 
21 MDNode *
22 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
23  LLVMContext &Ctx = Header->getContext();
24  SmallVector<Metadata *, 4> NewLoopProperties;
25  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
26  NewLoopProperties.push_back(TempNode.get());
27  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
28 
29  MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
30  LoopID->replaceOperandWith(0, LoopID);
31  return LoopID;
32 }
33 
34 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
35  ArrayRef<Metadata *> LoopProperties,
36  bool &HasUserTransforms) {
37  LLVMContext &Ctx = Header->getContext();
38 
39  Optional<bool> Enabled;
40  if (Attrs.PipelineDisabled)
41  Enabled = false;
42  else if (Attrs.PipelineInitiationInterval != 0)
43  Enabled = true;
44 
45  if (Enabled != true) {
46  SmallVector<Metadata *, 4> NewLoopProperties;
47  if (Enabled == false) {
48  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
49  NewLoopProperties.push_back(
50  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
51  ConstantAsMetadata::get(ConstantInt::get(
52  llvm::Type::getInt1Ty(Ctx), 1))}));
53  LoopProperties = NewLoopProperties;
54  }
55  return createLoopPropertiesMetadata(LoopProperties);
56  }
57 
59  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
60  Args.push_back(TempNode.get());
61  Args.append(LoopProperties.begin(), LoopProperties.end());
62 
63  if (Attrs.PipelineInitiationInterval > 0) {
64  Metadata *Vals[] = {
65  MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66  ConstantAsMetadata::get(ConstantInt::get(
67  llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
68  Args.push_back(MDNode::get(Ctx, Vals));
69  }
70 
71  // No follow-up: This is the last transformation.
72 
73  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74  LoopID->replaceOperandWith(0, LoopID);
75  HasUserTransforms = true;
76  return LoopID;
77 }
78 
79 MDNode *
80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81  ArrayRef<Metadata *> LoopProperties,
82  bool &HasUserTransforms) {
83  LLVMContext &Ctx = Header->getContext();
84 
85  Optional<bool> Enabled;
87  Enabled = false;
88  else if (Attrs.UnrollEnable == LoopAttributes::Full)
89  Enabled = None;
90  else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91  Attrs.UnrollCount != 0)
92  Enabled = true;
93 
94  if (Enabled != true) {
95  // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96  // if unrolling is disabled.
97  return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98  }
99 
100  SmallVector<Metadata *, 4> FollowupLoopProperties;
101 
102  // Apply all loop properties to the unrolled loop.
103  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104 
105  // Don't unroll an already unrolled loop.
106  FollowupLoopProperties.push_back(
107  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108 
109  bool FollowupHasTransforms = false;
110  MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111  FollowupHasTransforms);
112 
114  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
115  Args.push_back(TempNode.get());
116  Args.append(LoopProperties.begin(), LoopProperties.end());
117 
118  // Setting unroll.count
119  if (Attrs.UnrollCount > 0) {
120  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
121  ConstantAsMetadata::get(ConstantInt::get(
122  llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
123  Args.push_back(MDNode::get(Ctx, Vals));
124  }
125 
126  // Setting unroll.full or unroll.disable
127  if (Attrs.UnrollEnable == LoopAttributes::Enable) {
128  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
129  Args.push_back(MDNode::get(Ctx, Vals));
130  }
131 
132  if (FollowupHasTransforms)
133  Args.push_back(MDNode::get(
134  Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
135 
136  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137  LoopID->replaceOperandWith(0, LoopID);
138  HasUserTransforms = true;
139  return LoopID;
140 }
141 
142 MDNode *
143 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
144  ArrayRef<Metadata *> LoopProperties,
145  bool &HasUserTransforms) {
146  LLVMContext &Ctx = Header->getContext();
147 
148  Optional<bool> Enabled;
150  Enabled = false;
151  else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
152  Attrs.UnrollAndJamCount != 0)
153  Enabled = true;
154 
155  if (Enabled != true) {
156  SmallVector<Metadata *, 4> NewLoopProperties;
157  if (Enabled == false) {
158  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159  NewLoopProperties.push_back(MDNode::get(
160  Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
161  LoopProperties = NewLoopProperties;
162  }
163  return createPartialUnrollMetadata(Attrs, LoopProperties,
164  HasUserTransforms);
165  }
166 
167  SmallVector<Metadata *, 4> FollowupLoopProperties;
168  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169  FollowupLoopProperties.push_back(
170  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
171 
172  bool FollowupHasTransforms = false;
173  MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174  FollowupHasTransforms);
175 
177  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
178  Args.push_back(TempNode.get());
179  Args.append(LoopProperties.begin(), LoopProperties.end());
180 
181  // Setting unroll_and_jam.count
182  if (Attrs.UnrollAndJamCount > 0) {
183  Metadata *Vals[] = {
184  MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
185  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
186  Attrs.UnrollAndJamCount))};
187  Args.push_back(MDNode::get(Ctx, Vals));
188  }
189 
191  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
192  Args.push_back(MDNode::get(Ctx, Vals));
193  }
194 
195  if (FollowupHasTransforms)
196  Args.push_back(MDNode::get(
197  Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
198  Followup}));
199 
200  if (UnrollAndJamInnerFollowup)
201  Args.push_back(MDNode::get(
202  Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
203  UnrollAndJamInnerFollowup}));
204 
205  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
206  LoopID->replaceOperandWith(0, LoopID);
207  HasUserTransforms = true;
208  return LoopID;
209 }
210 
211 MDNode *
212 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
213  ArrayRef<Metadata *> LoopProperties,
214  bool &HasUserTransforms) {
215  LLVMContext &Ctx = Header->getContext();
216 
217  Optional<bool> Enabled;
219  Enabled = false;
220  else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
221  Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
222  Enabled = true;
223 
224  if (Enabled != true) {
225  SmallVector<Metadata *, 4> NewLoopProperties;
226  if (Enabled == false) {
227  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
228  NewLoopProperties.push_back(
229  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
230  ConstantAsMetadata::get(ConstantInt::get(
231  llvm::Type::getInt1Ty(Ctx), 0))}));
232  LoopProperties = NewLoopProperties;
233  }
234  return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
235  }
236 
237  // Apply all loop properties to the vectorized loop.
238  SmallVector<Metadata *, 4> FollowupLoopProperties;
239  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
240 
241  // Don't vectorize an already vectorized loop.
242  FollowupLoopProperties.push_back(
243  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
244 
245  bool FollowupHasTransforms = false;
246  MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
247  FollowupHasTransforms);
248 
250  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
251  Args.push_back(TempNode.get());
252  Args.append(LoopProperties.begin(), LoopProperties.end());
253 
254  // Setting vectorize.width
255  if (Attrs.VectorizeWidth > 0) {
256  Metadata *Vals[] = {
257  MDString::get(Ctx, "llvm.loop.vectorize.width"),
258  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
259  Attrs.VectorizeWidth))};
260  Args.push_back(MDNode::get(Ctx, Vals));
261  }
262 
263  // Setting interleave.count
264  if (Attrs.InterleaveCount > 0) {
265  Metadata *Vals[] = {
266  MDString::get(Ctx, "llvm.loop.interleave.count"),
267  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
268  Attrs.InterleaveCount))};
269  Args.push_back(MDNode::get(Ctx, Vals));
270  }
271 
272  // Setting vectorize.enable
274  Metadata *Vals[] = {
275  MDString::get(Ctx, "llvm.loop.vectorize.enable"),
276  ConstantAsMetadata::get(ConstantInt::get(
277  llvm::Type::getInt1Ty(Ctx),
279  Args.push_back(MDNode::get(Ctx, Vals));
280  }
281 
282  if (FollowupHasTransforms)
283  Args.push_back(MDNode::get(
284  Ctx,
285  {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
286 
287  MDNode *LoopID = MDNode::get(Ctx, Args);
288  LoopID->replaceOperandWith(0, LoopID);
289  HasUserTransforms = true;
290  return LoopID;
291 }
292 
293 MDNode *
294 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
295  ArrayRef<Metadata *> LoopProperties,
296  bool &HasUserTransforms) {
297  LLVMContext &Ctx = Header->getContext();
298 
299  Optional<bool> Enabled;
301  Enabled = false;
303  Enabled = true;
304 
305  if (Enabled != true) {
306  SmallVector<Metadata *, 4> NewLoopProperties;
307  if (Enabled == false) {
308  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
309  NewLoopProperties.push_back(
310  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
311  ConstantAsMetadata::get(ConstantInt::get(
312  llvm::Type::getInt1Ty(Ctx), 0))}));
313  LoopProperties = NewLoopProperties;
314  }
315  return createLoopVectorizeMetadata(Attrs, LoopProperties,
316  HasUserTransforms);
317  }
318 
319  bool FollowupHasTransforms = false;
320  MDNode *Followup =
321  createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
322 
324  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
325  Args.push_back(TempNode.get());
326  Args.append(LoopProperties.begin(), LoopProperties.end());
327 
328  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
329  ConstantAsMetadata::get(ConstantInt::get(
330  llvm::Type::getInt1Ty(Ctx),
332  Args.push_back(MDNode::get(Ctx, Vals));
333 
334  if (FollowupHasTransforms)
335  Args.push_back(MDNode::get(
336  Ctx,
337  {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
338 
339  MDNode *LoopID = MDNode::get(Ctx, Args);
340  LoopID->replaceOperandWith(0, LoopID);
341  HasUserTransforms = true;
342  return LoopID;
343 }
344 
345 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
346  ArrayRef<Metadata *> LoopProperties,
347  bool &HasUserTransforms) {
348  LLVMContext &Ctx = Header->getContext();
349 
350  Optional<bool> Enabled;
352  Enabled = false;
353  else if (Attrs.UnrollEnable == LoopAttributes::Full)
354  Enabled = true;
355 
356  if (Enabled != true) {
357  SmallVector<Metadata *, 4> NewLoopProperties;
358  if (Enabled == false) {
359  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
360  NewLoopProperties.push_back(
361  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
362  LoopProperties = NewLoopProperties;
363  }
364  return createLoopDistributeMetadata(Attrs, LoopProperties,
365  HasUserTransforms);
366  }
367 
369  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
370  Args.push_back(TempNode.get());
371  Args.append(LoopProperties.begin(), LoopProperties.end());
372  Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
373 
374  // No follow-up: there is no loop after full unrolling.
375  // TODO: Warn if there are transformations after full unrolling.
376 
377  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
378  LoopID->replaceOperandWith(0, LoopID);
379  HasUserTransforms = true;
380  return LoopID;
381 }
382 
383 MDNode *LoopInfo::createMetadata(
384  const LoopAttributes &Attrs,
385  llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
386  bool &HasUserTransforms) {
387  SmallVector<Metadata *, 3> LoopProperties;
388 
389  // If we have a valid start debug location for the loop, add it.
390  if (StartLoc) {
391  LoopProperties.push_back(StartLoc.getAsMDNode());
392 
393  // If we also have a valid end debug location for the loop, add it.
394  if (EndLoc)
395  LoopProperties.push_back(EndLoc.getAsMDNode());
396  }
397 
398  assert(!!AccGroup == Attrs.IsParallel &&
399  "There must be an access group iff the loop is parallel");
400  if (Attrs.IsParallel) {
401  LLVMContext &Ctx = Header->getContext();
402  LoopProperties.push_back(MDNode::get(
403  Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
404  }
405 
406  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
407  AdditionalLoopProperties.end());
408  return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
409 }
410 
412  : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
413  UnrollEnable(LoopAttributes::Unspecified),
414  UnrollAndJamEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
415  InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
416  DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
417  PipelineInitiationInterval(0) {}
418 
420  IsParallel = false;
421  VectorizeWidth = 0;
422  InterleaveCount = 0;
423  UnrollCount = 0;
424  UnrollAndJamCount = 0;
429  PipelineDisabled = false;
431 }
432 
433 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
434  const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
435  LoopInfo *Parent)
436  : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
437  Parent(Parent) {
438 
439  if (Attrs.IsParallel) {
440  // Create an access group for this loop.
441  LLVMContext &Ctx = Header->getContext();
442  AccGroup = MDNode::getDistinct(Ctx, {});
443  }
444 
445  if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
446  Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
447  Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
448  Attrs.PipelineInitiationInterval == 0 &&
452  Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
453  !EndLoc)
454  return;
455 
456  TempLoopID = MDNode::getTemporary(Header->getContext(), None);
457 }
458 
460  // We did not annotate the loop body instructions because there are no
461  // attributes for this loop.
462  if (!TempLoopID)
463  return;
464 
465  MDNode *LoopID;
466  LoopAttributes CurLoopAttr = Attrs;
467  LLVMContext &Ctx = Header->getContext();
468 
469  if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
470  Parent->Attrs.UnrollAndJamCount != 0)) {
471  // Parent unroll-and-jams this loop.
472  // Split the transformations in those that happens before the unroll-and-jam
473  // and those after.
474 
475  LoopAttributes BeforeJam, AfterJam;
476 
477  BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
478 
479  BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
480  BeforeJam.InterleaveCount = Attrs.InterleaveCount;
481  BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
482  BeforeJam.DistributeEnable = Attrs.DistributeEnable;
483 
484  switch (Attrs.UnrollEnable) {
487  BeforeJam.UnrollEnable = Attrs.UnrollEnable;
488  AfterJam.UnrollEnable = Attrs.UnrollEnable;
489  break;
492  break;
495  break;
496  }
497 
498  AfterJam.UnrollCount = Attrs.UnrollCount;
499  AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
501 
502  // If this loop is subject of an unroll-and-jam by the parent loop, and has
503  // an unroll-and-jam annotation itself, we have to decide whether to first
504  // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
505  // UnrollAndJam pass processes loops from inner to outer, so we apply the
506  // inner first.
507  BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
508  BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
509 
510  // Set the inner followup metadata to process by the outer loop. Only
511  // consider the first inner loop.
512  if (!Parent->UnrollAndJamInnerFollowup) {
513  // Splitting the attributes into a BeforeJam and an AfterJam part will
514  // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
515  // to be forwarded to the AfterJam part. We detect the situation here and
516  // add it manually.
517  SmallVector<Metadata *, 1> BeforeLoopProperties;
518  if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
519  BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
520  BeforeLoopProperties.push_back(
521  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
522 
523  bool InnerFollowupHasTransform = false;
524  MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
525  InnerFollowupHasTransform);
526  if (InnerFollowupHasTransform)
527  Parent->UnrollAndJamInnerFollowup = InnerFollowup;
528  }
529 
530  CurLoopAttr = BeforeJam;
531  }
532 
533  bool HasUserTransforms = false;
534  LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
535  TempLoopID->replaceAllUsesWith(LoopID);
536 }
537 
538 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
539  const llvm::DebugLoc &EndLoc) {
540  Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
541  Active.empty() ? nullptr : &Active.back()));
542  // Clear the attributes so nested loops do not inherit them.
543  StagedAttrs.clear();
544 }
545 
546 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
548  const llvm::DebugLoc &StartLoc,
549  const llvm::DebugLoc &EndLoc) {
550 
551  // Identify loop hint attributes from Attrs.
552  for (const auto *Attr : Attrs) {
553  const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
554  const OpenCLUnrollHintAttr *OpenCLHint =
555  dyn_cast<OpenCLUnrollHintAttr>(Attr);
556 
557  // Skip non loop hint attributes
558  if (!LH && !OpenCLHint) {
559  continue;
560  }
561 
562  LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
563  LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
564  unsigned ValueInt = 1;
565  // Translate opencl_unroll_hint attribute argument to
566  // equivalent LoopHintAttr enums.
567  // OpenCL v2.0 s6.11.5:
568  // 0 - enable unroll (no argument).
569  // 1 - disable unroll.
570  // other positive integer n - unroll by n.
571  if (OpenCLHint) {
572  ValueInt = OpenCLHint->getUnrollHint();
573  if (ValueInt == 0) {
574  State = LoopHintAttr::Enable;
575  } else if (ValueInt != 1) {
576  Option = LoopHintAttr::UnrollCount;
577  State = LoopHintAttr::Numeric;
578  }
579  } else if (LH) {
580  auto *ValueExpr = LH->getValue();
581  if (ValueExpr) {
582  llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
583  ValueInt = ValueAPS.getSExtValue();
584  }
585 
586  Option = LH->getOption();
587  State = LH->getState();
588  }
589  switch (State) {
590  case LoopHintAttr::Disable:
591  switch (Option) {
592  case LoopHintAttr::Vectorize:
593  // Disable vectorization by specifying a width of 1.
594  setVectorizeWidth(1);
595  break;
596  case LoopHintAttr::Interleave:
597  // Disable interleaving by speciyfing a count of 1.
598  setInterleaveCount(1);
599  break;
600  case LoopHintAttr::Unroll:
601  setUnrollState(LoopAttributes::Disable);
602  break;
603  case LoopHintAttr::UnrollAndJam:
604  setUnrollAndJamState(LoopAttributes::Disable);
605  break;
606  case LoopHintAttr::Distribute:
607  setDistributeState(false);
608  break;
609  case LoopHintAttr::PipelineDisabled:
610  setPipelineDisabled(true);
611  break;
612  case LoopHintAttr::UnrollCount:
613  case LoopHintAttr::UnrollAndJamCount:
614  case LoopHintAttr::VectorizeWidth:
615  case LoopHintAttr::InterleaveCount:
616  case LoopHintAttr::PipelineInitiationInterval:
617  llvm_unreachable("Options cannot be disabled.");
618  break;
619  }
620  break;
621  case LoopHintAttr::Enable:
622  switch (Option) {
623  case LoopHintAttr::Vectorize:
624  case LoopHintAttr::Interleave:
625  setVectorizeEnable(true);
626  break;
627  case LoopHintAttr::Unroll:
628  setUnrollState(LoopAttributes::Enable);
629  break;
630  case LoopHintAttr::UnrollAndJam:
631  setUnrollAndJamState(LoopAttributes::Enable);
632  break;
633  case LoopHintAttr::Distribute:
634  setDistributeState(true);
635  break;
636  case LoopHintAttr::UnrollCount:
637  case LoopHintAttr::UnrollAndJamCount:
638  case LoopHintAttr::VectorizeWidth:
639  case LoopHintAttr::InterleaveCount:
640  case LoopHintAttr::PipelineDisabled:
641  case LoopHintAttr::PipelineInitiationInterval:
642  llvm_unreachable("Options cannot enabled.");
643  break;
644  }
645  break;
646  case LoopHintAttr::AssumeSafety:
647  switch (Option) {
648  case LoopHintAttr::Vectorize:
649  case LoopHintAttr::Interleave:
650  // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
651  setParallel(true);
652  setVectorizeEnable(true);
653  break;
654  case LoopHintAttr::Unroll:
655  case LoopHintAttr::UnrollAndJam:
656  case LoopHintAttr::UnrollCount:
657  case LoopHintAttr::UnrollAndJamCount:
658  case LoopHintAttr::VectorizeWidth:
659  case LoopHintAttr::InterleaveCount:
660  case LoopHintAttr::Distribute:
661  case LoopHintAttr::PipelineDisabled:
662  case LoopHintAttr::PipelineInitiationInterval:
663  llvm_unreachable("Options cannot be used to assume mem safety.");
664  break;
665  }
666  break;
667  case LoopHintAttr::Full:
668  switch (Option) {
669  case LoopHintAttr::Unroll:
670  setUnrollState(LoopAttributes::Full);
671  break;
672  case LoopHintAttr::UnrollAndJam:
673  setUnrollAndJamState(LoopAttributes::Full);
674  break;
675  case LoopHintAttr::Vectorize:
676  case LoopHintAttr::Interleave:
677  case LoopHintAttr::UnrollCount:
678  case LoopHintAttr::UnrollAndJamCount:
679  case LoopHintAttr::VectorizeWidth:
680  case LoopHintAttr::InterleaveCount:
681  case LoopHintAttr::Distribute:
682  case LoopHintAttr::PipelineDisabled:
683  case LoopHintAttr::PipelineInitiationInterval:
684  llvm_unreachable("Options cannot be used with 'full' hint.");
685  break;
686  }
687  break;
688  case LoopHintAttr::Numeric:
689  switch (Option) {
690  case LoopHintAttr::VectorizeWidth:
691  setVectorizeWidth(ValueInt);
692  break;
693  case LoopHintAttr::InterleaveCount:
694  setInterleaveCount(ValueInt);
695  break;
696  case LoopHintAttr::UnrollCount:
697  setUnrollCount(ValueInt);
698  break;
699  case LoopHintAttr::UnrollAndJamCount:
700  setUnrollAndJamCount(ValueInt);
701  break;
702  case LoopHintAttr::PipelineInitiationInterval:
703  setPipelineInitiationInterval(ValueInt);
704  break;
705  case LoopHintAttr::Unroll:
706  case LoopHintAttr::UnrollAndJam:
707  case LoopHintAttr::Vectorize:
708  case LoopHintAttr::Interleave:
709  case LoopHintAttr::Distribute:
710  case LoopHintAttr::PipelineDisabled:
711  llvm_unreachable("Options cannot be assigned a value.");
712  break;
713  }
714  break;
715  }
716  }
717 
718  /// Stage the attributes.
719  push(Header, StartLoc, EndLoc);
720 }
721 
723  assert(!Active.empty() && "No active loops to pop");
724  Active.back().finish();
725  Active.pop_back();
726 }
727 
728 void LoopInfoStack::InsertHelper(Instruction *I) const {
729  if (I->mayReadOrWriteMemory()) {
730  SmallVector<Metadata *, 4> AccessGroups;
731  for (const LoopInfo &AL : Active) {
732  // Here we assume that every loop that has an access group is parallel.
733  if (MDNode *Group = AL.getAccessGroup())
734  AccessGroups.push_back(Group);
735  }
736  MDNode *UnionMD = nullptr;
737  if (AccessGroups.size() == 1)
738  UnionMD = cast<MDNode>(AccessGroups[0]);
739  else if (AccessGroups.size() >= 2)
740  UnionMD = MDNode::get(I->getContext(), AccessGroups);
741  I->setMetadata("llvm.access.group", UnionMD);
742  }
743 
744  if (!hasInfo())
745  return;
746 
747  const LoopInfo &L = getInfo();
748  if (!L.getLoopID())
749  return;
750 
751  if (I->isTerminator()) {
752  for (BasicBlock *Succ : successors(I))
753  if (Succ == L.getHeader()) {
754  I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
755  break;
756  }
757  return;
758  }
759 }
Defines the clang::ASTContext interface.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
Definition: Dominators.h:30
Attributes that may be specified on loops.
Definition: CGLoopInfo.h:35
unsigned UnrollAndJamCount
llvm.unroll.
Definition: CGLoopInfo.h:64
Information used when generating a structured loop.
Definition: CGLoopInfo.h:77
LoopAttributes(bool IsParallel=false)
Definition: CGLoopInfo.cpp:411
LVEnableState UnrollEnable
Value for llvm.loop.unroll.* metadata (enable, disable, or full).
Definition: CGLoopInfo.h:49
unsigned PipelineInitiationInterval
Value for llvm.loop.pipeline.iicount metadata.
Definition: CGLoopInfo.h:73
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:154
LineState State
unsigned InterleaveCount
Value for llvm.loop.interleave.count metadata.
Definition: CGLoopInfo.h:58
LVEnableState VectorizeEnable
Value for llvm.loop.vectorize.enable metadata.
Definition: CGLoopInfo.h:46
void pop()
End the current loop.
Definition: CGLoopInfo.cpp:722
Whether values of this type can be null is (explicitly) unspecified.
LVEnableState UnrollAndJamEnable
Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full).
Definition: CGLoopInfo.h:52
void finish()
Create the loop&#39;s metadata.
Definition: CGLoopInfo.cpp:459
NodeId Parent
Definition: ASTDiff.cpp:191
void InsertHelper(llvm::Instruction *I) const
Function called by the CodeGenFunction when an instruction is created.
Definition: CGLoopInfo.cpp:728
bool IsParallel
Generate llvm.loop.parallel metadata for loads and stores.
Definition: CGLoopInfo.h:40
unsigned UnrollCount
llvm.unroll.
Definition: CGLoopInfo.h:61
LVEnableState DistributeEnable
Value for llvm.loop.distribute.enable metadata.
Definition: CGLoopInfo.h:67
#define false
Definition: stdbool.h:17
llvm::MDNode * getLoopID() const
Get the loop id metadata for this loop.
Definition: CGLoopInfo.h:85
constexpr XRayInstrMask None
Definition: XRayInstr.h:37
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
bool PipelineDisabled
Value for llvm.loop.pipeline.disable metadata.
Definition: CGLoopInfo.h:70
static const TypeInfo & getInfo(unsigned id)
Definition: Types.cpp:33
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, LoopInfo *Parent)
Construct a new LoopInfo for the loop with entry Header.
Definition: CGLoopInfo.cpp:433
unsigned VectorizeWidth
Value for llvm.loop.vectorize.width metadata.
Definition: CGLoopInfo.h:55
llvm::BasicBlock * getHeader() const
Get the header block of this loop.
Definition: CGLoopInfo.h:88
Attr - This represents one attribute.
Definition: Attr.h:43