forked from OSchip/llvm-project
[CodeGen] Generate follow-up metadata for loops with more than one transformation.
Before this patch, CGLoop would dump all transformations for a loop into a single LoopID without encoding any order in which to apply them. rL348944 added the possibility to encode a transformation order using followup-attributes. When a loop has more than one transformation, use the follow-up attribute define the order in which they are applied. The emitted order is the defacto order as defined by the current LLVM pass pipeline, which is: LoopFullUnrollPass LoopDistributePass LoopVectorizePass LoopUnrollAndJamPass LoopUnrollPass MachinePipeliner This patch should therefore not change the assembly output, assuming that all explicit transformations can be applied, and no implicit transformations in-between. In the former case, WarnMissedTransformationsPass should emit a warning (except for MachinePipeliner which is not implemented yet). The latter could be avoided by adding 'llvm.loop.disable_nonforced' attributes. Because LoopUnrollAndJamPass processes a loop nest, generation of the MDNode is delayed to after the inner loop metadata have been processed. A temporary LoopID is therefore used to annotate instructions and RAUW'ed by the actual LoopID later. Differential Revision: https://reviews.llvm.org/D57978 llvm-svn: 357415
This commit is contained in:
Michael Kruse
parent
f6c04ad486
commit
58e7642669
|
|
@ -18,138 +18,396 @@
|
|||
using namespace clang::CodeGen;
|
||||
using namespace llvm;
|
||||
|
||||
static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
|
||||
const llvm::DebugLoc &StartLoc,
|
||||
const llvm::DebugLoc &EndLoc, MDNode *&AccGroup) {
|
||||
MDNode *
|
||||
LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
|
||||
LLVMContext &Ctx = Header->getContext();
|
||||
SmallVector<Metadata *, 4> NewLoopProperties;
|
||||
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
||||
NewLoopProperties.push_back(TempNode.get());
|
||||
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
||||
|
||||
if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
|
||||
Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
|
||||
Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
|
||||
Attrs.PipelineInitiationInterval == 0 &&
|
||||
Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
|
||||
Attrs.UnrollEnable == LoopAttributes::Unspecified &&
|
||||
Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
|
||||
Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
|
||||
!EndLoc)
|
||||
return nullptr;
|
||||
MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
|
||||
LoopID->replaceOperandWith(0, LoopID);
|
||||
return LoopID;
|
||||
}
|
||||
|
||||
MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
|
||||
ArrayRef<Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms) {
|
||||
LLVMContext &Ctx = Header->getContext();
|
||||
|
||||
Optional<bool> Enabled;
|
||||
if (Attrs.PipelineDisabled)
|
||||
Enabled = false;
|
||||
else if (Attrs.PipelineInitiationInterval != 0)
|
||||
Enabled = true;
|
||||
|
||||
if (Enabled != true) {
|
||||
SmallVector<Metadata *, 4> NewLoopProperties;
|
||||
if (Enabled == false) {
|
||||
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
||||
NewLoopProperties.push_back(
|
||||
MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
llvm::Type::getInt1Ty(Ctx), 1))}));
|
||||
LoopProperties = NewLoopProperties;
|
||||
}
|
||||
return createLoopPropertiesMetadata(LoopProperties);
|
||||
}
|
||||
|
||||
SmallVector<Metadata *, 4> Args;
|
||||
// Reserve operand 0 for loop id self reference.
|
||||
auto TempNode = MDNode::getTemporary(Ctx, None);
|
||||
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
||||
Args.push_back(TempNode.get());
|
||||
|
||||
// If we have a valid start debug location for the loop, add it.
|
||||
if (StartLoc) {
|
||||
Args.push_back(StartLoc.getAsMDNode());
|
||||
|
||||
// If we also have a valid end debug location for the loop, add it.
|
||||
if (EndLoc)
|
||||
Args.push_back(EndLoc.getAsMDNode());
|
||||
}
|
||||
|
||||
// Setting vectorize.width
|
||||
if (Attrs.VectorizeWidth > 0) {
|
||||
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
// Setting interleave.count
|
||||
if (Attrs.InterleaveCount > 0) {
|
||||
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
// Setting unroll.count
|
||||
if (Attrs.UnrollCount > 0) {
|
||||
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
// Setting unroll_and_jam.count
|
||||
if (Attrs.UnrollAndJamCount > 0) {
|
||||
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
Type::getInt32Ty(Ctx), Attrs.UnrollAndJamCount))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
// Setting vectorize.enable
|
||||
if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
|
||||
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
|
||||
LoopAttributes::Enable)))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
// Setting unroll.full or unroll.disable
|
||||
if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
|
||||
std::string Name;
|
||||
if (Attrs.UnrollEnable == LoopAttributes::Enable)
|
||||
Name = "llvm.loop.unroll.enable";
|
||||
else if (Attrs.UnrollEnable == LoopAttributes::Full)
|
||||
Name = "llvm.loop.unroll.full";
|
||||
else
|
||||
Name = "llvm.loop.unroll.disable";
|
||||
Metadata *Vals[] = {MDString::get(Ctx, Name)};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
// Setting unroll_and_jam.full or unroll_and_jam.disable
|
||||
if (Attrs.UnrollAndJamEnable != LoopAttributes::Unspecified) {
|
||||
std::string Name;
|
||||
if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable)
|
||||
Name = "llvm.loop.unroll_and_jam.enable";
|
||||
else if (Attrs.UnrollAndJamEnable == LoopAttributes::Full)
|
||||
Name = "llvm.loop.unroll_and_jam.full";
|
||||
else
|
||||
Name = "llvm.loop.unroll_and_jam.disable";
|
||||
Metadata *Vals[] = {MDString::get(Ctx, Name)};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
if (Attrs.DistributeEnable != LoopAttributes::Unspecified) {
|
||||
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
Type::getInt1Ty(Ctx), (Attrs.DistributeEnable ==
|
||||
LoopAttributes::Enable)))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
if (Attrs.IsParallel) {
|
||||
AccGroup = MDNode::getDistinct(Ctx, {});
|
||||
Args.push_back(MDNode::get(
|
||||
Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
|
||||
}
|
||||
|
||||
if (Attrs.PipelineDisabled) {
|
||||
Metadata *Vals[] = {
|
||||
MDString::get(Ctx, "llvm.loop.pipeline.disable"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
Type::getInt1Ty(Ctx), (Attrs.PipelineDisabled == true)))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
Args.append(LoopProperties.begin(), LoopProperties.end());
|
||||
|
||||
if (Attrs.PipelineInitiationInterval > 0) {
|
||||
Metadata *Vals[] = {
|
||||
MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
|
||||
llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
// Set the first operand to itself.
|
||||
// No follow-up: This is the last transformation.
|
||||
|
||||
MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
|
||||
LoopID->replaceOperandWith(0, LoopID);
|
||||
HasUserTransforms = true;
|
||||
return LoopID;
|
||||
}
|
||||
|
||||
MDNode *
|
||||
LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
|
||||
ArrayRef<Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms) {
|
||||
LLVMContext &Ctx = Header->getContext();
|
||||
|
||||
Optional<bool> Enabled;
|
||||
if (Attrs.UnrollEnable == LoopAttributes::Disable)
|
||||
Enabled = false;
|
||||
else if (Attrs.UnrollEnable == LoopAttributes::Full)
|
||||
Enabled = None;
|
||||
else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
|
||||
Attrs.UnrollCount != 0)
|
||||
Enabled = true;
|
||||
|
||||
if (Enabled != true) {
|
||||
// createFullUnrollMetadata will already have added llvm.loop.unroll.disable
|
||||
// if unrolling is disabled.
|
||||
return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
|
||||
}
|
||||
|
||||
SmallVector<Metadata *, 4> FollowupLoopProperties;
|
||||
|
||||
// Apply all loop properties to the unrolled loop.
|
||||
FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
||||
|
||||
// Don't unroll an already unrolled loop.
|
||||
FollowupLoopProperties.push_back(
|
||||
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
|
||||
|
||||
bool FollowupHasTransforms = false;
|
||||
MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
|
||||
FollowupHasTransforms);
|
||||
|
||||
SmallVector<Metadata *, 4> Args;
|
||||
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
||||
Args.push_back(TempNode.get());
|
||||
Args.append(LoopProperties.begin(), LoopProperties.end());
|
||||
|
||||
// Setting unroll.count
|
||||
if (Attrs.UnrollCount > 0) {
|
||||
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
// Setting unroll.full or unroll.disable
|
||||
if (Attrs.UnrollEnable == LoopAttributes::Enable) {
|
||||
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
if (FollowupHasTransforms)
|
||||
Args.push_back(MDNode::get(
|
||||
Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
|
||||
|
||||
MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
|
||||
LoopID->replaceOperandWith(0, LoopID);
|
||||
HasUserTransforms = true;
|
||||
return LoopID;
|
||||
}
|
||||
|
||||
MDNode *
|
||||
LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
|
||||
ArrayRef<Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms) {
|
||||
LLVMContext &Ctx = Header->getContext();
|
||||
|
||||
Optional<bool> Enabled;
|
||||
if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
|
||||
Enabled = false;
|
||||
else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
|
||||
Attrs.UnrollAndJamCount != 0)
|
||||
Enabled = true;
|
||||
|
||||
if (Enabled != true) {
|
||||
SmallVector<Metadata *, 4> NewLoopProperties;
|
||||
if (Enabled == false) {
|
||||
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
||||
NewLoopProperties.push_back(MDNode::get(
|
||||
Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
|
||||
LoopProperties = NewLoopProperties;
|
||||
}
|
||||
return createPartialUnrollMetadata(Attrs, LoopProperties,
|
||||
HasUserTransforms);
|
||||
}
|
||||
|
||||
SmallVector<Metadata *, 4> FollowupLoopProperties;
|
||||
FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
||||
FollowupLoopProperties.push_back(
|
||||
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
|
||||
|
||||
bool FollowupHasTransforms = false;
|
||||
MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
|
||||
FollowupHasTransforms);
|
||||
|
||||
SmallVector<Metadata *, 4> Args;
|
||||
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
||||
Args.push_back(TempNode.get());
|
||||
Args.append(LoopProperties.begin(), LoopProperties.end());
|
||||
|
||||
// Setting unroll_and_jam.count
|
||||
if (Attrs.UnrollAndJamCount > 0) {
|
||||
Metadata *Vals[] = {
|
||||
MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
|
||||
Attrs.UnrollAndJamCount))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
|
||||
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
if (FollowupHasTransforms)
|
||||
Args.push_back(MDNode::get(
|
||||
Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
|
||||
Followup}));
|
||||
|
||||
if (UnrollAndJamInnerFollowup)
|
||||
Args.push_back(MDNode::get(
|
||||
Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
|
||||
UnrollAndJamInnerFollowup}));
|
||||
|
||||
MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
|
||||
LoopID->replaceOperandWith(0, LoopID);
|
||||
HasUserTransforms = true;
|
||||
return LoopID;
|
||||
}
|
||||
|
||||
MDNode *
|
||||
LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
|
||||
ArrayRef<Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms) {
|
||||
LLVMContext &Ctx = Header->getContext();
|
||||
|
||||
Optional<bool> Enabled;
|
||||
if (Attrs.VectorizeEnable == LoopAttributes::Disable)
|
||||
Enabled = false;
|
||||
else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
|
||||
Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
|
||||
Enabled = true;
|
||||
|
||||
if (Enabled != true) {
|
||||
SmallVector<Metadata *, 4> NewLoopProperties;
|
||||
if (Enabled == false) {
|
||||
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
||||
NewLoopProperties.push_back(
|
||||
MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
llvm::Type::getInt1Ty(Ctx), 0))}));
|
||||
LoopProperties = NewLoopProperties;
|
||||
}
|
||||
return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
|
||||
}
|
||||
|
||||
// Apply all loop properties to the vectorized loop.
|
||||
SmallVector<Metadata *, 4> FollowupLoopProperties;
|
||||
FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
||||
|
||||
// Don't vectorize an already vectorized loop.
|
||||
FollowupLoopProperties.push_back(
|
||||
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
|
||||
|
||||
bool FollowupHasTransforms = false;
|
||||
MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
|
||||
FollowupHasTransforms);
|
||||
|
||||
SmallVector<Metadata *, 4> Args;
|
||||
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
||||
Args.push_back(TempNode.get());
|
||||
Args.append(LoopProperties.begin(), LoopProperties.end());
|
||||
|
||||
// Setting vectorize.width
|
||||
if (Attrs.VectorizeWidth > 0) {
|
||||
Metadata *Vals[] = {
|
||||
MDString::get(Ctx, "llvm.loop.vectorize.width"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
|
||||
Attrs.VectorizeWidth))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
// Setting interleave.count
|
||||
if (Attrs.InterleaveCount > 0) {
|
||||
Metadata *Vals[] = {
|
||||
MDString::get(Ctx, "llvm.loop.interleave.count"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
|
||||
Attrs.InterleaveCount))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
// Setting vectorize.enable
|
||||
if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
|
||||
Metadata *Vals[] = {
|
||||
MDString::get(Ctx, "llvm.loop.vectorize.enable"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
llvm::Type::getInt1Ty(Ctx),
|
||||
(Attrs.VectorizeEnable == LoopAttributes::Enable)))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
}
|
||||
|
||||
if (FollowupHasTransforms)
|
||||
Args.push_back(MDNode::get(
|
||||
Ctx,
|
||||
{MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
|
||||
|
||||
MDNode *LoopID = MDNode::get(Ctx, Args);
|
||||
LoopID->replaceOperandWith(0, LoopID);
|
||||
HasUserTransforms = true;
|
||||
return LoopID;
|
||||
}
|
||||
|
||||
MDNode *
|
||||
LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
|
||||
ArrayRef<Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms) {
|
||||
LLVMContext &Ctx = Header->getContext();
|
||||
|
||||
Optional<bool> Enabled;
|
||||
if (Attrs.DistributeEnable == LoopAttributes::Disable)
|
||||
Enabled = false;
|
||||
if (Attrs.DistributeEnable == LoopAttributes::Enable)
|
||||
Enabled = true;
|
||||
|
||||
if (Enabled != true) {
|
||||
SmallVector<Metadata *, 4> NewLoopProperties;
|
||||
if (Enabled == false) {
|
||||
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
||||
NewLoopProperties.push_back(
|
||||
MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
llvm::Type::getInt1Ty(Ctx), 0))}));
|
||||
LoopProperties = NewLoopProperties;
|
||||
}
|
||||
return createLoopVectorizeMetadata(Attrs, LoopProperties,
|
||||
HasUserTransforms);
|
||||
}
|
||||
|
||||
bool FollowupHasTransforms = false;
|
||||
MDNode *Followup =
|
||||
createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
|
||||
|
||||
SmallVector<Metadata *, 4> Args;
|
||||
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
||||
Args.push_back(TempNode.get());
|
||||
Args.append(LoopProperties.begin(), LoopProperties.end());
|
||||
|
||||
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
|
||||
ConstantAsMetadata::get(ConstantInt::get(
|
||||
llvm::Type::getInt1Ty(Ctx),
|
||||
(Attrs.DistributeEnable == LoopAttributes::Enable)))};
|
||||
Args.push_back(MDNode::get(Ctx, Vals));
|
||||
|
||||
if (FollowupHasTransforms)
|
||||
Args.push_back(MDNode::get(
|
||||
Ctx,
|
||||
{MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
|
||||
|
||||
MDNode *LoopID = MDNode::get(Ctx, Args);
|
||||
LoopID->replaceOperandWith(0, LoopID);
|
||||
HasUserTransforms = true;
|
||||
return LoopID;
|
||||
}
|
||||
|
||||
MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
|
||||
ArrayRef<Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms) {
|
||||
LLVMContext &Ctx = Header->getContext();
|
||||
|
||||
Optional<bool> Enabled;
|
||||
if (Attrs.UnrollEnable == LoopAttributes::Disable)
|
||||
Enabled = false;
|
||||
else if (Attrs.UnrollEnable == LoopAttributes::Full)
|
||||
Enabled = true;
|
||||
|
||||
if (Enabled != true) {
|
||||
SmallVector<Metadata *, 4> NewLoopProperties;
|
||||
if (Enabled == false) {
|
||||
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
||||
NewLoopProperties.push_back(
|
||||
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
|
||||
LoopProperties = NewLoopProperties;
|
||||
}
|
||||
return createLoopDistributeMetadata(Attrs, LoopProperties,
|
||||
HasUserTransforms);
|
||||
}
|
||||
|
||||
SmallVector<Metadata *, 4> Args;
|
||||
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
||||
Args.push_back(TempNode.get());
|
||||
Args.append(LoopProperties.begin(), LoopProperties.end());
|
||||
Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
|
||||
|
||||
// No follow-up: there is no loop after full unrolling.
|
||||
// TODO: Warn if there are transformations after full unrolling.
|
||||
|
||||
MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
|
||||
LoopID->replaceOperandWith(0, LoopID);
|
||||
HasUserTransforms = true;
|
||||
return LoopID;
|
||||
}
|
||||
|
||||
MDNode *LoopInfo::createMetadata(
|
||||
const LoopAttributes &Attrs,
|
||||
llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
|
||||
bool &HasUserTransforms) {
|
||||
SmallVector<Metadata *, 3> LoopProperties;
|
||||
|
||||
// If we have a valid start debug location for the loop, add it.
|
||||
if (StartLoc) {
|
||||
LoopProperties.push_back(StartLoc.getAsMDNode());
|
||||
|
||||
// If we also have a valid end debug location for the loop, add it.
|
||||
if (EndLoc)
|
||||
LoopProperties.push_back(EndLoc.getAsMDNode());
|
||||
}
|
||||
|
||||
assert(!!AccGroup == Attrs.IsParallel &&
|
||||
"There must be an access group iff the loop is parallel");
|
||||
if (Attrs.IsParallel) {
|
||||
LLVMContext &Ctx = Header->getContext();
|
||||
LoopProperties.push_back(MDNode::get(
|
||||
Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
|
||||
}
|
||||
|
||||
LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
|
||||
AdditionalLoopProperties.end());
|
||||
return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
|
||||
}
|
||||
|
||||
LoopAttributes::LoopAttributes(bool IsParallel)
|
||||
: IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
|
||||
UnrollEnable(LoopAttributes::Unspecified),
|
||||
|
|
@ -173,15 +431,114 @@ void LoopAttributes::clear() {
|
|||
}
|
||||
|
||||
LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
|
||||
const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
|
||||
: LoopID(nullptr), Header(Header), Attrs(Attrs) {
|
||||
LoopID =
|
||||
createMetadata(Header->getContext(), Attrs, StartLoc, EndLoc, AccGroup);
|
||||
const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
|
||||
LoopInfo *Parent)
|
||||
: Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
|
||||
Parent(Parent) {
|
||||
|
||||
if (Attrs.IsParallel) {
|
||||
// Create an access group for this loop.
|
||||
LLVMContext &Ctx = Header->getContext();
|
||||
AccGroup = MDNode::getDistinct(Ctx, {});
|
||||
}
|
||||
|
||||
if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
|
||||
Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
|
||||
Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
|
||||
Attrs.PipelineInitiationInterval == 0 &&
|
||||
Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
|
||||
Attrs.UnrollEnable == LoopAttributes::Unspecified &&
|
||||
Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
|
||||
Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
|
||||
!EndLoc)
|
||||
return;
|
||||
|
||||
TempLoopID = MDNode::getTemporary(Header->getContext(), None);
|
||||
}
|
||||
|
||||
void LoopInfo::finish() {
|
||||
// We did not annotate the loop body instructions because there are no
|
||||
// attributes for this loop.
|
||||
if (!TempLoopID)
|
||||
return;
|
||||
|
||||
MDNode *LoopID;
|
||||
LoopAttributes CurLoopAttr = Attrs;
|
||||
LLVMContext &Ctx = Header->getContext();
|
||||
|
||||
if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
|
||||
Parent->Attrs.UnrollAndJamCount != 0)) {
|
||||
// Parent unroll-and-jams this loop.
|
||||
// Split the transformations in those that happens before the unroll-and-jam
|
||||
// and those after.
|
||||
|
||||
LoopAttributes BeforeJam, AfterJam;
|
||||
|
||||
BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
|
||||
|
||||
BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
|
||||
BeforeJam.InterleaveCount = Attrs.InterleaveCount;
|
||||
BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
|
||||
BeforeJam.DistributeEnable = Attrs.DistributeEnable;
|
||||
|
||||
switch (Attrs.UnrollEnable) {
|
||||
case LoopAttributes::Unspecified:
|
||||
case LoopAttributes::Disable:
|
||||
BeforeJam.UnrollEnable = Attrs.UnrollEnable;
|
||||
AfterJam.UnrollEnable = Attrs.UnrollEnable;
|
||||
break;
|
||||
case LoopAttributes::Full:
|
||||
BeforeJam.UnrollEnable = LoopAttributes::Full;
|
||||
break;
|
||||
case LoopAttributes::Enable:
|
||||
AfterJam.UnrollEnable = LoopAttributes::Enable;
|
||||
break;
|
||||
}
|
||||
|
||||
AfterJam.UnrollCount = Attrs.UnrollCount;
|
||||
AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
|
||||
AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
|
||||
|
||||
// If this loop is subject of an unroll-and-jam by the parent loop, and has
|
||||
// an unroll-and-jam annotation itself, we have to decide whether to first
|
||||
// apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
|
||||
// UnrollAndJam pass processes loops from inner to outer, so we apply the
|
||||
// inner first.
|
||||
BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
|
||||
BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
|
||||
|
||||
// Set the inner followup metadata to process by the outer loop. Only
|
||||
// consider the first inner loop.
|
||||
if (!Parent->UnrollAndJamInnerFollowup) {
|
||||
// Splitting the attributes into a BeforeJam and an AfterJam part will
|
||||
// stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
|
||||
// to be forwarded to the AfterJam part. We detect the situation here and
|
||||
// add it manually.
|
||||
SmallVector<Metadata *, 1> BeforeLoopProperties;
|
||||
if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
|
||||
BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
|
||||
BeforeLoopProperties.push_back(
|
||||
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
|
||||
|
||||
bool InnerFollowupHasTransform = false;
|
||||
MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
|
||||
InnerFollowupHasTransform);
|
||||
if (InnerFollowupHasTransform)
|
||||
Parent->UnrollAndJamInnerFollowup = InnerFollowup;
|
||||
}
|
||||
|
||||
CurLoopAttr = BeforeJam;
|
||||
}
|
||||
|
||||
bool HasUserTransforms = false;
|
||||
LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
|
||||
TempLoopID->replaceAllUsesWith(LoopID);
|
||||
}
|
||||
|
||||
void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
|
||||
const llvm::DebugLoc &EndLoc) {
|
||||
Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc));
|
||||
Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
|
||||
Active.empty() ? nullptr : &Active.back()));
|
||||
// Clear the attributes so nested loops do not inherit them.
|
||||
StagedAttrs.clear();
|
||||
}
|
||||
|
|
@ -364,6 +721,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
|
|||
|
||||
void LoopInfoStack::pop() {
|
||||
assert(!Active.empty() && "No active loops to pop");
|
||||
Active.back().finish();
|
||||
Active.pop_back();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -78,10 +78,11 @@ class LoopInfo {
|
|||
public:
|
||||
/// Construct a new LoopInfo for the loop with entry Header.
|
||||
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs,
|
||||
const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc);
|
||||
const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
|
||||
LoopInfo *Parent);
|
||||
|
||||
/// Get the loop id metadata for this loop.
|
||||
llvm::MDNode *getLoopID() const { return LoopID; }
|
||||
llvm::MDNode *getLoopID() const { return TempLoopID.get(); }
|
||||
|
||||
/// Get the header block of this loop.
|
||||
llvm::BasicBlock *getHeader() const { return Header; }
|
||||
|
|
@ -92,15 +93,92 @@ public:
|
|||
/// Return this loop's access group or nullptr if it does not have one.
|
||||
llvm::MDNode *getAccessGroup() const { return AccGroup; }
|
||||
|
||||
/// Create the loop's metadata. Must be called after its nested loops have
|
||||
/// been processed.
|
||||
void finish();
|
||||
|
||||
private:
|
||||
/// Loop ID metadata.
|
||||
llvm::MDNode *LoopID;
|
||||
llvm::TempMDTuple TempLoopID;
|
||||
/// Header block of this loop.
|
||||
llvm::BasicBlock *Header;
|
||||
/// The attributes for this loop.
|
||||
LoopAttributes Attrs;
|
||||
/// The access group for memory accesses parallel to this loop.
|
||||
llvm::MDNode *AccGroup = nullptr;
|
||||
/// Start location of this loop.
|
||||
llvm::DebugLoc StartLoc;
|
||||
/// End location of this loop.
|
||||
llvm::DebugLoc EndLoc;
|
||||
/// The next outer loop, or nullptr if this is the outermost loop.
|
||||
LoopInfo *Parent;
|
||||
/// If this loop has unroll-and-jam metadata, this can be set by the inner
|
||||
/// loop's LoopInfo to set the llvm.loop.unroll_and_jam.followup_inner
|
||||
/// metadata.
|
||||
llvm::MDNode *UnrollAndJamInnerFollowup = nullptr;
|
||||
|
||||
/// Create a LoopID without any transformations.
|
||||
llvm::MDNode *
|
||||
createLoopPropertiesMetadata(llvm::ArrayRef<llvm::Metadata *> LoopProperties);
|
||||
|
||||
/// Create a LoopID for transformations.
|
||||
///
|
||||
/// The methods call each other in case multiple transformations are applied
|
||||
/// to a loop. The transformation first to be applied will use LoopID of the
|
||||
/// next transformation in its followup attribute.
|
||||
///
|
||||
/// @param Attrs The loop's transformations.
|
||||
/// @param LoopProperties Non-transformation properties such as debug
|
||||
/// location, parallel accesses and disabled
|
||||
/// transformations. These are added to the returned
|
||||
/// LoopID.
|
||||
/// @param HasUserTransforms [out] Set to true if the returned MDNode encodes
|
||||
/// at least one transformation.
|
||||
///
|
||||
/// @return A LoopID (metadata node) that can be used for the llvm.loop
|
||||
/// annotation or followup-attribute.
|
||||
/// @{
|
||||
llvm::MDNode *
|
||||
createPipeliningMetadata(const LoopAttributes &Attrs,
|
||||
llvm::ArrayRef<llvm::Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms);
|
||||
llvm::MDNode *
|
||||
createPartialUnrollMetadata(const LoopAttributes &Attrs,
|
||||
llvm::ArrayRef<llvm::Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms);
|
||||
llvm::MDNode *
|
||||
createUnrollAndJamMetadata(const LoopAttributes &Attrs,
|
||||
llvm::ArrayRef<llvm::Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms);
|
||||
llvm::MDNode *
|
||||
createLoopVectorizeMetadata(const LoopAttributes &Attrs,
|
||||
llvm::ArrayRef<llvm::Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms);
|
||||
llvm::MDNode *
|
||||
createLoopDistributeMetadata(const LoopAttributes &Attrs,
|
||||
llvm::ArrayRef<llvm::Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms);
|
||||
llvm::MDNode *
|
||||
createFullUnrollMetadata(const LoopAttributes &Attrs,
|
||||
llvm::ArrayRef<llvm::Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms);
|
||||
/// @}
|
||||
|
||||
/// Create a LoopID for this loop, including transformation-unspecific
|
||||
/// metadata such as debug location.
|
||||
///
|
||||
/// @param Attrs This loop's attributes and transformations.
|
||||
/// @param LoopProperties Additional non-transformation properties to add
|
||||
/// to the LoopID, such as transformation-specific
|
||||
/// metadata that are not covered by @p Attrs.
|
||||
/// @param HasUserTransforms [out] Set to true if the returned MDNode encodes
|
||||
/// at least one transformation.
|
||||
///
|
||||
/// @return A LoopID (metadata node) that can be used for the llvm.loop
|
||||
/// annotation.
|
||||
llvm::MDNode *createMetadata(const LoopAttributes &Attrs,
|
||||
llvm::ArrayRef<llvm::Metadata *> LoopProperties,
|
||||
bool &HasUserTransforms);
|
||||
};
|
||||
|
||||
/// A stack of loop information corresponding to loop nesting levels.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c++11 -emit-llvm -o - %s | FileCheck %s
|
||||
|
||||
extern "C" void followup_inner(int n, int *x) {
|
||||
#pragma unroll_and_jam(4)
|
||||
for(int j = 0; j < n; j++) {
|
||||
#pragma clang loop pipeline_initiation_interval(10)
|
||||
#pragma clang loop unroll_count(4)
|
||||
#pragma clang loop vectorize(assume_safety)
|
||||
#pragma clang loop distribute(enable)
|
||||
for(int i = 0; i < n; i++)
|
||||
x[j+i*n] = 10;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// CHECK-LABEL: define void @followup_inner
|
||||
// CHECK: br label %for.cond1, !llvm.loop ![[INNERLOOP_3:[0-9]+]]
|
||||
// CHECK: br label %for.cond, !llvm.loop ![[OUTERLOOP_9:[0-9]+]]
|
||||
|
||||
// CHECK-DAG: ![[ACCESSGROUP_2:[0-9]+]] = distinct !{}
|
||||
|
||||
// CHECK-DAG: ![[INNERLOOP_3:[0-9]+]] = distinct !{![[INNERLOOP_3:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[DISTRIBUTE_5:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]]}
|
||||
// CHECK-DAG: ![[PARALLEL_ACCESSES_4:[0-9]+]] = !{!"llvm.loop.parallel_accesses", !2}
|
||||
// CHECK-DAG: ![[DISTRIBUTE_5:[0-9]+]] = !{!"llvm.loop.distribute.enable", i1 true}
|
||||
// CHECK-DAG: ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]] = !{!"llvm.loop.distribute.followup_all", ![[LOOP_7:[0-9]+]]}
|
||||
|
||||
// CHECK-DAG: ![[LOOP_7:[0-9]+]] = distinct !{![[LOOP_7:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[VECTORIZE_8:[0-9]+]]}
|
||||
// CHECK-DAG: ![[VECTORIZE_8:[0-9]+]] = !{!"llvm.loop.vectorize.enable", i1 true}
|
||||
|
||||
// CHECK-DAG: ![[OUTERLOOP_9:[0-9]+]] = distinct !{![[OUTERLOOP_9:[0-9]+]], ![[UNROLLANDJAM_COUNT_10:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUPINNER_11:[0-9]+]]}
|
||||
// CHECK-DAG: ![[UNROLLANDJAM_COUNT_10:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.count", i32 4}
|
||||
// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUPINNER_11:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.followup_inner", !12}
|
||||
|
||||
// CHECK-DAG: ![[LOOP_12:[0-9]+]] = distinct !{![[LOOP_12:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_13:[0-9]+]], ![[UNROLL_COUNT_13:[0-9]+]], ![[UNROLL_FOLLOWUP_14:[0-9]+]]}
|
||||
// CHECK-DAG: ![[ISVECTORIZED_13:[0-9]+]] = !{!"llvm.loop.isvectorized"}
|
||||
// CHECK-DAG: ![[UNROLL_COUNT_13:[0-9]+]] = !{!"llvm.loop.unroll.count", i32 4}
|
||||
// CHECK-DAG: ![[UNROLL_FOLLOWUP_14:[0-9]+]] = !{!"llvm.loop.unroll.followup_all", ![[LOOP_15:[0-9]+]]}
|
||||
|
||||
// CHECK-DAG: ![[LOOP_15:[0-9]+]] = distinct !{![[LOOP_15:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_13:[0-9]+]], ![[UNROLL_DISABLE_16:[0-9]+]], ![[PIPELINE_17:[0-9]+]]}
|
||||
// CHECK-DAG: ![[UNROLL_DISABLE_16:[0-9]+]] = !{!"llvm.loop.unroll.disable"}
|
||||
// CHECK-DAG: ![[PIPELINE_17:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 10}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c++11 -emit-llvm -o - %s | FileCheck %s
|
||||
|
||||
extern "C" void followup_outer(int n, int *x) {
|
||||
#pragma clang loop pipeline_initiation_interval(10)
|
||||
#pragma clang loop unroll_count(4)
|
||||
#pragma unroll_and_jam
|
||||
#pragma clang loop vectorize(assume_safety)
|
||||
#pragma clang loop distribute(enable)
|
||||
for(int j = 0; j < n; j++) {
|
||||
x[j] = 10;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// CHECK-LABEL: define void @followup_outer
|
||||
// CHECK: br label %for.cond, !llvm.loop ![[LOOP_3:[0-9]+]]
|
||||
|
||||
// CHECK-DAG: ![[ACCESSGROUP_2:[0-9]+]] = distinct !{}
|
||||
|
||||
// CHECK-DAG: ![[LOOP_3:[0-9]+]] = distinct !{![[LOOP_3:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[DISTRIBUTE_5:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]]}
|
||||
// CHECK-DAG: ![[PARALLEL_ACCESSES_4:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[ACCESSGROUP_2]]}
|
||||
// CHECK-DAG: ![[DISTRIBUTE_5:[0-9]+]] = !{!"llvm.loop.distribute.enable", i1 true}
|
||||
// CHECK-DAG: ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]] = !{!"llvm.loop.distribute.followup_all", ![[LOOP_7:[0-9]+]]}
|
||||
|
||||
// CHECK-DAG: ![[LOOP_7:[0-9]+]] = distinct !{![[LOOP_7:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[VECTORIZE_8:[0-9]+]], ![[VECTORIZE_FOLLOWUP_9:[0-9]+]]}
|
||||
// CHECK-DAG: ![[VECTORIZE_8:[0-9]+]] = !{!"llvm.loop.vectorize.enable", i1 true}
|
||||
// CHECK-DAG: ![[VECTORIZE_FOLLOWUP_9:[0-9]+]] = !{!"llvm.loop.vectorize.followup_all", ![[LOOP_10:[0-9]+]]}
|
||||
|
||||
// CHECK-DAG: ![[LOOP_10:[0-9]+]] = distinct !{![[LOOP_10:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_11:[0-9]+]], ![[UNROLLANDJAM_12:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUPOUTER_13:[0-9]+]]}
|
||||
// CHECK-DAG: ![[ISVECTORIZED_11:[0-9]+]] = !{!"llvm.loop.isvectorized"}
|
||||
// CHECK-DAG: ![[UNROLLANDJAM_12:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.enable"}
|
||||
// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUPOUTER_13:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[LOOP_14:[0-9]+]]}
|
||||
|
||||
// CHECK-DAG: ![[LOOP_14:[0-9]+]] = distinct !{![[LOOP_14:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_11:[0-9]+]], ![[UNROLLANDJAM_DISABLE_15:[0-9]+]], ![[UNROLL_COUNT_16:[0-9]+]], ![[UNROLL_FOLLOWUP_17:[0-9]+]]}
|
||||
// CHECK-DAG: ![[UNROLLANDJAM_DISABLE_15:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.disable"}
|
||||
// CHECK-DAG: ![[UNROLL_COUNT_16:[0-9]+]] = !{!"llvm.loop.unroll.count", i32 4}
|
||||
// CHECK-DAG: ![[UNROLL_FOLLOWUP_17:[0-9]+]] = !{!"llvm.loop.unroll.followup_all", ![[LOOP_18:[0-9]+]]}
|
||||
|
||||
// CHECK-DAG: ![[LOOP_18:[0-9]+]] = distinct !{![[LOOP_18:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_11:[0-9]+]], ![[UNROLLANDJAM_DISABLE_15:[0-9]+]], ![[UNROLL_DISABLE_19:[0-9]+]], ![[INITIATIONINTERVAL_20:[0-9]+]]}
|
||||
// CHECK-DAG: ![[UNROLL_DISABLE_19:[0-9]+]] = !{!"llvm.loop.unroll.disable"}
|
||||
// CHECK-DAG: ![[INITIATIONINTERVAL_20:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 10}
|
||||
|
|
@ -26,7 +26,7 @@ void vectorize_imperfectly_nested_test(int *List, int Length) {
|
|||
// CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
|
||||
// CHECK: ![[ACCESS_GROUP_LIST_3:[0-9]+]] = !{![[ACCESS_GROUP_2]], ![[ACCESS_GROUP_4:[0-9]+]]}
|
||||
// CHECK: ![[ACCESS_GROUP_4]] = distinct !{}
|
||||
// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_8:[0-9]+]]}
|
||||
// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], ![[PARALLEL_ACCESSES_8:[0-9]+]]
|
||||
// CHECK: ![[PARALLEL_ACCESSES_8]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_4]]}
|
||||
// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_10:[0-9]+]]}
|
||||
// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], ![[PARALLEL_ACCESSES_10:[0-9]+]]
|
||||
// CHECK: ![[PARALLEL_ACCESSES_10]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ void vectorize_nested_test(int *List, int Length) {
|
|||
// CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
|
||||
// CHECK: ![[ACCESS_GROUP_LIST_3]] = !{![[ACCESS_GROUP_2]], ![[ACCESS_GROUP_4:[0-9]+]]}
|
||||
// CHECK: ![[ACCESS_GROUP_4]] = distinct !{}
|
||||
// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_8:[0-9]+]]}
|
||||
// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], ![[PARALLEL_ACCESSES_8:[0-9]+]]
|
||||
// CHECK: ![[PARALLEL_ACCESSES_8]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_4]]}
|
||||
// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_10:[0-9]+]]}
|
||||
// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], ![[PARALLEL_ACCESSES_10:[0-9]+]]
|
||||
// CHECK: ![[PARALLEL_ACCESSES_10]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
|
||||
|
|
|
|||
|
|
@ -18,5 +18,5 @@ void vectorize_outer_test(int *List, int Length) {
|
|||
|
||||
// CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
|
||||
// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]],
|
||||
// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_9:[0-9]+]]}
|
||||
// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], ![[PARALLEL_ACCESSES_9:[0-9]+]]
|
||||
// CHECK: ![[PARALLEL_ACCESSES_9]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
|
||||
|
|
|
|||
|
|
@ -47,12 +47,12 @@ void interleave_test(int *List, int Length) {
|
|||
}
|
||||
|
||||
// CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
|
||||
// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], ![[INTERLEAVE_1:[0-9]+]], ![[INTENABLE_1:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[PARALLEL_ACCESSES_7:[0-9]+]]}
|
||||
// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], ![[PARALLEL_ACCESSES_7:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[INTERLEAVE_1:[0-9]+]], ![[INTENABLE_1:[0-9]+]]}
|
||||
// CHECK: ![[PARALLEL_ACCESSES_7]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
|
||||
// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
|
||||
// CHECK: ![[INTERLEAVE_1]] = !{!"llvm.loop.interleave.count", i32 1}
|
||||
// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true}
|
||||
// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
|
||||
// CHECK: ![[PARALLEL_ACCESSES_7]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
|
||||
// CHECK: ![[ACCESS_GROUP_8]] = distinct !{}
|
||||
// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], ![[WIDTH_1:[0-9]+]], ![[INTENABLE_1]], ![[UNROLL_DISABLE]], ![[PARALLEL_ACCESSES_11:[0-9]+]]}
|
||||
// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
|
||||
// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], ![[PARALLEL_ACCESSES_11:[0-9]+]], ![[UNROLL_DISABLE]], ![[WIDTH_1:[0-9]+]], ![[INTENABLE_1]]}
|
||||
// CHECK: ![[PARALLEL_ACCESSES_11]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_8]]}
|
||||
// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
|
||||
|
|
|
|||
|
|
@ -158,37 +158,60 @@ void template_test(double *List, int Length) {
|
|||
for_template_constant_expression_test<double, 2, 4, 8>(List, Length);
|
||||
}
|
||||
|
||||
// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[WIDTH_4:.*]], ![[INTERLEAVE_4:.*]], ![[INTENABLE_1:.*]], ![[UNROLL_FULL:.*]], ![[DISTRIBUTE_ENABLE:.*]]}
|
||||
// CHECK: ![[WIDTH_4]] = !{!"llvm.loop.vectorize.width", i32 4}
|
||||
// CHECK: ![[INTERLEAVE_4]] = !{!"llvm.loop.interleave.count", i32 4}
|
||||
// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true}
|
||||
// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[UNROLL_FULL:.*]]}
|
||||
// CHECK: ![[UNROLL_FULL]] = !{!"llvm.loop.unroll.full"}
|
||||
// CHECK: ![[DISTRIBUTE_ENABLE]] = !{!"llvm.loop.distribute.enable", i1 true}
|
||||
// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2:.*]], ![[WIDTH_8:.*]], ![[INTERLEAVE_4:.*]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]]}
|
||||
// CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8}
|
||||
|
||||
// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_8:.*]], ![[INTERLEAVE_4:.*]]}
|
||||
// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
|
||||
// CHECK: ![[DISTRIBUTE_DISABLE]] = !{!"llvm.loop.distribute.enable", i1 false}
|
||||
// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[INTERLEAVE_4:.*]], ![[UNROLL_8:.*]], ![[INTENABLE_1:.*]]}
|
||||
// CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8}
|
||||
// CHECK: ![[INTERLEAVE_4]] = !{!"llvm.loop.interleave.count", i32 4}
|
||||
|
||||
// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[INTERLEAVE_4:.*]], ![[INTENABLE_1:.*]], ![[FOLLOWUP_VECTOR_3:.*]]}
|
||||
// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true}
|
||||
// CHECK: ![[FOLLOWUP_VECTOR_3]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_3:.*]]}
|
||||
// CHECK: ![[AFTER_VECTOR_3]] = distinct !{![[AFTER_VECTOR_3]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
|
||||
// CHECK: ![[ISVECTORIZED]] = !{!"llvm.loop.isvectorized"}
|
||||
// CHECK: ![[UNROLL_8]] = !{!"llvm.loop.unroll.count", i32 8}
|
||||
|
||||
// CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]]}
|
||||
// CHECK: ![[WIDTH_2]] = !{!"llvm.loop.vectorize.width", i32 2}
|
||||
// CHECK: ![[INTERLEAVE_2]] = !{!"llvm.loop.interleave.count", i32 2}
|
||||
// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[WIDTH_1:.*]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]]}
|
||||
|
||||
// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1:.*]]}
|
||||
// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
|
||||
// CHECK: ![[LOOP_6]] = distinct !{![[LOOP_6]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[UNROLL_8:.*]]}
|
||||
|
||||
// CHECK: ![[LOOP_6]] = distinct !{![[LOOP_6]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[FOLLOWUP_VECTOR_6:.*]]}
|
||||
// CHECK: ![[FOLLOWUP_VECTOR_6]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_6:.*]]}
|
||||
// CHECK: ![[AFTER_VECTOR_6]] = distinct !{![[AFTER_VECTOR_6]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
|
||||
|
||||
// CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[WIDTH_5:.*]]}
|
||||
// CHECK: ![[WIDTH_5]] = !{!"llvm.loop.vectorize.width", i32 5}
|
||||
|
||||
// CHECK: ![[LOOP_8]] = distinct !{![[LOOP_8]], ![[WIDTH_5:.*]]}
|
||||
// CHECK: ![[LOOP_9]] = distinct !{![[LOOP_9]], ![[WIDTH_8:.*]], ![[INTERLEAVE_8:.*]], ![[UNROLL_8:.*]]}
|
||||
// CHECK: ![[INTERLEAVE_8]] = !{!"llvm.loop.interleave.count", i32 8}
|
||||
// CHECK: ![[LOOP_10]] = distinct !{![[LOOP_10]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[UNROLL_8:.*]]}
|
||||
// CHECK: ![[LOOP_11]] = distinct !{![[LOOP_11]], ![[WIDTH_2:.*]], ![[INTERLEAVE_4:.*]], ![[UNROLL_8:.*]]}
|
||||
// CHECK: ![[LOOP_12]] = distinct !{![[LOOP_12]], ![[WIDTH_6:.*]], ![[INTERLEAVE_10:.*]], ![[UNROLL_24:.*]]}
|
||||
// CHECK: ![[WIDTH_6]] = !{!"llvm.loop.vectorize.width", i32 6}
|
||||
// CHECK: ![[INTERLEAVE_10]] = !{!"llvm.loop.interleave.count", i32 10}
|
||||
|
||||
// CHECK: ![[LOOP_9]] = distinct !{![[LOOP_9]], ![[WIDTH_8:.*]], ![[INTERLEAVE_8:.*]], ![[FOLLOWUP_VECTOR_9:.*]]}
|
||||
// CHECK: ![[FOLLOWUP_VECTOR_9]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_9:.*]]}
|
||||
// CHECK: ![[AFTER_VECTOR_9]] = distinct !{![[AFTER_VECTOR_9]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
|
||||
|
||||
// CHECK: ![[LOOP_10]] = distinct !{![[LOOP_10]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[FOLLOWUP_VECTOR_10:.*]]}
|
||||
// CHECK: ![[FOLLOWUP_VECTOR_10]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_10:.*]]}
|
||||
// CHECK: ![[AFTER_VECTOR_10]] = distinct !{![[AFTER_VECTOR_10]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
|
||||
|
||||
// CHECK: ![[LOOP_11]] = distinct !{![[LOOP_11]], ![[WIDTH_2:.*]], ![[INTERLEAVE_4:.*]], ![[FOLLOWUP_VECTOR_11:.*]]}
|
||||
// CHECK: ![[FOLLOWUP_VECTOR_11]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_11:.*]]}
|
||||
// CHECK: ![[AFTER_VECTOR_11]] = distinct !{![[AFTER_VECTOR_11]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
|
||||
|
||||
// CHECK: ![[LOOP_12]] = distinct !{![[LOOP_12]], ![[WIDTH_6:.*]], ![[INTERLEAVE_10:.*]], ![[FOLLOWUP_VECTOR_12:.*]]}
|
||||
// CHECK: ![[FOLLOWUP_VECTOR_12]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_12:.*]]}
|
||||
// CHECK: ![[AFTER_VECTOR_12]] = distinct !{![[AFTER_VECTOR_12]], ![[ISVECTORIZED:.*]], ![[UNROLL_24:.*]]}
|
||||
// CHECK: ![[UNROLL_24]] = !{!"llvm.loop.unroll.count", i32 24}
|
||||
// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[UNROLL_32:.*]]}
|
||||
|
||||
// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[FOLLOWUP_VECTOR_13:.*]]}
|
||||
// CHECK: ![[INTERLEAVE_16]] = !{!"llvm.loop.interleave.count", i32 16}
|
||||
// CHECK: ![[FOLLOWUP_VECTOR_13]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_13:.*]]}
|
||||
// CHECK: ![[AFTER_VECTOR_13]] = distinct !{![[AFTER_VECTOR_13]], ![[ISVECTORIZED:.*]], ![[UNROLL_32:.*]]}
|
||||
// CHECK: ![[UNROLL_32]] = !{!"llvm.loop.unroll.count", i32 32}
|
||||
|
||||
// CHECK: ![[LOOP_14]] = distinct !{![[LOOP_14]], ![[WIDTH_10:.*]]}
|
||||
// CHECK: ![[WIDTH_10]] = !{!"llvm.loop.vectorize.width", i32 10}
|
||||
|
|
|
|||
|
|
@ -51,5 +51,5 @@ void clang_unroll_plus_nounroll_and_jam(int *List, int Length, int Value) {
|
|||
// CHECK: ![[UNJ_4]] = !{!"llvm.loop.unroll_and_jam.count", i32 4}
|
||||
// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[UNJ_DISABLE:.*]]}
|
||||
// CHECK: ![[UNJ_DISABLE]] = !{!"llvm.loop.unroll_and_jam.disable"}
|
||||
// CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[UNROLL_4:.*]], ![[UNJ_DISABLE:.*]]}
|
||||
// CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[UNJ_DISABLE:.*]], ![[UNROLL_4:.*]]}
|
||||
// CHECK: ![[UNROLL_4]] = !{!"llvm.loop.unroll.count", i32 4}
|
||||
|
|
|
|||
|
|
@ -147,16 +147,16 @@ void h3(float *c, float *a, float *b, int size)
|
|||
// CHECK: [[LOOP_H1_HEADER:![0-9]+]] = distinct !{[[LOOP_H1_HEADER]], [[LOOP_WIDTH_8:![0-9]+]], [[LOOP_VEC_ENABLE]]}
|
||||
// CHECK: [[LOOP_WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8}
|
||||
// CHECK: ![[ACCESS_GROUP_7]] = distinct !{}
|
||||
// CHECK: [[LOOP_H1_HEADER:![0-9]+]] = distinct !{[[LOOP_H1_HEADER]], [[LOOP_WIDTH_8]], [[LOOP_VEC_ENABLE]], ![[PARALLEL_ACCESSES_9:[0-9]+]]}
|
||||
// CHECK: [[LOOP_H1_HEADER:![0-9]+]] = distinct !{[[LOOP_H1_HEADER]], ![[PARALLEL_ACCESSES_9:[0-9]+]], [[LOOP_WIDTH_8]], [[LOOP_VEC_ENABLE]]}
|
||||
// CHECK: ![[PARALLEL_ACCESSES_9]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_7]]}
|
||||
//
|
||||
// Metadata for h2:
|
||||
// CHECK: ![[ACCESS_GROUP_10]] = distinct !{}
|
||||
// CHECK: [[LOOP_H2_HEADER]] = distinct !{[[LOOP_H2_HEADER]], [[LOOP_VEC_ENABLE]], ![[PARALLEL_ACCESSES_12:[0-9]+]]}
|
||||
// CHECK: [[LOOP_H2_HEADER]] = distinct !{[[LOOP_H2_HEADER]], ![[PARALLEL_ACCESSES_12:[0-9]+]], [[LOOP_VEC_ENABLE]]}
|
||||
// CHECK: ![[PARALLEL_ACCESSES_12]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_10]]}
|
||||
//
|
||||
// Metadata for h3:
|
||||
// CHECK: ![[ACCESS_GROUP_13]] = distinct !{}
|
||||
// CHECK: [[LOOP_H3_HEADER]] = distinct !{[[LOOP_H3_HEADER]], [[LOOP_VEC_ENABLE]], ![[PARALLEL_ACCESSES_15:[0-9]+]]}
|
||||
// CHECK: [[LOOP_H3_HEADER]] = distinct !{[[LOOP_H3_HEADER]], ![[PARALLEL_ACCESSES_15:[0-9]+]], [[LOOP_VEC_ENABLE]]}
|
||||
// CHECK: ![[PARALLEL_ACCESSES_15]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_13]]}
|
||||
//
|
||||
|
|
|
|||
Loading…
Reference in New Issue