maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[mlir] Change vector.transfer_read/write "masked" attribute to "in_bounds". 

This is in preparation for adding a new "mask" operand. The existing "masked" attribute was used to specify dimensions that may be out-of-bounds. Such transfers can be lowered to masked load/stores. The new "in_bounds" attribute is used to specify dimensions that are guaranteed to be within bounds. (Semantics is inverted.)

Differential Revision: https://reviews.llvm.org/D99639
This commit is contained in:
Matthias Springer 2021-03-31 14:59:30 +09:00
parent b5995fced4
commit 95f8135043
26 changed files with 315 additions and 302 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
mlir/include/mlir/Dialect/Vector/VectorOps.h
View File

@ -121,18 +121,18 @@ enum class VectorTransposeLowering {
/// intrinsics.
Flat = 1,
};
/// Enum to control the splitting of `vector.transfer` operations into masked
/// and unmasked variants.
/// Enum to control the splitting of `vector.transfer` operations into
/// in-bounds and out-of-bounds variants.
enum class VectorTransferSplit {
/// Do not split vector transfer operations.
None = 0,
/// Split using masked + unmasked vector.transfer operations.
/// Split using in-bounds + out-of-bounds vector.transfer operations.
VectorTransfer = 1,
/// Split using a unmasked vector.transfer + linalg.fill + linalg.copy
/// Split using an in-bounds vector.transfer + linalg.fill + linalg.copy
/// operations.
LinalgCopy = 2,
/// Do not split vector transfer operation but instead mark it as "unmasked".
ForceUnmasked = 3
/// Do not split vector transfer operation but instead mark it as "in-bounds".
ForceInBounds = 3
};
/// Structure to control the behavior of vector transform patterns.
struct VectorTransformsOptions {

37
mlir/include/mlir/Dialect/Vector/VectorOps.td
View File

@ -1139,7 +1139,7 @@ def Vector_TransferReadOp :
]>,
Arguments<(ins AnyShaped:$source, Variadic<Index>:$indices,
AffineMapAttr:$permutation_map, AnyType:$padding,
OptionalAttr<BoolArrayAttr>:$masked)>,
OptionalAttr<BoolArrayAttr>:$in_bounds)>,
Results<(outs AnyVector:$vector)> {
let summary = "Reads a supervector from memory into an SSA vector value.";
@ -1166,15 +1166,14 @@ def Vector_TransferReadOp :
The size of the slice is specified by the size of the vector, given as the
return type.
An `ssa-value` of the same elemental type as the MemRef/Tensor is provided
as the last operand to specify padding in the case of out-of-bounds
accesses.
An SSA value `padding` of the same elemental type as the MemRef/Tensor is
provided to specify a fallback value in the case of out-of-bounds accesses.
An optional boolean array attribute is provided to specify which dimensions
of the transfer need masking. When a dimension is specified as not requiring
masking, the `vector.transfer_read` may be lowered to simple loads. The
absence of this `masked` attribute signifies that all dimensions of the
transfer need to be masked.
of the transfer are guaranteed to be within bounds. The absence of this
`in_bounds` attribute signifies that any dimension of the transfer may be
out-of-bounds. A `vector.transfer_read` can be lowered to a simple load if
all dimensions are specified to be within bounds.
This operation is called 'read' by opposition to 'load' because the
super-vector granularity is generally not representable with a single
@ -1291,15 +1290,15 @@ def Vector_TransferReadOp :
// Builder that sets padding to zero.
OpBuilder<(ins "VectorType":$vector, "Value":$source,
"ValueRange":$indices, "AffineMap":$permutationMap,
CArg<"ArrayRef<bool>", "{}">:$maybeMasked)>,
CArg<"ArrayRef<bool>", "{}">:$inBounds)>,
// Builder that sets padding to 'getMinorIdentityMap'.
OpBuilder<(ins "VectorType":$vector, "Value":$source,
"ValueRange":$indices, "Value":$padding,
CArg<"ArrayRef<bool>", "{}">:$maybeMasked)>,
CArg<"ArrayRef<bool>", "{}">:$inBounds)>,
// Builder that sets permutation map (resp. padding) to
// 'getMinorIdentityMap' (resp. zero).
OpBuilder<(ins "VectorType":$vector, "Value":$source,
"ValueRange":$indices, CArg<"ArrayRef<bool>", "{}">:$maybeMasked)>
"ValueRange":$indices, CArg<"ArrayRef<bool>", "{}">:$inBounds)>,
];
let hasFolder = 1;
@ -1314,7 +1313,7 @@ def Vector_TransferWriteOp :
Arguments<(ins AnyVector:$vector, AnyShaped:$source,
Variadic<Index>:$indices,
AffineMapAttr:$permutation_map,
OptionalAttr<BoolArrayAttr>:$masked)>,
OptionalAttr<BoolArrayAttr>:$in_bounds)>,
Results<(outs Optional<AnyRankedTensor>:$result)> {
let summary = "The vector.transfer_write op writes a supervector to memory.";
@ -1343,10 +1342,10 @@ def Vector_TransferWriteOp :
The size of the slice is specified by the size of the vector.
An optional boolean array attribute is provided to specify which dimensions
of the transfer need masking. When a dimension is specified as not requiring
masking, the `vector.transfer_write` may be lowered to simple stores. The
absence of this `mask` attribute signifies that all dimensions of the
transfer need to be masked.
of the transfer are guaranteed to be within bounds. The absence of this
`in_bounds` attribute signifies that any dimension of the transfer may be
out-of-bounds. A `vector.transfer_write` can be lowered to a simple store
if all dimensions are specified to be within bounds.
This operation is called 'write' by opposition to 'store' because the
super-vector granularity is generally not representable with a single
@ -1387,13 +1386,13 @@ def Vector_TransferWriteOp :
let builders = [
// Builder that sets permutation map to 'getMinorIdentityMap'.
OpBuilder<(ins "Value":$vector, "Value":$source, "ValueRange":$indices,
CArg<"ArrayRef<bool>", "{}">:$maybeMasked)>,
CArg<"ArrayRef<bool>", "{}">:$inBounds)>,
OpBuilder<(ins "Value":$vector, "Value":$source, "ValueRange":$indices,
"AffineMap":$permutationMap)>,
OpBuilder<(ins "Value":$vector, "Value":$source, "ValueRange":$indices,
"AffineMapAttr":$permutationMap, "ArrayAttr":$masked)>,
"AffineMapAttr":$permutationMap, "ArrayAttr":$inBounds)>,
OpBuilder<(ins "Value":$vector, "Value":$source, "ValueRange":$indices,
"AffineMap":$permutationMap, "ArrayAttr":$masked)>,
"AffineMap":$permutationMap, "ArrayAttr":$inBounds)>,
];
let hasFolder = 1;

7
mlir/include/mlir/Dialect/Vector/VectorTransforms.h
View File

@ -166,7 +166,8 @@ private:
UnrollVectorOptions options;
};
/// Split a vector.transfer operation into an unmasked fastpath and a slowpath.
/// Split a vector.transfer operation into an in-bounds (i.e., no out-of-bounds
/// masking) fastpath and a slowpath.
/// If `ifOp` is not null and the result is `success, the `ifOp` points to the
/// newly created conditional upon function return.
/// To accomodate for the fact that the original vector.transfer indexing may be
@ -185,11 +186,11 @@ private:
/// memref.cast %A: memref<A...> to compatibleMemRefType
/// scf.yield %view : compatibleMemRefType, index, index
/// } else {
/// // slowpath, masked vector.transfer or linalg.copy.
/// // slowpath, not in-bounds vector.transfer or linalg.copy.
/// memref.cast %alloc: memref<B...> to compatibleMemRefType
/// scf.yield %4 : compatibleMemRefType, index, index
// }
/// %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
/// %0 = vector.transfer_read %1#0[%1#1, %1#2] {in_bounds = [true ... true]}
/// ```
/// where `alloc` is a top of the function alloca'ed buffer of one vector.
///

31
mlir/include/mlir/Interfaces/VectorInterfaces.td
View File

@ -53,12 +53,12 @@ def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
let methods = [
StaticInterfaceMethod<
/*desc=*/"Return the `masked` attribute name.",
/*desc=*/"Return the `in_bounds` attribute name.",
/*retTy=*/"StringRef",
/*methodName=*/"getMaskedAttrName",
/*methodName=*/"getInBoundsAttrName",
/*args=*/(ins),
/*methodBody=*/"",
/*defaultImplementation=*/ [{ return "masked"; }]
/*defaultImplementation=*/ [{ return "in_bounds"; }]
>,
StaticInterfaceMethod<
/*desc=*/"Return the `permutation_map` attribute name.",
@ -70,16 +70,16 @@ def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
>,
InterfaceMethod<
/*desc=*/[{
Return `false` when the `masked` attribute at dimension
`dim` is set to `false`. Return `true` otherwise.}],
Return `true` when the `in_bounds` attribute at dimension
`dim` is set to `true`. Return `false` otherwise.}],
/*retTy=*/"bool",
/*methodName=*/"isMaskedDim",
/*methodName=*/"isDimInBounds",
/*args=*/(ins "unsigned":$dim),
/*methodBody=*/"",
/*defaultImplementation=*/[{
return !$_op.masked() ||
$_op.masked()->template cast<ArrayAttr>()[dim]
.template cast<BoolAttr>().getValue();
return $_op.in_bounds() &&
$_op.in_bounds()->template cast<ArrayAttr>()[dim]
.template cast<BoolAttr>().getValue();
}]
>,
InterfaceMethod<
@ -115,11 +115,11 @@ def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
/*defaultImplementation=*/
>,
InterfaceMethod<
/*desc=*/"Return the `masked` boolean ArrayAttr.",
/*desc=*/"Return the `in_bounds` boolean ArrayAttr.",
/*retTy=*/"Optional<ArrayAttr>",
/*methodName=*/"masked",
/*methodName=*/"in_bounds",
/*args=*/(ins),
/*methodBody=*/"return $_op.masked();"
/*methodBody=*/"return $_op.in_bounds();"
/*defaultImplementation=*/
>,
InterfaceMethod<
@ -162,14 +162,15 @@ def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
"return $_op.getShapedType().getRank() - $_op.getTransferRank();"
>,
InterfaceMethod<
/*desc=*/[{ Returns true if at least one of the dimensions is masked.}],
/*desc=*/[{ Returns true if at least one of the dimensions may be
out-of-bounds.}],
/*retTy=*/"bool",
/*methodName=*/"hasMaskedDim",
/*methodName=*/"hasOutOfBoundsDim",
/*args=*/(ins),
/*methodBody=*/"",
/*defaultImplementation=*/[{
for (unsigned idx = 0, e = $_op.getTransferRank(); idx < e; ++idx)
if ($_op.isMaskedDim(idx))
if (!$_op.isDimInBounds(idx))
return true;
return false;
}]

10
mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp
View File

@ -79,7 +79,7 @@ void LoadOpOfSubViewFolder<vector::TransferReadOp>::replaceOp(
ArrayRef<Value> sourceIndices, PatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<vector::TransferReadOp>(
loadOp, loadOp.getVectorType(), subViewOp.source(), sourceIndices,
loadOp.permutation_map(), loadOp.padding(), loadOp.maskedAttr());
loadOp.permutation_map(), loadOp.padding(), loadOp.in_boundsAttr());
}
template <>
@ -92,12 +92,12 @@ void StoreOpOfSubViewFolder<memref::StoreOp>::replaceOp(
template <>
void StoreOpOfSubViewFolder<vector::TransferWriteOp>::replaceOp(
vector::TransferWriteOp tranferWriteOp, memref::SubViewOp subViewOp,
vector::TransferWriteOp transferWriteOp, memref::SubViewOp subViewOp,
ArrayRef<Value> sourceIndices, PatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
tranferWriteOp, tranferWriteOp.vector(), subViewOp.source(),
sourceIndices, tranferWriteOp.permutation_map(),
tranferWriteOp.maskedAttr());
transferWriteOp, transferWriteOp.vector(), subViewOp.source(),
sourceIndices, transferWriteOp.permutation_map(),
transferWriteOp.in_boundsAttr());
}
} // namespace

5
mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
View File

@ -1248,14 +1248,15 @@ public:
Value vectorDataPtr =
castDataPtr(rewriter, loc, dataPtr, memRefType, toLLVMTy(vtp));
if (!xferOp.isMaskedDim(0))
if (xferOp.isDimInBounds(0))
return replaceTransferOpWithLoadOrStore(rewriter,
*this->getTypeConverter(), loc,
xferOp, operands, vectorDataPtr);
// 2. Create a vector with linear indices [ 0 .. vector_length - 1 ].
// 3. Create offsetVector = [ offset + 0 .. offset + vector_length - 1 ].
// 4. Let dim the memref dimension, compute the vector comparison mask:
// 4. Let dim the memref dimension, compute the vector comparison mask
// (in-bounds mask):
// [ offset + 0 .. offset + vector_length - 1 ] < [ dim .. dim ]
//
// TODO: when the leaf transfer rank is k > 1, we need the last `k`

2
mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp
View File

@ -72,7 +72,7 @@ public:
return failure();
// Have it handled in vector->llvm conversion pass.
if (!xferOp.isMaskedDim(0))
if (xferOp.isDimInBounds(0))
return failure();
auto toLLVMTy = [&](Type t) {

18
mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
View File

@ -230,7 +230,7 @@ emitInBoundsCondition(PatternRewriter &rewriter,
Value iv = std::get<0>(it), off = std::get<1>(it), ub = std::get<2>(it);
using namespace mlir::edsc::op;
majorIvsPlusOffsets.push_back(iv + off);
if (xferOp.isMaskedDim(leadingRank + idx)) {
if (!xferOp.isDimInBounds(leadingRank + idx)) {
Value inBoundsCond = onTheFlyFoldSLT(majorIvsPlusOffsets.back(), ub);
if (inBoundsCond)
inBoundsCondition = (inBoundsCondition)
@ -276,14 +276,14 @@ LogicalResult NDTransferOpHelper<TransferReadOp>::doReplace() {
Value memref = xferOp.source();
auto map =
getTransferMinorIdentityMap(xferOp.getShapedType(), minorVectorType);
ArrayAttr masked;
if (!xferOp.isMaskedDim(xferOp.getVectorType().getRank() - 1)) {
ArrayAttr inBounds;
if (xferOp.isDimInBounds(xferOp.getVectorType().getRank() - 1)) {
OpBuilder &b = ScopedContext::getBuilderRef();
masked = b.getBoolArrayAttr({false});
inBounds = b.getBoolArrayAttr({true});
}
return vector_transfer_read(minorVectorType, memref, indexing,
AffineMapAttr::get(map), xferOp.padding(),
masked);
inBounds);
};
// 1. Compute the inBoundsCondition in the current loops ivs + offset
@ -382,13 +382,13 @@ LogicalResult NDTransferOpHelper<TransferWriteOp>::doReplace() {
result = memref_load(alloc, majorIvs);
auto map =
getTransferMinorIdentityMap(xferOp.getShapedType(), minorVectorType);
ArrayAttr masked;
if (!xferOp.isMaskedDim(xferOp.getVectorType().getRank() - 1)) {
ArrayAttr inBounds;
if (xferOp.isDimInBounds(xferOp.getVectorType().getRank() - 1)) {
OpBuilder &b = ScopedContext::getBuilderRef();
masked = b.getBoolArrayAttr({false});
inBounds = b.getBoolArrayAttr({true});
}
vector_transfer_write(result, xferOp.source(), indexing,
AffineMapAttr::get(map), masked);
AffineMapAttr::get(map), inBounds);
};
// 1. Compute the inBoundsCondition in the current loops ivs + offset

103
mlir/lib/Dialect/Vector/VectorOps.cpp
View File

@ -2192,7 +2192,12 @@ static LogicalResult verifyPermutationMap(AffineMap permutationMap,
static LogicalResult verifyTransferOp(Operation *op, ShapedType shapedType,
VectorType vectorType,
AffineMap permutationMap,
ArrayAttr optionalMasked) {
ArrayAttr inBounds) {
if (op->hasAttr("masked")) {
return op->emitOpError("masked attribute has been removed. "
"Use in_bounds instead.");
}
if (!shapedType.isa<MemRefType, RankedTensorType>())
return op->emitOpError(
"requires source to be a memref or ranked tensor type");
@ -2239,11 +2244,10 @@ static LogicalResult verifyTransferOp(Operation *op, ShapedType shapedType,
return op->emitOpError("requires a permutation_map with input dims of the "
"same rank as the source type");
if (optionalMasked) {
if (permutationMap.getNumResults() !=
static_cast<int64_t>(optionalMasked.size()))
return op->emitOpError("expects the optional masked attr of same rank as "
"permutation_map results: ")
if (inBounds) {
if (permutationMap.getNumResults() != static_cast<int64_t>(inBounds.size()))
return op->emitOpError("expects the optional in_bounds attr of same rank "
"as permutation_map results: ")
<< AffineMapAttr::get(permutationMap);
}
@ -2254,58 +2258,58 @@ static LogicalResult verifyTransferOp(Operation *op, ShapedType shapedType,
void TransferReadOp::build(OpBuilder &builder, OperationState &result,
VectorType vectorType, Value source,
ValueRange indices, AffineMap permutationMap,
ArrayRef<bool> maybeMasked) {
ArrayRef<bool> inBounds) {
Type elemType = source.getType().cast<ShapedType>().getElementType();
Value padding = builder.create<ConstantOp>(result.location, elemType,
builder.getZeroAttr(elemType));
if (maybeMasked.empty())
if (inBounds.empty())
return build(builder, result, vectorType, source, indices, permutationMap,
padding, ArrayAttr());
ArrayAttr maskedArrayAttr = builder.getBoolArrayAttr(maybeMasked);
ArrayAttr inBoundsArrayAttr = builder.getBoolArrayAttr(inBounds);
build(builder, result, vectorType, source, indices, permutationMap, padding,
maskedArrayAttr);
inBoundsArrayAttr);
}
/// Builder that sets permutation map to 'getMinorIdentityMap'.
void TransferReadOp::build(OpBuilder &builder, OperationState &result,
VectorType vectorType, Value source,
ValueRange indices, Value padding,
ArrayRef<bool> maybeMasked) {
ArrayRef<bool> inBounds) {
auto permMap = getTransferMinorIdentityMap(
source.getType().cast<ShapedType>(), vectorType);
if (maybeMasked.empty())
if (inBounds.empty())
return build(builder, result, vectorType, source, indices, permMap, padding,
ArrayAttr());
ArrayAttr maskedArrayAttr = builder.getBoolArrayAttr(maybeMasked);
ArrayAttr inBoundsArrayAttr = builder.getBoolArrayAttr(inBounds);
build(builder, result, vectorType, source, indices, permMap, padding,
maskedArrayAttr);
inBoundsArrayAttr);
}
/// Builder that sets permutation map (resp. padding) to 'getMinorIdentityMap'
/// (resp. zero).
void TransferReadOp::build(OpBuilder &builder, OperationState &result,
VectorType vectorType, Value source,
ValueRange indices, ArrayRef<bool> maybeMasked) {
ValueRange indices, ArrayRef<bool> inBounds) {
auto permMap = getTransferMinorIdentityMap(
source.getType().cast<ShapedType>(), vectorType);
build(builder, result, vectorType, source, indices, permMap, maybeMasked);
build(builder, result, vectorType, source, indices, permMap, inBounds);
}
static void printTransferAttrs(OpAsmPrinter &p, VectorTransferOpInterface op) {
SmallVector<StringRef, 2> elidedAttrs;
if (op.permutation_map().isMinorIdentity())
elidedAttrs.push_back(op.getPermutationMapAttrName());
bool elideMasked = true;
if (auto maybeMasked = op.masked()) {
for (auto attr : *maybeMasked) {
if (!attr.template cast<BoolAttr>().getValue()) {
elideMasked = false;
bool elideInBounds = true;
if (auto inBounds = op.in_bounds()) {
for (auto attr : *inBounds) {
if (attr.template cast<BoolAttr>().getValue()) {
elideInBounds = false;
break;
}
}
}
if (elideMasked)
elidedAttrs.push_back(op.getMaskedAttrName());
if (elideInBounds)
elidedAttrs.push_back(op.getInBoundsAttrName());
p.printOptionalAttrDict(op->getAttrs(), elidedAttrs);
}
@ -2366,7 +2370,7 @@ static LogicalResult verify(TransferReadOp op) {
if (failed(verifyTransferOp(op.getOperation(), shapedType, vectorType,
permutationMap,
op.masked() ? *op.masked() : ArrayAttr())))
op.in_bounds() ? *op.in_bounds() : ArrayAttr())))
return failure();
if (auto sourceVectorElementType = sourceElementType.dyn_cast<VectorType>()) {
@ -2438,23 +2442,23 @@ static bool isInBounds(TransferOp op, int64_t resultIdx, int64_t indicesIdx) {
}
template <typename TransferOp>
static LogicalResult foldTransferMaskAttribute(TransferOp op) {
static LogicalResult foldTransferInBoundsAttribute(TransferOp op) {
AffineMap permutationMap = op.permutation_map();
if (!permutationMap.isMinorIdentity())
return failure();
bool changed = false;
SmallVector<bool, 4> isMasked;
isMasked.reserve(op.getTransferRank());
SmallVector<bool, 4> newInBounds;
newInBounds.reserve(op.getTransferRank());
op.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) {
// Already marked unmasked, nothing to see here.
if (!op.isMaskedDim(resultIdx)) {
isMasked.push_back(false);
// Already marked as in-bounds, nothing to see here.
if (op.isDimInBounds(resultIdx)) {
newInBounds.push_back(true);
return;
}
// Currently masked, check whether we can statically determine it is
// Currently out-of-bounds, check whether we can statically determine it is
// inBounds.
auto inBounds = isInBounds(op, resultIdx, indicesIdx);
isMasked.push_back(!inBounds);
newInBounds.push_back(inBounds);
// We commit the pattern if it is "more inbounds".
changed |= inBounds;
});
@ -2462,13 +2466,14 @@ static LogicalResult foldTransferMaskAttribute(TransferOp op) {
return failure();
// OpBuilder is only used as a helper to build an I64ArrayAttr.
OpBuilder b(op.getContext());
op->setAttr(TransferOp::getMaskedAttrName(), b.getBoolArrayAttr(isMasked));
op->setAttr(TransferOp::getInBoundsAttrName(),
b.getBoolArrayAttr(newInBounds));
return success();
}
OpFoldResult TransferReadOp::fold(ArrayRef<Attribute>) {
/// transfer_read(memrefcast) -> transfer_read
if (succeeded(foldTransferMaskAttribute(*this)))
if (succeeded(foldTransferInBoundsAttribute(*this)))
return getResult();
if (succeeded(foldMemRefCast(*this)))
return getResult();
@ -2496,40 +2501,40 @@ void TransferReadOp::getEffects(
/// Builder that sets permutation map to 'getMinorIdentityMap'.
void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
Value vector, Value source, ValueRange indices,
ArrayRef<bool> maybeMasked) {
ArrayRef<bool> inBounds) {
auto vectorType = vector.getType().cast<VectorType>();
auto permMap = getTransferMinorIdentityMap(
source.getType().cast<ShapedType>(), vectorType);
if (maybeMasked.empty())
if (inBounds.empty())
return build(builder, result, vector, source, indices, permMap,
ArrayAttr());
ArrayAttr maskedArrayAttr = builder.getBoolArrayAttr(maybeMasked);
build(builder, result, vector, source, indices, permMap, maskedArrayAttr);
ArrayAttr inBoundsArrayAttr = builder.getBoolArrayAttr(inBounds);
build(builder, result, vector, source, indices, permMap, inBoundsArrayAttr);
}
void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
Value vector, Value source, ValueRange indices,
AffineMap permutationMap) {
build(builder, result, vector, source, indices, permutationMap,
/*maybeMasked=*/ArrayAttr());
/*inBounds=*/ArrayAttr());
}
void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
Value vector, Value source, ValueRange indices,
AffineMapAttr permutationMap,
/*optional*/ ArrayAttr masked) {
/*optional*/ ArrayAttr inBounds) {
Type resultType = source.getType().dyn_cast<RankedTensorType>();
build(builder, result, resultType, vector, source, indices, permutationMap,
masked);
inBounds);
}
void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
Value vector, Value source, ValueRange indices,
AffineMap permutationMap,
/*optional*/ ArrayAttr masked) {
/*optional*/ ArrayAttr inBounds) {
Type resultType = source.getType().dyn_cast<RankedTensorType>();
build(builder, result, resultType, vector, source, indices, permutationMap,
masked);
inBounds);
}
static ParseResult parseTransferWriteOp(OpAsmParser &parser,
@ -2585,7 +2590,7 @@ static LogicalResult verify(TransferWriteOp op) {
if (failed(verifyTransferOp(op.getOperation(), shapedType, vectorType,
permutationMap,
op.masked() ? *op.masked() : ArrayAttr())))
op.in_bounds() ? *op.in_bounds() : ArrayAttr())))
return failure();
return verifyPermutationMap(permutationMap,
@ -2595,9 +2600,9 @@ static LogicalResult verify(TransferWriteOp op) {
/// Fold:
/// ```
/// %t1 = ...
/// %v = vector.transfer_read %t0[%c0...], {masked = [false...]} :
/// %v = vector.transfer_read %t0[%c0...], {in_bounds = [true...]} :
/// tensor<static_sizesxf32>, vector<static_sizesxf32>
/// %t2 = vector.transfer_write %v, %t1[%c0...] {masked = [false...]} :
/// %t2 = vector.transfer_write %v, %t1[%c0...] {in_bounds = [true...]} :
/// vector<static_sizesxf32>, tensor<static_sizesxf32>
/// ```
///
@ -2627,8 +2632,8 @@ static LogicalResult foldReadInitWrite(TransferWriteOp write,
// Bail on mismatching ranks.
if (read.getTransferRank() != write.getTransferRank())
return failure();
// Bail on masked.
if (read.hasMaskedDim() || write.hasMaskedDim())
// Bail on potential out-of-bounds accesses.
if (read.hasOutOfBoundsDim() || write.hasOutOfBoundsDim())
return failure();
// Tensor types must be the same.
if (read.source().getType() != rankedTensorType)
@ -2656,7 +2661,7 @@ LogicalResult TransferWriteOp::fold(ArrayRef<Attribute> operands,
SmallVectorImpl<OpFoldResult> &results) {
if (succeeded(foldReadInitWrite(*this, operands, results)))
return success();
if (succeeded(foldTransferMaskAttribute(*this)))
if (succeeded(foldTransferInBoundsAttribute(*this)))
return success();
return foldMemRefCast(*this);
}

5
mlir/lib/Dialect/Vector/VectorTransferOpTransforms.cpp
View File

@ -38,7 +38,8 @@ static Operation *findAncestorOpInRegion(Region *region, Operation *op) {
/// transfer_read.
static bool transferEncompasses(vector::TransferWriteOp defWrite,
vector::TransferReadOp read) {
return !defWrite.hasMaskedDim() && defWrite.indices() == read.indices() &&
return !defWrite.hasOutOfBoundsDim() &&
defWrite.indices() == read.indices() &&
defWrite.getVectorType() == read.getVectorType() &&
defWrite.permutation_map() == read.permutation_map();
}
@ -175,7 +176,7 @@ void TransferOptimization::deadStoreOp(vector::TransferWriteOp write) {
/// potentially aliasing ops that may reach the transfer_read are post-dominated
/// by the transfer_write.
void TransferOptimization::storeToLoadForwarding(vector::TransferReadOp read) {
if (read.hasMaskedDim())
if (read.hasOutOfBoundsDim())
return;
LLVM_DEBUG(DBGS() << "Candidate for Forwarding: " << *read.getOperation()
<< "\n");

96
mlir/lib/Dialect/Vector/VectorTransforms.cpp
View File

@ -613,12 +613,12 @@ static Value unrollTransferReadOp(vector::TransferReadOp readOp,
// Get VectorType for slice 'i'.
auto sliceVectorType = tupleType.getType(index);
// Create split TransferReadOp for 'sliceUser'.
// `masked` attribute propagates conservatively: if the coarse op didn't
// need masking, the fine op doesn't either.
// `in_bounds` attribute propagates conservatively: if the coarse op didn't
// need out-of-bounds masking, the fine op doesn't either.
vectorTupleValues[index] = builder.create<vector::TransferReadOp>(
loc, sliceVectorType, readOp.source(), sliceIndices,
readOp.permutation_map(), readOp.padding(),
readOp.masked() ? *readOp.masked() : ArrayAttr());
readOp.in_bounds() ? *readOp.in_bounds() : ArrayAttr());
};
generateTransferOpSlices(shapedElementType, sourceVectorType, tupleType,
targetShape, strides, indices, builder, createSlice);
@ -662,7 +662,7 @@ mlir::vector::unrollTransferWriteOp(OpBuilder &builder, Operation *op,
loc, element.getResult(),
resultTensor ? resultTensor : writeOp.source(), sliceIndices,
writeOp.permutation_map(),
writeOp.masked() ? *writeOp.masked() : ArrayAttr());
writeOp.in_bounds() ? *writeOp.in_bounds() : ArrayAttr());
if (!write->getResults().empty())
resultTensor = write->getResult(0);
};
@ -800,13 +800,13 @@ public:
Value resultTensor;
auto createSlice = [&](unsigned index, ArrayRef<Value> sliceIndices) {
// Create split TransferWriteOp for source vector 'tupleOp.operand[i]'.
// 'masked' attribute propagates conservatively: if the coarse op didn't
// need masking, the fine op doesn't either.
// 'in_bounds' attribute propagates conservatively: if the coarse op
// didn't need out-of-bounds masking, the fine op doesn't either.
Operation *write = rewriter.create<vector::TransferWriteOp>(
loc, tupleOp.getOperand(index),
resultTensor ? resultTensor : writeOp.source(), sliceIndices,
writeOp.permutation_map(),
writeOp.masked() ? *writeOp.masked() : ArrayAttr());
writeOp.in_bounds() ? *writeOp.in_bounds() : ArrayAttr());
if (!write->getResults().empty())
resultTensor = write->getResult(0);
};
@ -2267,16 +2267,16 @@ static Value createScopedFoldedSLE(Value v, Value ub) {
}
// Operates under a scoped context to build the condition to ensure that a
// particular VectorTransferOpInterface is unmasked.
// particular VectorTransferOpInterface is in-bounds.
static Value createScopedInBoundsCond(VectorTransferOpInterface xferOp) {
assert(xferOp.permutation_map().isMinorIdentity() &&
"Expected minor identity map");
Value inBoundsCond;
xferOp.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) {
// Zip over the resulting vector shape and memref indices.
// If the dimension is known to be unmasked, it does not participate in the
// construction of `inBoundsCond`.
if (!xferOp.isMaskedDim(resultIdx))
// If the dimension is known to be in-bounds, it does not participate in
// the construction of `inBoundsCond`.
if (xferOp.isDimInBounds(resultIdx))
return;
int64_t vectorSize = xferOp.getVectorType().getDimSize(resultIdx);
using namespace edsc::op;
@ -2298,8 +2298,8 @@ LogicalResult mlir::vector::splitFullAndPartialTransferPrecondition(
// TODO: expand support to these 2 cases.
if (!xferOp.permutation_map().isMinorIdentity())
return failure();
// Must have some masked dimension to be a candidate for splitting.
if (!xferOp.hasMaskedDim())
// Must have some out-of-bounds dimension to be a candidate for splitting.
if (!xferOp.hasOutOfBoundsDim())
return failure();
// Don't split transfer operations directly under IfOp, this avoids applying
// the pattern recursively.
@ -2492,7 +2492,8 @@ static scf::IfOp createScopedFullPartialVectorTransferRead(
return fullPartialIfOp;
}
/// Split a vector.transfer operation into an unmasked fastpath and a slowpath.
/// Split a vector.transfer operation into an in-bounds (i.e., no out-of-bounds
/// masking) fastpath and a slowpath.
/// If `ifOp` is not null and the result is `success, the `ifOp` points to the
/// newly created conditional upon function return.
/// To accomodate for the fact that the original vector.transfer indexing may be
@ -2511,11 +2512,11 @@ static scf::IfOp createScopedFullPartialVectorTransferRead(
/// memref.cast %A: memref<A...> to compatibleMemRefType
/// scf.yield %view : compatibleMemRefType, index, index
/// } else {
/// // slowpath, masked vector.transfer or linalg.copy.
/// // slowpath, not in-bounds vector.transfer or linalg.copy.
/// memref.cast %alloc: memref<B...> to compatibleMemRefType
/// scf.yield %4 : compatibleMemRefType, index, index
// }
/// %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
/// %0 = vector.transfer_read %1#0[%1#1, %1#2] {in_bounds = [true ... true]}
/// ```
/// where `alloc` is a top of the function alloca'ed buffer of one vector.
///
@ -2533,10 +2534,11 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
if (options.vectorTransferSplit == VectorTransferSplit::None)
return failure();
SmallVector<bool, 4> bools(xferOp.getTransferRank(), false);
auto unmaskedAttr = b.getBoolArrayAttr(bools);
if (options.vectorTransferSplit == VectorTransferSplit::ForceUnmasked) {
xferOp->setAttr(vector::TransferReadOp::getMaskedAttrName(), unmaskedAttr);
SmallVector<bool, 4> bools(xferOp.getTransferRank(), true);
auto inBoundsAttr = b.getBoolArrayAttr(bools);
if (options.vectorTransferSplit == VectorTransferSplit::ForceInBounds) {
xferOp->setAttr(vector::TransferReadOp::getInBoundsAttrName(),
inBoundsAttr);
return success();
}
@ -2575,7 +2577,7 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
getCastCompatibleMemRefType(xferOp.getShapedType().cast<MemRefType>(),
alloc.getType().cast<MemRefType>());
// Read case: full fill + partial copy -> unmasked vector.xfer_read.
// Read case: full fill + partial copy -> in-bounds vector.xfer_read.
SmallVector<Type, 4> returnTypes(1 + xferOp.getTransferRank(),
b.getIndexType());
returnTypes[0] = compatibleMemRefType;
@ -2590,10 +2592,10 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
if (ifOp)
*ifOp = fullPartialIfOp;
// Unmask the existing read op, it always reads from a full buffer.
// Set existing read op to in-bounds, it always reads from a full buffer.
for (unsigned i = 0, e = returnTypes.size(); i != e; ++i)
xferReadOp.setOperand(i, fullPartialIfOp.getResult(i));
xferOp->setAttr(vector::TransferReadOp::getMaskedAttrName(), unmaskedAttr);
xferOp->setAttr(vector::TransferReadOp::getInBoundsAttrName(), inBoundsAttr);
return success();
}
@ -2691,7 +2693,7 @@ struct TransferReadExtractPattern
}
Value newRead = vector_transfer_read(extract.getType(), read.source(),
indices, read.permutation_map(),
read.padding(), read.maskedAttr());
read.padding(), read.in_boundsAttr());
Value dest = rewriter.create<ConstantOp>(
read.getLoc(), read.getType(), rewriter.getZeroAttr(read.getType()));
newRead = rewriter.create<vector::InsertMapOp>(read.getLoc(), newRead, dest,
@ -2726,7 +2728,7 @@ struct TransferWriteInsertPattern
std_constant_index(insert.getSourceVectorType().getDimSize(pos));
}
vector_transfer_write(insert.vector(), write.source(), indices,
write.permutation_map(), write.maskedAttr());
write.permutation_map(), write.in_boundsAttr());
rewriter.eraseOp(write);
return success();
}
@ -2736,7 +2738,7 @@ struct TransferWriteInsertPattern
/// `vector.transfer_read` to a combination of `vector.load` and
/// `vector.broadcast` if all of the following hold:
/// - The op reads from a memref with the default layout.
/// - Masking is not required.
/// - Out-of-bounds masking is not required.
/// - If the memref's element type is a vector type then it coincides with the
/// result type.
/// - The permutation map doesn't perform permutation (broadcasting is allowed).
@ -2774,8 +2776,9 @@ struct TransferReadToVectorLoadLowering
// TODO: Support non-default layouts.
if (!memRefType.getAffineMaps().empty())
return failure();
// TODO: When masking is required, we can create a MaskedLoadOp
if (read.hasMaskedDim())
// TODO: When out-of-bounds masking is required, we can create a
// MaskedLoadOp.
if (read.hasOutOfBoundsDim())
return failure();
Operation *loadOp;
@ -2807,7 +2810,7 @@ struct TransferReadToVectorLoadLowering
/// Progressive lowering of transfer_write. This pattern supports lowering of
/// `vector.transfer_write` to `vector.store` if all of the following hold:
/// - The op writes to a memref with the default layout.
/// - Masking is not required.
/// - Out-of-bounds masking is not required.
/// - If the memref's element type is a vector type then it coincides with the
/// type of the written value.
/// - The permutation map is the minor identity map (neither permutation nor
@ -2833,8 +2836,9 @@ struct TransferWriteToVectorStoreLowering
// TODO: Support non-default layouts.
if (!memRefType.getAffineMaps().empty())
return failure();
// TODO: When masking is required, we can create a MaskedStoreOp
if (write.hasMaskedDim())
// TODO: When out-of-bounds masking is required, we can create a
// MaskedStoreOp.
if (write.hasOutOfBoundsDim())
return failure();
rewriter.replaceOpWithNewOp<vector::StoreOp>(
write, write.vector(), write.source(), write.indices());
@ -2889,7 +2893,7 @@ struct TransferReadPermutationLowering
VectorType::get(newVectorShape, op.getVectorType().getElementType());
Value newRead = rewriter.create<vector::TransferReadOp>(
op.getLoc(), newReadType, op.source(), op.indices(), newMap,
op.padding(), op.masked() ? *op.masked() : ArrayAttr());
op.padding(), op.in_bounds() ? *op.in_bounds() : ArrayAttr());
SmallVector<int64_t> transposePerm(permutation.begin(), permutation.end());
rewriter.replaceOpWithNewOp<vector::TransposeOp>(op, newRead,
transposePerm);
@ -2935,14 +2939,14 @@ struct TransferOpReduceRank : public OpRewritePattern<vector::TransferReadOp> {
originalVecType.getShape().take_back(reducedShapeRank));
VectorType newReadType =
VectorType::get(newShape, originalVecType.getElementType());
ArrayAttr newMask =
op.masked()
ArrayAttr newInBounds =
op.in_bounds()
? rewriter.getArrayAttr(
op.maskedAttr().getValue().take_back(reducedShapeRank))
op.in_boundsAttr().getValue().take_back(reducedShapeRank))
: ArrayAttr();
Value newRead = rewriter.create<vector::TransferReadOp>(
op.getLoc(), newReadType, op.source(), op.indices(), newMap,
op.padding(), newMask);
op.padding(), newInBounds);
rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, originalVecType,
newRead);
return success();
@ -3075,14 +3079,14 @@ struct CastAwayTransferReadLeadingOneDim
AffineMap::get(oldMap.getNumDims(), oldMap.getNumSymbols(), newResults,
rewriter.getContext());
ArrayAttr mask;
if (read.masked())
mask = rewriter.getArrayAttr(
read.maskedAttr().getValue().take_back(newType.getRank()));
ArrayAttr inBounds;
if (read.in_bounds())
inBounds = rewriter.getArrayAttr(
read.in_boundsAttr().getValue().take_back(newType.getRank()));
auto newRead = rewriter.create<vector::TransferReadOp>(
read.getLoc(), newType, read.source(), read.indices(), newMap,
read.padding(), mask);
read.padding(), inBounds);
rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(read, oldType, newRead);
return success();
@ -3115,15 +3119,15 @@ struct CastAwayTransferWriteLeadingOneDim
AffineMap::get(oldMap.getNumDims(), oldMap.getNumSymbols(), newResults,
rewriter.getContext());
ArrayAttr mask;
if (write.masked())
mask = rewriter.getArrayAttr(
write.maskedAttr().getValue().take_back(newType.getRank()));
ArrayAttr inBounds;
if (write.in_bounds())
inBounds = rewriter.getArrayAttr(
write.in_boundsAttr().getValue().take_back(newType.getRank()));
auto newVector = rewriter.create<vector::ShapeCastOp>(
write.getLoc(), newType, write.vector());
rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
write, newVector, write.source(), write.indices(), newMap, mask);
write, newVector, write.source(), write.indices(), newMap, inBounds);
return success();
}

4
mlir/test/Conversion/LinalgToVector/linalg-to-vector.mlir
View File

@ -38,8 +38,8 @@ func @conv_1d(%arg0: memref<?xf32>, %arg1: memref<?xf32>, %arg2: memref<?xf32>)
// CHECK: %[[v15:.*]] = affine.min #[[$map4]](%arg4)[%0]
// CHECK: %[[v16:.*]] = subview %[[arg1]][%[[arg4]]] [%[[v15]]] [1] : memref<?xf32> to memref<?xf32, #[[$map1]]>
// CHECK: %[[v17:.*]] = subview %[[v6]][0] [%[[v13]]] [1] : memref<3xf32> to memref<?xf32>
// CHECK: %[[v19:.*]] = vector.transfer_read %[[v6]][%[[c0]]], %[[cst]] {masked = [false]} : memref<3xf32>, vector<3xf32>
// CHECK: %[[v20:.*]] = vector.transfer_read %[[v7]][%[[c0]]], %[[cst]] {masked = [false]} : memref<3xf32>, vector<3xf32>
// CHECK: %[[v19:.*]] = vector.transfer_read %[[v6]][%[[c0]]], %[[cst]] {in_bounds = [true]} : memref<3xf32>, vector<3xf32>
// CHECK: %[[v20:.*]] = vector.transfer_read %[[v7]][%[[c0]]], %[[cst]] {in_bounds = [true]} : memref<3xf32>, vector<3xf32>
// CHECK: %[[v21:.*]] = mulf %[[v19]], %[[v20]] : vector<3xf32>
// CHECK: %[[v22:.*]] = vector.reduction "add", %[[v21]], %[[cst]] : vector<3xf32> into f32
// CHECK: store %[[v22]], %[[v8]][%[[c0]]] : memref<1xf32>

8
mlir/test/Conversion/StandardToSPIRV/legalization.mlir
View File

@ -74,10 +74,10 @@ func @fold_static_stride_subview_with_transfer_read(%arg0 : memref<12x32xf32>, %
// CHECK: [[INDEX1:%.*]] = addi [[ARG1]], [[STRIDE1]] : index
// CHECK: [[STRIDE2:%.*]] = muli [[ARG4]], [[C3]] : index
// CHECK: [[INDEX2:%.*]] = addi [[ARG2]], [[STRIDE2]] : index
// CHECK: vector.transfer_read [[ARG0]]{{\[}}[[INDEX1]], [[INDEX2]]{{\]}}, [[F1]] {masked = [false]}
// CHECK: vector.transfer_read [[ARG0]]{{\[}}[[INDEX1]], [[INDEX2]]{{\]}}, [[F1]] {in_bounds = [true]}
%f1 = constant 1.0 : f32
%0 = memref.subview %arg0[%arg1, %arg2][4, 4][2, 3] : memref<12x32xf32> to memref<4x4xf32, offset:?, strides: [64, 3]>
%1 = vector.transfer_read %0[%arg3, %arg4], %f1 {masked = [false]} : memref<4x4xf32, offset:?, strides: [64, 3]>, vector<4xf32>
%1 = vector.transfer_read %0[%arg3, %arg4], %f1 {in_bounds = [true]} : memref<4x4xf32, offset:?, strides: [64, 3]>, vector<4xf32>
return %1 : vector<4xf32>
}
@ -91,9 +91,9 @@ func @fold_static_stride_subview_with_transfer_write(%arg0 : memref<12x32xf32>,
// CHECK: [[INDEX1:%.*]] = addi [[ARG1]], [[STRIDE1]] : index
// CHECK: [[STRIDE2:%.*]] = muli [[ARG4]], [[C3]] : index
// CHECK: [[INDEX2:%.*]] = addi [[ARG2]], [[STRIDE2]] : index
// CHECK: vector.transfer_write [[ARG5]], [[ARG0]]{{\[}}[[INDEX1]], [[INDEX2]]{{\]}} {masked = [false]}
// CHECK: vector.transfer_write [[ARG5]], [[ARG0]]{{\[}}[[INDEX1]], [[INDEX2]]{{\]}} {in_bounds = [true]}
%0 = memref.subview %arg0[%arg1, %arg2][4, 4][2, 3] :
memref<12x32xf32> to memref<4x4xf32, offset:?, strides: [64, 3]>
vector.transfer_write %arg5, %0[%arg3, %arg4] {masked = [false]} : vector<4xf32>, memref<4x4xf32, offset:?, strides: [64, 3]>
vector.transfer_write %arg5, %0[%arg3, %arg4] {in_bounds = [true]} : vector<4xf32>, memref<4x4xf32, offset:?, strides: [64, 3]>
return
}

8
mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
View File

@ -1162,13 +1162,13 @@ func @transfer_read_1d_non_zero_addrspace(%A : memref<?xf32, 3>, %base: index) -
// -----
func @transfer_read_1d_not_masked(%A : memref<?xf32>, %base: index) -> vector<17xf32> {
func @transfer_read_1d_inbounds(%A : memref<?xf32>, %base: index) -> vector<17xf32> {
%f7 = constant 7.0: f32
%f = vector.transfer_read %A[%base], %f7 {masked = [false]} :
%f = vector.transfer_read %A[%base], %f7 {in_bounds = [true]} :
memref<?xf32>, vector<17xf32>
return %f: vector<17xf32>
}
// CHECK-LABEL: func @transfer_read_1d_not_masked
// CHECK-LABEL: func @transfer_read_1d_inbounds
// CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<17xf32>
//
// 1. Bitcast to vector form.
@ -1184,7 +1184,7 @@ func @transfer_read_1d_not_masked(%A : memref<?xf32>, %base: index) -> vector<17
func @transfer_read_1d_cast(%A : memref<?xi32>, %base: index) -> vector<12xi8> {
%c0 = constant 0: i32
%v = vector.transfer_read %A[%base], %c0 {masked = [false]} :
%v = vector.transfer_read %A[%base], %c0 {in_bounds = [true]} :
memref<?xi32>, vector<12xi8>
return %v: vector<12xi8>
}

16
mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir
View File

@ -318,15 +318,15 @@ func @transfer_write_progressive(%A : memref<?x?xf32>, %base: index, %vec: vecto
// FULL-UNROLL-DAG: #[[$MAP1:.*]] = affine_map<()[s0] -> (s0 + 1)>
// FULL-UNROLL-DAG: #[[$MAP2:.*]] = affine_map<()[s0] -> (s0 + 2)>
// CHECK-LABEL: transfer_write_progressive_unmasked(
// CHECK-LABEL: transfer_write_progressive_inbounds(
// CHECK-SAME: %[[A:[a-zA-Z0-9]+]]: memref<?x?xf32>,
// CHECK-SAME: %[[base:[a-zA-Z0-9]+]]: index,
// CHECK-SAME: %[[vec:[a-zA-Z0-9]+]]: vector<3x15xf32>
// FULL-UNROLL-LABEL: transfer_write_progressive_unmasked(
// FULL-UNROLL-LABEL: transfer_write_progressive_inbounds(
// FULL-UNROLL-SAME: %[[A:[a-zA-Z0-9]+]]: memref<?x?xf32>,
// FULL-UNROLL-SAME: %[[base:[a-zA-Z0-9]+]]: index,
// FULL-UNROLL-SAME: %[[vec:[a-zA-Z0-9]+]]: vector<3x15xf32>
func @transfer_write_progressive_unmasked(%A : memref<?x?xf32>, %base: index, %vec: vector<3x15xf32>) {
func @transfer_write_progressive_inbounds(%A : memref<?x?xf32>, %base: index, %vec: vector<3x15xf32>) {
// CHECK-NOT: scf.if
// CHECK-NEXT: %[[alloc:.*]] = memref.alloca() : memref<3xvector<15xf32>>
// CHECK-NEXT: %[[vmemref:.*]] = vector.type_cast %[[alloc]] : memref<3xvector<15xf32>> to memref<vector<3x15xf32>>
@ -334,17 +334,17 @@ func @transfer_write_progressive_unmasked(%A : memref<?x?xf32>, %base: index, %v
// CHECK-NEXT: affine.for %[[I:.*]] = 0 to 3 {
// CHECK-NEXT: %[[add:.*]] = affine.apply #[[$MAP0]](%[[I]])[%[[base]]]
// CHECK-NEXT: %[[vec_1d:.*]] = memref.load %0[%[[I]]] : memref<3xvector<15xf32>>
// CHECK-NEXT: vector.transfer_write %[[vec_1d]], %[[A]][%[[add]], %[[base]]] {masked = [false]} : vector<15xf32>, memref<?x?xf32>
// CHECK-NEXT: vector.transfer_write %[[vec_1d]], %[[A]][%[[add]], %[[base]]] {in_bounds = [true]} : vector<15xf32>, memref<?x?xf32>
// FULL-UNROLL: %[[VEC0:.*]] = vector.extract %[[vec]][0] : vector<3x15xf32>
// FULL-UNROLL: vector.transfer_write %[[VEC0]], %[[A]][%[[base]], %[[base]]] {masked = [false]} : vector<15xf32>, memref<?x?xf32>
// FULL-UNROLL: vector.transfer_write %[[VEC0]], %[[A]][%[[base]], %[[base]]] {in_bounds = [true]} : vector<15xf32>, memref<?x?xf32>
// FULL-UNROLL: %[[I1:.*]] = affine.apply #[[$MAP1]]()[%[[base]]]
// FULL-UNROLL: %[[VEC1:.*]] = vector.extract %[[vec]][1] : vector<3x15xf32>
// FULL-UNROLL: vector.transfer_write %2, %[[A]][%[[I1]], %[[base]]] {masked = [false]} : vector<15xf32>, memref<?x?xf32>
// FULL-UNROLL: vector.transfer_write %2, %[[A]][%[[I1]], %[[base]]] {in_bounds = [true]} : vector<15xf32>, memref<?x?xf32>
// FULL-UNROLL: %[[I2:.*]] = affine.apply #[[$MAP2]]()[%[[base]]]
// FULL-UNROLL: %[[VEC2:.*]] = vector.extract %[[vec]][2] : vector<3x15xf32>
// FULL-UNROLL: vector.transfer_write %[[VEC2:.*]], %[[A]][%[[I2]], %[[base]]] {masked = [false]} : vector<15xf32>, memref<?x?xf32>
vector.transfer_write %vec, %A[%base, %base] {masked = [false, false]} :
// FULL-UNROLL: vector.transfer_write %[[VEC2:.*]], %[[A]][%[[I2]], %[[base]]] {in_bounds = [true]} : vector<15xf32>, memref<?x?xf32>
vector.transfer_write %vec, %A[%base, %base] {in_bounds = [true, true]} :
vector<3x15xf32>, memref<?x?xf32>
return
}

24
mlir/test/Dialect/Linalg/forward-vector-transfers.mlir
View File

@ -6,14 +6,14 @@
// CHECK-NOT: linalg.copy
// CHECK: %[[ALLOC:.*]] = memref.alloc
// CHECK: vector.transfer_read %[[ARG0]]
// CHECK-NOT: masked
// CHECK-NOT: in_bounds
func @testAllocRead(%in: memref<? x f32>) -> vector<32 x f32> {
%c0 = constant 0: index
%f0 = constant 0.0: f32
%alloc = memref.alloc() : memref<32 x f32>
%subview = memref.subview %alloc[0][16][1] : memref<32 x f32> to memref<16 x f32>
linalg.copy(%in, %subview): memref<? x f32>, memref<16 x f32>
%0 = vector.transfer_read %alloc[%c0], %f0 {masked = [false]} : memref<32 x f32>, vector<32 x f32>
%0 = vector.transfer_read %alloc[%c0], %f0 {in_bounds = [true]} : memref<32 x f32>, vector<32 x f32>
memref.dealloc %alloc : memref<32 x f32>
return %0: vector<32 x f32>
}
@ -24,7 +24,7 @@ func @testAllocRead(%in: memref<? x f32>) -> vector<32 x f32> {
// CHECK-NOT: linalg.copy
// CHECK: %[[ALLOC:.*]] = memref.alloc
// CHECK: vector.transfer_read %[[ARG0]]
// CHECK-NOT: masked
// CHECK-NOT: in_bounds
func @testAllocFillRead(%in: memref<? x f32>) -> vector<32 x f32> {
%c0 = constant 0: index
%f0 = constant 0.0: f32
@ -32,7 +32,7 @@ func @testAllocFillRead(%in: memref<? x f32>) -> vector<32 x f32> {
linalg.fill(%alloc, %f0): memref<32 x f32>, f32
%subview = memref.subview %alloc[0][16][1] : memref<32 x f32> to memref<16 x f32>
linalg.copy(%in, %subview): memref<? x f32>, memref<16 x f32>
%0 = vector.transfer_read %alloc[%c0], %f0 {masked = [false]} : memref<32 x f32>, vector<32 x f32>
%0 = vector.transfer_read %alloc[%c0], %f0 {in_bounds = [true]} : memref<32 x f32>, vector<32 x f32>
memref.dealloc %alloc : memref<32 x f32>
return %0: vector<32 x f32>
}
@ -43,7 +43,7 @@ func @testAllocFillRead(%in: memref<? x f32>) -> vector<32 x f32> {
// CHECK-NOT: linalg.copy
// CHECK: %[[ALLOC:.*]] = memref.alloc
// CHECK: vector.transfer_read %[[ARG0]]
// CHECK-NOT: masked
// CHECK-NOT: in_bounds
func @testViewRead(%in: memref<? x f32>) -> vector<32 x f32> {
%c0 = constant 0: index
%f0 = constant 0.0: f32
@ -51,7 +51,7 @@ func @testViewRead(%in: memref<? x f32>) -> vector<32 x f32> {
%view = memref.view %alloc[%c0][] : memref<128 x i8> to memref<32 x f32>
%subview = memref.subview %view[0][16][1] : memref<32 x f32> to memref<16 x f32>
linalg.copy(%in, %subview): memref<? x f32>, memref<16 x f32>
%0 = vector.transfer_read %view[%c0], %f0 {masked = [false]} : memref<32 x f32>, vector<32 x f32>
%0 = vector.transfer_read %view[%c0], %f0 {in_bounds = [true]} : memref<32 x f32>, vector<32 x f32>
memref.dealloc %alloc : memref<128 x i8>
return %0: vector<32 x f32>
}
@ -62,7 +62,7 @@ func @testViewRead(%in: memref<? x f32>) -> vector<32 x f32> {
// CHECK-NOT: linalg.copy
// CHECK: %[[ALLOC:.*]] = memref.alloc
// CHECK: vector.transfer_read %[[ARG0]]
// CHECK-NOT: masked
// CHECK-NOT: in_bounds
func @testViewFillRead(%in: memref<? x f32>) -> vector<32 x f32> {
%c0 = constant 0: index
%f0 = constant 0.0: f32
@ -71,7 +71,7 @@ func @testViewFillRead(%in: memref<? x f32>) -> vector<32 x f32> {
%subview = memref.subview %view[0][16][1] : memref<32 x f32> to memref<16 x f32>
linalg.fill(%view, %f0): memref<32 x f32>, f32
linalg.copy(%in, %subview): memref<? x f32>, memref<16 x f32>
%0 = vector.transfer_read %view[%c0], %f0 {masked = [false]} : memref<32 x f32>, vector<32 x f32>
%0 = vector.transfer_read %view[%c0], %f0 {in_bounds = [true]} : memref<32 x f32>, vector<32 x f32>
memref.dealloc %alloc : memref<128 x i8>
return %0: vector<32 x f32>
}
@ -82,13 +82,13 @@ func @testViewFillRead(%in: memref<? x f32>) -> vector<32 x f32> {
// CHECK-NOT: linalg.copy
// CHECK: %[[ALLOC:.*]] = memref.alloc
// CHECK: vector.transfer_write %[[ARG0]], %[[ARG1]]
// CHECK-NOT: masked
// CHECK-NOT: in_bounds
func @testAllocWrite(%vec: vector<32 x f32>, %out: memref<? x f32>) {
%c0 = constant 0: index
%f0 = constant 0.0: f32
%alloc = memref.alloc() : memref<32 x f32>
%subview = memref.subview %alloc[0][16][1] : memref<32 x f32> to memref<16 x f32>
vector.transfer_write %vec, %alloc[%c0] {masked = [false]} : vector<32 x f32>, memref<32 x f32>
vector.transfer_write %vec, %alloc[%c0] {in_bounds = [true]} : vector<32 x f32>, memref<32 x f32>
linalg.copy(%subview, %out): memref<16 x f32>, memref<? x f32>
memref.dealloc %alloc : memref<32 x f32>
return
@ -100,14 +100,14 @@ func @testAllocWrite(%vec: vector<32 x f32>, %out: memref<? x f32>) {
// CHECK-NOT: linalg.copy
// CHECK: %[[ALLOC:.*]] = memref.alloc
// CHECK: vector.transfer_write %[[ARG0]], %[[ARG1]]
// CHECK-NOT: masked
// CHECK-NOT: in_bounds
func @testViewWrite(%vec: vector<32 x f32>, %out: memref<? x f32>) {
%c0 = constant 0: index
%f0 = constant 0.0: f32
%alloc = memref.alloc() : memref<128 x i8>
%view = memref.view %alloc[%c0][] : memref<128 x i8> to memref<32 x f32>
%subview = memref.subview %view[0][16][1] : memref<32 x f32> to memref<16 x f32>
vector.transfer_write %vec, %view[%c0] {masked = [false]} : vector<32 x f32>, memref<32 x f32>
vector.transfer_write %vec, %view[%c0] {in_bounds = [true]} : vector<32 x f32>, memref<32 x f32>
linalg.copy(%subview, %out): memref<16 x f32>, memref<? x f32>
memref.dealloc %alloc : memref<128 x i8>
return

6
mlir/test/Dialect/Linalg/vectorization.mlir
View File

@ -393,7 +393,7 @@ func @matmul_tensors(
// a later canonicalization fuses the add into vector.contract.
// CHECK: %[[C:.*]] = vector.contract {{.*}} iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %[[V0]], %[[V1]], %[[VEC_C0]] : vector<8x4xf32>, vector<4x12xf32> into vector<8x12xf32>
// CHECK: %[[C2:.*]] = addf %[[V2]], %[[C]] : vector<8x12xf32>
// CHECK: %[[W:.*]] = vector.transfer_write %[[C2]], %[[ARG2]][%[[C0]], %[[C0]]] {masked = [false, false]} : vector<8x12xf32>, tensor<8x12xf32>
// CHECK: %[[W:.*]] = vector.transfer_write %[[C2]], %[[ARG2]][%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<8x12xf32>, tensor<8x12xf32>
%0 = linalg.matmul ins(%arg0, %arg1: tensor<8x4xf32>, tensor<4x12xf32>)
outs(%arg2: tensor<8x12xf32>)
-> tensor<8x12xf32>
@ -421,7 +421,7 @@ func @matmul_i8_i8_i32(%a: memref<4x6xi8>, %b: memref<6x12xi8>, %c: memref<4x12x
// CHECK: %[[C:.*]] = vector.contract {{.*}} iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %[[V0_32]], %[[V1_32]], %[[VEC_C0]]
// CHECK-SAME: vector<4x6xi32>, vector<6x12xi32> into vector<4x12xi32>
// CHECK: %[[RES:.*]] = addi %[[V2]], %[[C]] : vector<4x12xi32>
// CHECK: vector.transfer_write %[[RES]], %[[ARG2]][%[[C0]], %[[C0]]] {masked = [false, false]}
// CHECK: vector.transfer_write %[[RES]], %[[ARG2]][%[[C0]], %[[C0]]] {in_bounds = [true, true]}
// CHECK-SAME: vector<4x12xi32>, memref<4x12xi32>
linalg.matmul_i8_i8_i32 ins(%a, %b : memref<4x6xi8>, memref<6x12xi8>)
outs(%c: memref<4x12xi32>)
@ -438,7 +438,7 @@ func @pad_static(%arg0: tensor<?x?x?xf32>, %pad_value: f32) -> tensor<2x3x4xf32>
// CHECK-SAME: : tensor<?x?x?xf32>, vector<2x3x4xf32>
// CHECK: %[[INIT:.*]] = linalg.init_tensor [2, 3, 4] : tensor<2x3x4xf32>
// CHECK: %[[WRITTEN:.*]] = vector.transfer_write %[[READ]], %[[INIT]][%[[C0]], %[[C0]], %[[C0]]]
// CHECK-SAME: {masked = [false, false, false]} : vector<2x3x4xf32>, tensor<2x3x4xf32>
// CHECK-SAME: {in_bounds = [true, true, true]} : vector<2x3x4xf32>, tensor<2x3x4xf32>
%c0 = constant 0 : index
%0 = linalg.pad_tensor %arg0 low[0, %c0, 0] high[0, 0, %c0] {
^bb0(%arg1: index, %arg2: index, %arg3: index):

26
mlir/test/Dialect/Vector/canonicalize.mlir
View File

@ -257,10 +257,10 @@ func @cast_transfers(%A: memref<4x8xf32>) -> (vector<4x8xf32>) {
%f0 = constant 0.0 : f32
%0 = memref.cast %A : memref<4x8xf32> to memref<?x?xf32>
// CHECK: vector.transfer_read %{{.*}} {masked = [false, false]} : memref<4x8xf32>, vector<4x8xf32>
// CHECK: vector.transfer_read %{{.*}} {in_bounds = [true, true]} : memref<4x8xf32>, vector<4x8xf32>
%1 = vector.transfer_read %0[%c0, %c0], %f0 : memref<?x?xf32>, vector<4x8xf32>
// CHECK: vector.transfer_write %{{.*}} {masked = [false, false]} : vector<4x8xf32>, memref<4x8xf32>
// CHECK: vector.transfer_write %{{.*}} {in_bounds = [true, true]} : vector<4x8xf32>, memref<4x8xf32>
vector.transfer_write %1, %0[%c0, %c0] : vector<4x8xf32>, memref<?x?xf32>
return %1 : vector<4x8xf32>
}
@ -273,7 +273,7 @@ func @cast_transfers(%A: tensor<4x8xf32>) -> (vector<4x8xf32>) {
%f0 = constant 0.0 : f32
%0 = tensor.cast %A : tensor<4x8xf32> to tensor<?x?xf32>
// CHECK: vector.transfer_read %{{.*}} {masked = [false, false]} : tensor<4x8xf32>, vector<4x8xf32>
// CHECK: vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<4x8xf32>, vector<4x8xf32>
%1 = vector.transfer_read %0[%c0, %c0], %f0 : tensor<?x?xf32>, vector<4x8xf32>
return %1 : vector<4x8xf32>
@ -537,20 +537,20 @@ func @fold_vector_transfers(%A: memref<?x8xf32>) -> (vector<4x8xf32>, vector<4x9
%c0 = constant 0 : index
%f0 = constant 0.0 : f32
// CHECK: vector.transfer_read %{{.*}} {masked = [true, false]}
// CHECK: vector.transfer_read %{{.*}} {in_bounds = [false, true]}
%1 = vector.transfer_read %A[%c0, %c0], %f0 : memref<?x8xf32>, vector<4x8xf32>
// CHECK: vector.transfer_write %{{.*}} {masked = [true, false]}
// CHECK: vector.transfer_write %{{.*}} {in_bounds = [false, true]}
vector.transfer_write %1, %A[%c0, %c0] : vector<4x8xf32>, memref<?x8xf32>
// Both dims masked, attribute is elided.
// Both dims may be out-of-bounds, attribute is elided.
// CHECK: vector.transfer_read %{{.*}}
// CHECK-NOT: masked
// CHECK-NOT: in_bounds
%2 = vector.transfer_read %A[%c0, %c0], %f0 : memref<?x8xf32>, vector<4x9xf32>
// Both dims masked, attribute is elided.
// Both dims may be out-of-bounds, attribute is elided.
// CHECK: vector.transfer_write %{{.*}}
// CHECK-NOT: masked
// CHECK-NOT: in_bounds
vector.transfer_write %2, %A[%c0, %c0] : vector<4x9xf32>, memref<?x8xf32>
// CHECK: return
@ -780,20 +780,20 @@ func @transfer_folding_1(%t0: tensor<2x3x4xf32>, %t1: tensor<2x3x4xf32>)
{
%c0 = constant 0 : index
%pad = constant 0.0 : f32
%v = vector.transfer_read %t0[%c0, %c0, %c0], %pad {masked = [false, false, false]} :
%v = vector.transfer_read %t0[%c0, %c0, %c0], %pad {in_bounds = [true, true, true]} :
tensor<2x3x4xf32>, vector<2x3x4xf32>
%r0 = vector.transfer_write %v, %t1[%c0, %c0, %c0] {masked = [false, false, false]} :
%r0 = vector.transfer_write %v, %t1[%c0, %c0, %c0] {in_bounds = [true, true, true]} :
vector<2x3x4xf32>, tensor<2x3x4xf32>
%t2 = "test.constant"() { value = dense<6.0> : tensor<2x3x4xf32>} : () -> (tensor<2x3x4xf32>)
%r1 = vector.transfer_write %v, %t2[%c0, %c0, %c0] {masked = [false, false, false]} :
%r1 = vector.transfer_write %v, %t2[%c0, %c0, %c0] {in_bounds = [true, true, true]} :
vector<2x3x4xf32>, tensor<2x3x4xf32>
// CHECK-NEXT: some_op_that_may_have_side_effects
%t3 = "some_op_that_may_have_side_effects"() : () -> (tensor<2x3x4xf32>)
%r2 = vector.transfer_write %v, %t0[%c0, %c0, %c0] {masked = [false, false, false]} :
%r2 = vector.transfer_write %v, %t0[%c0, %c0, %c0] {in_bounds = [true, true, true]} :
vector<2x3x4xf32>, tensor<2x3x4xf32>
// CHECK-NEXT: return %[[T0]], %[[T0]], %[[T0]]

4
mlir/test/Dialect/Vector/invalid.mlir
View File

@ -363,8 +363,8 @@ func @test_vector.transfer_read(%arg0: memref<?x?xvector<2x3xf32>>) {
%c3 = constant 3 : index
%f0 = constant 0.0 : f32
%vf0 = splat %f0 : vector<2x3xf32>
// expected-error@+1 {{ expects the optional masked attr of same rank as permutation_map results: affine_map<(d0, d1) -> (d0, d1)>}}
%0 = vector.transfer_read %arg0[%c3, %c3], %vf0 {masked = [false], permutation_map = affine_map<(d0, d1)->(d0, d1)>} : memref<?x?xvector<2x3xf32>>, vector<1x1x2x3xf32>
// expected-error@+1 {{ expects the optional in_bounds attr of same rank as permutation_map results: affine_map<(d0, d1) -> (d0, d1)>}}
%0 = vector.transfer_read %arg0[%c3, %c3], %vf0 {in_bounds = [true], permutation_map = affine_map<(d0, d1)->(d0, d1)>} : memref<?x?xvector<2x3xf32>>, vector<1x1x2x3xf32>
}
// -----

12
mlir/test/Dialect/Vector/ops.mlir
View File

@ -23,8 +23,8 @@ func @vector_transfer_ops(%arg0: memref<?x?xf32>,
%3 = vector.transfer_read %arg0[%c3, %c3], %cst {permutation_map = affine_map<(d0, d1)->(d1)>} : memref<?x?xf32>, vector<128xf32>
// CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} : memref<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
%4 = vector.transfer_read %arg1[%c3, %c3], %vf0 {permutation_map = affine_map<(d0, d1)->(d0, d1)>} : memref<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
// CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} {masked = [true, false]} : memref<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
%5 = vector.transfer_read %arg1[%c3, %c3], %vf0 {masked = [true, false]} : memref<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
// CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} {in_bounds = [false, true]} : memref<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
%5 = vector.transfer_read %arg1[%c3, %c3], %vf0 {in_bounds = [false, true]} : memref<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
// CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} : memref<?x?xvector<4x3xi32>>, vector<5x24xi8>
%6 = vector.transfer_read %arg2[%c3, %c3], %v0 : memref<?x?xvector<4x3xi32>>, vector<5x24xi8>
@ -36,7 +36,7 @@ func @vector_transfer_ops(%arg0: memref<?x?xf32>,
// CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<1x1x4x3xf32>, memref<?x?xvector<4x3xf32>>
vector.transfer_write %4, %arg1[%c3, %c3] {permutation_map = affine_map<(d0, d1)->(d0, d1)>} : vector<1x1x4x3xf32>, memref<?x?xvector<4x3xf32>>
// CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<1x1x4x3xf32>, memref<?x?xvector<4x3xf32>>
vector.transfer_write %5, %arg1[%c3, %c3] {masked = [true, true]} : vector<1x1x4x3xf32>, memref<?x?xvector<4x3xf32>>
vector.transfer_write %5, %arg1[%c3, %c3] {in_bounds = [false, false]} : vector<1x1x4x3xf32>, memref<?x?xvector<4x3xf32>>
// CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<5x24xi8>, memref<?x?xvector<4x3xi32>>
vector.transfer_write %6, %arg2[%c3, %c3] : vector<5x24xi8>, memref<?x?xvector<4x3xi32>>
@ -69,8 +69,8 @@ func @vector_transfer_ops_tensor(%arg0: tensor<?x?xf32>,
%3 = vector.transfer_read %arg0[%c3, %c3], %cst {permutation_map = affine_map<(d0, d1)->(d1)>} : tensor<?x?xf32>, vector<128xf32>
// CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} : tensor<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
%4 = vector.transfer_read %arg1[%c3, %c3], %vf0 {permutation_map = affine_map<(d0, d1)->(d0, d1)>} : tensor<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
// CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} {masked = [true, false]} : tensor<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
%5 = vector.transfer_read %arg1[%c3, %c3], %vf0 {masked = [true, false]} : tensor<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
// CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} {in_bounds = [false, true]} : tensor<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
%5 = vector.transfer_read %arg1[%c3, %c3], %vf0 {in_bounds = [false, true]} : tensor<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
// CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}} : tensor<?x?xvector<4x3xi32>>, vector<5x24xi8>
%6 = vector.transfer_read %arg2[%c3, %c3], %v0 : tensor<?x?xvector<4x3xi32>>, vector<5x24xi8>
@ -82,7 +82,7 @@ func @vector_transfer_ops_tensor(%arg0: tensor<?x?xf32>,
// CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<1x1x4x3xf32>, tensor<?x?xvector<4x3xf32>>
%9 = vector.transfer_write %4, %arg1[%c3, %c3] {permutation_map = affine_map<(d0, d1)->(d0, d1)>} : vector<1x1x4x3xf32>, tensor<?x?xvector<4x3xf32>>
// CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<1x1x4x3xf32>, tensor<?x?xvector<4x3xf32>>
%10 = vector.transfer_write %5, %arg1[%c3, %c3] {masked = [true, true]} : vector<1x1x4x3xf32>, tensor<?x?xvector<4x3xf32>>
%10 = vector.transfer_write %5, %arg1[%c3, %c3] {in_bounds = [false, false]} : vector<1x1x4x3xf32>, tensor<?x?xvector<4x3xf32>>
// CHECK: vector.transfer_write %{{.*}}, %{{.*}}[%[[C3]], %[[C3]]] : vector<5x24xi8>, tensor<?x?xvector<4x3xi32>>
%11 = vector.transfer_write %6, %arg2[%c3, %c3] : vector<5x24xi8>, tensor<?x?xvector<4x3xi32>>

8
mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
View File

@ -54,7 +54,7 @@ func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -
// CHECK-SAME: memref<?x8xf32>, index, index
// CHECK: }
// CHECK: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %cst
// CHECK_SAME: {masked = [false, false]} : memref<?x8xf32>, vector<4x8xf32>
// CHECK_SAME: {in_bounds = [true, true]} : memref<?x8xf32>, vector<4x8xf32>
// LINALG-DAG: %[[c0:.*]] = constant 0 : index
// LINALG-DAG: %[[c4:.*]] = constant 4 : index
@ -88,7 +88,7 @@ func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -
// LINALG-SAME: memref<?x8xf32>, index, index
// LINALG: }
// LINALG: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %cst
// LINALG_SAME: {masked = [false, false]} : memref<?x8xf32>, vector<4x8xf32>
// LINALG_SAME: {in_bounds = [true, true]} : memref<?x8xf32>, vector<4x8xf32>
%1 = vector.transfer_read %A[%i, %j], %f0 : memref<?x8xf32>, vector<4x8xf32>
// LINALG: return %[[res]] : vector<4x8xf32>
@ -142,7 +142,7 @@ func @split_vector_transfer_read_strided_2d(
// CHECK: scf.yield %[[yielded]], %[[c0]], %[[c0]] :
// CHECK-SAME: memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index
// CHECK: }
// CHECK: %[[res:.*]] = vector.transfer_read {{.*}} {masked = [false, false]} :
// CHECK: %[[res:.*]] = vector.transfer_read {{.*}} {in_bounds = [true, true]} :
// CHECK-SAME: memref<?x8xf32, #[[$map_2d_stride_1]]>, vector<4x8xf32>
// LINALG-DAG: %[[c0:.*]] = constant 0 : index
@ -178,7 +178,7 @@ func @split_vector_transfer_read_strided_2d(
// LINALG: scf.yield %[[yielded]], %[[c0]], %[[c0]] :
// LINALG-SAME: memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index
// LINALG: }
// LINALG: %[[res:.*]] = vector.transfer_read {{.*}} {masked = [false, false]} :
// LINALG: %[[res:.*]] = vector.transfer_read {{.*}} {in_bounds = [true, true]} :
// LINALG-SAME: memref<?x8xf32, #[[$map_2d_stride_1]]>, vector<4x8xf32>
%1 = vector.transfer_read %A[%i, %j], %f0 :
memref<7x8xf32, offset:?, strides:[?, 1]>, vector<4x8xf32>

65
mlir/test/Dialect/Vector/vector-transfer-lowering.mlir
View File

@ -11,8 +11,8 @@
func @transfer_to_load(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> {
%cf0 = constant 0.0 : f32
%res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false]} : memref<8x8xf32>, vector<4xf32>
vector.transfer_write %res, %mem[%i, %i] {masked = [false]} : vector<4xf32>, memref<8x8xf32>
%res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true]} : memref<8x8xf32>, vector<4xf32>
vector.transfer_write %res, %mem[%i, %i] {in_bounds = [true]} : vector<4xf32>, memref<8x8xf32>
return %res : vector<4xf32>
}
@ -29,8 +29,8 @@ func @transfer_to_load(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> {
func @transfer_2D(%mem : memref<8x8xf32>, %i : index) -> vector<2x4xf32> {
%cf0 = constant 0.0 : f32
%res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false, false]} : memref<8x8xf32>, vector<2x4xf32>
vector.transfer_write %res, %mem[%i, %i] {masked = [false, false]} : vector<2x4xf32>, memref<8x8xf32>
%res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true, true]} : memref<8x8xf32>, vector<2x4xf32>
vector.transfer_write %res, %mem[%i, %i] {in_bounds = [true, true]} : vector<2x4xf32>, memref<8x8xf32>
return %res : vector<2x4xf32>
}
@ -60,42 +60,43 @@ func @transfer_vector_element(%mem : memref<8x8xvector<2x4xf32>>, %i : index) ->
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xvector<2x4xf32>>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<1x2x4xf32> {
// CHECK-NEXT: %[[CF0:.*]] = constant dense<0.000000e+00> : vector<2x4xf32>
// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {masked = [false]} : memref<8x8xvector<2x4xf32>>, vector<1x2x4xf32>
// CHECK-NEXT: vector.transfer_write %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] {masked = [false]} : vector<1x2x4xf32>, memref<8x8xvector<2x4xf32>>
// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {in_bounds = [true]} : memref<8x8xvector<2x4xf32>>, vector<1x2x4xf32>
// CHECK-NEXT: vector.transfer_write %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] {in_bounds = [true]} : vector<1x2x4xf32>, memref<8x8xvector<2x4xf32>>
// CHECK-NEXT: return %[[RES]] : vector<1x2x4xf32>
// CHECK-NEXT: }
func @transfer_vector_element_different_types(%mem : memref<8x8xvector<2x4xf32>>, %i : index) -> vector<1x2x4xf32> {
%cf0 = constant dense<0.0> : vector<2x4xf32>
%res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false]} : memref<8x8xvector<2x4xf32>>, vector<1x2x4xf32>
vector.transfer_write %res, %mem[%i, %i] {masked = [false]} : vector<1x2x4xf32>, memref<8x8xvector<2x4xf32>>
%res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true]} : memref<8x8xvector<2x4xf32>>, vector<1x2x4xf32>
vector.transfer_write %res, %mem[%i, %i] {in_bounds = [true]} : vector<1x2x4xf32>, memref<8x8xvector<2x4xf32>>
return %res : vector<1x2x4xf32>
}
// -----
// TODO: transfer_read/write cannot be lowered because there is an unmasked
// dimension.
// CHECK-LABEL: func @transfer_2D_masked(
// TODO: transfer_read/write cannot be lowered because there is a dimension
// that is not guaranteed to be in-bounds.
// CHECK-LABEL: func @transfer_2D_not_inbounds(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<2x4xf32> {
// CHECK-NEXT: %[[CF0:.*]] = constant 0.000000e+00 : f32
// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {masked = [false, true]} : memref<8x8xf32>, vector<2x4xf32>
// CHECK-NEXT: vector.transfer_write %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {masked = [true, false]} : vector<2x4xf32>, memref<8x8xf32>
// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {in_bounds = [true, false]} : memref<8x8xf32>, vector<2x4xf32>
// CHECK-NEXT: vector.transfer_write %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {in_bounds = [false, true]} : vector<2x4xf32>, memref<8x8xf32>
// CHECK-NEXT: return %[[RES]] : vector<2x4xf32>
// CHECK-NEXT: }
func @transfer_2D_masked(%mem : memref<8x8xf32>, %i : index) -> vector<2x4xf32> {
func @transfer_2D_not_inbounds(%mem : memref<8x8xf32>, %i : index) -> vector<2x4xf32> {
%cf0 = constant 0.0 : f32
%res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false, true]} : memref<8x8xf32>, vector<2x4xf32>
vector.transfer_write %res, %mem[%i, %i] {masked = [true, false]} : vector<2x4xf32>, memref<8x8xf32>
%res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true, false]} : memref<8x8xf32>, vector<2x4xf32>
vector.transfer_write %res, %mem[%i, %i] {in_bounds = [false, true]} : vector<2x4xf32>, memref<8x8xf32>
return %res : vector<2x4xf32>
}
// -----
// TODO: transfer_read/write cannot be lowered because they are masked.
// CHECK-LABEL: func @transfer_masked(
// TODO: transfer_read/write cannot be lowered because they are not guaranteed
// to be in-bounds.
// CHECK-LABEL: func @transfer_not_inbounds(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<4xf32> {
// CHECK-NEXT: %[[CF0:.*]] = constant 0.000000e+00 : f32
@ -104,7 +105,7 @@ func @transfer_2D_masked(%mem : memref<8x8xf32>, %i : index) -> vector<2x4xf32>
// CHECK-NEXT: return %[[RES]] : vector<4xf32>
// CHECK-NEXT: }
func @transfer_masked(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> {
func @transfer_not_inbounds(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> {
%cf0 = constant 0.0 : f32
%res = vector.transfer_read %mem[%i, %i], %cf0 : memref<8x8xf32>, vector<4xf32>
vector.transfer_write %res, %mem[%i, %i] : vector<4xf32>, memref<8x8xf32>
@ -119,16 +120,16 @@ func @transfer_masked(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> {
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32, #{{.*}}>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<4xf32> {
// CHECK-NEXT: %[[CF0:.*]] = constant 0.000000e+00 : f32
// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {masked = [false]} : memref<8x8xf32, #{{.*}}>, vector<4xf32>
// CHECK-NEXT: vector.transfer_write %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {masked = [false]} : vector<4xf32>, memref<8x8xf32, #{{.*}}>
// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {in_bounds = [true]} : memref<8x8xf32, #{{.*}}>, vector<4xf32>
// CHECK-NEXT: vector.transfer_write %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {in_bounds = [true]} : vector<4xf32>, memref<8x8xf32, #{{.*}}>
// CHECK-NEXT: return %[[RES]] : vector<4xf32>
// CHECK-NEXT: }
#layout = affine_map<(d0, d1) -> (d0*16 + d1)>
func @transfer_nondefault_layout(%mem : memref<8x8xf32, #layout>, %i : index) -> vector<4xf32> {
%cf0 = constant 0.0 : f32
%res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false]} : memref<8x8xf32, #layout>, vector<4xf32>
vector.transfer_write %res, %mem[%i, %i] {masked = [false]} : vector<4xf32>, memref<8x8xf32, #layout>
%res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true]} : memref<8x8xf32, #layout>, vector<4xf32>
vector.transfer_write %res, %mem[%i, %i] {in_bounds = [true]} : vector<4xf32>, memref<8x8xf32, #layout>
return %res : vector<4xf32>
}
@ -140,15 +141,15 @@ func @transfer_nondefault_layout(%mem : memref<8x8xf32, #layout>, %i : index) ->
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<4xf32> {
// CHECK-NEXT: %[[CF0:.*]] = constant 0.000000e+00 : f32
// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {masked = [false], permutation_map = #{{.*}}} : memref<8x8xf32>, vector<4xf32>
// CHECK-NEXT: vector.transfer_write %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {masked = [false], permutation_map = #{{.*}}} : vector<4xf32>, memref<8x8xf32>
// CHECK-NEXT: %[[RES:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]]], %[[CF0]] {in_bounds = [true], permutation_map = #{{.*}}} : memref<8x8xf32>, vector<4xf32>
// CHECK-NEXT: vector.transfer_write %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {in_bounds = [true], permutation_map = #{{.*}}} : vector<4xf32>, memref<8x8xf32>
// CHECK-NEXT: return %[[RES]] : vector<4xf32>
// CHECK-NEXT: }
func @transfer_perm_map(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> {
%cf0 = constant 0.0 : f32
%res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false], permutation_map = affine_map<(d0, d1) -> (d0)>} : memref<8x8xf32>, vector<4xf32>
vector.transfer_write %res, %mem[%i, %i] {masked = [false], permutation_map = affine_map<(d0, d1) -> (d0)>} : vector<4xf32>, memref<8x8xf32>
%res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true], permutation_map = affine_map<(d0, d1) -> (d0)>} : memref<8x8xf32>, vector<4xf32>
vector.transfer_write %res, %mem[%i, %i] {in_bounds = [true], permutation_map = affine_map<(d0, d1) -> (d0)>} : vector<4xf32>, memref<8x8xf32>
return %res : vector<4xf32>
}
@ -167,7 +168,7 @@ func @transfer_perm_map(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> {
#broadcast = affine_map<(d0, d1) -> (0)>
func @transfer_broadcasting(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32> {
%cf0 = constant 0.0 : f32
%res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false], permutation_map = #broadcast} : memref<8x8xf32>, vector<4xf32>
%res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true], permutation_map = #broadcast} : memref<8x8xf32>, vector<4xf32>
return %res : vector<4xf32>
}
@ -185,7 +186,7 @@ func @transfer_broadcasting(%mem : memref<8x8xf32>, %i : index) -> vector<4xf32>
#broadcast = affine_map<(d0, d1) -> (0, 0)>
func @transfer_broadcasting_2D(%mem : memref<8x8xf32>, %i : index) -> vector<4x4xf32> {
%cf0 = constant 0.0 : f32
%res = vector.transfer_read %mem[%i, %i], %cf0 {masked = [false, false], permutation_map = #broadcast} : memref<8x8xf32>, vector<4x4xf32>
%res = vector.transfer_read %mem[%i, %i], %cf0 {in_bounds = [true, true], permutation_map = #broadcast} : memref<8x8xf32>, vector<4x4xf32>
return %res : vector<4x4xf32>
}
@ -203,7 +204,7 @@ func @transfer_broadcasting_2D(%mem : memref<8x8xf32>, %i : index) -> vector<4x4
#broadcast = affine_map<(d0, d1, d2, d3, d4) -> (d1, 0, 0, d4)>
func @transfer_broadcasting_complex(%mem : memref<10x20x30x8x8xf32>, %i : index) -> vector<3x2x4x5xf32> {
%cf0 = constant 0.0 : f32
%res = vector.transfer_read %mem[%i, %i, %i, %i, %i], %cf0 {masked = [false, false, false, false], permutation_map = #broadcast} : memref<10x20x30x8x8xf32>, vector<3x2x4x5xf32>
%res = vector.transfer_read %mem[%i, %i, %i, %i, %i], %cf0 {in_bounds = [true, true, true, true], permutation_map = #broadcast} : memref<10x20x30x8x8xf32>, vector<3x2x4x5xf32>
return %res : vector<3x2x4x5xf32>
}
@ -236,8 +237,8 @@ func @transfer_read_permutations(%arg0 : memref<?x?xf32>, %arg1 : memref<?x?x?x?
// CHECK: vector.transfer_read {{.*}} {permutation_map = #[[$MAP0]]} : memref<?x?x?x?xf32>, vector<16x14x7x8xf32>
// CHECK: vector.transpose %{{.*}}, [2, 1, 3, 0] : vector<16x14x7x8xf32> to vector<7x14x8x16xf32>
%2 = vector.transfer_read %arg1[%c0, %c0, %c0, %c0], %cst {masked = [false, false, true, false], permutation_map = #map2} : memref<?x?x?x?xf32>, vector<7x14x8x16xf32>
// CHECK: vector.transfer_read {{.*}} {masked = [false, true, false], permutation_map = #[[$MAP1]]} : memref<?x?x?x?xf32>, vector<14x16x7xf32>
%2 = vector.transfer_read %arg1[%c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, false, true], permutation_map = #map2} : memref<?x?x?x?xf32>, vector<7x14x8x16xf32>
// CHECK: vector.transfer_read {{.*}} {in_bounds = [true, false, true], permutation_map = #[[$MAP1]]} : memref<?x?x?x?xf32>, vector<14x16x7xf32>
// CHECK: vector.broadcast %{{.*}} : vector<14x16x7xf32> to vector<8x14x16x7xf32>
// CHECK: vector.transpose %{{.*}}, [3, 1, 0, 2] : vector<8x14x16x7xf32> to vector<7x14x8x16xf32>

58
mlir/test/Dialect/Vector/vector-transferop-opt.mlir
View File

@ -13,16 +13,16 @@ func @forward_dead_store(%arg0: i1, %arg1 : memref<4x4xf32>,
%c4 = constant 4 : index
%c0 = constant 0 : index
%cf0 = constant 0.0 : f32
vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} :
vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
%0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {masked = [false, false]} :
%0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {in_bounds = [true, true]} :
memref<4x4xf32>, vector<1x4xf32>
%x = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%acc = %0)
-> (vector<1x4xf32>) {
%1 = addf %acc, %acc : vector<1x4xf32>
scf.yield %1 : vector<1x4xf32>
}
vector.transfer_write %x, %arg1[%c1, %c0] {masked = [false, false]} :
vector.transfer_write %x, %arg1[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
return
}
@ -40,18 +40,18 @@ func @forward_nested(%arg0: i1, %arg1 : memref<4x4xf32>, %v0 : vector<1x4xf32>,
%c0 = constant 0 : index
%c1 = constant 1 : index
%cf0 = constant 0.0 : f32
vector.transfer_write %v1, %arg1[%i, %c0] {masked = [false, false]} :
vector.transfer_write %v1, %arg1[%i, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} :
vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
%x = scf.if %arg0 -> (vector<1x4xf32>) {
%0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {masked = [false, false]} :
%0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {in_bounds = [true, true]} :
memref<4x4xf32>, vector<1x4xf32>
scf.yield %0 : vector<1x4xf32>
} else {
scf.yield %v1 : vector<1x4xf32>
}
vector.transfer_write %x, %arg1[%c0, %c0] {masked = [false, false]} :
vector.transfer_write %x, %arg1[%c0, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
return
}
@ -73,18 +73,18 @@ func @forward_nested_negative(%arg0: i1, %arg1 : memref<4x4xf32>,
%c0 = constant 0 : index
%c1 = constant 1 : index
%cf0 = constant 0.0 : f32
vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} :
vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
%x = scf.if %arg0 -> (vector<1x4xf32>) {
%0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {masked = [false, false]} :
%0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {in_bounds = [true, true]} :
memref<4x4xf32>, vector<1x4xf32>
scf.yield %0 : vector<1x4xf32>
} else {
vector.transfer_write %v1, %arg1[%i, %c0] {masked = [false, false]} :
vector.transfer_write %v1, %arg1[%i, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
scf.yield %v1 : vector<1x4xf32>
}
vector.transfer_write %x, %arg1[%c0, %i] {masked = [false, false]} :
vector.transfer_write %x, %arg1[%c0, %i] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
return
}
@ -108,24 +108,24 @@ func @dead_store_region(%arg0: i1, %arg1 : memref<4x4xf32>,
%c0 = constant 0 : index
%c1 = constant 1 : index
%cf0 = constant 0.0 : f32
vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} :
vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
%x = scf.if %arg0 -> (vector<1x4xf32>) {
scf.yield %v1 : vector<1x4xf32>
} else {
%0 = vector.transfer_read %arg1[%i, %c0], %cf0 {masked = [false, false]} :
%0 = vector.transfer_read %arg1[%i, %c0], %cf0 {in_bounds = [true, true]} :
memref<4x4xf32>, vector<1x4xf32>
scf.yield %0 : vector<1x4xf32>
}
scf.if %arg0 {
vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} :
vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
}
vector.transfer_write %x, %arg1[%c1, %c0] {masked = [false, false]} :
vector.transfer_write %x, %arg1[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
vector.transfer_write %x, %arg1[%c1, %c0] {masked = [false, false]} :
vector.transfer_write %x, %arg1[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
%1 = vector.transfer_read %arg1[%i, %c0], %cf0 {masked = [false, false]} :
%1 = vector.transfer_read %arg1[%i, %c0], %cf0 {in_bounds = [true, true]} :
memref<4x4xf32>, vector<1x4xf32>
return %1 : vector<1x4xf32>
}
@ -144,15 +144,15 @@ func @dead_store_negative(%arg0: i1, %arg1 : memref<4x4xf32>,
%c1 = constant 1 : index
%cf0 = constant 0.0 : f32
%x = scf.if %arg0 -> (vector<1x4xf32>) {
vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} :
vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
%0 = vector.transfer_read %arg1[%i, %c0], %cf0 {masked = [false, false]} :
%0 = vector.transfer_read %arg1[%i, %c0], %cf0 {in_bounds = [true, true]} :
memref<4x4xf32>, vector<1x4xf32>
scf.yield %0 : vector<1x4xf32>
} else {
scf.yield %v1 : vector<1x4xf32>
}
vector.transfer_write %x, %arg1[%c1, %c0] {masked = [false, false]} :
vector.transfer_write %x, %arg1[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
return
}
@ -172,13 +172,13 @@ func @dead_store_nested_region(%arg0: i1, %arg1: i1, %arg2 : memref<4x4xf32>,
%c1 = constant 1 : index
%cf0 = constant 0.0 : f32
scf.if %arg0 {
%0 = vector.transfer_read %arg2[%i, %c0], %cf0 {masked = [false, false]} :
%0 = vector.transfer_read %arg2[%i, %c0], %cf0 {in_bounds = [true, true]} :
memref<4x4xf32>, vector<1x4xf32>
scf.if %arg1 {
vector.transfer_write %v1, %arg2[%c1, %c0] {masked = [false, false]} :
vector.transfer_write %v1, %arg2[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
}
vector.transfer_write %v0, %arg2[%c1, %c0] {masked = [false, false]} :
vector.transfer_write %v0, %arg2[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, memref<4x4xf32>
}
return
@ -197,16 +197,16 @@ func @forward_dead_store_tensor(%arg0: i1, %arg1 : tensor<4x4xf32>,
%c4 = constant 4 : index
%c0 = constant 0 : index
%cf0 = constant 0.0 : f32
%w0 = vector.transfer_write %v0, %arg1[%c1, %c0] {masked = [false, false]} :
%w0 = vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, tensor<4x4xf32>
%0 = vector.transfer_read %w0[%c1, %c0], %cf0 {masked = [false, false]} :
%0 = vector.transfer_read %w0[%c1, %c0], %cf0 {in_bounds = [true, true]} :
tensor<4x4xf32>, vector<1x4xf32>
%x = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%acc = %0)
-> (vector<1x4xf32>) {
%1 = addf %acc, %acc : vector<1x4xf32>
scf.yield %1 : vector<1x4xf32>
}
%w1 = vector.transfer_write %x, %w0[%c1, %c0] {masked = [false, false]} :
%w1 = vector.transfer_write %x, %w0[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, tensor<4x4xf32>
return %w1 : tensor<4x4xf32>
}
@ -224,16 +224,16 @@ func @forward_dead_store_negative_tensor(%arg0: i1, %arg1 : tensor<4x4xf32>,
%c4 = constant 4 : index
%c0 = constant 0 : index
%cf0 = constant 0.0 : f32
%w0 = vector.transfer_write %v0, %arg1[%c1, %i] {masked = [false, false]} :
%w0 = vector.transfer_write %v0, %arg1[%c1, %i] {in_bounds = [true, true]} :
vector<1x4xf32>, tensor<4x4xf32>
%0 = vector.transfer_read %w0[%c1, %c0], %cf0 {masked = [false, false]} :
%0 = vector.transfer_read %w0[%c1, %c0], %cf0 {in_bounds = [true, true]} :
tensor<4x4xf32>, vector<1x4xf32>
%x = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%acc = %0)
-> (vector<1x4xf32>) {
%1 = addf %acc, %acc : vector<1x4xf32>
scf.yield %1 : vector<1x4xf32>
}
%w1 = vector.transfer_write %x, %w0[%c1, %c0] {masked = [false, false]} :
%w1 = vector.transfer_write %x, %w0[%c1, %c0] {in_bounds = [true, true]} :
vector<1x4xf32>, tensor<4x4xf32>
return %w1 : tensor<4x4xf32>
}

36
mlir/test/Dialect/Vector/vector-transforms.mlir
View File

@ -246,10 +246,10 @@ func @contraction4x4_ikj(%arg0 : vector<4x2xf32>, %arg1 : vector<2x4xf32>,
// CHECK-NEXT: %[[R2:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"], kind = #vector.kind<add>} %[[VTR1]], %[[VTR2]], %[[VTR6]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32>
// CHECK-NEXT: %[[R3:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"], kind = #vector.kind<add>} %[[VTR1]], %[[VTR3]], %[[VTR7]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32>
// CHECK-NEXT: vector.transfer_write %[[R0]], %{{.*}}[%[[C0]], %[[C0]]] {masked = [false, false]} : vector<2x2xf32>, memref<4x4xf32>
// CHECK-NEXT: vector.transfer_write %[[R1]], %{{.*}}[%[[C0]], %[[C2]]] {masked = [false, false]} : vector<2x2xf32>, memref<4x4xf32>
// CHECK-NEXT: vector.transfer_write %[[R2]], %{{.*}}[%[[C2]], %[[C0]]] {masked = [false, false]} : vector<2x2xf32>, memref<4x4xf32>
// CHECK-NEXT: vector.transfer_write %[[R3]], %{{.*}}[%[[C2]], %[[C2]]] {masked = [false, false]} : vector<2x2xf32>, memref<4x4xf32>
// CHECK-NEXT: vector.transfer_write %[[R0]], %{{.*}}[%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<2x2xf32>, memref<4x4xf32>
// CHECK-NEXT: vector.transfer_write %[[R1]], %{{.*}}[%[[C0]], %[[C2]]] {in_bounds = [true, true]} : vector<2x2xf32>, memref<4x4xf32>
// CHECK-NEXT: vector.transfer_write %[[R2]], %{{.*}}[%[[C2]], %[[C0]]] {in_bounds = [true, true]} : vector<2x2xf32>, memref<4x4xf32>
// CHECK-NEXT: vector.transfer_write %[[R3]], %{{.*}}[%[[C2]], %[[C2]]] {in_bounds = [true, true]} : vector<2x2xf32>, memref<4x4xf32>
// CHECK-NEXT: return
func @contraction4x4_ikj_xfer_read(%arg0 : memref<4x2xf32>,
@ -424,10 +424,10 @@ func @cancelling_shape_cast_ops(%arg0 : vector<2x4xf32>) -> vector<2x4xf32> {
// CHECK-LABEL: func @vector_transfers_vector_element_type
// CHECK-DAG: %[[C1:.*]] = constant 1 : index
// CHECK-DAG: %[[C0:.*]] = constant 0 : index
// CHECK: %[[VTR0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]]], %{{.*}} {masked = [false, false]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32>
// CHECK-NEXT: %[[VTR1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C1]], %[[C0]]], %{{.*}} {masked = [false, false]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32>
// CHECK-NEXT: vector.transfer_write %[[VTR0]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]]] {masked = [false, false]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>>
// CHECK-NEXT: vector.transfer_write %[[VTR1]], %{{.*}}[%[[C0]], %[[C1]], %[[C0]]] {masked = [false, false]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>>
// CHECK: %[[VTR0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]]], %{{.*}} {in_bounds = [true, true]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32>
// CHECK-NEXT: %[[VTR1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C1]], %[[C0]]], %{{.*}} {in_bounds = [true, true]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32>
// CHECK-NEXT: vector.transfer_write %[[VTR0]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>>
// CHECK-NEXT: vector.transfer_write %[[VTR1]], %{{.*}}[%[[C0]], %[[C1]], %[[C0]]] {in_bounds = [true, true]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>>
func @vector_transfers_vector_element_type() {
%c0 = constant 0 : index
@ -577,10 +577,10 @@ func @elementwise_unroll(%arg0 : memref<4x4xf32>, %arg1 : memref<4x4xf32>) {
// CHECK-NEXT: %[[R2:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"], kind = #vector.kind<add>} %[[VTR1]], %[[VTR2]], %[[VTR6]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32>
// CHECK-NEXT: %[[R3:.*]] = vector.contract {indexing_maps = [#map2, #map3, #map0], iterator_types = ["parallel", "reduction", "parallel"], kind = #vector.kind<add>} %[[VTR1]], %[[VTR3]], %[[VTR7]] : vector<2x2xf32>, vector<2x2xf32> into vector<2x2xf32>
// CHECK-NEXT: %[[VTW0:.*]] = vector.transfer_write %[[R0]], %{{.*}}[%[[C0]], %[[C0]]] {masked = [false, false]} : vector<2x2xf32>, tensor<4x4xf32>
// CHECK-NEXT: %[[VTW1:.*]] = vector.transfer_write %[[R1]], %[[VTW0]][%[[C0]], %[[C2]]] {masked = [false, false]} : vector<2x2xf32>, tensor<4x4xf32>
// CHECK-NEXT: %[[VTW2:.*]] = vector.transfer_write %[[R2]], %[[VTW1]][%[[C2]], %[[C0]]] {masked = [false, false]} : vector<2x2xf32>, tensor<4x4xf32>
// CHECK-NEXT: %[[VTW3:.*]] = vector.transfer_write %[[R3]], %[[VTW2]][%[[C2]], %[[C2]]] {masked = [false, false]} : vector<2x2xf32>, tensor<4x4xf32>
// CHECK-NEXT: %[[VTW0:.*]] = vector.transfer_write %[[R0]], %{{.*}}[%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<2x2xf32>, tensor<4x4xf32>
// CHECK-NEXT: %[[VTW1:.*]] = vector.transfer_write %[[R1]], %[[VTW0]][%[[C0]], %[[C2]]] {in_bounds = [true, true]} : vector<2x2xf32>, tensor<4x4xf32>
// CHECK-NEXT: %[[VTW2:.*]] = vector.transfer_write %[[R2]], %[[VTW1]][%[[C2]], %[[C0]]] {in_bounds = [true, true]} : vector<2x2xf32>, tensor<4x4xf32>
// CHECK-NEXT: %[[VTW3:.*]] = vector.transfer_write %[[R3]], %[[VTW2]][%[[C2]], %[[C2]]] {in_bounds = [true, true]} : vector<2x2xf32>, tensor<4x4xf32>
// CHECK-NEXT: return %[[VTW3]] : tensor<4x4xf32>
func @contraction4x4_ikj_xfer_read_tensor(%arg0 : tensor<4x2xf32>,
@ -637,9 +637,9 @@ func @cast_away_transfer_read_leading_one_dims(%arg0: memref<1x4x8x16xf16>) -> v
%c0 = constant 0 : index
// CHECK: %[[F0:.+]] = constant 0.000000e+00 : f16
%f0 = constant 0. : f16
// CHECK: %[[READ:.+]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]], %[[C0]]], %[[F0]] {masked = [false]} : memref<1x4x8x16xf16>, vector<4xf16>
// CHECK: %[[READ:.+]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]], %[[C0]]], %[[F0]] {in_bounds = [true]} : memref<1x4x8x16xf16>, vector<4xf16>
// CHECK: %[[CAST:.+]] = vector.shape_cast %[[READ]] : vector<4xf16> to vector<1x4xf16>
%0 = vector.transfer_read %arg0[%c0, %c0, %c0, %c0], %f0 {masked = [false, false]} : memref<1x4x8x16xf16>, vector<1x4xf16>
%0 = vector.transfer_read %arg0[%c0, %c0, %c0, %c0], %f0 {in_bounds = [true, true]} : memref<1x4x8x16xf16>, vector<1x4xf16>
// CHECK: return %[[CAST]]
return %0: vector<1x4xf16>
}
@ -649,7 +649,7 @@ func @cast_away_transfer_read_leading_one_dims_one_element(%arg0: memref<1x1x1x1
%c0 = constant 0 : index
%f0 = constant 0. : f16
// CHECK: vector.shape_cast %{{.+}} : vector<1xf16> to vector<1x1xf16>
%0 = vector.transfer_read %arg0[%c0, %c0, %c0, %c0], %f0 {masked = [false, false]} : memref<1x1x1x1xf16>, vector<1x1xf16>
%0 = vector.transfer_read %arg0[%c0, %c0, %c0, %c0], %f0 {in_bounds = [true, true]} : memref<1x1x1x1xf16>, vector<1x1xf16>
return %0: vector<1x1xf16>
}
@ -658,9 +658,9 @@ func @cast_away_transfer_write_leading_one_dims(%arg0: memref<1x4x8x16xf16>, %ar
// CHECK: %[[C0:.+]] = constant 0 : index
%c0 = constant 0 : index
// CHECK: %[[CAST:.+]] = vector.shape_cast %{{.*}} : vector<1x4xf16> to vector<4xf16>
// CHECK: vector.transfer_write %[[CAST]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]], %[[C0]]] {masked = [false]} : vector<4xf16>, memref<1x4x8x16xf16>
// CHECK: vector.transfer_write %[[CAST]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true]} : vector<4xf16>, memref<1x4x8x16xf16>
vector.transfer_write %arg1, %arg0[%c0, %c0, %c0, %c0] {masked = [false, false]} : vector<1x4xf16>, memref<1x4x8x16xf16>
vector.transfer_write %arg1, %arg0[%c0, %c0, %c0, %c0] {in_bounds = [true, true]} : vector<1x4xf16>, memref<1x4x8x16xf16>
return
}
@ -668,7 +668,7 @@ func @cast_away_transfer_write_leading_one_dims(%arg0: memref<1x4x8x16xf16>, %ar
func @cast_away_transfer_write_leading_one_dims_one_element(%arg0: memref<1x1x1x1xf16>, %arg1: vector<1x1xf16>) {
%c0 = constant 0 : index
// CHECK: vector.shape_cast %{{.+}} : vector<1x1xf16> to vector<1xf16>
vector.transfer_write %arg1, %arg0[%c0, %c0, %c0, %c0] {masked = [false, false]} : vector<1x1xf16>, memref<1x1x1x1xf16>
vector.transfer_write %arg1, %arg0[%c0, %c0, %c0, %c0] {in_bounds = [true, true]} : vector<1x1xf16>, memref<1x1x1x1xf16>
return
}

8
mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir
View File

@ -12,10 +12,10 @@ func @transfer_read_1d(%A : memref<?xf32>, %base: index) {
return
}
func @transfer_read_unmasked_4(%A : memref<?xf32>, %base: index) {
func @transfer_read_inbounds_4(%A : memref<?xf32>, %base: index) {
%fm42 = constant -42.0: f32
%f = vector.transfer_read %A[%base], %fm42
{permutation_map = affine_map<(d0) -> (d0)>, masked = [false]} :
{permutation_map = affine_map<(d0) -> (d0)>, in_bounds = [true]} :
memref<?xf32>, vector<4xf32>
vector.print %f: vector<4xf32>
return
@ -53,9 +53,9 @@ func @entry() {
// Read shifted by 0 and pad with -42:
// ( 0, 1, 2, 0, 0, -42, ..., -42)
call @transfer_read_1d(%A, %c0) : (memref<?xf32>, index) -> ()
// Read unmasked 4 @ 1, guaranteed to not overflow.
// Read in-bounds 4 @ 1, guaranteed to not overflow.
// Exercises proper alignment.
call @transfer_read_unmasked_4(%A, %c1) : (memref<?xf32>, index) -> ()
call @transfer_read_inbounds_4(%A, %c1) : (memref<?xf32>, index) -> ()
return
}

8
mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir
View File

@ -3,11 +3,11 @@
// RUN: -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
// RUN: FileCheck %s
func @transfer_write16_unmasked_1d(%A : memref<?xf32>, %base: index) {
func @transfer_write16_inbounds_1d(%A : memref<?xf32>, %base: index) {
%f = constant 16.0 : f32
%v = splat %f : vector<16xf32>
vector.transfer_write %v, %A[%base]
{permutation_map = affine_map<(d0) -> (d0)>, masked = [false]}
{permutation_map = affine_map<(d0) -> (d0)>, in_bounds = [true]}
: vector<16xf32>, memref<?xf32>
return
}
@ -54,9 +54,9 @@ func @entry() {
vector.print %0 : vector<32xf32>
// Overwrite with 16 values of 16 at base 3.
// Statically guaranteed to be unmasked. Exercises proper alignment.
// Statically guaranteed to be in-bounds. Exercises proper alignment.
%c3 = constant 3: index
call @transfer_write16_unmasked_1d(%A, %c3) : (memref<?xf32>, index) -> ()
call @transfer_write16_inbounds_1d(%A, %c3) : (memref<?xf32>, index) -> ()
%1 = call @transfer_read_1d(%A) : (memref<?xf32>) -> (vector<32xf32>)
vector.print %1 : vector<32xf32>