maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[mlir][vector][NFC] Split into IR, Transforms and Utils 

This reduces the dependencies of the MLIRVector target and makes the dialect consistent with other dialects.

Differential Revision: https://reviews.llvm.org/D118533
This commit is contained in:
Matthias Springer 2022-01-31 19:10:51 +09:00
parent b8290ffa9f
commit 99ef9eebad
72 changed files with 452 additions and 302 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mlir/docs/Dialects/Vector.md
View File

@ -119,7 +119,7 @@ d2, d3) -> (d3, d1, d0)} : vector<5x4x3xf32>, memref<?x?x?x?xf32>
The list of Vector is currently undergoing evolutions and is best kept track of
by following the evolution of the
[VectorOps.td](https://github.com/llvm/llvm-project/blob/main/mlir/include/mlir/Dialect/Vector/VectorOps.td)
[VectorOps.td](https://github.com/llvm/llvm-project/blob/main/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td)
ODS file (markdown documentation is automatically generated locally when
building and populates the
[Vector doc](https://github.com/llvm/llvm-project/blob/main/mlir/docs/Dialects/Vector.md)).

2
mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
View File

@ -17,7 +17,7 @@
#include "mlir/Dialect/SCF/Utils/Utils.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Utils/StaticValueUtils.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
#include "mlir/Dialect/X86Vector/Transforms.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Transforms/DialectConversion.h"

39
mlir/include/mlir/Dialect/Utils/IndexingUtils.h Normal file
View File

@ -0,0 +1,39 @@
//===- IndexingUtils.h - Helpers related to index computations --*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This header file defines utilities and common canonicalization patterns for
// reshape operations.
//
//===----------------------------------------------------------------------===//
#ifndef MLIR_DIALECT_UTILS_INDEXINGUTILS_H
#define MLIR_DIALECT_UTILS_INDEXINGUTILS_H
#include "mlir/Support/LLVM.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallVector.h"
namespace mlir {
class ArrayAttr;
/// Computes and returns the linearized index of 'offsets' w.r.t. 'basis'.
int64_t linearize(ArrayRef<int64_t> offsets, ArrayRef<int64_t> basis);
/// Given the strides together with a linear index in the dimension
/// space, returns the vector-space offsets in each dimension for a
/// de-linearized index.
SmallVector<int64_t, 4> delinearize(ArrayRef<int64_t> strides,
int64_t linearIndex);
/// Helper that returns a subset of `arrayAttr` as a vector of int64_t.
SmallVector<int64_t, 4> getI64SubArray(ArrayAttr arrayAttr,
unsigned dropFront = 0,
unsigned dropBack = 0);
} // namespace mlir
#endif // MLIR_DIALECT_UTILS_INDEXINGUTILS_H

10
mlir/include/mlir/Dialect/Vector/CMakeLists.txt
View File

@ -1,8 +1,2 @@
add_mlir_dialect(VectorOps vector)
add_mlir_doc(VectorOps VectorOps Dialects/ -gen-op-doc)
set(LLVM_TARGET_DEFINITIONS VectorOps.td)
mlir_tablegen(VectorOpsEnums.h.inc -gen-enum-decls)
mlir_tablegen(VectorOpsEnums.cpp.inc -gen-enum-defs)
add_public_tablegen_target(MLIRVectorOpsEnumsIncGen)
add_dependencies(mlir-headers MLIRVectorOpsEnumsIncGen)
add_subdirectory(IR)
add_subdirectory(Transforms)

8
mlir/include/mlir/Dialect/Vector/IR/CMakeLists.txt Normal file
View File

@ -0,0 +1,8 @@
add_mlir_dialect(VectorOps vector)
add_mlir_doc(VectorOps VectorOps Dialects/ -gen-op-doc)
set(LLVM_TARGET_DEFINITIONS VectorOps.td)
mlir_tablegen(VectorOpsEnums.h.inc -gen-enum-decls)
mlir_tablegen(VectorOpsEnums.cpp.inc -gen-enum-defs)
add_public_tablegen_target(MLIRVectorOpsEnumsIncGen)
add_dependencies(mlir-headers MLIRVectorOpsEnumsIncGen)

35
mlir/include/mlir/Dialect/Vector/VectorOps.h → mlir/include/mlir/Dialect/Vector/IR/VectorOps.h
View File

@ -10,8 +10,8 @@
//
//===----------------------------------------------------------------------===//
#ifndef MLIR_DIALECT_VECTOR_VECTOROPS_H
#define MLIR_DIALECT_VECTOR_VECTOROPS_H
#ifndef MLIR_DIALECT_VECTOR_IR_VECTOROPS_H
#define MLIR_DIALECT_VECTOR_IR_VECTOROPS_H
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
@ -27,13 +27,15 @@
#include "llvm/ADT/StringExtras.h"
// Pull in all enum type definitions and utility function declarations.
#include "mlir/Dialect/Vector/VectorOpsEnums.h.inc"
#include "mlir/Dialect/Vector/IR/VectorOpsEnums.h.inc"
namespace mlir {
class MLIRContext;
class RewritePatternSet;
namespace vector {
class TransferReadOp;
class TransferWriteOp;
class VectorDialect;
namespace detail {
@ -152,18 +154,35 @@ Value getVectorReductionOp(arith::AtomicRMWKind op, OpBuilder &builder,
/// return true for memrefs with no strides.
bool isLastMemrefDimUnitStride(MemRefType type);
namespace impl {
/// Build the default minor identity map suitable for a vector transfer. This
/// also handles the case memref<... x vector<...>> -> vector<...> in which the
/// rank of the identity map must take the vector element type into account.
AffineMap getTransferMinorIdentityMap(ShapedType shapedType,
VectorType vectorType);
} // namespace impl
/// Return true if the transfer_write fully writes the data accessed by the
/// transfer_read.
bool checkSameValueRAW(TransferWriteOp defWrite, TransferReadOp read);
/// Return true if the write op fully over-write the priorWrite transfer_write
/// op.
bool checkSameValueWAW(TransferWriteOp write, TransferWriteOp priorWrite);
/// Same behavior as `isDisjointTransferSet` but doesn't require the operations
/// to have the same tensor/memref. This allows comparing operations accessing
/// different tensors.
bool isDisjointTransferIndices(VectorTransferOpInterface transferA,
VectorTransferOpInterface transferB);
/// Return true if we can prove that the transfer operations access disjoint
/// memory.
bool isDisjointTransferSet(VectorTransferOpInterface transferA,
VectorTransferOpInterface transferB);
} // namespace vector
} // namespace mlir
#define GET_OP_CLASSES
#include "mlir/Dialect/Vector/VectorOps.h.inc"
#include "mlir/Dialect/Vector/VectorOpsDialect.h.inc"
#include "mlir/Dialect/Vector/IR/VectorOps.h.inc"
#include "mlir/Dialect/Vector/IR/VectorOpsDialect.h.inc"
#endif // MLIR_DIALECT_VECTOR_VECTOROPS_H
#endif // MLIR_DIALECT_VECTOR_IR_VECTOROPS_H

0
mlir/include/mlir/Dialect/Vector/VectorOps.td → mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
View File

1
mlir/include/mlir/Dialect/Vector/Transforms/CMakeLists.txt Normal file
View File

@ -0,0 +1 @@
# This dialect does currently not have any passes.

10
mlir/include/mlir/Dialect/Vector/VectorRewritePatterns.h → mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
View File

@ -6,13 +6,13 @@
//
//===----------------------------------------------------------------------===//
#ifndef MLIR_DIALECT_VECTOR_VECTORREWRITEPATTERNS_H
#define MLIR_DIALECT_VECTOR_VECTORREWRITEPATTERNS_H
#ifndef MLIR_DIALECT_VECTOR_TRANSFORMS_VECTORREWRITEPATTERNS_H
#define MLIR_DIALECT_VECTOR_TRANSFORMS_VECTORREWRITEPATTERNS_H
#include <utility>
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/PatternMatch.h"
@ -513,4 +513,4 @@ private:
} // namespace vector
} // namespace mlir
#endif // MLIR_DIALECT_VECTOR_VECTORREWRITEPATTERNS_H
#endif // MLIR_DIALECT_VECTOR_TRANSFORMS_VECTORREWRITEPATTERNS_H

10
mlir/include/mlir/Dialect/Vector/VectorTransforms.h → mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h
View File

@ -6,11 +6,11 @@
//
//===----------------------------------------------------------------------===//
#ifndef MLIR_DIALECT_VECTOR_VECTORTRANSFORMS_H
#define MLIR_DIALECT_VECTOR_VECTORTRANSFORMS_H
#ifndef MLIR_DIALECT_VECTOR_TRANSFORMS_VECTORTRANSFORMS_H
#define MLIR_DIALECT_VECTOR_TRANSFORMS_VECTORTRANSFORMS_H
#include "mlir/Dialect/Vector/VectorRewritePatterns.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
namespace mlir {
class MLIRContext;
@ -94,4 +94,4 @@ void transferOpflowOpt(FuncOp func);
} // namespace vector
} // namespace mlir
#endif // MLIR_DIALECT_VECTOR_VECTORTRANSFORMS_H
#endif // MLIR_DIALECT_VECTOR_TRANSFORMS_VECTORTRANSFORMS_H

47
mlir/include/mlir/Dialect/Vector/VectorUtils.h → mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h
View File

@ -6,8 +6,8 @@
//
//===----------------------------------------------------------------------===//
#ifndef MLIR_DIALECT_VECTOR_VECTORUTILS_H_
#define MLIR_DIALECT_VECTOR_VECTORUTILS_H_
#ifndef MLIR_DIALECT_VECTOR_UTILS_VECTORUTILS_H_
#define MLIR_DIALECT_VECTOR_UTILS_VECTORUTILS_H_
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/Support/LLVM.h"
@ -47,15 +47,6 @@ int64_t computeMaxLinearIndex(ArrayRef<int64_t> basis);
SmallVector<int64_t, 4> computeStrides(ArrayRef<int64_t> shape,
ArrayRef<int64_t> sizes);
/// Computes and returns the linearized index of 'offsets' w.r.t. 'basis'.
int64_t linearize(ArrayRef<int64_t> offsets, ArrayRef<int64_t> basis);
/// Given the strides together with a linear index in the dimension
/// space, returns the vector-space offsets in each dimension for a
/// de-linearized index.
SmallVector<int64_t, 4> delinearize(ArrayRef<int64_t> strides,
int64_t linearIndex);
/// Given the target sizes of a vector, together with vector-space offsets,
/// returns the element-space offsets for each dimension.
SmallVector<int64_t, 4>
@ -158,38 +149,6 @@ AffineMap
makePermutationMap(Operation *insertPoint, ArrayRef<Value> indices,
const DenseMap<Operation *, unsigned> &loopToVectorDim);
/// Build the default minor identity map suitable for a vector transfer. This
/// also handles the case memref<... x vector<...>> -> vector<...> in which the
/// rank of the identity map must take the vector element type into account.
AffineMap getTransferMinorIdentityMap(ShapedType shapedType,
VectorType vectorType);
/// Return true if we can prove that the transfer operations access disjoint
/// memory.
bool isDisjointTransferSet(VectorTransferOpInterface transferA,
VectorTransferOpInterface transferB);
/// Same behavior as `isDisjointTransferSet` but doesn't require the operations
/// to have the same tensor/memref. This allows comparing operations accessing
/// different tensors.
bool isDisjointTransferIndices(VectorTransferOpInterface transferA,
VectorTransferOpInterface transferB);
/// Return true if the transfer_write fully writes the data accessed by the
/// transfer_read.
bool checkSameValueRAW(vector::TransferWriteOp defWrite,
vector::TransferReadOp read);
/// Return true if the write op fully over-write the priorWrite transfer_write
/// op.
bool checkSameValueWAW(vector::TransferWriteOp write,
vector::TransferWriteOp priorWrite);
// Helper that returns a subset of `arrayAttr` as a vector of int64_t.
SmallVector<int64_t, 4> getI64SubArray(ArrayAttr arrayAttr,
unsigned dropFront = 0,
unsigned dropBack = 0);
namespace matcher {
/// Matches vector.transfer_read, vector.transfer_write and ops that return a
@ -205,4 +164,4 @@ bool operatesOnSuperVectorsOf(Operation &op, VectorType subVectorType);
} // namespace matcher
} // namespace mlir
#endif // MLIR_DIALECT_VECTOR_VECTORUTILS_H_
#endif // MLIR_DIALECT_VECTOR_UTILS_VECTORUTILS_H_

2
mlir/include/mlir/InitAllDialects.h
View File

@ -45,7 +45,7 @@
#include "mlir/Dialect/Tensor/IR/TensorInferTypeOpInterfaceImpl.h"
#include "mlir/Dialect/Tensor/IR/TensorTilingInterfaceImpl.h"
#include "mlir/Dialect/Tosa/IR/TosaOps.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/X86Vector/X86VectorDialect.h"
#include "mlir/IR/Dialect.h"

2
mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
View File

@ -19,7 +19,7 @@
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/SCF/SCF.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/AffineExprVisitor.h"
#include "mlir/IR/BlockAndValueMapping.h"
#include "mlir/IR/Builders.h"

2
mlir/lib/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.cpp
View File

@ -9,7 +9,7 @@
#include "mlir/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.h"
#include "../PassDetail.h"
#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Pass/PassRegistry.h"

2
mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
View File

@ -25,7 +25,7 @@
#include "mlir/Dialect/GPU/Passes.h"
#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
#include "mlir/Dialect/Math/IR/Math.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/DialectConversion.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"

2
mlir/lib/Conversion/MathToLibm/CMakeLists.txt
View File

@ -12,7 +12,9 @@ add_mlir_conversion_library(MLIRMathToLibm
LINK_LIBS PUBLIC
MLIRArithmetic
MLIRDialectUtils
MLIRMath
MLIRStandardOpsTransforms
MLIRVector
MLIRVectorUtils
)

5
mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
View File

@ -12,8 +12,9 @@
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/Math/IR/Math.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Utils/IndexingUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/BuiltinDialect.h"
#include "mlir/IR/PatternMatch.h"

1
mlir/lib/Conversion/VectorToGPU/CMakeLists.txt
View File

@ -14,4 +14,5 @@ add_mlir_conversion_library(MLIRVectorToGPU
MLIRMemRef
MLIRTransforms
MLIRVector
MLIRVectorUtils
)

4
mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
View File

@ -21,8 +21,8 @@
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/SCF/SCF.h"
#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/Builders.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"

1
mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
View File

@ -25,6 +25,7 @@ add_mlir_conversion_library(MLIRVectorToLLVM
MLIRTargetLLVMIRExport
MLIRTransforms
MLIRVector
MLIRVectorTransforms
MLIRX86Vector
MLIRX86VectorTransforms
)

2
mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
View File

@ -14,7 +14,7 @@
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/Support/MathExtras.h"
#include "mlir/Target/LLVMIR/TypeToLLVM.h"

2
mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
View File

@ -21,7 +21,7 @@
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Vector/VectorRewritePatterns.h"
#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
#include "mlir/Dialect/X86Vector/Transforms.h"
#include "mlir/Dialect/X86Vector/X86VectorDialect.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"

2
mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp
View File

@ -22,7 +22,7 @@
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/DialectConversion.h"

1
mlir/lib/Conversion/VectorToSCF/CMakeLists.txt
View File

@ -13,4 +13,5 @@ add_mlir_conversion_library(MLIRVectorToSCF
MLIRMemRef
MLIRTransforms
MLIRVector
MLIRVectorTransforms
)

2
mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
View File

@ -19,7 +19,7 @@
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/SCF/SCF.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/ImplicitLocOpBuilder.h"
#include "mlir/Pass/Pass.h"

2
mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
View File

@ -17,7 +17,7 @@
#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
#include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Transforms/DialectConversion.h"
#include <numeric>

1
mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
View File

@ -34,6 +34,7 @@ add_mlir_dialect_library(MLIRAffineTransforms
MLIRStandard
MLIRTransformUtils
MLIRVector
MLIRVectorUtils
MLIRVectorToLLVM
)

4
mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
View File

@ -18,8 +18,8 @@
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/BlockAndValueMapping.h"
#include "mlir/Support/LLVM.h"
#include "llvm/ADT/STLExtras.h"

2
mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
View File

@ -11,7 +11,7 @@
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/OpImplementation.h"
#include "mlir/IR/TypeUtilities.h"

2
mlir/lib/Dialect/Linalg/ComprehensiveBufferize/VectorInterfaceImpl.cpp
View File

@ -8,7 +8,7 @@
#include "mlir/Dialect/Linalg/ComprehensiveBufferize/VectorInterfaceImpl.h"
#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/Dialect.h"
#include "mlir/IR/Operation.h"

2
mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp
View File

@ -19,7 +19,7 @@
#include "mlir/Dialect/StandardOps/Transforms/Passes.h"
#include "mlir/Dialect/StandardOps/Utils/Utils.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/BuiltinDialect.h"
#include "mlir/IR/Operation.h"
#include "mlir/Pass/Pass.h"

2
mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
View File

@ -59,6 +59,8 @@ add_mlir_dialect_library(MLIRLinalgTransforms
MLIRTransformUtils
MLIRVector
MLIRVectorBufferizableOpInterfaceImpl
MLIRVectorTransforms
MLIRVectorUtils
MLIRX86VectorTransforms
MLIRVectorToSCF
)

4
mlir/lib/Dialect/Linalg/Transforms/CodegenStrategy.cpp
View File

@ -15,8 +15,8 @@
#include "mlir/Dialect/Linalg/Passes.h"
#include "mlir/Dialect/Linalg/Transforms/Hoisting.h"
#include "mlir/Dialect/SCF/Transforms.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "mlir/Transforms/Passes.h"

4
mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
View File

@ -18,8 +18,8 @@
#include "mlir/Dialect/SCF/Utils/Utils.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/AsmState.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/Dominance.h"

10
mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
View File

@ -21,8 +21,8 @@
#include "mlir/Dialect/SCF/Utils/Utils.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/Dominance.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
@ -202,7 +202,7 @@ static bool tensorChunkAccessedByUnknownOp(HoistableWrite write,
continue;
}
auto read = dyn_cast<vector::TransferReadOp>(user);
if (!read || !isDisjointTransferIndices(
if (!read || !vector::isDisjointTransferIndices(
cast<VectorTransferOpInterface>(read.getOperation()),
cast<VectorTransferOpInterface>(
write.transferWriteOp.getOperation()))) {
@ -464,14 +464,14 @@ void mlir::linalg::hoistRedundantVectorTransfers(FuncOp func) {
continue;
if (auto transferWriteUse =
dyn_cast<vector::TransferWriteOp>(use.getOwner())) {
if (!isDisjointTransferSet(
if (!vector::isDisjointTransferSet(
cast<VectorTransferOpInterface>(transferWrite.getOperation()),
cast<VectorTransferOpInterface>(
transferWriteUse.getOperation())))
return WalkResult::advance();
} else if (auto transferReadUse =
dyn_cast<vector::TransferReadOp>(use.getOwner())) {
if (!isDisjointTransferSet(
if (!vector::isDisjointTransferSet(
cast<VectorTransferOpInterface>(transferWrite.getOperation()),
cast<VectorTransferOpInterface>(
transferReadUse.getOperation())))

2
mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
View File

@ -15,7 +15,7 @@
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Dialect/Linalg/Utils/Utils.h"
#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/PatternMatch.h"

2
mlir/lib/Dialect/Linalg/Transforms/LinalgStrategyPasses.cpp
View File

@ -25,7 +25,7 @@
#include "mlir/Dialect/Linalg/Utils/Utils.h"
#include "mlir/Dialect/SCF/Transforms.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/Pass/PassManager.h"

2
mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
View File

@ -21,7 +21,7 @@
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Utils/StaticValueUtils.h"
#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/Matchers.h"
#include "mlir/Pass/Pass.h"

4
mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
View File

@ -19,8 +19,8 @@
#include "mlir/Dialect/Linalg/Utils/Utils.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/PatternMatch.h"

2
mlir/lib/Dialect/Math/Transforms/AlgebraicSimplification.cpp
View File

@ -15,7 +15,7 @@
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/Math/IR/Math.h"
#include "mlir/Dialect/Math/Transforms/Passes.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/TypeUtilities.h"

2
mlir/lib/Dialect/Math/Transforms/CMakeLists.txt
View File

@ -8,6 +8,7 @@ add_mlir_dialect_library(MLIRMathTransforms
LINK_LIBS PUBLIC
MLIRArithmetic
MLIRDialectUtils
MLIRIR
MLIRMath
MLIRPass
@ -15,4 +16,5 @@ add_mlir_dialect_library(MLIRMathTransforms
MLIRTransforms
MLIRX86Vector
MLIRVector
MLIRVectorUtils
)

5
mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
View File

@ -18,8 +18,9 @@
#include "mlir/Dialect/Math/IR/Math.h"
#include "mlir/Dialect/Math/Transforms/Approximation.h"
#include "mlir/Dialect/Math/Transforms/Passes.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Utils/IndexingUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/Dialect/X86Vector/X86VectorDialect.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/ImplicitLocOpBuilder.h"

2
mlir/lib/Dialect/MemRef/Transforms/FoldSubViewOps.cpp
View File

@ -16,7 +16,7 @@
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/MemRef/Transforms/Passes.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"

2
mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
View File

@ -24,7 +24,7 @@
#include "mlir/Dialect/SparseTensor/Transforms/Passes.h"
#include "mlir/Dialect/SparseTensor/Utils/Merger.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/TensorEncoding.h"
#include "llvm/ADT/SmallBitVector.h"

1
mlir/lib/Dialect/Utils/CMakeLists.txt
View File

@ -1,4 +1,5 @@
add_mlir_library(MLIRDialectUtils
IndexingUtils.cpp
ReshapeOpsUtils.cpp
StructuredOpsUtils.cpp
StaticValueUtils.cpp

44
mlir/lib/Dialect/Utils/IndexingUtils.cpp Normal file
View File

@ -0,0 +1,44 @@
//===- IndexingUtils.cpp - Helpers related to index computations ----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/Utils/IndexingUtils.h"
#include "mlir/IR/BuiltinAttributes.h"
int64_t mlir::linearize(ArrayRef<int64_t> offsets, ArrayRef<int64_t> basis) {
assert(offsets.size() == basis.size());
int64_t linearIndex = 0;
for (unsigned idx = 0, e = basis.size(); idx < e; ++idx)
linearIndex += offsets[idx] * basis[idx];
return linearIndex;
}
llvm::SmallVector<int64_t, 4> mlir::delinearize(ArrayRef<int64_t> sliceStrides,
int64_t index) {
int64_t rank = sliceStrides.size();
SmallVector<int64_t, 4> vectorOffsets(rank);
for (int64_t r = 0; r < rank; ++r) {
assert(sliceStrides[r] > 0);
vectorOffsets[r] = index / sliceStrides[r];
index %= sliceStrides[r];
}
return vectorOffsets;
}
llvm::SmallVector<int64_t, 4> mlir::getI64SubArray(ArrayAttr arrayAttr,
unsigned dropFront,
unsigned dropBack) {
assert(arrayAttr.size() > dropFront + dropBack && "Out of bounds");
auto range = arrayAttr.getAsRange<IntegerAttr>();
SmallVector<int64_t, 4> res;
res.reserve(arrayAttr.size() - dropFront - dropBack);
for (auto it = range.begin() + dropFront, eit = range.end() - dropBack;
it != eit; ++it)
res.push_back((*it).getValue().getSExtValue());
return res;
}

37
mlir/lib/Dialect/Vector/CMakeLists.txt
View File

@ -1,34 +1,3 @@
add_mlir_dialect_library(MLIRVector
VectorDropLeadUnitDim.cpp
VectorInsertExtractStridedSliceRewritePatterns.cpp
VectorMultiDimReductionTransforms.cpp
VectorOps.cpp
VectorTransferOpTransforms.cpp
VectorTransferSplitRewritePatterns.cpp
VectorTransferPermutationMapRewritePatterns.cpp
VectorTransforms.cpp
VectorUnrollDistribute.cpp
VectorUtils.cpp
ADDITIONAL_HEADER_DIRS
${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Vector
DEPENDS
MLIRVectorOpsIncGen
MLIRVectorOpsEnumsIncGen
LINK_LIBS PUBLIC
MLIRArithmetic
MLIRDialectUtils
MLIRIR
MLIRStandard
MLIRAffine
MLIRAffineUtils
MLIRLinalg
MLIRMemRef
MLIRSCF
MLIRAffineAnalysis
MLIRDataLayoutInterfaces
MLIRSideEffectInterfaces
MLIRVectorInterfaces
)
add_subdirectory(IR)
add_subdirectory(Transforms)
add_subdirectory(Utils)

22
mlir/lib/Dialect/Vector/IR/CMakeLists.txt Normal file
View File

@ -0,0 +1,22 @@
add_mlir_dialect_library(MLIRVector
VectorOps.cpp
ADDITIONAL_HEADER_DIRS
${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Vector/IR
DEPENDS
MLIRVectorOpsIncGen
MLIRVectorOpsEnumsIncGen
LINK_LIBS PUBLIC
MLIRAffine
MLIRAffineAnalysis
MLIRAffineUtils
MLIRArithmetic
MLIRDialectUtils
MLIRIR
MLIRMemRef
MLIRStandard
MLIRSideEffectInterfaces
MLIRVectorInterfaces
)

87
mlir/lib/Dialect/Vector/VectorOps.cpp → mlir/lib/Dialect/Vector/IR/VectorOps.cpp
View File

@ -11,14 +11,15 @@
//
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/StandardOps/Utils/Utils.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Utils/IndexingUtils.h"
#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/BlockAndValueMapping.h"
@ -35,9 +36,9 @@
#include "llvm/ADT/bit.h"
#include <numeric>
#include "mlir/Dialect/Vector/VectorOpsDialect.cpp.inc"
#include "mlir/Dialect/Vector/IR/VectorOpsDialect.cpp.inc"
// Pull in all enum type and utility function definitions.
#include "mlir/Dialect/Vector/VectorOpsEnums.cpp.inc"
#include "mlir/Dialect/Vector/IR/VectorOpsEnums.cpp.inc"
using namespace mlir;
using namespace mlir::vector;
@ -119,6 +120,80 @@ bool mlir::vector::isLastMemrefDimUnitStride(MemRefType type) {
return succeeded(successStrides) && (strides.empty() || strides.back() == 1);
}
AffineMap mlir::vector::getTransferMinorIdentityMap(ShapedType shapedType,
VectorType vectorType) {
int64_t elementVectorRank = 0;
VectorType elementVectorType =
shapedType.getElementType().dyn_cast<VectorType>();
if (elementVectorType)
elementVectorRank += elementVectorType.getRank();
// 0-d transfers are to/from tensor<t>/memref<t> and vector<1xt>.
// TODO: replace once we have 0-d vectors.
if (shapedType.getRank() == 0 &&
vectorType.getShape() == ArrayRef<int64_t>{1})
return AffineMap::get(
/*numDims=*/0, /*numSymbols=*/0,
getAffineConstantExpr(0, shapedType.getContext()));
return AffineMap::getMinorIdentityMap(
shapedType.getRank(), vectorType.getRank() - elementVectorRank,
shapedType.getContext());
}
bool mlir::vector::checkSameValueRAW(vector::TransferWriteOp defWrite,
vector::TransferReadOp read) {
return !defWrite.hasOutOfBoundsDim() && !defWrite.mask() && !read.mask() &&
defWrite.indices() == read.indices() &&
defWrite.getVectorType() == read.getVectorType() &&
defWrite.permutation_map() == read.permutation_map();
}
bool mlir::vector::checkSameValueWAW(vector::TransferWriteOp write,
vector::TransferWriteOp priorWrite) {
return priorWrite.indices() == write.indices() &&
priorWrite.mask() == write.mask() &&
priorWrite.getVectorType() == write.getVectorType() &&
priorWrite.permutation_map() == write.permutation_map();
}
bool mlir::vector::isDisjointTransferIndices(
VectorTransferOpInterface transferA, VectorTransferOpInterface transferB) {
// For simplicity only look at transfer of same type.
if (transferA.getVectorType() != transferB.getVectorType())
return false;
unsigned rankOffset = transferA.getLeadingShapedRank();
for (unsigned i = 0, e = transferA.indices().size(); i < e; i++) {
auto indexA = transferA.indices()[i].getDefiningOp<arith::ConstantOp>();
auto indexB = transferB.indices()[i].getDefiningOp<arith::ConstantOp>();
// If any of the indices are dynamic we cannot prove anything.
if (!indexA || !indexB)
continue;
if (i < rankOffset) {
// For leading dimensions, if we can prove that index are different we
// know we are accessing disjoint slices.
if (indexA.getValue().cast<IntegerAttr>().getInt() !=
indexB.getValue().cast<IntegerAttr>().getInt())
return true;
} else {
// For this dimension, we slice a part of the memref we need to make sure
// the intervals accessed don't overlap.
int64_t distance =
std::abs(indexA.getValue().cast<IntegerAttr>().getInt() -
indexB.getValue().cast<IntegerAttr>().getInt());
if (distance >= transferA.getVectorType().getDimSize(i - rankOffset))
return true;
}
}
return false;
}
bool mlir::vector::isDisjointTransferSet(VectorTransferOpInterface transferA,
VectorTransferOpInterface transferB) {
if (transferA.source() != transferB.source())
return false;
return isDisjointTransferIndices(transferA, transferB);
}
//===----------------------------------------------------------------------===//
// CombiningKindAttr
//===----------------------------------------------------------------------===//
@ -233,7 +308,7 @@ void VectorDialect::initialize() {
addOperations<
#define GET_OP_LIST
#include "mlir/Dialect/Vector/VectorOps.cpp.inc"
#include "mlir/Dialect/Vector/IR/VectorOps.cpp.inc"
>();
}
@ -4311,4 +4386,4 @@ void mlir::vector::populateVectorToVectorCanonicalizationPatterns(
}
#define GET_OP_CLASSES
#include "mlir/Dialect/Vector/VectorOps.cpp.inc"
#include "mlir/Dialect/Vector/IR/VectorOps.cpp.inc"

28
mlir/lib/Dialect/Vector/Transforms/CMakeLists.txt Normal file
View File

@ -0,0 +1,28 @@
add_mlir_dialect_library(MLIRVectorTransforms
VectorDropLeadUnitDim.cpp
VectorInsertExtractStridedSliceRewritePatterns.cpp
VectorMultiDimReductionTransforms.cpp
VectorTransferOpTransforms.cpp
VectorTransferSplitRewritePatterns.cpp
VectorTransferPermutationMapRewritePatterns.cpp
VectorTransforms.cpp
VectorUnrollDistribute.cpp
ADDITIONAL_HEADER_DIRS
${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Vector/Transforms
LINK_LIBS PUBLIC
MLIRAffine
MLIRAffineAnalysis
MLIRAffineUtils
MLIRArithmetic
MLIRDialectUtils
MLIRIR
MLIRLinalg
MLIRMemRef
MLIRSCF
MLIRStandard
MLIRVector
MLIRVectorInterfaces
MLIRVectorUtils
)

4
mlir/lib/Dialect/Vector/VectorDropLeadUnitDim.cpp → mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
View File

@ -6,8 +6,8 @@
//
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/Vector/VectorRewritePatterns.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/ImplicitLocOpBuilder.h"
#include "mlir/IR/TypeUtilities.h"

7
mlir/lib/Dialect/Vector/VectorInsertExtractStridedSliceRewritePatterns.cpp → mlir/lib/Dialect/Vector/Transforms/VectorInsertExtractStridedSliceRewritePatterns.cpp
View File

@ -9,9 +9,10 @@
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorRewritePatterns.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Utils/IndexingUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/BuiltinTypes.h"
using namespace mlir;

4
mlir/lib/Dialect/Vector/VectorMultiDimReductionTransforms.cpp → mlir/lib/Dialect/Vector/Transforms/VectorMultiDimReductionTransforms.cpp
View File

@ -10,8 +10,8 @@
//
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/Vector/VectorRewritePatterns.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/ImplicitLocOpBuilder.h"
#include "mlir/IR/TypeUtilities.h"

10
mlir/lib/Dialect/Vector/VectorTransferOpTransforms.cpp → mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
View File

@ -12,9 +12,9 @@
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/Dominance.h"
#include "llvm/ADT/STLExtras.h"
@ -113,7 +113,7 @@ void TransferOptimization::deadStoreOp(vector::TransferWriteOp write) {
} else {
if (auto read = dyn_cast<vector::TransferReadOp>(user)) {
// Don't need to consider disjoint reads.
if (isDisjointTransferSet(
if (vector::isDisjointTransferSet(
cast<VectorTransferOpInterface>(write.getOperation()),
cast<VectorTransferOpInterface>(read.getOperation())))
continue;
@ -169,7 +169,7 @@ void TransferOptimization::storeToLoadForwarding(vector::TransferReadOp read) {
if (auto write = dyn_cast<vector::TransferWriteOp>(user)) {
// If there is a write, but we can prove that it is disjoint we can ignore
// the write.
if (isDisjointTransferSet(
if (vector::isDisjointTransferSet(
cast<VectorTransferOpInterface>(write.getOperation()),
cast<VectorTransferOpInterface>(read.getOperation())))
continue;

2
mlir/lib/Dialect/Vector/VectorTransferPermutationMapRewritePatterns.cpp → mlir/lib/Dialect/Vector/Transforms/VectorTransferPermutationMapRewritePatterns.cpp
View File

@ -14,7 +14,7 @@
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
#include "mlir/Interfaces/VectorInterfaces.h"
using namespace mlir;

2
mlir/lib/Dialect/Vector/VectorTransferSplitRewritePatterns.cpp → mlir/lib/Dialect/Vector/Transforms/VectorTransferSplitRewritePatterns.cpp
View File

@ -21,7 +21,7 @@
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Interfaces/VectorInterfaces.h"

2
mlir/lib/Dialect/Vector/VectorTransforms.cpp → mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
View File

@ -20,7 +20,7 @@
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
#include "mlir/IR/ImplicitLocOpBuilder.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/PatternMatch.h"

3
mlir/lib/Dialect/Vector/VectorUnrollDistribute.cpp → mlir/lib/Dialect/Vector/Transforms/VectorUnrollDistribute.cpp
View File

@ -11,7 +11,8 @@
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Dialect/Utils/IndexingUtils.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
#include "mlir/IR/ImplicitLocOpBuilder.h"
#include "mlir/Interfaces/VectorInterfaces.h"
#include "llvm/ADT/MapVector.h"

18
mlir/lib/Dialect/Vector/Utils/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
add_mlir_dialect_library(MLIRVectorUtils
VectorUtils.cpp
ADDITIONAL_HEADER_DIRS
${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Vector/Utils
LINK_LIBS PUBLIC
MLIRAffine
MLIRAffineAnalysis
MLIRArithmetic
MLIRDialectUtils
MLIRIR
MLIRMemRef
MLIRStandard
MLIRTensor
MLIRVector
MLIRVectorInterfaces
)

112
mlir/lib/Dialect/Vector/VectorUtils.cpp → mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
View File

@ -10,14 +10,15 @@
//
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/IntegerSet.h"
#include "mlir/IR/Operation.h"
@ -64,26 +65,6 @@ SmallVector<int64_t, 4> mlir::computeStrides(ArrayRef<int64_t> shape,
return sliceStrides;
}
int64_t mlir::linearize(ArrayRef<int64_t> offsets, ArrayRef<int64_t> basis) {
assert(offsets.size() == basis.size());
int64_t linearIndex = 0;
for (unsigned idx = 0, e = basis.size(); idx < e; ++idx)
linearIndex += offsets[idx] * basis[idx];
return linearIndex;
}
SmallVector<int64_t, 4> mlir::delinearize(ArrayRef<int64_t> sliceStrides,
int64_t index) {
int64_t rank = sliceStrides.size();
SmallVector<int64_t, 4> vectorOffsets(rank);
for (int64_t r = 0; r < rank; ++r) {
assert(sliceStrides[r] > 0);
vectorOffsets[r] = index / sliceStrides[r];
index %= sliceStrides[r];
}
return vectorOffsets;
}
SmallVector<int64_t, 4> mlir::computeElementOffsetsFromVectorSliceOffsets(
ArrayRef<int64_t> sizes, ArrayRef<int64_t> vectorOffsets) {
SmallVector<int64_t, 4> result;
@ -233,25 +214,6 @@ AffineMap mlir::makePermutationMap(
return makePermutationMap(op->getBlock(), indices, loopToVectorDim);
}
AffineMap mlir::getTransferMinorIdentityMap(ShapedType shapedType,
VectorType vectorType) {
int64_t elementVectorRank = 0;
VectorType elementVectorType =
shapedType.getElementType().dyn_cast<VectorType>();
if (elementVectorType)
elementVectorRank += elementVectorType.getRank();
// 0-d transfers are to/from tensor<t>/memref<t> and vector<1xt>.
// TODO: replace once we have 0-d vectors.
if (shapedType.getRank() == 0 &&
vectorType.getShape() == ArrayRef<int64_t>{1})
return AffineMap::get(
/*numDims=*/0, /*numSymbols=*/0,
getAffineConstantExpr(0, shapedType.getContext()));
return AffineMap::getMinorIdentityMap(
shapedType.getRank(), vectorType.getRank() - elementVectorRank,
shapedType.getContext());
}
bool matcher::operatesOnSuperVectorsOf(Operation &op,
VectorType subVectorType) {
// First, extract the vector type and distinguish between:
@ -304,71 +266,3 @@ bool matcher::operatesOnSuperVectorsOf(Operation &op,
// between parallel, reduction and possibly other cases.
return ratio.hasValue();
}
bool mlir::isDisjointTransferIndices(VectorTransferOpInterface transferA,
VectorTransferOpInterface transferB) {
// For simplicity only look at transfer of same type.
if (transferA.getVectorType() != transferB.getVectorType())
return false;
unsigned rankOffset = transferA.getLeadingShapedRank();
for (unsigned i = 0, e = transferA.indices().size(); i < e; i++) {
auto indexA = transferA.indices()[i].getDefiningOp<arith::ConstantOp>();
auto indexB = transferB.indices()[i].getDefiningOp<arith::ConstantOp>();
// If any of the indices are dynamic we cannot prove anything.
if (!indexA || !indexB)
continue;
if (i < rankOffset) {
// For leading dimensions, if we can prove that index are different we
// know we are accessing disjoint slices.
if (indexA.getValue().cast<IntegerAttr>().getInt() !=
indexB.getValue().cast<IntegerAttr>().getInt())
return true;
} else {
// For this dimension, we slice a part of the memref we need to make sure
// the intervals accessed don't overlap.
int64_t distance =
std::abs(indexA.getValue().cast<IntegerAttr>().getInt() -
indexB.getValue().cast<IntegerAttr>().getInt());
if (distance >= transferA.getVectorType().getDimSize(i - rankOffset))
return true;
}
}
return false;
}
bool mlir::isDisjointTransferSet(VectorTransferOpInterface transferA,
VectorTransferOpInterface transferB) {
if (transferA.source() != transferB.source())
return false;
return isDisjointTransferIndices(transferA, transferB);
}
bool mlir::checkSameValueRAW(vector::TransferWriteOp defWrite,
vector::TransferReadOp read) {
return !defWrite.hasOutOfBoundsDim() && !defWrite.mask() && !read.mask() &&
defWrite.indices() == read.indices() &&
defWrite.getVectorType() == read.getVectorType() &&
defWrite.permutation_map() == read.permutation_map();
}
bool mlir::checkSameValueWAW(vector::TransferWriteOp write,
vector::TransferWriteOp priorWrite) {
return priorWrite.indices() == write.indices() &&
priorWrite.mask() == write.mask() &&
priorWrite.getVectorType() == write.getVectorType() &&
priorWrite.permutation_map() == write.permutation_map();
}
SmallVector<int64_t, 4> mlir::getI64SubArray(ArrayAttr arrayAttr,
unsigned dropFront,
unsigned dropBack) {
assert(arrayAttr.size() > dropFront + dropBack && "Out of bounds");
auto range = arrayAttr.getAsRange<IntegerAttr>();
SmallVector<int64_t, 4> res;
res.reserve(arrayAttr.size() - dropFront - dropBack);
for (auto it = range.begin() + dropFront, eit = range.end() - dropBack;
it != eit; ++it)
res.push_back((*it).getValue().getSExtValue());
return res;
}

2
mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp
View File

@ -12,7 +12,7 @@
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/X86Vector/Transforms.h"
#include "mlir/IR/ImplicitLocOpBuilder.h"
#include "mlir/IR/Matchers.h"

2
mlir/python/mlir/dialects/VectorOps.td
View File

@ -10,6 +10,6 @@
#define PYTHON_BINDINGS_VECTOR_OPS
include "mlir/Bindings/Python/Attributes.td"
include "mlir/Dialect/Vector/VectorOps.td"
include "mlir/Dialect/Vector/IR/VectorOps.td"
#endif

5
mlir/test/lib/Dialect/Affine/CMakeLists.txt
View File

@ -18,9 +18,10 @@ add_mlir_library(MLIRAffineTransformsTestPasses
Core
LINK_LIBS PUBLIC
MLIRAffineTransforms
MLIRAffineUtils
MLIRIR
MLIRPass
MLIRAffineTransforms
MLIRSupport
MLIRAffineUtils
MLIRVectorUtils
)

4
mlir/test/lib/Dialect/Affine/TestVectorizationUtils.cpp
View File

@ -16,8 +16,8 @@
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/VectorUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/Diagnostics.h"

2
mlir/test/lib/Dialect/Linalg/TestComprehensiveBufferize.cpp
View File

@ -25,7 +25,7 @@
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/StandardOps/Transforms/BufferizableOpInterfaceImpl.h"
#include "mlir/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Transforms/Passes.h"

2
mlir/test/lib/Dialect/Linalg/TestLinalgCodegenStrategy.cpp
View File

@ -18,7 +18,7 @@
#include "mlir/Dialect/Linalg/Transforms/CodegenStrategy.h"
#include "mlir/Dialect/Linalg/Utils/Utils.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Pass/Pass.h"

2
mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp
View File

@ -20,7 +20,7 @@
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Dialect/Linalg/Utils/Utils.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"

2
mlir/test/lib/Dialect/Math/TestAlgebraicSimplification.cpp
View File

@ -12,7 +12,7 @@
#include "mlir/Dialect/Math/IR/Math.h"
#include "mlir/Dialect/Math/Transforms/Passes.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"

2
mlir/test/lib/Dialect/Math/TestPolynomialApproximation.cpp
View File

@ -14,7 +14,7 @@
#include "mlir/Dialect/Arithmetic/IR/Arithmetic.h"
#include "mlir/Dialect/Math/IR/Math.h"
#include "mlir/Dialect/Math/Transforms/Passes.h"
#include "mlir/Dialect/Vector/VectorOps.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/X86Vector/X86VectorDialect.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"

2
mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
View File

@ -17,7 +17,7 @@
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/SCF/SCF.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/Vector/VectorTransforms.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"

94
utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
View File

@ -2264,6 +2264,7 @@ cc_library(
":Support",
":Transforms",
":VectorOps",
":VectorUtils",
"//llvm:Support",
],
)
@ -2791,26 +2792,20 @@ cc_library(
name = "VectorOps",
srcs = glob(
[
"lib/Dialect/Vector/*.cpp",
"lib/Dialect/Vector/*.h",
"lib/Dialect/Vector/IR/*.cpp",
],
),
hdrs = glob([
"include/mlir/Dialect/Vector/*.h",
"include/mlir/Dialect/Vector/IR/*.h",
]),
includes = ["include"],
deps = [
":Affine",
":AffineAnalysis",
":AffineUtils",
":Analysis",
":ArithmeticDialect",
":DataLayoutInterfaces",
":DialectUtils",
":IR",
":LinalgOps",
":MemRefDialect",
":SCFDialect",
":SideEffectInterfaces",
":StandardOps",
":Support",
@ -2822,6 +2817,64 @@ cc_library(
],
)
cc_library(
name = "VectorTransforms",
srcs = glob(
[
"lib/Dialect/Vector/Transforms/*.cpp",
],
),
hdrs = glob([
"include/mlir/Dialect/Vector/Transforms/*.h",
]),
includes = ["include"],
deps = [
":Affine",
":AffineAnalysis",
":Analysis",
":ArithmeticDialect",
":DialectUtils",
":IR",
":LinalgOps",
":MemRefDialect",
":SCFDialect",
":StandardOps",
":Support",
":TensorDialect",
":VectorInterfaces",
":VectorOps",
":VectorUtils",
"//llvm:Support",
],
)
cc_library(
name = "VectorUtils",
srcs = glob(
[
"lib/Dialect/Vector/Utils/*.cpp",
],
),
hdrs = glob([
"include/mlir/Dialect/Vector/Utils/*.h",
]),
includes = ["include"],
deps = [
":Affine",
":AffineAnalysis",
":ArithmeticDialect",
":DialectUtils",
":IR",
":MemRefDialect",
":StandardOps",
":Support",
":TensorDialect",
":VectorInterfaces",
":VectorOps",
"//llvm:Support",
],
)
cc_library(
name = "Support",
srcs = glob(
@ -6906,6 +6959,8 @@ cc_library(
":VectorBufferizableOpInterfaceImpl",
":VectorOps",
":VectorToSCF",
":VectorTransforms",
":VectorUtils",
":X86VectorTransforms",
"//llvm:Support",
],
@ -6948,7 +7003,7 @@ cc_library(
td_library(
name = "VectorOpsTdFiles",
srcs = ["include/mlir/Dialect/Vector/VectorOps.td"],
srcs = ["include/mlir/Dialect/Vector/IR/VectorOps.td"],
includes = ["include"],
deps = [
":OpBaseTdFiles",
@ -6964,33 +7019,33 @@ gentbl_cc_library(
tbl_outs = [
(
["-gen-op-decls"],
"include/mlir/Dialect/Vector/VectorOps.h.inc",
"include/mlir/Dialect/Vector/IR/VectorOps.h.inc",
),
(
["-gen-op-defs"],
"include/mlir/Dialect/Vector/VectorOps.cpp.inc",
"include/mlir/Dialect/Vector/IR/VectorOps.cpp.inc",
),
(
[
"-gen-dialect-decls",
"-dialect=vector",
],
"include/mlir/Dialect/Vector/VectorOpsDialect.h.inc",
"include/mlir/Dialect/Vector/IR/VectorOpsDialect.h.inc",
),
(
[
"-gen-dialect-defs",
"-dialect=vector",
],
"include/mlir/Dialect/Vector/VectorOpsDialect.cpp.inc",
"include/mlir/Dialect/Vector/IR/VectorOpsDialect.cpp.inc",
),
(
["-gen-enum-decls"],
"include/mlir/Dialect/Vector/VectorOpsEnums.h.inc",
"include/mlir/Dialect/Vector/IR/VectorOpsEnums.h.inc",
),
(
["-gen-enum-defs"],
"include/mlir/Dialect/Vector/VectorOpsEnums.cpp.inc",
"include/mlir/Dialect/Vector/IR/VectorOpsEnums.cpp.inc",
),
(
["-gen-op-doc"],
@ -6998,7 +7053,7 @@ gentbl_cc_library(
),
],
tblgen = ":mlir-tblgen",
td_file = "include/mlir/Dialect/Vector/VectorOps.td",
td_file = "include/mlir/Dialect/Vector/IR/VectorOps.td",
deps = [":VectorOpsTdFiles"],
)
@ -7031,6 +7086,7 @@ cc_library(
":ToLLVMIRTranslation",
":Transforms",
":VectorOps",
":VectorTransforms",
":X86Vector",
":X86VectorTransforms",
"//llvm:Core",
@ -7065,6 +7121,7 @@ cc_library(
":Support",
":Transforms",
":VectorOps",
":VectorUtils",
"//llvm:Core",
"//llvm:Support",
],
@ -7095,6 +7152,7 @@ cc_library(
":Support",
":Transforms",
":VectorOps",
":VectorTransforms",
"//llvm:Core",
"//llvm:Support",
],
@ -7679,6 +7737,7 @@ cc_library(
includes = ["include"],
deps = [
":ArithmeticDialect",
":DialectUtils",
":IR",
":MathDialect",
":Pass",
@ -7687,6 +7746,7 @@ cc_library(
":Support",
":Transforms",
":VectorOps",
":VectorUtils",
":X86Vector",
"//llvm:Support",
],
@ -7705,6 +7765,7 @@ cc_library(
deps = [
":ArithmeticDialect",
":ConversionPassIncGen",
":DialectUtils",
":IR",
":LLVMDialect",
":MathDialect",
@ -7713,6 +7774,7 @@ cc_library(
":Support",
":Transforms",
":VectorOps",
":VectorUtils",
"//llvm:Core",
"//llvm:Support",
],

2
utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
View File

@ -338,6 +338,7 @@ cc_library(
"//mlir:Support",
"//mlir:Transforms",
"//mlir:VectorOps",
"//mlir:VectorUtils",
],
)
@ -526,6 +527,7 @@ cc_library(
"//mlir:TransformUtils",
"//mlir:VectorOps",
"//mlir:VectorToSCF",
"//mlir:VectorTransforms",
"//mlir:X86Vector",
],
)