[mlir] Refactor LICM into a utility

LICM is refactored into a utility that is application on any region. The implementation is moved to Transform/Utils.

mlir/include/mlir/IR/Region.h

@@ -341,6 +341,12 @@ public:
   RegionRange(Arg &&arg)
       : RegionRange(ArrayRef<std::unique_ptr<Region>>(std::forward<Arg>(arg))) {
   }
+  template <typename Arg>
+  RegionRange(
+      Arg &&arg,
+      typename std::enable_if_t<
+          std::is_constructible<ArrayRef<Region *>, Arg>::value> * = nullptr)
+      : RegionRange(ArrayRef<Region *>(std::forward<Arg>(arg))) {}
   RegionRange(ArrayRef<std::unique_ptr<Region>> regions);
   RegionRange(ArrayRef<Region *> regions);
 

mlir/include/mlir/Interfaces/LoopLikeInterface.h

@@ -15,20 +15,7 @@
 
 #include "mlir/IR/OpDefinition.h"
 
-//===----------------------------------------------------------------------===//
-// LoopLike Interfaces
-//===----------------------------------------------------------------------===//
-
 /// Include the generated interface declarations.
 #include "mlir/Interfaces/LoopLikeInterface.h.inc"
 
-//===----------------------------------------------------------------------===//
-// LoopLike Utilities
-//===----------------------------------------------------------------------===//
-
-namespace mlir {
-/// Move loop invariant code out of a `looplike` operation.
-void moveLoopInvariantCode(LoopLikeOpInterface looplike);
-} // namespace mlir
-
 #endif // MLIR_INTERFACES_LOOPLIKEINTERFACE_H_

mlir/include/mlir/Transforms/ControlFlowSinkUtils.h

@@ -17,6 +17,7 @@ class DominanceInfo;
 class Operation;
 class Region;
 class RegionBranchOpInterface;
+class RegionRange;
 
 /// Given a list of regions, perform control flow sinking on them. For each
 /// region, control-flow sinking moves operations that dominate the region but
@@ -61,7 +62,7 @@ class RegionBranchOpInterface;
 ///
 /// Returns the number of operations sunk.
 size_t
-controlFlowSink(ArrayRef<Region *> regions, DominanceInfo &domInfo,
+controlFlowSink(RegionRange regions, DominanceInfo &domInfo,
                 function_ref<bool(Operation *, Region *)> shouldMoveIntoRegion,
                 function_ref<void(Operation *, Region *)> moveIntoRegion);
 

mlir/include/mlir/Transforms/LoopInvariantCodeMotionUtils.h

@@ -0,0 +1,75 @@
+//===- LoopInvariantCodeMotionUtils.h - LICM Utils --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TRANSFORMS_LOOPINVARIANTCODEMOTIONUTILS_H
+#define MLIR_TRANSFORMS_LOOPINVARIANTCODEMOTIONUTILS_H
+
+#include "mlir/Support/LLVM.h"
+
+namespace mlir {
+
+class LoopLikeOpInterface;
+class Operation;
+class Region;
+class RegionRange;
+class Value;
+
+/// Given a list of regions, perform loop-invariant code motion. An operation is
+/// loop-invariant if it depends only of values defined outside of the loop.
+/// LICM moves these operations out of the loop body so that they are not
+/// computed more than once.
+///
+/// Example:
+///
+/// ```mlir
+/// affine.for %arg0 = 0 to 10 {
+///   affine.for %arg1 = 0 to 10 {
+///     %v0 = arith.addi %arg0, %arg0 : i32
+///     %v1 = arith.addi %v0, %arg1 : i32
+///   }
+/// }
+/// ```
+///
+/// After LICM:
+///
+/// ```mlir
+/// affine.for %arg0 = 0 to 10 {
+///   %v0 = arith.addi %arg0, %arg0 : i32
+///   affine.for %arg1 = 0 to 10 {
+///     %v1 = arith.addi %v0, %arg1 : i32
+///   }
+/// }
+/// ```
+///
+/// Users must supply three callbacks.
+///
+/// - `isDefinedOutsideRegion` returns true if the given value is invariant with
+///   respect to the given region. A common implementation might be:
+///   `value.getParentRegion()->isProperAncestor(region)`.
+/// - `shouldMoveOutOfRegion` returns true if the provided operation can be
+///   moved of the given region, e.g. if it is side-effect free.
+/// - `moveOutOfRegion` moves the operation out of the given region. A common
+///   implementation might be: `op->moveBefore(region->getParentOp())`.
+///
+/// An operation is moved if all of its operands satisfy
+/// `isDefinedOutsideRegion` and it satisfies `shouldMoveOutOfRegion`.
+///
+/// Returns the number of operations moved.
+size_t moveLoopInvariantCode(
+    RegionRange regions,
+    function_ref<bool(Value, Region *)> isDefinedOutsideRegion,
+    function_ref<bool(Operation *, Region *)> shouldMoveOutOfRegion,
+    function_ref<void(Operation *, Region *)> moveOutOfRegion);
+
+/// Move side-effect free loop invariant code out of a loop-like op using
+/// methods provided by the interface.
+size_t moveLoopInvariantCode(LoopLikeOpInterface loopLike);
+
+} // end namespace mlir
+
+#endif // MLIR_TRANSFORMS_LOOPINVARIANTCODEMOTIONUTILS_H

mlir/include/mlir/Transforms/SideEffectUtils.h

@@ -0,0 +1,30 @@
+//===- SideEffectUtils.h - Side Effect Utils --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TRANSFORMS_SIDEFFECTUTILS_H
+#define MLIR_TRANSFORMS_SIDEFFECTUTILS_H
+
+namespace mlir {
+
+class Operation;
+
+/// Returns true if the given operation is side-effect free.
+///
+/// An operation is side-effect free if its implementation of
+/// `MemoryEffectOpInterface` indicates that it has no memory effects. For
+/// example, it may implement `NoSideEffect` in ODS. Alternatively, if the
+/// operation `HasRecursiveSideEffects`, then it is side-effect free if all of
+/// its nested operations are side-effect free.
+///
+/// If the operation has both, then it is side-effect free if both conditions
+/// are satisfied.
+bool isSideEffectFree(Operation *op);
+
+} // end namespace mlir
+
+#endif // MLIR_TRANSFORMS_SIDEFFECTUTILS_H

mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp

@@ -26,6 +26,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Dominance.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Debug.h"
 

mlir/lib/Dialect/Linalg/Transforms/LinalgStrategyPasses.cpp

@@ -31,6 +31,7 @@
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
 #include "mlir/Transforms/Passes.h"
 
 using namespace mlir;

mlir/lib/Interfaces/CMakeLists.txt

@@ -40,17 +40,4 @@ add_mlir_interface_library(SideEffectInterfaces)
 add_mlir_interface_library(TilingInterface)
 add_mlir_interface_library(VectorInterfaces)
 add_mlir_interface_library(ViewLikeInterface)
-
-add_mlir_library(MLIRLoopLikeInterface
-  LoopLikeInterface.cpp
-
-  ADDITIONAL_HEADER_DIRS
-  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Interfaces
-
-  DEPENDS
-  MLIRLoopLikeInterfaceIncGen
-
-  LINK_LIBS PUBLIC
-  MLIRIR
-  MLIRSideEffectInterfaces
-  )
+add_mlir_interface_library(LoopLikeInterface)

mlir/lib/Interfaces/LoopLikeInterface.cpp

@@ -7,108 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Interfaces/LoopLikeInterface.h"
-#include "mlir/Interfaces/SideEffectInterfaces.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/Support/Debug.h"
-#include <queue>
 
 using namespace mlir;
 
-#define DEBUG_TYPE "loop-like"
-
-//===----------------------------------------------------------------------===//
-// LoopLike Interfaces
-//===----------------------------------------------------------------------===//
-
 /// Include the definitions of the loop-like interfaces.
 #include "mlir/Interfaces/LoopLikeInterface.cpp.inc"
-
-//===----------------------------------------------------------------------===//
-// LoopLike Utilities
-//===----------------------------------------------------------------------===//
-
-/// Returns true if the given operation is side-effect free as are all of its
-/// nested operations.
-///
-/// TODO: There is a duplicate function in ControlFlowSink. Move
-/// `moveLoopInvariantCode` to TransformUtils and then factor out this function.
-static bool isSideEffectFree(Operation *op) {
-  if (auto memInterface = dyn_cast<MemoryEffectOpInterface>(op)) {
-    // If the op has side-effects, it cannot be moved.
-    if (!memInterface.hasNoEffect())
-      return false;
-    // If the op does not have recursive side effects, then it can be moved.
-    if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>())
-      return true;
-  } else if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>()) {
-    // Otherwise, if the op does not implement the memory effect interface and
-    // it does not have recursive side effects, then it cannot be known that the
-    // op is moveable.
-    return false;
-  }
-
-  // Recurse into the regions and ensure that all nested ops can also be moved.
-  for (Region &region : op->getRegions())
-    for (Operation &op : region.getOps())
-      if (!isSideEffectFree(&op))
-        return false;
-  return true;
-}
-
-/// Checks whether the given op can be hoisted by checking that
-/// - the op and none of its contained operations depend on values inside of the
-///   loop (by means of calling definedOutside).
-/// - the op has no side-effects.
-static bool canBeHoisted(Operation *op,
-                         function_ref<bool(Value)> definedOutside) {
-  if (!isSideEffectFree(op))
-    return false;
-
-  // Do not move terminators.
-  if (op->hasTrait<OpTrait::IsTerminator>())
-    return false;
-
-  // Walk the nested operations and check that all used values are either
-  // defined outside of the loop or in a nested region, but not at the level of
-  // the loop body.
-  auto walkFn = [&](Operation *child) {
-    for (Value operand : child->getOperands()) {
-      // Ignore values defined in a nested region.
-      if (op->isAncestor(operand.getParentRegion()->getParentOp()))
-        continue;
-      if (!definedOutside(operand))
-        return WalkResult::interrupt();
-    }
-    return WalkResult::advance();
-  };
-  return !op->walk(walkFn).wasInterrupted();
-}
-
-void mlir::moveLoopInvariantCode(LoopLikeOpInterface looplike) {
-  Region *loopBody = &looplike.getLoopBody();
-
-  std::queue<Operation *> worklist;
-  // Add top-level operations in the loop body to the worklist.
-  for (Operation &op : loopBody->getOps())
-    worklist.push(&op);
-
-  auto definedOutside = [&](Value value) {
-    return looplike.isDefinedOutsideOfLoop(value);
-  };
-
-  while (!worklist.empty()) {
-    Operation *op = worklist.front();
-    worklist.pop();
-    // Skip ops that have already been moved. Check if the op can be hoisted.
-    if (op->getParentRegion() != loopBody || !canBeHoisted(op, definedOutside))
-      continue;
-
-    looplike.moveOutOfLoop(op);
-
-    // Since the op has been moved, we need to check its users within the
-    // top-level of the loop body.
-    for (Operation *user : op->getUsers())
-      if (user->getParentRegion() == loopBody)
-        worklist.push(user);
-  }
-}

mlir/lib/Transforms/ControlFlowSink.cpp

@@ -19,6 +19,7 @@
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Transforms/ControlFlowSinkUtils.h"
 #include "mlir/Transforms/Passes.h"
+#include "mlir/Transforms/SideEffectUtils.h"
 
 using namespace mlir;
 
@@ -29,31 +30,6 @@ struct ControlFlowSink : public ControlFlowSinkBase<ControlFlowSink> {
 };
 } // end anonymous namespace
 
-/// Returns true if the given operation is side-effect free as are all of its
-/// nested operations.
-static bool isSideEffectFree(Operation *op) {
-  if (auto memInterface = dyn_cast<MemoryEffectOpInterface>(op)) {
-    // If the op has side-effects, it cannot be moved.
-    if (!memInterface.hasNoEffect())
-      return false;
-    // If the op does not have recursive side effects, then it can be moved.
-    if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>())
-      return true;
-  } else if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>()) {
-    // Otherwise, if the op does not implement the memory effect interface and
-    // it does not have recursive side effects, then it cannot be known that the
-    // op is moveable.
-    return false;
-  }
-
-  // Recurse into the regions and ensure that all nested ops can also be moved.
-  for (Region &region : op->getRegions())
-    for (Operation &op : region.getOps())
-      if (!isSideEffectFree(&op))
-        return false;
-  return true;
-}
-
 void ControlFlowSink::runOnOperation() {
   auto &domInfo = getAnalysis<DominanceInfo>();
   getOperation()->walk([&](RegionBranchOpInterface branch) {

mlir/lib/Transforms/LoopInvariantCodeMotion.cpp

@@ -11,15 +11,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "PassDetail.h"
-#include "mlir/IR/Builders.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
-#include "mlir/Interfaces/SideEffectInterfaces.h"
+#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
 #include "mlir/Transforms/Passes.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-
-#define DEBUG_TYPE "licm"
+#include "mlir/Transforms/SideEffectUtils.h"
 
 using namespace mlir;
 
@@ -35,10 +30,8 @@ void LoopInvariantCodeMotion::runOnOperation() {
   // Walk through all loops in a function in innermost-loop-first order. This
   // way, we first LICM from the inner loop, and place the ops in
   // the outer loop, which in turn can be further LICM'ed.
-  getOperation()->walk([&](LoopLikeOpInterface loopLike) {
-    LLVM_DEBUG(loopLike.print(llvm::dbgs() << "\nOriginal loop:\n"));
-    moveLoopInvariantCode(loopLike);
-  });
+  getOperation()->walk(
+      [&](LoopLikeOpInterface loopLike) { moveLoopInvariantCode(loopLike); });
 }
 
 std::unique_ptr<Pass> mlir::createLoopInvariantCodeMotionPass() {

mlir/lib/Transforms/Utils/CMakeLists.txt

@@ -4,6 +4,7 @@ add_mlir_library(MLIRTransformUtils
   FoldUtils.cpp
   GreedyPatternRewriteDriver.cpp
   InliningUtils.cpp
+  LoopInvariantCodeMotionUtils.cpp
   RegionUtils.cpp
 
   ADDITIONAL_HEADER_DIRS

mlir/lib/Transforms/Utils/ControlFlowSinkUtils.cpp

@@ -41,7 +41,7 @@ public:
 
   /// Given a list of regions, find operations to sink and sink them. Return the
   /// number of operations sunk.
-  size_t sinkRegions(ArrayRef<Region *> regions);
+  size_t sinkRegions(RegionRange regions);
 
 private:
   /// Given a region and an op which dominates the region, returns true if all
@@ -117,7 +117,7 @@ void Sinker::sinkRegion(Region *region) {
   }
 }
 
-size_t Sinker::sinkRegions(ArrayRef<Region *> regions) {
+size_t Sinker::sinkRegions(RegionRange regions) {
   for (Region *region : regions)
     if (!region->empty())
       sinkRegion(region);
@@ -125,7 +125,7 @@ size_t Sinker::sinkRegions(ArrayRef<Region *> regions) {
 }
 
 size_t mlir::controlFlowSink(
-    ArrayRef<Region *> regions, DominanceInfo &domInfo,
+    RegionRange regions, DominanceInfo &domInfo,
     function_ref<bool(Operation *, Region *)> shouldMoveIntoRegion,
     function_ref<void(Operation *, Region *)> moveIntoRegion) {
   return Sinker(shouldMoveIntoRegion, moveIntoRegion, domInfo)

mlir/lib/Transforms/Utils/LoopInvariantCodeMotionUtils.cpp

@@ -0,0 +1,104 @@
+//===- LoopInvariantCodeMotionUtils.cpp - LICM Utils ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the implementation of the core LICM algorithm.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/Interfaces/LoopLikeInterface.h"
+#include "mlir/Transforms/SideEffectUtils.h"
+#include "llvm/Support/Debug.h"
+#include <queue>
+
+#define DEBUG_TYPE "licm"
+
+using namespace mlir;
+
+/// Checks whether the given op can be hoisted by checking that
+/// - the op and none of its contained operations depend on values inside of the
+///   loop (by means of calling definedOutside).
+/// - the op has no side-effects.
+static bool canBeHoisted(Operation *op,
+                         function_ref<bool(Value)> definedOutside) {
+  // Do not move terminators.
+  if (op->hasTrait<OpTrait::IsTerminator>())
+    return false;
+
+  // Walk the nested operations and check that all used values are either
+  // defined outside of the loop or in a nested region, but not at the level of
+  // the loop body.
+  auto walkFn = [&](Operation *child) {
+    for (Value operand : child->getOperands()) {
+      // Ignore values defined in a nested region.
+      if (op->isAncestor(operand.getParentRegion()->getParentOp()))
+        continue;
+      if (!definedOutside(operand))
+        return WalkResult::interrupt();
+    }
+    return WalkResult::advance();
+  };
+  return !op->walk(walkFn).wasInterrupted();
+}
+
+size_t mlir::moveLoopInvariantCode(
+    RegionRange regions,
+    function_ref<bool(Value, Region *)> isDefinedOutsideRegion,
+    function_ref<bool(Operation *, Region *)> shouldMoveOutOfRegion,
+    function_ref<void(Operation *, Region *)> moveOutOfRegion) {
+  size_t numMoved = 0;
+
+  for (Region *region : regions) {
+    LLVM_DEBUG(llvm::dbgs() << "Original loop:\n" << *region->getParentOp());
+
+    std::queue<Operation *> worklist;
+    // Add top-level operations in the loop body to the worklist.
+    for (Operation &op : region->getOps())
+      worklist.push(&op);
+
+    auto definedOutside = [&](Value value) {
+      return isDefinedOutsideRegion(value, region);
+    };
+
+    while (!worklist.empty()) {
+      Operation *op = worklist.front();
+      worklist.pop();
+      // Skip ops that have already been moved. Check if the op can be hoisted.
+      if (op->getParentRegion() != region)
+        continue;
+
+      LLVM_DEBUG(llvm::dbgs() << "Checking op: " << *op);
+      if (!shouldMoveOutOfRegion(op, region) ||
+          !canBeHoisted(op, definedOutside))
+        continue;
+
+      LLVM_DEBUG(llvm::dbgs() << "Moving loop-invariant op: " << *op);
+      moveOutOfRegion(op, region);
+      ++numMoved;
+
+      // Since the op has been moved, we need to check its users within the
+      // top-level of the loop body.
+      for (Operation *user : op->getUsers())
+        if (user->getParentRegion() == region)
+          worklist.push(user);
+    }
+  }
+
+  return numMoved;
+}
+
+size_t mlir::moveLoopInvariantCode(LoopLikeOpInterface loopLike) {
+  return moveLoopInvariantCode(
+      &loopLike.getLoopBody(),
+      [&](Value value, Region *) {
+        return loopLike.isDefinedOutsideOfLoop(value);
+      },
+      [&](Operation *op, Region *) { return isSideEffectFree(op); },
+      [&](Operation *op, Region *) { loopLike.moveOutOfLoop(op); });
+}

mlir/lib/Transforms/Utils/SideEffectUtils.cpp

@@ -0,0 +1,36 @@
+//===- SideEffectUtils.cpp - Side Effect Utils ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Transforms/SideEffectUtils.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
+
+using namespace mlir;
+
+bool mlir::isSideEffectFree(Operation *op) {
+  if (auto memInterface = dyn_cast<MemoryEffectOpInterface>(op)) {
+    // If the op has side-effects, it cannot be moved.
+    if (!memInterface.hasNoEffect())
+      return false;
+    // If the op does not have recursive side effects, then it can be moved.
+    if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>())
+      return true;
+  } else if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>()) {
+    // Otherwise, if the op does not implement the memory effect interface and
+    // it does not have recursive side effects, then it cannot be known that the
+    // op is moveable.
+    return false;
+  }
+
+  // Recurse into the regions and ensure that all nested ops can also be moved.
+  for (Region &region : op->getRegions())
+    for (Operation &op : region.getOps())
+      if (!isSideEffectFree(&op))
+        return false;
+  return true;
+}

utils/bazel/llvm-project-overlay/mlir/BUILD.bazel

@@ -4906,6 +4906,7 @@ cc_library(
     deps = [
         ":ControlFlowInterfaces",
         ":IR",
+        ":LoopLikeInterface",
         ":Rewrite",
         ":SideEffectInterfaces",
         ":Support",