Define an detail::OperandStorage class to handle managing instruction operands. This class stores operands in a similar way to SmallVector except for two key differences. The first is the inline storage, which is a trailing objects array. The second is that being able to dynamically resize the operand list is optional. This means that we can enable the cases where operations need to change the number of operands after construction without losing the spatial locality benefits of the common case (operation instructions / non-control flow instructions with a lifetime fixed number of operands).

PiperOrigin-RevId: 231910497

mlir/include/mlir/IR/Instructions.h

@@ -42,21 +42,157 @@ template <typename ObjectType, typename ElementType> class ResultIterator;
 template <typename ObjectType, typename ElementType> class ResultTypeIterator;
 class Function;
 
+namespace detail {
+/// A utility class holding the information necessary to dynamically resize
+/// operands.
+struct ResizableStorage {
+  ResizableStorage(InstOperand *opBegin, unsigned numOperands)
+      : firstOpAndIsDynamic(opBegin, false), capacity(numOperands) {}
+
+  ~ResizableStorage() { cleanupStorage(); }
+
+  /// Cleanup any allocated storage.
+  void cleanupStorage() {
+    // If the storage is dynamic, then we need to free the storage.
+    if (isStorageDynamic())
+      free(firstOpAndIsDynamic.getPointer());
+  }
+
+  /// Sets the storage pointer to a new dynamically allocated block.
+  void setDynamicStorage(InstOperand *opBegin) {
+    /// Cleanup the old storage if necessary.
+    cleanupStorage();
+    firstOpAndIsDynamic.setPointerAndInt(opBegin, true);
+  }
+
+  /// Returns the current storage pointer.
+  InstOperand *getPointer() { return firstOpAndIsDynamic.getPointer(); }
+  const InstOperand *getPointer() const {
+    return firstOpAndIsDynamic.getPointer();
+  }
+
+  /// Returns if the current storage of operands is in the trailing objects is
+  /// in a dynamically allocated memory block.
+  bool isStorageDynamic() const { return firstOpAndIsDynamic.getInt(); }
+
+  /// A pointer to the first operand element. This is either to the trailing
+  /// objects storage, or a dynamically allocated block of memory.
+  llvm::PointerIntPair<InstOperand *, 1, bool> firstOpAndIsDynamic;
+
+  // The maximum number of operands that can be currently held by the storage.
+  unsigned capacity;
+};
+
+/// This class handles the management of instruction operands. Operands are
+/// stored similarly to the elements of a SmallVector except for two key
+/// differences. The first is the inline storage, which is a trailing objects
+/// array. The second is that being able to dynamically resize the operand list
+/// is optional.
+class OperandStorage final
+    : private llvm::TrailingObjects<OperandStorage, ResizableStorage,
+                                    InstOperand> {
+public:
+  OperandStorage(unsigned numOperands, bool resizable)
+      : numOperands(numOperands), resizable(resizable) {
+    // Initialize the resizable storage.
+    if (resizable)
+      new (&getResizableStorage())
+          ResizableStorage(getTrailingObjects<InstOperand>(), numOperands);
+  }
+
+  ~OperandStorage() {
+    // Manually destruct the operands.
+    for (auto &operand : getInstOperands())
+      operand.~InstOperand();
+
+    // If the storage is resizable then destruct the utility.
+    if (resizable)
+      getResizableStorage().~ResizableStorage();
+  }
+
+  /// Replace the operands contained in the storage with the ones provided in
+  /// 'operands'.
+  void setOperands(Instruction *owner, ArrayRef<Value *> operands);
+
+  /// Erase an operand held by the storage.
+  void eraseOperand(unsigned index);
+
+  /// Get the instruction operands held by the storage.
+  ArrayRef<InstOperand> getInstOperands() const {
+    return {getRawOperands(), size()};
+  }
+  MutableArrayRef<InstOperand> getInstOperands() {
+    return {getRawOperands(), size()};
+  }
+
+  /// Return the number of operands held in the storage.
+  unsigned size() const { return numOperands; }
+
+  /// Returns the additional size necessary for allocating this object.
+  static size_t additionalAllocSize(unsigned numOperands, bool resizable) {
+    return additionalSizeToAlloc<ResizableStorage, InstOperand>(
+        resizable ? 1 : 0, numOperands);
+  }
+
+  /// Returns if this storage is resizable.
+  bool isResizable() const { return resizable; }
+
+private:
+  /// Clear the storage and destroy the current operands held by the storage.
+  void clear() { numOperands = 0; }
+
+  /// Returns the current pointer for the raw operands array.
+  InstOperand *getRawOperands() {
+    return resizable ? getResizableStorage().getPointer()
+                     : getTrailingObjects<InstOperand>();
+  }
+  const InstOperand *getRawOperands() const {
+    return resizable ? getResizableStorage().getPointer()
+                     : getTrailingObjects<InstOperand>();
+  }
+
+  /// Returns the resizable operand utility class.
+  ResizableStorage &getResizableStorage() {
+    assert(resizable);
+    return *getTrailingObjects<ResizableStorage>();
+  }
+  const ResizableStorage &getResizableStorage() const {
+    assert(resizable);
+    return *getTrailingObjects<ResizableStorage>();
+  }
+
+  /// Grow the internal resizable operand storage.
+  void grow(ResizableStorage &resizeUtil, size_t minSize);
+
+  /// The current number of operands, and the current max operand capacity.
+  unsigned numOperands : 31;
+
+  /// Whether this storage is resizable or not.
+  bool resizable : 1;
+
+  // This stuff is used by the TrailingObjects template.
+  friend llvm::TrailingObjects<OperandStorage, ResizableStorage, InstOperand>;
+  size_t numTrailingObjects(OverloadToken<ResizableStorage>) const {
+    return resizable ? 1 : 0;
+  }
+};
+} // end namespace detail
+
 /// Operations represent all of the arithmetic and other basic computation in
 /// MLIR.
 ///
 class OperationInst final
     : public Instruction,
       private llvm::TrailingObjects<OperationInst, InstResult, BlockOperand,
-                                    unsigned, InstOperand, BlockList> {
+                                    unsigned, BlockList,
+                                    detail::OperandStorage> {
 public:
   /// Create a new OperationInst with the specific fields.
-  static OperationInst *create(Location location, OperationName name,
-                               ArrayRef<Value *> operands,
-                               ArrayRef<Type> resultTypes,
-                               ArrayRef<NamedAttribute> attributes,
-                               ArrayRef<Block *> successors,
-                               unsigned numBlockLists, MLIRContext *context);
+  static OperationInst *
+  create(Location location, OperationName name, ArrayRef<Value *> operands,
+         ArrayRef<Type> resultTypes, ArrayRef<NamedAttribute> attributes,
+         ArrayRef<Block *> successors, unsigned numBlockLists,
+         bool resizableOperandList, MLIRContext *context);
 
   /// Return the context this operation is associated with.
   MLIRContext *getContext() const;
@@ -77,7 +213,13 @@ public:
   // Operands
   //===--------------------------------------------------------------------===//
 
-  unsigned getNumOperands() const { return numOperands; }
+  /// Returns if the operation has a resizable operation list, i.e. operands can
+  /// be added.
+  bool hasResizableOperandsList() const {
+    return getOperandStorage().isResizable();
+  }
+
+  unsigned getNumOperands() const { return getOperandStorage().size(); }
 
   Value *getOperand(unsigned idx) { return getInstOperand(idx).get(); }
   const Value *getOperand(unsigned idx) const {
@@ -119,10 +261,10 @@ public:
   }
 
   ArrayRef<InstOperand> getInstOperands() const {
-    return {getTrailingObjects<InstOperand>(), numOperands};
+    return getOperandStorage().getInstOperands();
   }
   MutableArrayRef<InstOperand> getInstOperands() {
-    return {getTrailingObjects<InstOperand>(), numOperands};
+    return getOperandStorage().getInstOperands();
   }
 
   InstOperand &getInstOperand(unsigned idx) { return getInstOperands()[idx]; }
@@ -302,7 +444,8 @@ public:
   void eraseSuccessorOperand(unsigned succIndex, unsigned opIndex) {
     assert(succIndex < getNumSuccessors());
     assert(opIndex < getNumSuccessorOperands(succIndex));
-    eraseOperand(getSuccessorOperandIndex(succIndex) + opIndex);
+    getOperandStorage().eraseOperand(getSuccessorOperandIndex(succIndex) +
+                                     opIndex);
     --getTrailingObjects<unsigned>()[succIndex];
   }
 
@@ -429,7 +572,6 @@ public:
   }
 
 private:
-  unsigned numOperands;
   const unsigned numResults, numSuccs, numBlockLists;
 
   /// This holds the name of the operation.
@@ -438,21 +580,22 @@ private:
   /// This holds general named attributes for the operation.
   AttributeListStorage *attrs;
 
-  OperationInst(Location location, OperationName name, unsigned numOperands,
-                unsigned numResults, unsigned numSuccessors,
-                unsigned numBlockLists, ArrayRef<NamedAttribute> attributes,
-                MLIRContext *context);
+  OperationInst(Location location, OperationName name, unsigned numResults,
+                unsigned numSuccessors, unsigned numBlockLists,
+                ArrayRef<NamedAttribute> attributes, MLIRContext *context);
   ~OperationInst();
 
-  /// Erase the operand at 'index'.
-  void eraseOperand(unsigned index);
+  /// Returns the operand storage object.
+  detail::OperandStorage &getOperandStorage() {
+    return *getTrailingObjects<detail::OperandStorage>();
+  }
+  const detail::OperandStorage &getOperandStorage() const {
+    return *getTrailingObjects<detail::OperandStorage>();
+  }
 
   // This stuff is used by the TrailingObjects template.
   friend llvm::TrailingObjects<OperationInst, InstResult, BlockOperand,
-                               unsigned, InstOperand, BlockList>;
-  size_t numTrailingObjects(OverloadToken<InstOperand>) const {
-    return numOperands;
-  }
+                               unsigned, BlockList, detail::OperandStorage>;
   size_t numTrailingObjects(OverloadToken<InstResult>) const {
     return numResults;
   }
@@ -555,7 +698,9 @@ inline auto OperationInst::getResultTypes() const
 }
 
 /// For instruction represents an affine loop nest.
-class ForInst : public Instruction {
+class ForInst final
+    : public Instruction,
+      private llvm::TrailingObjects<ForInst, detail::OperandStorage> {
 public:
   static ForInst *create(Location location, ArrayRef<Value *> lbOperands,
                          AffineMap lbMap, ArrayRef<Value *> ubOperands,
@@ -648,7 +793,7 @@ public:
   // Operands
   //===--------------------------------------------------------------------===//
 
-  unsigned getNumOperands() const { return operands.size(); }
+  unsigned getNumOperands() const { return getOperandStorage().size(); }
 
   Value *getOperand(unsigned idx) { return getInstOperand(idx).get(); }
   const Value *getOperand(unsigned idx) const {
@@ -670,8 +815,12 @@ public:
     return const_operand_iterator(this, getNumOperands());
   }
 
-  ArrayRef<InstOperand> getInstOperands() const { return operands; }
-  MutableArrayRef<InstOperand> getInstOperands() { return operands; }
+  ArrayRef<InstOperand> getInstOperands() const {
+    return getOperandStorage().getInstOperands();
+  }
+  MutableArrayRef<InstOperand> getInstOperands() {
+    return getOperandStorage().getInstOperands();
+  }
   InstOperand &getInstOperand(unsigned idx) { return getInstOperands()[idx]; }
   const InstOperand &getInstOperand(unsigned idx) const {
     return getInstOperands()[idx];
@@ -711,6 +860,8 @@ public:
     return const_cast<ForInst *>(this)->getInductionVar();
   }
 
+  void destroy();
+
 private:
   // The Block for the body. By construction, this list always contains exactly
   // one block.
@@ -723,13 +874,21 @@ private:
   // Positive constant step. Since index is stored as an int64_t, we restrict
   // step to the set of positive integers that int64_t can represent.
   int64_t step;
-  // Operands for the lower and upper bounds, with the former followed by the
-  // latter. Dimensional operands are followed by symbolic operands for each
-  // bound.
-  std::vector<InstOperand> operands;
 
-  explicit ForInst(Location location, unsigned numOperands, AffineMap lbMap,
-                   AffineMap ubMap, int64_t step);
+  explicit ForInst(Location location, AffineMap lbMap, AffineMap ubMap,
+                   int64_t step);
+  ~ForInst();
+
+  /// Returns the operand storage object.
+  detail::OperandStorage &getOperandStorage() {
+    return *getTrailingObjects<detail::OperandStorage>();
+  }
+  const detail::OperandStorage &getOperandStorage() const {
+    return *getTrailingObjects<detail::OperandStorage>();
+  }
+
+  // This stuff is used by the TrailingObjects template.
+  friend llvm::TrailingObjects<ForInst, detail::OperandStorage>;
 };
 
 /// Returns if the provided value is the induction variable of a ForInst.

mlir/include/mlir/IR/OperationSupport.h

@@ -234,6 +234,8 @@ struct OperationState {
   /// Successors of this operation and their respective operands.
   SmallVector<Block *, 1> successors;
   unsigned numBlockLists = 0;
+  /// If the operation has a resizable operand list.
+  bool resizableOperandList = false;
 
 public:
   OperationState(MLIRContext *context, Location location, StringRef name)
@@ -245,7 +247,8 @@ public:
   OperationState(MLIRContext *context, Location location, StringRef name,
                  ArrayRef<Value *> operands, ArrayRef<Type> types,
                  ArrayRef<NamedAttribute> attributes,
-                 ArrayRef<Block *> successors = {}, unsigned numBlockLists = 0)
+                 ArrayRef<Block *> successors = {}, unsigned numBlockLists = 0,
+                 bool resizableOperandList = false)
       : context(context), location(location), name(name, context),
         operands(operands.begin(), operands.end()),
         types(types.begin(), types.end()),
@@ -282,6 +285,11 @@ public:
 
   /// Add a new block list with the specified blocks.
   void reserveBlockLists(unsigned numReserved) { numBlockLists += numReserved; }
+
+  /// Sets the operand list of the operation as resizable.
+  void setOperandListToResizable(bool isResizable = true) {
+    resizableOperandList = isResizable;
+  }
 };
 
 } // end namespace mlir

mlir/lib/IR/Builders.cpp

@@ -305,9 +305,10 @@ Block *FuncBuilder::createBlock(Block *insertBefore) {
 
 /// Create an operation given the fields represented as an OperationState.
 OperationInst *FuncBuilder::createOperation(const OperationState &state) {
-  auto *op = OperationInst::create(
-      state.location, state.name, state.operands, state.types, state.attributes,
-      state.successors, state.numBlockLists, context);
+  auto *op = OperationInst::create(state.location, state.name, state.operands,
+                                   state.types, state.attributes,
+                                   state.successors, state.numBlockLists,
+                                   state.resizableOperandList, context);
   block->getInstructions().insert(insertPoint, op);
   return op;
 }

mlir/lib/IR/Instruction.cpp

@@ -54,6 +54,79 @@ template <> unsigned BlockOperand::getOperandNumber() const {
   return this - &getOwner()->getBlockOperands()[0];
 }
 
+//===----------------------------------------------------------------------===//
+// OperandStorage
+//===----------------------------------------------------------------------===//
+
+/// Replace the operands contained in the storage with the ones provided in
+/// 'operands'.
+void detail::OperandStorage::setOperands(Instruction *owner,
+                                         ArrayRef<Value *> operands) {
+  // If the number of operands is less than or equal to the current amount, we
+  // can just update in place.
+  if (operands.size() <= numOperands) {
+    auto instOperands = getInstOperands();
+
+    // If the number of new operands is less than the current count, then remove
+    // any extra operands.
+    for (unsigned i = operands.size(); i != numOperands; ++i)
+      instOperands[i].~InstOperand();
+
+    // Set the operands in place.
+    numOperands = operands.size();
+    for (unsigned i = 0; i != numOperands; ++i)
+      instOperands[i].set(operands[i]);
+    return;
+  }
+
+  // Otherwise, we need to be resizable.
+  assert(resizable && "Only resizable operations may add operands");
+
+  // Grow the capacity if necessary.
+  auto &resizeUtil = getResizableStorage();
+  if (resizeUtil.capacity < operands.size())
+    grow(resizeUtil, operands.size());
+
+  // Set the operands.
+  InstOperand *opBegin = getRawOperands();
+  for (unsigned i = 0; i != numOperands; ++i)
+    opBegin[i].set(operands[i]);
+  for (unsigned e = operands.size(); numOperands != e; ++numOperands)
+    new (&opBegin[numOperands]) InstOperand(owner, operands[numOperands]);
+}
+
+/// Erase an operand held by the storage.
+void detail::OperandStorage::eraseOperand(unsigned index) {
+  assert(index < size());
+  auto Operands = getInstOperands();
+  --numOperands;
+
+  // Shift all operands down by 1 if the operand to remove is not at the end.
+  if (index != numOperands)
+    std::rotate(&Operands[index], &Operands[index + 1], &Operands[numOperands]);
+  Operands[numOperands].~InstOperand();
+}
+
+/// Grow the internal operand storage.
+void detail::OperandStorage::grow(ResizableStorage &resizeUtil,
+                                  size_t minSize) {
+  // Allocate a new storage array.
+  resizeUtil.capacity =
+      std::max(size_t(llvm::NextPowerOf2(resizeUtil.capacity + 2)), minSize);
+  InstOperand *newStorage = static_cast<InstOperand *>(
+      llvm::safe_malloc(resizeUtil.capacity * sizeof(InstOperand)));
+
+  // Move the current operands to the new storage.
+  auto operands = getInstOperands();
+  std::uninitialized_copy(std::make_move_iterator(operands.begin()),
+                          std::make_move_iterator(operands.end()), newStorage);
+
+  // Destroy the original operands and update the resizable storage pointer.
+  for (auto &operand : operands)
+    operand.~InstOperand();
+  resizeUtil.setDynamicStorage(newStorage);
+}
+
 //===----------------------------------------------------------------------===//
 // Instruction
 //===----------------------------------------------------------------------===//
@@ -71,7 +144,7 @@ void Instruction::destroy() {
     cast<OperationInst>(this)->destroy();
     break;
   case Kind::For:
-    delete cast<ForInst>(this);
+    cast<ForInst>(this)->destroy();
     break;
   }
 }
@@ -298,29 +371,32 @@ void Instruction::dropAllReferences() {
 //===----------------------------------------------------------------------===//
 
 /// Create a new OperationInst with the specific fields.
-OperationInst *OperationInst::create(Location location, OperationName name,
-                                     ArrayRef<Value *> operands,
-                                     ArrayRef<Type> resultTypes,
-                                     ArrayRef<NamedAttribute> attributes,
-                                     ArrayRef<Block *> successors,
-                                     unsigned numBlockLists,
-                                     MLIRContext *context) {
+OperationInst *
+OperationInst::create(Location location, OperationName name,
+                      ArrayRef<Value *> operands, ArrayRef<Type> resultTypes,
+                      ArrayRef<NamedAttribute> attributes,
+                      ArrayRef<Block *> successors, unsigned numBlockLists,
+                      bool resizableOperandList, MLIRContext *context) {
   unsigned numSuccessors = successors.size();
 
   // Input operands are nullptr-separated for each successors in the case of
   // terminators, the nullptr aren't actually stored.
   unsigned numOperands = operands.size() - numSuccessors;
 
-  auto byteSize =
-      totalSizeToAlloc<InstResult, BlockOperand, unsigned, InstOperand,
-                       BlockList>(resultTypes.size(), numSuccessors,
-                                  numSuccessors, numOperands, numBlockLists);
+  // Compute the byte size for the instruction and the operand storage.
+  auto byteSize = totalSizeToAlloc<InstResult, BlockOperand, unsigned,
+                                   BlockList, detail::OperandStorage>(
+      resultTypes.size(), numSuccessors, numSuccessors, numBlockLists,
+      /*detail::OperandStorage*/ 1);
+  byteSize += llvm::alignTo(detail::OperandStorage::additionalAllocSize(
+                                numOperands, resizableOperandList),
+                            alignof(OperationInst));
   void *rawMem = malloc(byteSize);
 
   // Initialize the OperationInst part of the instruction.
   auto inst = ::new (rawMem)
-      OperationInst(location, name, numOperands, resultTypes.size(),
-                    numSuccessors, numBlockLists, attributes, context);
+      OperationInst(location, name, resultTypes.size(), numSuccessors,
+                    numBlockLists, attributes, context);
 
   assert((numSuccessors == 0 || inst->isTerminator()) &&
          "unexpected successors in a non-terminator operation");
@@ -330,6 +406,9 @@ OperationInst *OperationInst::create(Location location, OperationName name,
     new (&inst->getBlockList(i)) BlockList(inst);
 
   // Initialize the results and operands.
+  new (&inst->getOperandStorage())
+      detail::OperandStorage(numOperands, resizableOperandList);
+
   auto instResults = inst->getInstResults();
   for (unsigned i = 0, e = resultTypes.size(); i != e; ++i)
     new (&instResults[i]) InstResult(resultTypes[i], inst);
@@ -395,13 +474,12 @@ OperationInst *OperationInst::create(Location location, OperationName name,
 }
 
 OperationInst::OperationInst(Location location, OperationName name,
-                             unsigned numOperands, unsigned numResults,
-                             unsigned numSuccessors, unsigned numBlockLists,
+                             unsigned numResults, unsigned numSuccessors,
+                             unsigned numBlockLists,
                              ArrayRef<NamedAttribute> attributes,
                              MLIRContext *context)
-    : Instruction(Kind::OperationInst, location), numOperands(numOperands),
-      numResults(numResults), numSuccs(numSuccessors),
-      numBlockLists(numBlockLists), name(name) {
+    : Instruction(Kind::OperationInst, location), numResults(numResults),
+      numSuccs(numSuccessors), numBlockLists(numBlockLists), name(name) {
 #ifndef NDEBUG
   for (auto elt : attributes)
     assert(elt.second != nullptr && "Attributes cannot have null entries");
@@ -412,8 +490,7 @@ OperationInst::OperationInst(Location location, OperationName name,
 
 OperationInst::~OperationInst() {
   // Explicitly run the destructors for the operands and results.
-  for (auto &operand : getInstOperands())
-    operand.~InstOperand();
+  getOperandStorage().~OperandStorage();
 
   for (auto &result : getInstResults())
     result.~InstResult();
@@ -468,17 +545,6 @@ void OperationInst::setSuccessor(Block *block, unsigned index) {
   getBlockOperands()[index].set(block);
 }
 
-void OperationInst::eraseOperand(unsigned index) {
-  assert(index < getNumOperands());
-  auto Operands = getInstOperands();
-  --numOperands;
-
-  // Shift all operands down by 1 if the operand to remove is not at the end.
-  if (index != numOperands)
-    std::rotate(&Operands[index], &Operands[index + 1], &Operands[numOperands]);
-  Operands[numOperands].~InstOperand();
-}
-
 auto OperationInst::getNonSuccessorOperands() const
     -> llvm::iterator_range<const_operand_iterator> {
   return {const_operand_iterator(this, 0),
@@ -604,21 +670,33 @@ ForInst *ForInst::create(Location location, ArrayRef<Value *> lbOperands,
              "upper bound operand count does not match the affine map");
   assert(step > 0 && "step has to be a positive integer constant");
 
+  // Compute the byte size for the instruction and the operand storage.
   unsigned numOperands = lbOperands.size() + ubOperands.size();
-  ForInst *inst = new ForInst(location, numOperands, lbMap, ubMap, step);
+  auto byteSize = totalSizeToAlloc<detail::OperandStorage>(
+      /*detail::OperandStorage*/ 1);
+  byteSize += llvm::alignTo(detail::OperandStorage::additionalAllocSize(
+                                numOperands, /*resizable=*/true),
+                            alignof(ForInst));
+  void *rawMem = malloc(byteSize);
 
+  // Initialize the OperationInst part of the instruction.
+  ForInst *inst = ::new (rawMem) ForInst(location, lbMap, ubMap, step);
+  new (&inst->getOperandStorage())
+      detail::OperandStorage(numOperands, /*resizable=*/true);
+
+  auto operands = inst->getInstOperands();
   unsigned i = 0;
   for (unsigned e = lbOperands.size(); i != e; ++i)
-    inst->operands.emplace_back(InstOperand(inst, lbOperands[i]));
+    new (&operands[i]) InstOperand(inst, lbOperands[i]);
 
   for (unsigned j = 0, e = ubOperands.size(); j != e; ++i, ++j)
-    inst->operands.emplace_back(InstOperand(inst, ubOperands[j]));
+    new (&operands[i]) InstOperand(inst, ubOperands[j]);
 
   return inst;
 }
 
-ForInst::ForInst(Location location, unsigned numOperands, AffineMap lbMap,
-                 AffineMap ubMap, int64_t step)
+ForInst::ForInst(Location location, AffineMap lbMap, AffineMap ubMap,
+                 int64_t step)
     : Instruction(Instruction::Kind::For, location), body(this), lbMap(lbMap),
       ubMap(ubMap), step(step) {
 
@@ -628,10 +706,10 @@ ForInst::ForInst(Location location, unsigned numOperands, AffineMap lbMap,
 
   // Add an argument to the block for the induction variable.
   bodyEntry->addArgument(Type::getIndex(lbMap.getResult(0).getContext()));
-
-  operands.reserve(numOperands);
 }
 
+ForInst::~ForInst() { getOperandStorage().~OperandStorage(); }
+
 const AffineBound ForInst::getLowerBound() const {
   return AffineBound(*this, 0, lbMap.getNumInputs(), lbMap);
 }
@@ -644,16 +722,12 @@ void ForInst::setLowerBound(ArrayRef<Value *> lbOperands, AffineMap map) {
   assert(lbOperands.size() == map.getNumInputs());
   assert(map.getNumResults() >= 1 && "bound map has at least one result");
 
-  SmallVector<Value *, 4> ubOperands(getUpperBoundOperands());
+  SmallVector<Value *, 4> newOperands(lbOperands.begin(), lbOperands.end());
+
+  auto ubOperands = getUpperBoundOperands();
+  newOperands.append(ubOperands.begin(), ubOperands.end());
+  getOperandStorage().setOperands(this, newOperands);
 
-  operands.clear();
-  operands.reserve(lbOperands.size() + ubMap.getNumInputs());
-  for (auto *operand : lbOperands) {
-    operands.emplace_back(InstOperand(this, operand));
-  }
-  for (auto *operand : ubOperands) {
-    operands.emplace_back(InstOperand(this, operand));
-  }
   this->lbMap = map;
 }
 
@@ -661,16 +735,10 @@ void ForInst::setUpperBound(ArrayRef<Value *> ubOperands, AffineMap map) {
   assert(ubOperands.size() == map.getNumInputs());
   assert(map.getNumResults() >= 1 && "bound map has at least one result");
 
-  SmallVector<Value *, 4> lbOperands(getLowerBoundOperands());
+  SmallVector<Value *, 4> newOperands(getLowerBoundOperands());
+  newOperands.append(ubOperands.begin(), ubOperands.end());
+  getOperandStorage().setOperands(this, newOperands);
 
-  operands.clear();
-  operands.reserve(lbOperands.size() + ubOperands.size());
-  for (auto *operand : lbOperands) {
-    operands.emplace_back(InstOperand(this, operand));
-  }
-  for (auto *operand : ubOperands) {
-    operands.emplace_back(InstOperand(this, operand));
-  }
   this->ubMap = map;
 }
 
@@ -767,6 +835,11 @@ void ForInst::walkOpsPostOrder(std::function<void(OperationInst *)> callback) {
 /// Returns the induction variable for this loop.
 Value *ForInst::getInductionVar() { return getBody()->getArgument(0); }
 
+void ForInst::destroy() {
+  this->~ForInst();
+  free(this);
+}
+
 /// Returns if the provided value is the induction variable of a ForInst.
 bool mlir::isForInductionVar(const Value *val) {
   return getForInductionVarOwner(val) != nullptr;
@@ -848,9 +921,9 @@ Instruction *Instruction::clone(BlockAndValueMapping &mapper,
       resultTypes.push_back(result->getType());
 
     unsigned numBlockLists = opInst->getNumBlockLists();
-    auto *newOp = OperationInst::create(getLoc(), opInst->getName(), operands,
-                                        resultTypes, opInst->getAttrs(),
-                                        successors, numBlockLists, context);
+    auto *newOp = OperationInst::create(
+        getLoc(), opInst->getName(), operands, resultTypes, opInst->getAttrs(),
+        successors, numBlockLists, opInst->hasResizableOperandsList(), context);
 
     // Clone the block lists.
     for (unsigned i = 0; i != numBlockLists; ++i)

mlir/lib/Parser/Parser.cpp

@@ -2406,7 +2406,8 @@ Value *FunctionParser::createForwardReferencePlaceholder(SMLoc loc, Type type) {
   auto name = OperationName("placeholder", getContext());
   auto *inst = OperationInst::create(
       getEncodedSourceLocation(loc), name, /*operands=*/{}, type,
-      /*attributes=*/{}, /*successors=*/{}, /*numBlockLists=*/0, getContext());
+      /*attributes=*/{}, /*successors=*/{}, /*numBlockLists=*/0,
+      /*resizableOperandList=*/true, getContext());
   forwardReferencePlaceholders[inst->getResult(0)] = loc;
   return inst->getResult(0);
 }
@@ -2787,6 +2788,9 @@ OperationInst *FunctionParser::parseGenericOperation() {
 
   OperationState result(builder.getContext(), srcLocation, name);
 
+  // Generic operations have a resizable operation list.
+  result.setOperandListToResizable();
+
   // Parse the operand list.
   SmallVector<SSAUseInfo, 8> operandInfos;
 

mlir/lib/Transforms/Utils/Utils.cpp

@@ -102,6 +102,7 @@ bool mlir::replaceAllMemRefUsesWith(const Value *oldMemRef, Value *newMemRef,
     // Construct the new operation instruction using this memref.
     OperationState state(opInst->getContext(), opInst->getLoc(),
                          opInst->getName());
+    state.setOperandListToResizable(opInst->hasResizableOperandsList());
     state.operands.reserve(opInst->getNumOperands() + extraIndices.size());
     // Insert the non-memref operands.
     state.operands.append(opInst->operand_begin(),

mlir/lib/Transforms/Vectorize.cpp

@@ -1122,6 +1122,7 @@ static OperationInst *vectorizeOneOperationInst(FuncBuilder *b,
   OperationState newOp(b->getContext(), opInst->getLoc(),
                        opInst->getName().getStringRef(), operands, types,
                        opInst->getAttrs());
+  newOp.setOperandListToResizable(opInst->hasResizableOperandsList());
   return b->createOperation(newOp);
 }