diff --git a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h index 90ebd94c4207..c37528178e83 100644 --- a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h +++ b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h @@ -77,7 +77,7 @@ public: /// supported LLVM IR type. In particular, if more than one value is /// returned, create an LLVM IR structure type with elements that correspond /// to each of the MLIR types converted with `convertType`. - Type packFunctionResults(ArrayRef types); + Type packFunctionResults(TypeRange types); /// Convert a type in the context of the default or bare pointer calling /// convention. Calling convention sensitive types, such as MemRefType and diff --git a/mlir/include/mlir/IR/Operation.h b/mlir/include/mlir/IR/Operation.h index 63a94d971da7..027c878929f4 100644 --- a/mlir/include/mlir/IR/Operation.h +++ b/mlir/include/mlir/IR/Operation.h @@ -272,10 +272,10 @@ public: //===--------------------------------------------------------------------===// /// Return the number of results held by this operation. - unsigned getNumResults(); + unsigned getNumResults() { return numResults; } /// Get the 'idx'th result of this operation. - OpResult getResult(unsigned idx) { return OpResult(this, idx); } + OpResult getResult(unsigned idx) { return OpResult(getOpResultImpl(idx)); } /// Support result iteration. using result_range = ResultRange; @@ -283,7 +283,10 @@ public: result_iterator result_begin() { return getResults().begin(); } result_iterator result_end() { return getResults().end(); } - result_range getResults() { return result_range(this); } + result_range getResults() { + return numResults == 0 ? result_range(nullptr, 0) + : result_range(getInlineOpResult(0), numResults); + } result_range getOpResults() { return getResults(); } OpResult getOpResult(unsigned idx) { return getResult(idx); } @@ -293,7 +296,7 @@ public: using result_type_range = result_range::type_range; result_type_iterator result_type_begin() { return getResultTypes().begin(); } result_type_iterator result_type_end() { return getResultTypes().end(); } - result_type_range getResultTypes(); + result_type_range getResultTypes() { return getResults().getTypes(); } //===--------------------------------------------------------------------===// // Attributes @@ -620,7 +623,7 @@ private: bool hasValidOrder() { return orderIndex != kInvalidOrderIdx; } private: - Operation(Location location, OperationName name, TypeRange resultTypes, + Operation(Location location, OperationName name, unsigned numResults, unsigned numSuccessors, unsigned numRegions, DictionaryAttr attributes, bool hasOperandStorage); @@ -630,17 +633,17 @@ private: /// Returns the additional size necessary for allocating the given objects /// before an Operation in-memory. - static size_t prefixAllocSize(unsigned numTrailingResults, + static size_t prefixAllocSize(unsigned numOutOfLineResults, unsigned numInlineResults) { - return sizeof(detail::TrailingOpResult) * numTrailingResults + - sizeof(detail::InLineOpResult) * numInlineResults; + return sizeof(detail::OutOfLineOpResult) * numOutOfLineResults + + sizeof(detail::InlineOpResult) * numInlineResults; } /// Returns the additional size allocated before this Operation in-memory. size_t prefixAllocSize() { unsigned numResults = getNumResults(); - unsigned numTrailingResults = OpResult::getNumTrailing(numResults); + unsigned numOutOfLineResults = OpResult::getNumTrailing(numResults); unsigned numInlineResults = OpResult::getNumInline(numResults); - return prefixAllocSize(numTrailingResults, numInlineResults); + return prefixAllocSize(numOutOfLineResults, numInlineResults); } /// Returns the operand storage object. @@ -649,20 +652,29 @@ private: return *getTrailingObjects(); } - /// Returns a pointer to the use list for the given trailing result. - detail::TrailingOpResult *getTrailingResult(unsigned resultNumber) { - // Trailing results are stored in reverse order after(before in memory) the - // inline results. - return reinterpret_cast( - getInlineResult(OpResult::getMaxInlineResults() - 1)) - + /// Returns a pointer to the use list for the given out-of-line result. + detail::OutOfLineOpResult *getOutOfLineOpResult(unsigned resultNumber) { + // Out-of-line results are stored in reverse order after (before in memory) + // the inline results. + return reinterpret_cast(getInlineOpResult( + detail::OpResultImpl::getMaxInlineResults() - 1)) - ++resultNumber; } /// Returns a pointer to the use list for the given inline result. - detail::InLineOpResult *getInlineResult(unsigned resultNumber) { + detail::InlineOpResult *getInlineOpResult(unsigned resultNumber) { // Inline results are stored in reverse order before the operation in // memory. - return reinterpret_cast(this) - ++resultNumber; + return reinterpret_cast(this) - ++resultNumber; + } + + /// Returns a pointer to the use list for the given result, which may be + /// either inline or out-of-line. + detail::OpResultImpl *getOpResultImpl(unsigned resultNumber) { + unsigned maxInlineResults = detail::OpResultImpl::getMaxInlineResults(); + if (resultNumber < maxInlineResults) + return getInlineOpResult(resultNumber); + return getOutOfLineOpResult(resultNumber - maxInlineResults); } /// Provide a 'getParent' method for ilist_node_with_parent methods. @@ -683,24 +695,15 @@ private: /// O(1) local dominance checks between operations. mutable unsigned orderIndex = 0; + const unsigned numResults; const unsigned numSuccs; - const unsigned numRegions : 30; + const unsigned numRegions : 31; /// This bit signals whether this operation has an operand storage or not. The /// operand storage may be elided for operations that are known to never have /// operands. bool hasOperandStorage : 1; - /// This holds the result types of the operation. There are three different - /// states recorded here: - /// - 0 results : The type below is null. - /// - 1 result : The single result type is held here. - /// - N results : The type here is a tuple holding the result types. - /// Note: We steal a bit for 'hasSingleResult' from somewhere else so that we - /// can use 'resultType` in an ArrayRef. - bool hasSingleResult : 1; - Type resultType; - /// This holds the name of the operation. OperationName name; diff --git a/mlir/include/mlir/IR/OperationSupport.h b/mlir/include/mlir/IR/OperationSupport.h index 4e2aef3f06b0..3aa710e5893c 100644 --- a/mlir/include/mlir/IR/OperationSupport.h +++ b/mlir/include/mlir/IR/OperationSupport.h @@ -555,36 +555,6 @@ private: }; } // end namespace detail -//===----------------------------------------------------------------------===// -// ResultStorage -//===----------------------------------------------------------------------===// - -namespace detail { -/// This class provides the implementation for an in-line operation result. This -/// is an operation result whose number can be stored inline inside of the bits -/// of an Operation*. -struct alignas(8) InLineOpResult : public IRObjectWithUseList {}; -/// This class provides the implementation for an out-of-line operation result. -/// This is an operation result whose number cannot be stored inline inside of -/// the bits of an Operation*. -struct alignas(8) TrailingOpResult : public IRObjectWithUseList { - TrailingOpResult(uint64_t trailingResultNumber) - : trailingResultNumber(trailingResultNumber) {} - - /// Returns the parent operation of this trailing result. - Operation *getOwner(); - - /// Return the proper result number of this op result. - unsigned getResultNumber() { - return trailingResultNumber + OpResult::getMaxInlineResults(); - } - - /// The trailing result number, or the offset from the beginning of the - /// trailing array. - uint64_t trailingResultNumber; -}; -} // end namespace detail - //===----------------------------------------------------------------------===// // OpPrintingFlags //===----------------------------------------------------------------------===// @@ -757,54 +727,36 @@ private: /// This class implements the result iterators for the Operation class. class ResultRange final - : public llvm::indexed_accessor_range { + : public llvm::detail::indexed_accessor_range_base< + ResultRange, detail::OpResultImpl *, OpResult, OpResult, OpResult> { public: - using indexed_accessor_range::indexed_accessor_range; - ResultRange(Operation *op); + using RangeBaseT::RangeBaseT; /// Returns the types of the values within this range. - using type_iterator = ArrayRef::iterator; - using type_range = ArrayRef; - type_range getTypes() const; + using type_iterator = ValueTypeIterator; + using type_range = ValueTypeRange; + type_range getTypes() const { return {begin(), end()}; } auto getType() const { return getTypes(); } private: - /// See `llvm::indexed_accessor_range` for details. - static OpResult dereference(Operation *op, ptrdiff_t index); + /// See `llvm::detail::indexed_accessor_range_base` for details. + static detail::OpResultImpl *offset_base(detail::OpResultImpl *object, + ptrdiff_t index) { + return object->getNextResultAtOffset(index); + } + /// See `llvm::detail::indexed_accessor_range_base` for details. + static OpResult dereference_iterator(detail::OpResultImpl *object, + ptrdiff_t index) { + return offset_base(object, index); + } - /// Allow access to `dereference_iterator`. - friend llvm::indexed_accessor_range; + /// Allow access to `offset_base` and `dereference_iterator`. + friend RangeBaseT; }; //===----------------------------------------------------------------------===// // ValueRange -namespace detail { -/// The type representing the owner of a ValueRange. This is either a list of -/// values, operands, or an Operation+start index for results. -struct ValueRangeOwner { - ValueRangeOwner(const Value *owner) : ptr(owner), startIndex(0) {} - ValueRangeOwner(OpOperand *owner) : ptr(owner), startIndex(0) {} - ValueRangeOwner(Operation *owner, unsigned startIndex) - : ptr(owner), startIndex(startIndex) {} - bool operator==(const ValueRangeOwner &rhs) const { return ptr == rhs.ptr; } - - /// The owner pointer of the range. The owner has represents three distinct - /// states: - /// const Value *: The owner is the base to a contiguous array of Value. - /// OpOperand * : The owner is the base to a contiguous array of operands. - /// void* : This owner is an Operation*. It is marked as void* here - /// because the definition of Operation is not visible here. - PointerUnion ptr; - - /// Ths start index into the range. This is only used for Operation* owners. - unsigned startIndex; -}; -} // end namespace detail - /// This class provides an abstraction over the different types of ranges over /// Values. In many cases, this prevents the need to explicitly materialize a /// SmallVector/std::vector. This class should be used in places that are not @@ -812,8 +764,15 @@ struct ValueRangeOwner { /// parameter. class ValueRange final : public llvm::detail::indexed_accessor_range_base< - ValueRange, detail::ValueRangeOwner, Value, Value, Value> { + ValueRange, + PointerUnion, + Value, Value, Value> { public: + /// The type representing the owner of a ValueRange. This is either a list of + /// values, operands, or results. + using OwnerT = + PointerUnion; + using RangeBaseT::RangeBaseT; template , Type, - Type, Type> { +class TypeRange : public llvm::detail::indexed_accessor_range_base< + TypeRange, + llvm::PointerUnion, + Type, Type, Type> { public: using RangeBaseT::RangeBaseT; TypeRange(ArrayRef types = llvm::None); @@ -64,7 +64,9 @@ private: /// * A pointer to the first element of an array of values. /// * A pointer to the first element of an array of types. /// * A pointer to the first element of an array of operands. - using OwnerT = llvm::PointerUnion; + /// * A pointer to the first element of an array of results. + using OwnerT = llvm::PointerUnion; /// See `llvm::detail::indexed_accessor_range_base` for details. static OwnerT offset_base(OwnerT object, ptrdiff_t index); @@ -111,6 +113,18 @@ public: template ValueTypeRange(Container &&c) : ValueTypeRange(c.begin(), c.end()) {} + /// Return the type at the given index. + Type operator[](size_t index) const { + assert(index < size() && "invalid index into type range"); + return *(this->begin() + index); + } + + /// Return the size of this range. + size_t size() const { return llvm::size(*this); } + + /// Return first type in the range. + Type front() { return (*this)[0]; } + /// Compare this range with another. template bool operator==(const OtherT &other) const { diff --git a/mlir/include/mlir/IR/TypeUtilities.h b/mlir/include/mlir/IR/TypeUtilities.h index f5e611124e70..52a4e497e621 100644 --- a/mlir/include/mlir/IR/TypeUtilities.h +++ b/mlir/include/mlir/IR/TypeUtilities.h @@ -57,8 +57,7 @@ LogicalResult verifyCompatibleShape(Type type1, Type type2); /// Returns success if the given two arrays have the same number of elements and /// each pair wise entries have compatible shape. -LogicalResult verifyCompatibleShapes(ArrayRef types1, - ArrayRef types2); +LogicalResult verifyCompatibleShapes(TypeRange types1, TypeRange types2); //===----------------------------------------------------------------------===// // Utility Iterators diff --git a/mlir/include/mlir/IR/Value.h b/mlir/include/mlir/IR/Value.h index 1e98891ff945..2a0bd67c1cb0 100644 --- a/mlir/include/mlir/IR/Value.h +++ b/mlir/include/mlir/IR/Value.h @@ -26,10 +26,52 @@ class OpResult; class Region; class Value; +//===----------------------------------------------------------------------===// +// Value +//===----------------------------------------------------------------------===// + namespace detail { -/// The internal implementation of a BlockArgument. -class BlockArgumentImpl; -} // end namespace detail + +/// The base class for all derived Value classes. It contains all of the +/// components that are shared across Value classes. +class alignas(8) ValueImpl : public IRObjectWithUseList { +public: + /// The enumeration represents the various different kinds of values the + /// internal representation may take. We use all of the bits from Type that we + /// can to store indices inline. + enum class Kind { + /// The first N kinds are all inline operation results. An inline operation + /// result means that the kind represents the result number. This removes + /// the need to store an additional index value. The derived class here is + /// an `OpResultImpl`. + InlineOpResult = 0, + + /// The next kind represents a 'out-of-line' operation result. This is for + /// results with numbers larger than we can represent inline. The derived + /// class here is an `OpResultImpl`. + OutOfLineOpResult = 6, + + /// The last kind represents a block argument. The derived class here is an + /// `BlockArgumentImpl`. + BlockArgument = 7 + }; + + /// Return the type of this value. + Type getType() const { return typeAndKind.getPointer(); } + + /// Set the type of this value. + void setType(Type type) { return typeAndKind.setPointer(type); } + + /// Return the kind of this value. + Kind getKind() const { return typeAndKind.getInt(); } + +protected: + ValueImpl(Type type, Kind kind) : typeAndKind(type, kind) {} + + /// The type of this result and the kind. + llvm::PointerIntPair typeAndKind; +}; +} // namespace detail /// This class represents an instance of an SSA value in the MLIR system, /// representing a computable value that has a type and a set of users. An SSA @@ -39,41 +81,7 @@ class BlockArgumentImpl; /// an Operation(in the case of an OpResult). class Value { public: - /// The enumeration represents the various different kinds of values the - /// internal representation may take. We steal 2 bits to support a total of 4 - /// possible values. - enum class Kind { - /// The first N kinds are all inline operation results. An inline operation - /// result means that the kind represents the result number, and the owner - /// pointer is the owning `Operation*`. Note: These are packed first to make - /// result number lookups more efficient. - OpResult0 = 0, - OpResult1 = 1, - - /// The next kind represents a 'trailing' operation result. This is for - /// results with numbers larger than we can represent inline. The owner here - /// is an `TrailingOpResult*` that points to a trailing storage on the - /// parent operation. - TrailingOpResult = 2, - - /// The last kind represents a block argument. The owner here is a - /// `BlockArgumentImpl*`. - BlockArgument = 3 - }; - - /// This value represents the 'owner' of the value and its kind. See the - /// 'Kind' enumeration above for a more detailed description of each kind of - /// owner. - struct ImplTypeTraits : public llvm::PointerLikeTypeTraits { - // We know that all pointers within the ImplType are aligned by 8-bytes, - // meaning that we can steal up to 3 bits for the different values. - static constexpr int NumLowBitsAvailable = 3; - }; - using ImplType = llvm::PointerIntPair; - -public: - constexpr Value(std::nullptr_t) : ownerAndKind() {} - Value(ImplType ownerAndKind = {}) : ownerAndKind(ownerAndKind) {} + Value(detail::ValueImpl *impl = nullptr) : impl(impl) {} Value(const Value &) = default; Value &operator=(const Value &) = default; @@ -86,26 +94,23 @@ public: bool isa() const { return isa() || isa(); } - template U dyn_cast() const { - return isa() ? U(ownerAndKind) : U(nullptr); + return isa() ? U(impl) : U(nullptr); } template U dyn_cast_or_null() const { - return (*this && isa()) ? U(ownerAndKind) : U(nullptr); + return (*this && isa()) ? U(impl) : U(nullptr); } template U cast() const { assert(isa()); - return U(ownerAndKind); + return U(impl); } - explicit operator bool() const { return ownerAndKind.getPointer(); } - bool operator==(const Value &other) const { - return ownerAndKind == other.ownerAndKind; - } + explicit operator bool() const { return impl; } + bool operator==(const Value &other) const { return impl == other.impl; } bool operator!=(const Value &other) const { return !(*this == other); } /// Return the type of this value. - Type getType() const; + Type getType() const { return impl->getType(); } /// Utility to get the associated MLIRContext that this value is defined in. MLIRContext *getContext() const { return getType().getContext(); } @@ -116,7 +121,7 @@ public: /// completely invalid IR very easily. It is strongly recommended that you /// recreate IR objects with the right types instead of mutating them in /// place. - void setType(Type newType); + void setType(Type newType) { impl->setType(newType); } /// If this value is the result of an operation, return the operation that /// defines it. @@ -144,10 +149,10 @@ public: //===--------------------------------------------------------------------===// /// Provide the use list that is attached to this value. - IRObjectWithUseList *getUseList() const; + IRObjectWithUseList *getUseList() const { return impl; } /// Drop all uses of this object from their respective owners. - void dropAllUses() const; + void dropAllUses() const { return getUseList()->dropAllUses(); } /// Replace all uses of 'this' value with the new value, updating anything in /// the IR that uses 'this' to use the other value instead. When this returns @@ -176,17 +181,17 @@ public: using use_iterator = ValueUseIterator; using use_range = iterator_range; - use_iterator use_begin() const; + use_iterator use_begin() const { return getUseList()->use_begin(); } use_iterator use_end() const { return use_iterator(); } /// Returns a range of all uses, which is useful for iterating over all uses. use_range getUses() const { return {use_begin(), use_end()}; } /// Returns true if this value has exactly one use. - bool hasOneUse() const; + bool hasOneUse() const { return getUseList()->hasOneUse(); } /// Returns true if this value has no uses. - bool use_empty() const; + bool use_empty() const { return getUseList()->use_empty(); } //===--------------------------------------------------------------------===// // Users @@ -201,9 +206,6 @@ public: //===--------------------------------------------------------------------===// // Utilities - /// Returns the kind of this value. - Kind getKind() const { return ownerAndKind.getInt(); } - void print(raw_ostream &os); void print(raw_ostream &os, AsmState &state); void dump(); @@ -212,29 +214,17 @@ public: void printAsOperand(raw_ostream &os, AsmState &state); /// Methods for supporting PointerLikeTypeTraits. - void *getAsOpaquePointer() const { return ownerAndKind.getOpaqueValue(); } + void *getAsOpaquePointer() const { return impl; } static Value getFromOpaquePointer(const void *pointer) { - Value value; - value.ownerAndKind.setFromOpaqueValue(const_cast(pointer)); - return value; + return reinterpret_cast(const_cast(pointer)); } + detail::ValueImpl *getImpl() const { return impl; } friend ::llvm::hash_code hash_value(Value arg); protected: - /// Returns true if the given operation result can be packed inline. - static bool canPackResultInline(unsigned resultNo) { - return resultNo < static_cast(Kind::TrailingOpResult); - } - - /// Construct a value. - Value(detail::BlockArgumentImpl *impl); - Value(Operation *op, unsigned resultNo); - - /// This value represents the 'owner' of the value and its kind. See the - /// 'Kind' enumeration above for a more detailed description of each kind of - /// owner. - ImplType ownerAndKind; + /// A pointer to the internal implementation of the value. + detail::ValueImpl *impl; }; inline raw_ostream &operator<<(raw_ostream &os, Value value) { @@ -248,12 +238,15 @@ inline raw_ostream &operator<<(raw_ostream &os, Value value) { namespace detail { /// The internal implementation of a BlockArgument. -class BlockArgumentImpl : public IRObjectWithUseList { - BlockArgumentImpl(Type type, Block *owner, int64_t index) - : type(type), owner(owner), index(index) {} +class BlockArgumentImpl : public ValueImpl { +public: + static bool classof(const ValueImpl *value) { + return value->getKind() == ValueImpl::Kind::BlockArgument; + } - /// The type of this argument. - Type type; +private: + BlockArgumentImpl(Type type, Block *owner, int64_t index) + : ValueImpl(type, Kind::BlockArgument), owner(owner), index(index) {} /// The owner of this argument. Block *owner; @@ -266,24 +259,18 @@ class BlockArgumentImpl : public IRObjectWithUseList { }; } // end namespace detail -/// Block arguments are values. +/// This class represents an argument of a Block. class BlockArgument : public Value { public: using Value::Value; static bool classof(Value value) { - return value.getKind() == Kind::BlockArgument; + return llvm::isa(value.getImpl()); } /// Returns the block that owns this argument. Block *getOwner() const { return getImpl()->owner; } - /// Return the type of this value. - Type getType() const { return getImpl()->type; } - - /// Set the type of this value. - void setType(Type newType) { getImpl()->type = newType; } - /// Returns the number of this argument. unsigned getArgNumber() const { return getImpl()->index; } @@ -298,8 +285,7 @@ private: /// Get a raw pointer to the internal implementation. detail::BlockArgumentImpl *getImpl() const { - return reinterpret_cast( - ownerAndKind.getPointer()); + return reinterpret_cast(impl); } /// Cache the position in the block argument list. @@ -316,27 +302,104 @@ private: // OpResult //===----------------------------------------------------------------------===// +namespace detail { +/// This class provides the implementation for an operation result. +class alignas(8) OpResultImpl : public ValueImpl { +public: + using ValueImpl::ValueImpl; + + static bool classof(const ValueImpl *value) { + return value->getKind() != ValueImpl::Kind::BlockArgument; + } + + /// Returns the parent operation of this result. + Operation *getOwner() const; + + /// Returns the result number of this op result. + unsigned getResultNumber() const; + + /// Returns the next operation result at `offset` after this result. This + /// method is useful when indexing the result storage of an operation, given + /// that there is more than one kind of operation result (with the different + /// kinds having different sizes) and that operations are stored in reverse + /// order. + OpResultImpl *getNextResultAtOffset(intptr_t offset); + + /// Returns the maximum number of results that can be stored inline. + static unsigned getMaxInlineResults() { + return static_cast(Kind::OutOfLineOpResult); + } +}; + +/// This class provides the implementation for an operation result whose index +/// can be represented "inline" in the underlying ValueImpl. +struct InlineOpResult : public OpResultImpl { +public: + InlineOpResult(Type type, unsigned resultNo) + : OpResultImpl(type, static_cast(resultNo)) { + assert(resultNo < getMaxInlineResults()); + } + + /// Return the result number of this op result. + unsigned getResultNumber() const { return static_cast(getKind()); } + + static bool classof(const OpResultImpl *value) { + return value->getKind() != ValueImpl::Kind::OutOfLineOpResult; + } +}; + +/// This class provides the implementation for an operation result whose index +/// cannot be represented "inline", and thus requires an additional index field. +class OutOfLineOpResult : public OpResultImpl { +public: + OutOfLineOpResult(Type type, uint64_t outOfLineIndex) + : OpResultImpl(type, Kind::OutOfLineOpResult), + outOfLineIndex(outOfLineIndex) {} + + static bool classof(const OpResultImpl *value) { + return value->getKind() == ValueImpl::Kind::OutOfLineOpResult; + } + + /// Return the result number of this op result. + unsigned getResultNumber() const { + return outOfLineIndex + getMaxInlineResults(); + } + + /// The trailing result number, or the offset from the beginning of the + /// `OutOfLineOpResult` array. + uint64_t outOfLineIndex; +}; + +/// Return the result number of this op result. +inline unsigned OpResultImpl::getResultNumber() const { + if (const auto *outOfLineResult = dyn_cast(this)) + return outOfLineResult->getResultNumber(); + return cast(this)->getResultNumber(); +} + +} // end namespace detail + /// This is a value defined by a result of an operation. class OpResult : public Value { public: using Value::Value; static bool classof(Value value) { - return value.getKind() != Kind::BlockArgument; + return llvm::isa(value.getImpl()); } /// Returns the operation that owns this result. - Operation *getOwner() const; + Operation *getOwner() const { return getImpl()->getOwner(); } /// Returns the number of this result. - unsigned getResultNumber() const; - - /// Returns the maximum number of results that can be stored inline. - static unsigned getMaxInlineResults() { - return static_cast(Kind::TrailingOpResult); - } + unsigned getResultNumber() const { return getImpl()->getResultNumber(); } private: + /// Get a raw pointer to the internal implementation. + detail::OpResultImpl *getImpl() const { + return reinterpret_cast(impl); + } + /// Given a number of operation results, returns the number that need to be /// stored inline. static unsigned getNumInline(unsigned numResults); @@ -351,7 +414,7 @@ private: /// Make Value hashable. inline ::llvm::hash_code hash_value(Value arg) { - return ::llvm::hash_value(arg.ownerAndKind.getOpaqueValue()); + return ::llvm::hash_value(arg.getImpl()); } } // namespace mlir @@ -360,11 +423,11 @@ namespace llvm { template <> struct DenseMapInfo { static mlir::Value getEmptyKey() { - auto pointer = llvm::DenseMapInfo::getEmptyKey(); + void *pointer = llvm::DenseMapInfo::getEmptyKey(); return mlir::Value::getFromOpaquePointer(pointer); } static mlir::Value getTombstoneKey() { - auto pointer = llvm::DenseMapInfo::getTombstoneKey(); + void *pointer = llvm::DenseMapInfo::getTombstoneKey(); return mlir::Value::getFromOpaquePointer(pointer); } static unsigned getHashValue(mlir::Value val) { @@ -372,55 +435,41 @@ template <> struct DenseMapInfo { } static bool isEqual(mlir::Value lhs, mlir::Value rhs) { return lhs == rhs; } }; +template <> +struct DenseMapInfo : public DenseMapInfo { + static mlir::BlockArgument getEmptyKey() { + void *pointer = llvm::DenseMapInfo::getEmptyKey(); + return reinterpret_cast(pointer); + } + static mlir::BlockArgument getTombstoneKey() { + void *pointer = llvm::DenseMapInfo::getTombstoneKey(); + return reinterpret_cast(pointer); + } +}; /// Allow stealing the low bits of a value. template <> struct PointerLikeTypeTraits { public: - static inline void *getAsVoidPointer(mlir::Value I) { - return const_cast(I.getAsOpaquePointer()); + static inline void *getAsVoidPointer(mlir::Value value) { + return const_cast(value.getAsOpaquePointer()); } - static inline mlir::Value getFromVoidPointer(void *P) { - return mlir::Value::getFromOpaquePointer(P); + static inline mlir::Value getFromVoidPointer(void *pointer) { + return mlir::Value::getFromOpaquePointer(pointer); } enum { NumLowBitsAvailable = - PointerLikeTypeTraits::NumLowBitsAvailable + PointerLikeTypeTraits::NumLowBitsAvailable }; }; - -template <> struct DenseMapInfo { - static mlir::BlockArgument getEmptyKey() { - auto pointer = llvm::DenseMapInfo::getEmptyKey(); - return mlir::BlockArgument( - mlir::Value::ImplType::getFromOpaqueValue(pointer)); - } - static mlir::BlockArgument getTombstoneKey() { - auto pointer = llvm::DenseMapInfo::getTombstoneKey(); - return mlir::BlockArgument( - mlir::Value::ImplType::getFromOpaqueValue(pointer)); - } - static unsigned getHashValue(mlir::BlockArgument val) { - return mlir::hash_value(val); - } - static bool isEqual(mlir::BlockArgument LHS, mlir::BlockArgument RHS) { - return LHS == RHS; - } -}; - -/// Allow stealing the low bits of a value. -template <> struct PointerLikeTypeTraits { +template <> +struct PointerLikeTypeTraits + : public PointerLikeTypeTraits { public: - static inline void *getAsVoidPointer(mlir::Value I) { - return const_cast(I.getAsOpaquePointer()); + static inline mlir::BlockArgument getFromVoidPointer(void *pointer) { + return reinterpret_cast(pointer); } - static inline mlir::BlockArgument getFromVoidPointer(void *P) { - return mlir::Value::getFromOpaquePointer(P).cast(); - } - enum { - NumLowBitsAvailable = - PointerLikeTypeTraits::NumLowBitsAvailable - }; }; + } // end namespace llvm #endif diff --git a/mlir/include/mlir/Interfaces/InferTypeOpInterface.td b/mlir/include/mlir/Interfaces/InferTypeOpInterface.td index f15d9b3e006a..4cec65a54844 100644 --- a/mlir/include/mlir/Interfaces/InferTypeOpInterface.td +++ b/mlir/include/mlir/Interfaces/InferTypeOpInterface.td @@ -50,8 +50,7 @@ def InferTypeOpInterface : OpInterface<"InferTypeOpInterface"> { " for an op.", /*retTy=*/"bool", /*methodName=*/"isCompatibleReturnTypes", - /*args=*/(ins "::llvm::ArrayRef<::mlir::Type>":$lhs, - "::llvm::ArrayRef<::mlir::Type>":$rhs), + /*args=*/(ins "::mlir::TypeRange":$lhs, "::mlir::TypeRange":$rhs), /*methodBody=*/[{ return ConcreteOp::isCompatibleReturnTypes(lhs, rhs); }], diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h index fa1752d96268..5cc5d8ae0586 100644 --- a/mlir/include/mlir/Transforms/DialectConversion.h +++ b/mlir/include/mlir/Transforms/DialectConversion.h @@ -161,8 +161,7 @@ public: /// Convert the given set of types, filling 'results' as necessary. This /// returns failure if the conversion of any of the types fails, success /// otherwise. - LogicalResult convertTypes(ArrayRef types, - SmallVectorImpl &results); + LogicalResult convertTypes(TypeRange types, SmallVectorImpl &results); /// Return true if the given type is legal for this type converter, i.e. the /// type converts to itself. diff --git a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp index 3e11a5ef1a14..663a922c5039 100644 --- a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp +++ b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp @@ -3866,7 +3866,7 @@ void mlir::populateStdToLLVMConversionPatterns( /// supported LLVM IR type. In particular, if more than one value is returned, /// create an LLVM IR structure type with elements that correspond to each of /// the MLIR types converted with `convertType`. -Type LLVMTypeConverter::packFunctionResults(ArrayRef types) { +Type LLVMTypeConverter::packFunctionResults(TypeRange types) { assert(!types.empty() && "expected non-empty list of type"); if (types.size() == 1) diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp index 83038c421b9a..23b17d287f63 100644 --- a/mlir/lib/Dialect/Async/IR/Async.cpp +++ b/mlir/lib/Dialect/Async/IR/Async.cpp @@ -134,7 +134,7 @@ static void print(OpAsmPrinter &p, ExecuteOp op) { } // -> (!async.value, ...) - p.printOptionalArrowTypeList(op.getResultTypes().drop_front(1)); + p.printOptionalArrowTypeList(llvm::drop_begin(op.getResultTypes())); p.printOptionalAttrDictWithKeyword(op->getAttrs(), {kOperandSegmentSizesAttr}); p.printRegion(op.body(), /*printEntryBlockArgs=*/false); diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp index f9ccda5a2544..c214e83b4f2d 100644 --- a/mlir/lib/IR/Operation.cpp +++ b/mlir/lib/IR/Operation.cpp @@ -105,11 +105,15 @@ Operation *Operation::create(Location location, OperationName name, TypeRange resultTypes, ValueRange operands, DictionaryAttr attributes, BlockRange successors, unsigned numRegions) { + assert(llvm::all_of(resultTypes, [](Type t) { return t; }) && + "unexpected null result type"); + // We only need to allocate additional memory for a subset of results. unsigned numTrailingResults = OpResult::getNumTrailing(resultTypes.size()); unsigned numInlineResults = OpResult::getNumInline(resultTypes.size()); unsigned numSuccessors = successors.size(); unsigned numOperands = operands.size(); + unsigned numResults = resultTypes.size(); // If the operation is known to have no operands, don't allocate an operand // storage. @@ -134,17 +138,20 @@ Operation *Operation::create(Location location, OperationName name, // Create the new Operation. Operation *op = - ::new (rawMem) Operation(location, name, resultTypes, numSuccessors, + ::new (rawMem) Operation(location, name, numResults, numSuccessors, numRegions, attributes, needsOperandStorage); assert((numSuccessors == 0 || op->mightHaveTrait()) && "unexpected successors in a non-terminator operation"); // Initialize the results. - for (unsigned i = 0; i < numInlineResults; ++i) - new (op->getInlineResult(i)) detail::InLineOpResult(); - for (unsigned i = 0; i < numTrailingResults; ++i) - new (op->getTrailingResult(i)) detail::TrailingOpResult(i); + auto resultTypeIt = resultTypes.begin(); + for (unsigned i = 0; i < numInlineResults; ++i, ++resultTypeIt) + new (op->getInlineOpResult(i)) detail::InlineOpResult(*resultTypeIt, i); + for (unsigned i = 0; i < numTrailingResults; ++i, ++resultTypeIt) { + new (op->getOutOfLineOpResult(i)) + detail::OutOfLineOpResult(*resultTypeIt, i); + } // Initialize the regions. for (unsigned i = 0; i != numRegions; ++i) @@ -162,24 +169,13 @@ Operation *Operation::create(Location location, OperationName name, return op; } -Operation::Operation(Location location, OperationName name, - TypeRange resultTypes, unsigned numSuccessors, - unsigned numRegions, DictionaryAttr attributes, - bool hasOperandStorage) - : location(location), numSuccs(numSuccessors), numRegions(numRegions), - hasOperandStorage(hasOperandStorage), hasSingleResult(false), name(name), +Operation::Operation(Location location, OperationName name, unsigned numResults, + unsigned numSuccessors, unsigned numRegions, + DictionaryAttr attributes, bool hasOperandStorage) + : location(location), numResults(numResults), numSuccs(numSuccessors), + numRegions(numRegions), hasOperandStorage(hasOperandStorage), name(name), attrs(attributes) { assert(attributes && "unexpected null attribute dictionary"); - assert(llvm::all_of(resultTypes, [](Type t) { return t; }) && - "unexpected null result type"); - if (!resultTypes.empty()) { - // If there is a single result it is stored in-place, otherwise use a tuple. - hasSingleResult = resultTypes.size() == 1; - if (hasSingleResult) - resultType = resultTypes.front(); - else - resultType = TupleType::get(location->getContext(), resultTypes); - } } // Operations are deleted through the destroy() member because they are @@ -543,21 +539,6 @@ void Operation::dropAllDefinedValueUses() { block.dropAllDefinedValueUses(); } -/// Return the number of results held by this operation. -unsigned Operation::getNumResults() { - if (!resultType) - return 0; - return hasSingleResult ? 1 : resultType.cast().size(); -} - -auto Operation::getResultTypes() -> result_type_range { - if (!resultType) - return llvm::None; - if (hasSingleResult) - return resultType; - return resultType.cast().getTypes(); -} - void Operation::setSuccessor(Block *block, unsigned index) { assert(index < getNumSuccessors()); getBlockOperands()[index].set(block); @@ -925,7 +906,7 @@ LogicalResult OpTrait::impl::verifySameOperandsAndResultType(Operation *op) { auto type = op->getResult(0).getType(); auto elementType = getElementTypeOrSelf(type); - for (auto resultType : op->getResultTypes().drop_front(1)) { + for (auto resultType : llvm::drop_begin(op->getResultTypes())) { if (getElementTypeOrSelf(resultType) != elementType || failed(verifyCompatibleShape(resultType, type))) return op->emitOpError() diff --git a/mlir/lib/IR/OperationSupport.cpp b/mlir/lib/IR/OperationSupport.cpp index df93e1039c3e..b939c5304e75 100644 --- a/mlir/lib/IR/OperationSupport.cpp +++ b/mlir/lib/IR/OperationSupport.cpp @@ -360,23 +360,6 @@ MutableArrayRef detail::OperandStorage::resize(Operation *owner, return newOperands; } -//===----------------------------------------------------------------------===// -// ResultStorage -//===----------------------------------------------------------------------===// - -/// Returns the parent operation of this trailing result. -Operation *detail::TrailingOpResult::getOwner() { - // We need to do some arithmetic to get the operation pointer. Trailing - // results are stored in reverse order before the inline results of the - // operation, so move the trailing owner up to the start of the array. - TrailingOpResult *trailingIt = this + (trailingResultNumber + 1); - - // Move the owner past the inline op results to get to the operation. - auto *inlineResultIt = reinterpret_cast(trailingIt) + - OpResult::getMaxInlineResults(); - return reinterpret_cast(inlineResultIt); -} - //===----------------------------------------------------------------------===// // Operation Value-Iterators //===----------------------------------------------------------------------===// @@ -483,21 +466,6 @@ void MutableOperandRange::updateLength(unsigned newLength) { } } -//===----------------------------------------------------------------------===// -// ResultRange - -ResultRange::ResultRange(Operation *op) - : ResultRange(op, /*startIndex=*/0, op->getNumResults()) {} - -ArrayRef ResultRange::getTypes() const { - return getBase()->getResultTypes().slice(getStartIndex(), size()); -} - -/// See `llvm::indexed_accessor_range` for details. -OpResult ResultRange::dereference(Operation *op, ptrdiff_t index) { - return op->getResult(index); -} - //===----------------------------------------------------------------------===// // ValueRange @@ -506,28 +474,24 @@ ValueRange::ValueRange(ArrayRef values) ValueRange::ValueRange(OperandRange values) : ValueRange(values.begin().getBase(), values.size()) {} ValueRange::ValueRange(ResultRange values) - : ValueRange( - {values.getBase(), static_cast(values.getStartIndex())}, - values.size()) {} + : ValueRange(values.getBase(), values.size()) {} /// See `llvm::detail::indexed_accessor_range_base` for details. ValueRange::OwnerT ValueRange::offset_base(const OwnerT &owner, ptrdiff_t index) { - if (auto *value = owner.ptr.dyn_cast()) + if (const auto *value = owner.dyn_cast()) return {value + index}; - if (auto *operand = owner.ptr.dyn_cast()) + if (auto *operand = owner.dyn_cast()) return {operand + index}; - Operation *operation = reinterpret_cast(owner.ptr.get()); - return {operation, owner.startIndex + static_cast(index)}; + return owner.get()->getNextResultAtOffset(index); } /// See `llvm::detail::indexed_accessor_range_base` for details. Value ValueRange::dereference_iterator(const OwnerT &owner, ptrdiff_t index) { - if (auto *value = owner.ptr.dyn_cast()) + if (const auto *value = owner.dyn_cast()) return value[index]; - if (auto *operand = owner.ptr.dyn_cast()) + if (auto *operand = owner.dyn_cast()) return operand[index].get(); - Operation *operation = reinterpret_cast(owner.ptr.get()); - return operation->getResult(owner.startIndex + index); + return owner.get()->getNextResultAtOffset(index); } //===----------------------------------------------------------------------===// @@ -538,26 +502,9 @@ llvm::hash_code OperationEquivalence::computeHash(Operation *op, Flags flags) { // Hash operations based upon their: // - Operation Name // - Attributes - llvm::hash_code hash = - llvm::hash_combine(op->getName(), op->getAttrDictionary()); - // - Result Types - ArrayRef resultTypes = op->getResultTypes(); - switch (resultTypes.size()) { - case 0: - // We don't need to add anything to the hash. - break; - case 1: - // Add in the result type. - hash = llvm::hash_combine(hash, resultTypes.front()); - break; - default: - // Use the type buffer as the hash, as we can guarantee it is the same for - // any given range of result types. This takes advantage of the fact the - // result types >1 are stored in a TupleType and uniqued. - hash = llvm::hash_combine(hash, resultTypes.data()); - break; - } + llvm::hash_code hash = llvm::hash_combine( + op->getName(), op->getAttrDictionary(), op->getResultTypes()); // - Operands bool ignoreOperands = flags & Flags::IgnoreOperands; @@ -584,26 +531,8 @@ bool OperationEquivalence::isEquivalentTo(Operation *lhs, Operation *rhs, if (lhs->getAttrDictionary() != rhs->getAttrDictionary()) return false; // Compare result types. - ArrayRef lhsResultTypes = lhs->getResultTypes(); - ArrayRef rhsResultTypes = rhs->getResultTypes(); - if (lhsResultTypes.size() != rhsResultTypes.size()) + if (lhs->getResultTypes() != rhs->getResultTypes()) return false; - switch (lhsResultTypes.size()) { - case 0: - break; - case 1: - // Compare the single result type. - if (lhsResultTypes.front() != rhsResultTypes.front()) - return false; - break; - default: - // Use the type buffer for the comparison, as we can guarantee it is the - // same for any given range of result types. This takes advantage of the - // fact the result types >1 are stored in a TupleType and uniqued. - if (lhsResultTypes.data() != rhsResultTypes.data()) - return false; - break; - } // Compare operands. bool ignoreOperands = flags & Flags::IgnoreOperands; if (ignoreOperands) diff --git a/mlir/lib/IR/TypeRange.cpp b/mlir/lib/IR/TypeRange.cpp index dadcb627b093..003cff885156 100644 --- a/mlir/lib/IR/TypeRange.cpp +++ b/mlir/lib/IR/TypeRange.cpp @@ -8,6 +8,7 @@ #include "mlir/IR/TypeRange.h" #include "mlir/IR/Operation.h" + using namespace mlir; //===----------------------------------------------------------------------===// @@ -21,18 +22,19 @@ TypeRange::TypeRange(ArrayRef types) TypeRange::TypeRange(OperandRange values) : TypeRange(values.begin().getBase(), values.size()) {} TypeRange::TypeRange(ResultRange values) - : TypeRange(values.getBase()->getResultTypes().slice(values.getStartIndex(), - values.size())) {} + : TypeRange(values.getBase(), values.size()) {} TypeRange::TypeRange(ArrayRef values) : TypeRange(values.data(), values.size()) {} TypeRange::TypeRange(ValueRange values) : TypeRange(OwnerT(), values.size()) { - detail::ValueRangeOwner owner = values.begin().getBase(); - if (auto *op = reinterpret_cast(owner.ptr.dyn_cast())) - this->base = op->getResultTypes().drop_front(owner.startIndex).data(); - else if (auto *operand = owner.ptr.dyn_cast()) + if (count == 0) + return; + ValueRange::OwnerT owner = values.begin().getBase(); + if (auto *result = owner.dyn_cast()) + this->base = result; + else if (auto *operand = owner.dyn_cast()) this->base = operand; else - this->base = owner.ptr.get(); + this->base = owner.get(); } /// See `llvm::detail::indexed_accessor_range_base` for details. @@ -41,13 +43,18 @@ TypeRange::OwnerT TypeRange::offset_base(OwnerT object, ptrdiff_t index) { return {value + index}; if (auto *operand = object.dyn_cast()) return {operand + index}; + if (auto *result = object.dyn_cast()) + return {result->getNextResultAtOffset(index)}; return {object.dyn_cast() + index}; } + /// See `llvm::detail::indexed_accessor_range_base` for details. Type TypeRange::dereference_iterator(OwnerT object, ptrdiff_t index) { if (const auto *value = object.dyn_cast()) return (value + index)->getType(); if (auto *operand = object.dyn_cast()) return (operand + index)->get().getType(); + if (auto *result = object.dyn_cast()) + return result->getNextResultAtOffset(index)->getType(); return object.dyn_cast()[index]; } diff --git a/mlir/lib/IR/TypeUtilities.cpp b/mlir/lib/IR/TypeUtilities.cpp index 7e96a69a1537..f5391e5ff521 100644 --- a/mlir/lib/IR/TypeUtilities.cpp +++ b/mlir/lib/IR/TypeUtilities.cpp @@ -88,11 +88,10 @@ LogicalResult mlir::verifyCompatibleShape(Type type1, Type type2) { /// Returns success if the given two arrays have the same number of elements and /// each pair wise entries have compatible shape. -LogicalResult mlir::verifyCompatibleShapes(ArrayRef types1, - ArrayRef types2) { +LogicalResult mlir::verifyCompatibleShapes(TypeRange types1, TypeRange types2) { if (types1.size() != types2.size()) return failure(); - for (auto it : zip_first(types1, types2)) + for (auto it : llvm::zip_first(types1, types2)) if (failed(verifyCompatibleShape(std::get<0>(it), std::get<1>(it)))) return failure(); return success(); diff --git a/mlir/lib/IR/Value.cpp b/mlir/lib/IR/Value.cpp index fd7e5b5d64e5..e28ab9ba470d 100644 --- a/mlir/lib/IR/Value.cpp +++ b/mlir/lib/IR/Value.cpp @@ -15,57 +15,6 @@ using namespace mlir; using namespace mlir::detail; -/// Construct a value. -Value::Value(BlockArgumentImpl *impl) - : ownerAndKind(impl, Kind::BlockArgument) {} -Value::Value(Operation *op, unsigned resultNo) { - assert(op->getNumResults() > resultNo && "invalid result number"); - if (LLVM_LIKELY(canPackResultInline(resultNo))) { - ownerAndKind = {op, static_cast(resultNo)}; - return; - } - - // If we can't pack the result directly, grab the use list from the parent op. - unsigned trailingNo = resultNo - OpResult::getMaxInlineResults(); - ownerAndKind = {op->getTrailingResult(trailingNo), Kind::TrailingOpResult}; -} - -/// Return the type of this value. -Type Value::getType() const { - if (BlockArgument arg = dyn_cast()) - return arg.getType(); - - // If this is an operation result, query the parent operation. - OpResult result = cast(); - Operation *owner = result.getOwner(); - if (owner->hasSingleResult) - return owner->resultType; - return owner->resultType.cast().getType(result.getResultNumber()); -} - -/// Mutate the type of this Value to be of the specified type. -void Value::setType(Type newType) { - if (BlockArgument arg = dyn_cast()) - return arg.setType(newType); - OpResult result = cast(); - - // If the owner has a single result, simply update it directly. - Operation *owner = result.getOwner(); - if (owner->hasSingleResult) { - owner->resultType = newType; - return; - } - unsigned resultNo = result.getResultNumber(); - - // Otherwise, rebuild the tuple if the new type is different from the current. - auto curTypes = owner->resultType.cast().getTypes(); - if (curTypes[resultNo] == newType) - return; - auto newTypes = llvm::to_vector<4>(curTypes); - newTypes[resultNo] = newType; - owner->resultType = TupleType::get(newType.getContext(), newTypes); -} - /// If this value is the result of an Operation, return the operation that /// defines it. Operation *Value::getDefiningOp() const { @@ -102,22 +51,6 @@ Block *Value::getParentBlock() { // Value::UseLists //===----------------------------------------------------------------------===// -/// Provide the use list that is attached to this value. -IRObjectWithUseList *Value::getUseList() const { - if (BlockArgument arg = dyn_cast()) - return arg.getImpl(); - if (getKind() != Kind::TrailingOpResult) { - OpResult result = cast(); - return result.getOwner()->getInlineResult(result.getResultNumber()); - } - - // Otherwise this is a trailing operation result, which contains a use list. - return reinterpret_cast(ownerAndKind.getPointer()); -} - -/// Drop all uses of this object from their respective owners. -void Value::dropAllUses() const { return getUseList()->dropAllUses(); } - /// Replace all uses of 'this' value with the new value, updating anything in /// the IR that uses 'this' to use the other value instead. When this returns /// there are zero uses of 'this'. @@ -152,55 +85,100 @@ bool Value::isUsedOutsideOfBlock(Block *block) { }); } -//===--------------------------------------------------------------------===// -// Uses - -auto Value::use_begin() const -> use_iterator { - return getUseList()->use_begin(); -} - -/// Returns true if this value has exactly one use. -bool Value::hasOneUse() const { return getUseList()->hasOneUse(); } - -/// Returns true if this value has no uses. -bool Value::use_empty() const { return getUseList()->use_empty(); } - //===----------------------------------------------------------------------===// // OpResult //===----------------------------------------------------------------------===// -/// Returns the operation that owns this result. -Operation *OpResult::getOwner() const { - // If the result is in-place, the `owner` is the operation. - void *owner = ownerAndKind.getPointer(); - if (LLVM_LIKELY(getKind() != Kind::TrailingOpResult)) - return static_cast(owner); +/// Returns the parent operation of this trailing result. +Operation *OpResultImpl::getOwner() const { + // We need to do some arithmetic to get the operation pointer. Results are + // stored in reverse order before the operation, so move the trailing owner up + // to the start of the array. A rough diagram of the memory layout is: + // + // | Out-of-Line results | Inline results | Operation | + // + // Given that the results are reverse order we use the result number to know + // how far to jump to get to the operation. So if we are currently the 0th + // result, the layout would be: + // + // | Inline result 0 | Operation + // + // ^-- To get the base address of the operation, we add the result count + 1. + if (const auto *result = dyn_cast(this)) { + result += result->getResultNumber() + 1; + return reinterpret_cast(const_cast(result)); + } - // Otherwise, query the trailing result for the owner. - return static_cast(owner)->getOwner(); + // Out-of-line results are stored in an array just before the inline results. + const OutOfLineOpResult *outOfLineIt = (const OutOfLineOpResult *)(this); + outOfLineIt += (outOfLineIt->outOfLineIndex + 1); + + // Move the owner past the inline results to get to the operation. + const auto *inlineIt = reinterpret_cast(outOfLineIt); + inlineIt += getMaxInlineResults(); + return reinterpret_cast(const_cast(inlineIt)); } -/// Return the result number of this result. -unsigned OpResult::getResultNumber() const { - // If the result is in-place, we can use the kind directly. - if (LLVM_LIKELY(getKind() != Kind::TrailingOpResult)) - return static_cast(ownerAndKind.getInt()); - // Otherwise, query the trailing result. - auto *result = static_cast(ownerAndKind.getPointer()); - return result->getResultNumber(); +OpResultImpl *OpResultImpl::getNextResultAtOffset(intptr_t offset) { + if (offset == 0) + return this; + // We need to do some arithmetic to get the next result given that results are + // in reverse order, and that we need to account for the different types of + // results. As a reminder, the rough diagram of the memory layout is: + // + // | Out-of-Line results | Inline results | Operation | + // + // So an example operation with two results would look something like: + // + // | Inline result 1 | Inline result 0 | Operation | + // + + // Handle the case where this result is an inline result. + OpResultImpl *result = this; + if (auto *inlineResult = dyn_cast(this)) { + // Check to see how many results there are after this one before the start + // of the out-of-line results. If the desired offset is less than the number + // remaining, we can directly use the offset from the current result + // pointer. The following diagrams highlight the two situations. + // + // | Out-of-Line results | Inline results | Operation | + // ^- Say we are here. + // ^- If our destination is here, we can use the + // offset directly. + // + intptr_t leftBeforeTrailing = + getMaxInlineResults() - inlineResult->getResultNumber() - 1; + if (leftBeforeTrailing >= offset) + return inlineResult - offset; + + // Otherwise, adjust the current result pointer to the end (start in memory) + // of the inline result array. + // + // | Out-of-Line results | Inline results | Operation | + // ^- Say we are here. + // ^- If our destination is here, we need to first jump to + // the end (start in memory) of the inline result array. + // + result = inlineResult - leftBeforeTrailing; + offset -= leftBeforeTrailing; + } + + // If we land here, the current result is an out-of-line result and we can + // offset directly. + return reinterpret_cast(result) - offset; } /// Given a number of operation results, returns the number that need to be /// stored inline. unsigned OpResult::getNumInline(unsigned numResults) { - return std::min(numResults, getMaxInlineResults()); + return std::min(numResults, OpResultImpl::getMaxInlineResults()); } /// Given a number of operation results, returns the number that need to be /// stored as trailing. unsigned OpResult::getNumTrailing(unsigned numResults) { // If we can pack all of the results, there is no need for additional storage. - unsigned maxInline = getMaxInlineResults(); + unsigned maxInline = OpResultImpl::getMaxInlineResults(); return numResults <= maxInline ? 0 : numResults - maxInline; } diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp index ae5b566f32d1..5c99c5812b54 100644 --- a/mlir/lib/Transforms/Utils/DialectConversion.cpp +++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp @@ -2499,9 +2499,9 @@ Type TypeConverter::convertType(Type t) { /// Convert the given set of types, filling 'results' as necessary. This /// returns failure if the conversion of any of the types fails, success /// otherwise. -LogicalResult TypeConverter::convertTypes(ArrayRef types, +LogicalResult TypeConverter::convertTypes(TypeRange types, SmallVectorImpl &results) { - for (auto type : types) + for (Type type : types) if (failed(convertType(type, results))) return failure(); return success();