From 99ac47e96c10922ca40e1267467bcfcbb51a51a0 Mon Sep 17 00:00:00 2001 From: Markus Pilman Date: Fri, 19 Mar 2021 18:08:09 -0600 Subject: [PATCH] documentation --- flow/WriteOnlySet.actor.cpp | 6 ++++ flow/WriteOnlySet.h | 65 +++++++++++++++++++++++++++++++++---- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/flow/WriteOnlySet.actor.cpp b/flow/WriteOnlySet.actor.cpp index 32023f5e24..93d9e99fc7 100644 --- a/flow/WriteOnlySet.actor.cpp +++ b/flow/WriteOnlySet.actor.cpp @@ -109,12 +109,14 @@ template class WriteOnlySet; // testing code namespace { +// Some statistics std::atomic instanceCounter = 0; std::atomic numInserts = 0; std::atomic numErase = 0; std::atomic numLockedErase = 0; std::atomic numCopied = 0; +// A simple object that counts the number of its instances. This is used to detect memory leaks. struct TestObject { mutable std::atomic _refCount = 1; TestObject() { instanceCounter.fetch_add(1); } @@ -130,6 +132,7 @@ struct TestObject { using TestSet = WriteOnlySet; using Clock = std::chrono::steady_clock; +// An actor that can join a set of threads in an async way. ACTOR Future threadjoiner(std::shared_ptr> threads, std::shared_ptr set) { loop { wait(delay(0.1)); @@ -156,6 +159,7 @@ ACTOR Future threadjoiner(std::shared_ptr> thread } } +// occasionally copy the contents of the past set. void testCopier(std::shared_ptr set, std::chrono::seconds runFor) { auto start = Clock::now(); while (true) { @@ -168,6 +172,7 @@ void testCopier(std::shared_ptr set, std::chrono::seconds runFor) { } } +// In a loop adds and removes a set of objects to the set void writer(std::shared_ptr set, std::chrono::seconds runFor) { auto start = Clock::now(); std::random_device rDev; @@ -203,6 +208,7 @@ void writer(std::shared_ptr set, std::chrono::seconds runFor) { } } +// This unit test creates 5 writer threads and one copier thread. TEST_CASE("/flow/WriteOnlySet") { if (g_network->isSimulated()) { // This test is not deterministic, so we shouldn't run it in simulation diff --git a/flow/WriteOnlySet.h b/flow/WriteOnlySet.h index 9d80795c68..a2589ec387 100644 --- a/flow/WriteOnlySet.h +++ b/flow/WriteOnlySet.h @@ -24,6 +24,21 @@ #include "flow/Trace.h" #include +/** + * This is a Write-Only set that supports copying the whole content. This data structure is lock-free and allows a user + * to insert and remove objects up to a given capacity (passed by a template). + * + * Template parameters: + * \param T The type to store. + * \param IndexType The type used as an index + * \param CAPACITY The maximum number of object this structure can store (if a user tries to store more, insert will + * fail gracefully) + * \pre T implements `void addref() const` and `void delref() const` + * \pre IndexType must have a copy constructor + * \pre IndexType must have a trivial assignment operator + * \pre IndexType must have a trivial destructor + * \pre IndexType can be used as an index into a std::vector + */ template class WriteOnlySet { public: @@ -37,25 +52,61 @@ public: WriteOnlySet(const WriteOnlySet&) = delete; WriteOnlySet& operator=(const WriteOnlySet&) = delete; - // Returns the number of elements at the time of calling. Keep in mind that this is a lockfree data structure, so - // the actual size might change anytime after or even during the call. This function only guarantees that the size - // was whatever the method returns at one point between the start and the end of the function call. The safest way - // to handle this is by assuming that this returns an estimate. - unsigned size(); + /** + * Attempts to insert \p lineage into the set. This method can fail if the set is full (its size is equal to its + * capacity). Calling insert on a full set is safe but the method will return \ref npos if the operation fails. + * + * \param lineage A reference to the object the user wants to insert. + * \ret An index that can later be used to erase the value again or \ref npos if the insert failed. + * \pre lineage.getPtr() % 2 == 0 (the memory for lineage has to be at least 2 byte aligned) + */ + [[nodiscard]] Index insert(const Reference& lineage); - Index insert(const Reference& lineage); + /** + * Erases the object associated with \p idx from the set. + * + * \ret Whether the reference count was decremented. Usually the return value is only interesting for testing and + * benchmarking purposes and will in most cases be ignored. If \ref delref wasn't called, it will be called + * later. Note that at the time the return value is checked, \ref delref might already have been called. + */ bool erase(Index idx); + /** + * Copies all elements that are stored in the set into a vector. This copy operation does NOT provide a snapshot of + * the data structure. The contract is weak: + * - All object that were in the set before copy is called and weren't removed until after copy returned are + * guaranteed to be in the result. + * - Any object that was inserted while copy is running might be in the result. + * - Any object that was erased while copy is running might be in the result. + */ std::vector> copy(); private: + // the implementation of erase -- the wrapper just makes the function a bit more readable. bool eraseImpl(Index idx); + // the last bit of a pointer within the set is used like a boolean and true means that the object is locked. Locking + // an object is only relevant for memory management. A locked pointer can still be erased from the set, but the + // erase won't call delref on the object. Instead it will push the pointer into the \ref freeList and copy will call + // delref later. static constexpr uintptr_t LOCK = 0b1; - std::atomic _size = 0; + + // The actual memory std::vector> _set; static_assert(std::atomic::is_always_lock_free, "Index type can't be used as a lock-free type"); static_assert(std::atomic::is_always_lock_free, "uintptr_t can't be used as a lock-free type"); + + // The freeQueue. On creation all indexes (0..capacity-1) are pushed into this queue. On insert one element from + // this queue is consumed and the resulting number is used as an index into the set. On erase the index is given + // back to the freeQueue. boost::lockfree::queue, boost::lockfree::capacity> freeQueue; + + // The freeList is used for memory management. Generally copying a shared pointer can't be done in a lock-free way. + // Instead, when we copy the data structure we first copy the address, then attempt to set the last bit to 1 and + // only if that succeeds we will increment the reference count. Whenever we attempt to remove an object + // in \ref erase we remove the object from the set (using an atomic compare and swap) and only decrement the + // reference count if the last bit is 0. If it's not we'll push the pointer into this free list. + // \ref copy will consume all elements from this freeList each time it runs and decrements the refcount for each + // element. boost::lockfree::queue, boost::lockfree::capacity> freeList; };