. /include/boost/corosio /detail /object_pool.hpp
94.20% Lines (65/69) 100.00% Functions (10/10)
[ ] prev/next file n p prev/next uncovered line
TLA Baseline Branch
Line Hits Code Line Hits Code
1   // 1   //
2   // Copyright (c) 2025 Steve Gerbino (steve@gerbino.co) 2   // Copyright (c) 2025 Steve Gerbino (steve@gerbino.co)
3   // 3   //
4   // Distributed under the Boost Software License, Version 1.0. (See accompanying 4   // Distributed under the Boost Software License, Version 1.0. (See accompanying
5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6   // 6   //
7   // Official repository: https://github.com/cppalliance/corosio 7   // Official repository: https://github.com/cppalliance/corosio
8   // 8   //
9   9  
10   #ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP 10   #ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
11   #define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP 11   #define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
12   12  
13   #include <boost/corosio/detail/config.hpp> 13   #include <boost/corosio/detail/config.hpp>
14   #include <boost/corosio/detail/intrusive.hpp> 14   #include <boost/corosio/detail/intrusive.hpp>
15   #include <boost/corosio/detail/object_ref.hpp> 15   #include <boost/corosio/detail/object_ref.hpp>
16   #include <atomic> 16   #include <atomic>
17   #include <cstddef> 17   #include <cstddef>
18   #include <initializer_list> 18   #include <initializer_list>
19   #include <limits> 19   #include <limits>
20   #include <mutex> 20   #include <mutex>
21   #include <utility> 21   #include <utility>
22   #include <vector> 22   #include <vector>
23   23  
24   namespace boost::corosio::detail { 24   namespace boost::corosio::detail {
25   25  
26   /** Per-service recycling pool for I/O implementations. 26   /** Per-service recycling pool for I/O implementations.
27   27  
28   Replaces per-impl `shared_ptr` ownership: a service news an impl 28   Replaces per-impl `shared_ptr` ownership: a service news an impl
29   once, then `recycle()` (the impl's `retire()` action) parks 29   once, then `recycle()` (the impl's `retire()` action) parks
30   it on the free list and `acquire()` hands it back out, so steady-state 30   it on the free list and `acquire()` hands it back out, so steady-state
31   construct/destroy allocates nothing. Unbounded; memory is released 31   construct/destroy allocates nothing. Unbounded; memory is released
32   only on pool destruction. Debug poisoning, where an impl does it, 32   only on pool destruction. Debug poisoning, where an impl does it,
33   happens in that impl's `retire()`, not here (the pool cannot know 33   happens in that impl's `retire()`, not here (the pool cannot know
34   which bytes are resettable). 34   which bytes are resettable).
35   35  
36   A service tears down its impls by calling `shutdown()`, which 36   A service tears down its impls by calling `shutdown()`, which
37   enters shutting-down mode and visits every live impl in one 37   enters shutting-down mode and visits every live impl in one
38   critical section; subsequent zero-ref crossings delete instead of 38   critical section; subsequent zero-ref crossings delete instead of
39   recycling. 39   recycling.
40   40  
41   `adopt()` and `remove()` are protected ownership-transfer 41   `adopt()` and `remove()` are protected ownership-transfer
42   primitives for sanctioned extensions — a derived pool exposing 42   primitives for sanctioned extensions — a derived pool exposing
43   them to a caller that tracks some of its live impls through its 43   them to a caller that tracks some of its live impls through its
44   own structure instead of this pool's `live_` list (the timer 44   own structure instead of this pool's `live_` list (the timer
45   service, via its expiry heap) — rather than part of the public 45   service, via its expiry heap) — rather than part of the public
46   surface every ordinary service uses. 46   surface every ordinary service uses.
47   47  
48   @par Thread Safety 48   @par Thread Safety
49   Distinct objects: Safe. Shared objects: Safe; all operations lock 49   Distinct objects: Safe. Shared objects: Safe; all operations lock
50   an internal mutex. 50   an internal mutex.
51   */ 51   */
52   template<class Impl> 52   template<class Impl>
53   class object_pool 53   class object_pool
54   { 54   {
55   std::mutex mutex_; 55   std::mutex mutex_;
56   intrusive_list<Impl> live_; 56   intrusive_list<Impl> live_;
57   intrusive_list<Impl> free_; 57   intrusive_list<Impl> free_;
58   bool shutting_down_ = false; 58   bool shutting_down_ = false;
59   59  
60   /** Marks `refs_` on an impl currently parked in `free_`. 60   /** Marks `refs_` on an impl currently parked in `free_`.
61   61  
62   `intrusive_list::remove()` cannot distinguish "linked in 62   `intrusive_list::remove()` cannot distinguish "linked in
63   `free_`" from "linked in `live_`" by inspecting the node 63   `free_`" from "linked in `live_`" by inspecting the node
64   alone — both lists share one `next_`/`prev_` pair per impl — 64   alone — both lists share one `next_`/`prev_` pair per impl —
65   so `recycle()` cannot safely call `live_.remove()` on an impl 65   so `recycle()` cannot safely call `live_.remove()` on an impl
66   it does not already know is live. Stamping this sentinel into 66   it does not already know is live. Stamping this sentinel into
67   `refs_` the moment an impl joins `free_`, and checking it 67   `refs_` the moment an impl joins `free_`, and checking it
68   before touching either list, is what lets `recycle()` and 68   before touching either list, is what lets `recycle()` and
69   `remove()` tell the two cases apart in every build mode, not 69   `remove()` tell the two cases apart in every build mode, not
70   just under asserts. `acquire()` always overwrites it with 1 70   just under asserts. `acquire()` always overwrites it with 1
71   before the impl is usable again. 71   before the impl is usable again.
72   */ 72   */
73   static constexpr std::size_t pooled_sentinel = pooled_refs; 73   static constexpr std::size_t pooled_sentinel = pooled_refs;
74   74  
75   /// Check whether @p impl is parked in `free_` (`refs_` reads the 75   /// Check whether @p impl is parked in `free_` (`refs_` reads the
76   /// sentinel). Callers must check this before any list operation — 76   /// sentinel). Callers must check this before any list operation —
77   /// see the class-level comment on `pooled_sentinel`. 77   /// see the class-level comment on `pooled_sentinel`.
HITCBC 78   84913 static bool is_pooled(Impl const* impl) noexcept 78   85923 static bool is_pooled(Impl const* impl) noexcept
79   { 79   {
HITCBC 80   84913 return impl->refs_.load(std::memory_order_relaxed) == 80   85923 return impl->refs_.load(std::memory_order_relaxed) ==
HITCBC 81   84913 pooled_sentinel; 81   85923 pooled_sentinel;
82   } 82   }
83   83  
84   protected: 84   protected:
85   /** Track a fresh or reset impl as live. 85   /** Track a fresh or reset impl as live.
86   86  
87   @pre @p impl is not parked in `free_` (`refs_` must not read 87   @pre @p impl is not parked in `free_` (`refs_` must not read
88   `pooled_sentinel`). The same list-corruption hazard 88   `pooled_sentinel`). The same list-corruption hazard
89   `recycle()`'s class-level comment documents applies here: a 89   `recycle()`'s class-level comment documents applies here: a
90   free-listed impl pushed onto `live_` directly, bypassing 90   free-listed impl pushed onto `live_` directly, bypassing
91   `acquire()`, would end up linked in both lists through one shared 91   `acquire()`, would end up linked in both lists through one shared
92   `next_`/`prev_` pair. 92   `next_`/`prev_` pair.
93   93  
94   @pre The caller is not racing `shutdown()`. Every caller adopts 94   @pre The caller is not racing `shutdown()`. Every caller adopts
95   while constructing an object (`acquire()`, or a derived pool's 95   while constructing an object (`acquire()`, or a derived pool's
96   cache handing a parked impl back out), and construction is 96   cache handing a parked impl back out), and construction is
97   expected to stop before the service's own `shutdown()` runs; 97   expected to stop before the service's own `shutdown()` runs;
98   this is checked only in debug builds because the failure mode 98   this is checked only in debug builds because the failure mode
99   is a silently orphaned impl (no close callback), not memory 99   is a silently orphaned impl (no close callback), not memory
100   corruption. 100   corruption.
101   */ 101   */
HITCBC 102   31619 void adopt(Impl* impl) 102   31900 void adopt(Impl* impl)
103   { 103   {
HITCBC 104   31619 std::lock_guard lock(mutex_); 104   31900 std::lock_guard lock(mutex_);
HITCBC 105   31619 if (is_pooled(impl)) 105   31900 if (is_pooled(impl))
106   { 106   {
MISUBC 107   ✗ BOOST_COROSIO_ASSERT(false); 107   ✗ BOOST_COROSIO_ASSERT(false);
108   return; 108   return;
109   } 109   }
HITCBC 110   31619 BOOST_COROSIO_ASSERT(!shutting_down_); 110   31900 BOOST_COROSIO_ASSERT(!shutting_down_);
HITCBC 111   31619 live_.push_back(impl); 111   31900 live_.push_back(impl);
HITCBC 112   31619 } 112   31900 }
113   113  
114   /** Unlink a live impl without recycling (ownership transfer). 114   /** Unlink a live impl without recycling (ownership transfer).
115   115  
116   Used by a caller that is taking an impl out of this pool's 116   Used by a caller that is taking an impl out of this pool's
117   bookkeeping entirely (a thread-local cache slot, a direct 117   bookkeeping entirely (a thread-local cache slot, a direct
118   delete during shutdown) rather than parking it on `free_`. 118   delete during shutdown) rather than parking it on `free_`.
119   119  
120   @pre @p impl is not parked in `free_` (`refs_` must not read 120   @pre @p impl is not parked in `free_` (`refs_` must not read
121   `pooled_sentinel`). The same list-corruption hazard 121   `pooled_sentinel`). The same list-corruption hazard
122   `recycle()`'s class-level comment documents applies here: a 122   `recycle()`'s class-level comment documents applies here: a
123   free-listed impl would be spliced out using `live_`'s 123   free-listed impl would be spliced out using `live_`'s
124   boundary pointers instead of `free_`'s. 124   boundary pointers instead of `free_`'s.
125   125  
126   @return True if @p impl was linked in `live_` and has now 126   @return True if @p impl was linked in `live_` and has now
127   been removed; false if it was already detached. 127   been removed; false if it was already detached.
128   */ 128   */
HITCBC 129   18358 bool remove(Impl* impl) noexcept 129   18425 bool remove(Impl* impl) noexcept
130   { 130   {
HITCBC 131   18358 std::lock_guard lock(mutex_); 131   18425 std::lock_guard lock(mutex_);
HITCBC 132   18358 if (is_pooled(impl)) 132   18425 if (is_pooled(impl))
133   { 133   {
MISUBC 134   ✗ BOOST_COROSIO_ASSERT(false); 134   ✗ BOOST_COROSIO_ASSERT(false);
135   return false; 135   return false;
136   } 136   }
HITCBC 137   18358 return live_.remove(impl); 137   18425 return live_.remove(impl);
HITCBC 138   18358 } 138   18425 }
139   139  
140   public: 140   public:
141   /** Destroy the pool, freeing every impl still parked or live. 141   /** Destroy the pool, freeing every impl still parked or live.
142   142  
143   A live impl is deleted whatever its count. Keepalives it holds 143   A live impl is deleted whatever its count. Keepalives it holds
144   on itself (an embedded op's `object_ref`) are released by its 144   on itself (an embedded op's `object_ref`) are released by its
145   own destructor; the count is pinned first so none of those 145   own destructor; the count is pinned first so none of those
146   releases reaches zero and calls `retire()` on an object that 146   releases reaches zero and calls `retire()` on an object that
147   is already being destroyed. 147   is already being destroyed.
148   148  
149   @pre `shutdown()` has already run, or no impls are live. 149   @pre `shutdown()` has already run, or no impls are live.
150   */ 150   */
HITCBC 151   8913 ~object_pool() 151   8917 ~object_pool()
152   { 152   {
HITCBC 153   8913 BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty()); 153   8917 BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty());
HITCBC 154   26739 for (auto* list : {&free_, &live_}) 154   26751 for (auto* list : {&free_, &live_})
155   { 155   {
HITCBC 156   25139 while (auto* impl = list->pop_front()) 156   25367 while (auto* impl = list->pop_front())
157   { 157   {
HITCBC 158   7313 impl->refs_.store(pooled_sentinel, std::memory_order_relaxed); 158   7533 impl->refs_.store(pooled_sentinel, std::memory_order_relaxed);
HITCBC 159   7313 delete impl; 159   7533 delete impl;
160   } 160   }
161   } 161   }
HITCBC 162   8913 } 162   8917 }
163   163  
HITCBC 164   8903 object_pool() = default; 164   8907 object_pool() = default;
165   object_pool(object_pool const&) = delete; 165   object_pool(object_pool const&) = delete;
166   object_pool& operator=(object_pool const&) = delete; 166   object_pool& operator=(object_pool const&) = delete;
167   167  
168   /** Pop-or-create an impl with the acquisition protocol applied. 168   /** Pop-or-create an impl with the acquisition protocol applied.
169   169  
170   Recycled impls are `reuse()`-reset and their reference count 170   Recycled impls are `reuse()`-reset and their reference count
171   restored from the pooled sentinel; fresh impls are constructed 171   restored from the pooled sentinel; fresh impls are constructed
172   in place from `args` on the cold path. Either way the impl is 172   in place from `args` on the cold path. Either way the impl is
173   adopted as live. This is the sequence every service's 173   adopted as live. This is the sequence every service's
174   `construct()` used to hand-roll. Making the pool itself the 174   `construct()` used to hand-roll. Making the pool itself the
175   only place that calls `new Impl` gives the whole library one 175   only place that calls `new Impl` gives the whole library one
176   audit point for allocation policy. 176   audit point for allocation policy.
177   177  
178   @pre Not called concurrently with, or after, `shutdown()` on 178   @pre Not called concurrently with, or after, `shutdown()` on
179   this pool. Every caller reaches this through a service's 179   this pool. Every caller reaches this through a service's
180   `construct()`, and service shutdown order guarantees no 180   `construct()`, and service shutdown order guarantees no
181   `construct()` outlives that service's own `shutdown()`; the 181   `construct()` outlives that service's own `shutdown()`; the
182   window is only asserted in debug builds (see `adopt()`) 182   window is only asserted in debug builds (see `adopt()`)
183   because the failure mode is an orphaned impl, not corruption. 183   because the failure mode is an orphaned impl, not corruption.
184   184  
185   @param args Constructor arguments for `Impl` (cold path only). 185   @param args Constructor arguments for `Impl` (cold path only).
186   186  
187   @return The acquired impl, `refs_ == 1`, tracked live. 187   @return The acquired impl, `refs_ == 1`, tracked live.
188   */ 188   */
189   template<class... Args> 189   template<class... Args>
HITCBC 190   38681 Impl* acquire(Args&&... args) 190   39348 Impl* acquire(Args&&... args)
191   { 191   {
192   Impl* impl; 192   Impl* impl;
193   { 193   {
HITCBC 194   38681 std::lock_guard lock(mutex_); 194   39348 std::lock_guard lock(mutex_);
HITCBC 195   38681 BOOST_COROSIO_ASSERT(!shutting_down_); 195   39348 BOOST_COROSIO_ASSERT(!shutting_down_);
HITCBC 196   38681 impl = free_.pop_front(); 196   39348 impl = free_.pop_front();
HITCBC 197   38681 if (impl) 197   39348 if (impl)
198   { 198   {
HITCBC 199   21691 impl->refs_.store(1, std::memory_order_relaxed); 199   22136 impl->refs_.store(1, std::memory_order_relaxed);
HITCBC 200   21691 live_.push_back(impl); 200   22136 live_.push_back(impl);
201   } 201   }
HITCBC 202   38681 } 202   39348 }
203   // reuse() runs unlocked: acquire()'s precondition excludes a 203   // reuse() runs unlocked: acquire()'s precondition excludes a
204   // concurrent shutdown() visiting the impl mid-reset. 204   // concurrent shutdown() visiting the impl mid-reset.
HITCBC 205   38681 if (impl) 205   39348 if (impl)
206   { 206   {
HITCBC 207   21691 impl->reuse(); 207   22136 impl->reuse();
HITCBC 208   21691 return impl; 208   22136 return impl;
209   } 209   }
HITCBC 210   16990 impl = new Impl(std::forward<Args>(args)...); 210   17212 impl = new Impl(std::forward<Args>(args)...);
HITCBC 211   16990 adopt(impl); 211   17212 adopt(impl);
HITCBC 212   16990 return impl; 212   17212 return impl;
213   } 213   }
214   214  
215   /** Move a zero-ref impl from live to the free list. 215   /** Move a zero-ref impl from live to the free list.
216   216  
217   Called from `retire()`, possibly on a scheduler thread. 217   Called from `retire()`, possibly on a scheduler thread.
218   Deletes instead when shutting down. The impl must not be 218   Deletes instead when shutting down. The impl must not be
219   touched after this returns. 219   touched after this returns.
220   220  
221   Two accounting bugs are possible here, neither from this 221   Two accounting bugs are possible here, neither from this
222   pool's own bookkeeping but from a caller's refcounting (e.g. 222   pool's own bookkeeping but from a caller's refcounting (e.g.
223   a duplicate `retire()` for one zero-crossing). Both are 223   a duplicate `retire()` for one zero-crossing). Both are
224   made safe in every build mode, not just under asserts, because 224   made safe in every build mode, not just under asserts, because
225   neither can be told apart from the correct case by a 225   neither can be told apart from the correct case by a
226   `BOOST_COROSIO_ASSERT` alone that still runs the list 226   `BOOST_COROSIO_ASSERT` alone that still runs the list
227   operations around it: 227   operations around it:
228   228  
229   - `impl` is already parked in `free_` — the list-corruption 229   - `impl` is already parked in `free_` — the list-corruption
230   hazard the class-level `pooled_sentinel` comment documents. 230   hazard the class-level `pooled_sentinel` comment documents.
231   The `is_pooled()` check runs first and returns before either 231   The `is_pooled()` check runs first and returns before either
232   list is touched. 232   list is touched.
233   - `impl` is fully detached from `live_` and is not the 233   - `impl` is fully detached from `live_` and is not the
234   sentinel (not shutting down): `live_.remove()` correctly 234   sentinel (not shutting down): `live_.remove()` correctly
235   returns false, but then the impl is reachable through 235   returns false, but then the impl is reachable through
236   neither `acquire()` nor `shutdown()`'s visit while still 236   neither `acquire()` nor `shutdown()`'s visit while still
237   allocated — stranded. Deleted instead of left to leak. 237   allocated — stranded. Deleted instead of left to leak.
238   238  
239   While shutting down, an impl that is no longer in `live_` is 239   While shutting down, an impl that is no longer in `live_` is
240   left alone rather than deleted: something else already took 240   left alone rather than deleted: something else already took
241   it out of the pool's bookkeeping (a derived pool's owner, or a 241   it out of the pool's bookkeeping (a derived pool's owner, or a
242   duplicate retirement of an impl already deleted), and deleting 242   duplicate retirement of an impl already deleted), and deleting
243   it again here could be a double free. 243   it again here could be a double free.
244   */ 244   */
HITCBC 245   34936 void recycle(Impl* impl) 245   35598 void recycle(Impl* impl)
246   { 246   {
HITCBC 247   34936 bool do_delete = false; 247   35598 bool do_delete = false;
248   { 248   {
HITCBC 249   34936 std::lock_guard lock(mutex_); 249   35598 std::lock_guard lock(mutex_);
HITCBC 250   34936 if (is_pooled(impl)) 250   35598 if (is_pooled(impl))
251   { 251   {
MISUBC 252   ✗ BOOST_COROSIO_ASSERT(false); 252   ✗ BOOST_COROSIO_ASSERT(false);
253   return; 253   return;
254   } 254   }
255   255  
HITCBC 256   34936 bool const was_live = live_.remove(impl); 256   35598 bool const was_live = live_.remove(impl);
HITCBC 257   34936 if (shutting_down_) 257   35598 if (shutting_down_)
258   { 258   {
HITCBC 259   5949 do_delete = was_live; 259   5943 do_delete = was_live;
260   } 260   }
HITCBC 261   28987 else if (was_live) 261   29655 else if (was_live)
262   { 262   {
HITCBC 263   28987 impl->refs_.store( 263   29655 impl->refs_.store(
264   pooled_sentinel, std::memory_order_relaxed); 264   pooled_sentinel, std::memory_order_relaxed);
265   // LIFO: the most recently retired impl is the 265   // LIFO: the most recently retired impl is the
266   // cache-warmest candidate for the next acquire(). 266   // cache-warmest candidate for the next acquire().
HITCBC 267   28987 free_.push_front(impl); 267   29655 free_.push_front(impl);
268   } 268   }
269   else 269   else
270   { 270   {
MISUBC 271   ✗ BOOST_COROSIO_ASSERT(false); 271   ✗ BOOST_COROSIO_ASSERT(false);
272   do_delete = true; 272   do_delete = true;
273   } 273   }
HITCBC 274   34936 } 274   35598 }
HITCBC 275   34936 if (do_delete) 275   35598 if (do_delete)
HITCBC 276   5949 delete impl; 276   5943 delete impl;
277   } 277   }
278   278  
279   /** Enter shutdown mode and visit every live impl. 279   /** Enter shutdown mode and visit every live impl.
280   280  
281   Sets shutting-down (subsequent zero-crossings delete instead 281   Sets shutting-down (subsequent zero-crossings delete instead
282   of recycling) and runs `f` on each live impl, all under one 282   of recycling) and runs `f` on each live impl, all under one
283   critical section. Each visited impl is pinned with a reference 283   critical section. Each visited impl is pinned with a reference
284   for the call and released after the critical section ends, so 284   for the call and released after the critical section ends, so
285   `f` may hold on to it; an impl whose count already reached zero 285   `f` may hold on to it; an impl whose count already reached zero
286   is retiring and is skipped rather than revived. 286   is retiring and is skipped rather than revived.
287   287  
288   @note The callback runs under the pool's non-recursive mutex; 288   @note The callback runs under the pool's non-recursive mutex;
289   it must not call recycle(), adopt(), or remove() on this pool. 289   it must not call recycle(), adopt(), or remove() on this pool.
290   */ 290   */
291   template<class F> 291   template<class F>
HITCBC 292   13321 void shutdown(F&& f) 292   13327 void shutdown(F&& f)
293   { 293   {
HITCBC 294   13321 std::vector<Impl*> pinned; 294   13327 std::vector<Impl*> pinned;
295   { 295   {
HITCBC 296   13321 std::lock_guard lock(mutex_); 296   13327 std::lock_guard lock(mutex_);
HITCBC 297   13321 shutting_down_ = true; 297   13327 shutting_down_ = true;
298   // Sized before any pin, so a failure here leaves none held. 298   // Sized before any pin, so a failure here leaves none held.
HITCBC 299   13321 std::size_t n = 0; 299   13327 std::size_t n = 0;
HITCBC 300   22531 live_.for_each([&](Impl*) { ++n; }); 300   22528 live_.for_each([&](Impl*) { ++n; });
HITCBC 301   13321 pinned.reserve(n); 301   13327 pinned.reserve(n);
HITCBC 302   13321 live_.for_each( 302   13327 live_.for_each(
HITCBC 303   18426 [&](Impl* impl) 303   18408 [&](Impl* impl)
304   { 304   {
HITCBC 305   9213 if (try_acquire(impl)) 305   9204 if (try_acquire(impl))
306   { 306   {
HITCBC 307   9212 pinned.push_back(impl); 307   9203 pinned.push_back(impl);
HITCBC 308   9212 f(impl); 308   9203 f(impl);
309   } 309   }
310   }); 310   });
HITCBC 311   13321 } 311   13327 }
HITCBC 312   22533 for (auto* impl : pinned) 312   22530 for (auto* impl : pinned)
HITCBC 313   9212 release(impl); 313   9203 release(impl);
HITCBC 314   13321 } 314   13327 }
315   }; 315   };
316   316  
317   } // namespace boost::corosio::detail 317   } // namespace boost::corosio::detail
318   318  
319   #endif 319   #endif