. /include/boost/corosio /detail /timer_service.hpp
96.23% Lines (230/239) 100.00% Functions (28/28)
[ ] prev/next file n p prev/next uncovered line
TLA Baseline Branch
Line Hits Code Line Hits Code
1   // 1   //
2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com) 2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com)
3   // Copyright (c) 2026 Steve Gerbino 3   // Copyright (c) 2026 Steve Gerbino
4   // 4   //
5   // Distributed under the Boost Software License, Version 1.0. (See accompanying 5   // Distributed under the Boost Software License, Version 1.0. (See accompanying
6   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 6   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7   // 7   //
8   // Official repository: https://github.com/cppalliance/corosio 8   // Official repository: https://github.com/cppalliance/corosio
9   // 9   //
10   10  
11   #ifndef BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP 11   #ifndef BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP
12   #define BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP 12   #define BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP
13   13  
14   #include <boost/corosio/detail/timer.hpp> 14   #include <boost/corosio/detail/timer.hpp>
15   #include <boost/corosio/detail/scheduler.hpp> 15   #include <boost/corosio/detail/scheduler.hpp>
16   #include <boost/corosio/detail/scheduler_op.hpp> 16   #include <boost/corosio/detail/scheduler_op.hpp>
17   #include <boost/corosio/detail/object_pool.hpp> 17   #include <boost/corosio/detail/object_pool.hpp>
18   #include <boost/corosio/detail/object_ref.hpp> 18   #include <boost/corosio/detail/object_ref.hpp>
19   #include <boost/corosio/detail/intrusive.hpp> 19   #include <boost/corosio/detail/intrusive.hpp>
20   #include <boost/corosio/detail/thread_local_ptr.hpp> 20   #include <boost/corosio/detail/thread_local_ptr.hpp>
21   #include <boost/capy/error.hpp> 21   #include <boost/capy/error.hpp>
22   #include <boost/capy/ex/execution_context.hpp> 22   #include <boost/capy/ex/execution_context.hpp>
23   #include <boost/capy/ex/abandon.hpp> 23   #include <boost/capy/ex/abandon.hpp>
24   #include <boost/capy/ex/executor_ref.hpp> 24   #include <boost/capy/ex/executor_ref.hpp>
25   #include <system_error> 25   #include <system_error>
26   26  
27   #include <atomic> 27   #include <atomic>
28   #include <chrono> 28   #include <chrono>
29   #include <coroutine> 29   #include <coroutine>
30   #include <cstddef> 30   #include <cstddef>
31   #include <limits> 31   #include <limits>
32   #include <mutex> 32   #include <mutex>
33   #include <thread> 33   #include <thread>
34   #include <stop_token> 34   #include <stop_token>
35   #include <utility> 35   #include <utility>
36   #include <vector> 36   #include <vector>
37   37  
38   namespace boost::corosio::detail { 38   namespace boost::corosio::detail {
39   39  
40   class scheduler; 40   class scheduler;
41   41  
42   /* 42   /*
43   Timer Service 43   Timer Service
44   ============= 44   =============
45   45  
46   Data Structures 46   Data Structures
47   --------------- 47   ---------------
48   waiter_node (defined in timer.hpp) holds per-waiter state: 48   waiter_node (defined in timer.hpp) holds per-waiter state:
49   coroutine handle, executor, error output, embedded 49   coroutine handle, executor, error output, embedded
50   completion_op. Each concurrent co_await t.wait() embeds one 50   completion_op. Each concurrent co_await t.wait() embeds one
51   waiter_node in the awaitable on the suspended coroutine's 51   waiter_node in the awaitable on the suspended coroutine's
52   frame — waits perform no allocation. 52   frame — waits perform no allocation.
53   53  
54   timer::implementation holds per-timer state: expiry, heap 54   timer::implementation holds per-timer state: expiry, heap
55   index, and the single published waiter. Each timer holds 55   index, and the single published waiter. Each timer holds
56   at most one waiter; process_expired's local cross-timer drain 56   at most one waiter; process_expired's local cross-timer drain
57   list still threads waiters through their intrusive hooks when 57   list still threads waiters through their intrusive hooks when
58   collecting several timers' waiters past the lock. 58   collecting several timers' waiters past the lock.
59   59  
60   timer_service owns a min-heap of active timers and recycles 60   timer_service owns a min-heap of active timers and recycles
61   retired impls through a per-service object_pool, fronted by a 61   retired impls through a per-service object_pool, fronted by a
62   thread-local single-slot cache. The heap is ordered by expiry 62   thread-local single-slot cache. The heap is ordered by expiry
63   time; the scheduler queries nearest_expiry() to set the 63   time; the scheduler queries nearest_expiry() to set the
64   epoll/timerfd timeout. 64   epoll/timerfd timeout.
65   65  
66   Optimization Strategy 66   Optimization Strategy
67   --------------------- 67   ---------------------
68   1. Deferred heap insertion — expires_after() stores the expiry 68   1. Deferred heap insertion — expires_after() stores the expiry
69   but does not insert into the heap. Insertion happens in wait(). 69   but does not insert into the heap. Insertion happens in wait().
70   2. Thread-local impl cache — single-slot per-thread cache, in 70   2. Thread-local impl cache — single-slot per-thread cache, in
71   front of the object_pool fallback; keeps a thread's hot impl 71   front of the object_pool fallback; keeps a thread's hot impl
72   out of the shared free list. 72   out of the shared free list.
73   3. Frame-resident waiter_node with embedded completion_op — 73   3. Frame-resident waiter_node with embedded completion_op —
74   eliminates heap allocation per wait/fire/cancel. 74   eliminates heap allocation per wait/fire/cancel.
75   4. Cached nearest expiry — atomic avoids mutex in nearest_expiry(). 75   4. Cached nearest expiry — atomic avoids mutex in nearest_expiry().
76   5. might_have_pending_waits_ flag — skips lock when no wait issued. 76   5. might_have_pending_waits_ flag — skips lock when no wait issued.
77   77  
78   Concurrency 78   Concurrency
79   ----------- 79   -----------
80   stop_token callbacks can fire from any thread. The impl_ 80   stop_token callbacks can fire from any thread. The impl_
81   pointer on waiter_node is used as a "still in list" marker. 81   pointer on waiter_node is used as a "still in list" marker.
82   A waiter_node's storage is the suspended coroutine's frame: 82   A waiter_node's storage is the suspended coroutine's frame:
83   every completion path must finish touching the node before 83   every completion path must finish touching the node before
84   posting the continuation or destroying the handle. 84   posting the continuation or destroying the handle.
85   */ 85   */
86   86  
87   inline void timer_service_invalidate_cache() noexcept; 87   inline void timer_service_invalidate_cache() noexcept;
88   88  
89   /** The timer service's recycling pool. 89   /** The timer service's recycling pool.
90   90  
91   Adds no behavior of its own — it exists only to re-expose 91   Adds no behavior of its own — it exists only to re-expose
92   `adopt()` and `remove()` as public, so the timer service's 92   `adopt()` and `remove()` as public, so the timer service's
93   thread-local cache can transfer a cached impl's `live_` tracking 93   thread-local cache can transfer a cached impl's `live_` tracking
94   on both sides of the handoff: `construct()` re-adopts a 94   on both sides of the handoff: `construct()` re-adopts a
95   TL-cached impl as live, `shutdown()` detaches one before its 95   TL-cached impl as live, `shutdown()` detaches one before its
96   direct delete. 96   direct delete.
97   */ 97   */
98   class timer_object_pool : public object_pool<timer::implementation> 98   class timer_object_pool : public object_pool<timer::implementation>
99   { 99   {
100   public: 100   public:
101   using object_pool::adopt; 101   using object_pool::adopt;
102   using object_pool::remove; 102   using object_pool::remove;
103   }; 103   };
104   104  
105   // timer_service class body — member function definitions are 105   // timer_service class body — member function definitions are
106   // out-of-class (after implementation and waiter_node are complete) 106   // out-of-class (after implementation and waiter_node are complete)
107   class BOOST_COROSIO_DECL timer_service final 107   class BOOST_COROSIO_DECL timer_service final
108   : public capy::execution_context::service 108   : public capy::execution_context::service
109   , public io_object::io_service 109   , public io_object::io_service
110   { 110   {
111   friend struct timer::implementation; 111   friend struct timer::implementation;
112   112  
113   public: 113   public:
114   using clock_type = std::chrono::steady_clock; 114   using clock_type = std::chrono::steady_clock;
115   using time_point = clock_type::time_point; 115   using time_point = clock_type::time_point;
116   116  
117   /// Type-erased callback for earliest-expiry-changed notifications. 117   /// Type-erased callback for earliest-expiry-changed notifications.
118   class callback 118   class callback
119   { 119   {
120   void* ctx_ = nullptr; 120   void* ctx_ = nullptr;
121   void (*fn_)(void*) = nullptr; 121   void (*fn_)(void*) = nullptr;
122   122  
123   public: 123   public:
124   /// Construct an empty callback. 124   /// Construct an empty callback.
HITCBC 125   4413 callback() = default; 125   4415 callback() = default;
126   126  
127   /// Construct a callback with the given context and function. 127   /// Construct a callback with the given context and function.
HITCBC 128   4413 callback(void* ctx, void (*fn)(void*)) noexcept : ctx_(ctx), fn_(fn) {} 128   4415 callback(void* ctx, void (*fn)(void*)) noexcept : ctx_(ctx), fn_(fn) {}
129   129  
130   /// Return true if the callback is non-empty. 130   /// Return true if the callback is non-empty.
131   explicit operator bool() const noexcept 131   explicit operator bool() const noexcept
132   { 132   {
133   return fn_ != nullptr; 133   return fn_ != nullptr;
134   } 134   }
135   135  
136   /// Invoke the callback. 136   /// Invoke the callback.
HITCBC 137   8036 void operator()() const 137   7920 void operator()() const
138   { 138   {
HITCBC 139   8036 if (fn_) 139   7920 if (fn_)
HITCBC 140   8036 fn_(ctx_); 140   7920 fn_(ctx_);
HITCBC 141   8036 } 141   7920 }
142   }; 142   };
143   143  
144   private: 144   private:
145   struct heap_entry 145   struct heap_entry
146   { 146   {
147   time_point time_; 147   time_point time_;
148   timer::implementation* timer_; 148   timer::implementation* timer_;
149   }; 149   };
150   150  
151   scheduler* sched_ = nullptr; 151   scheduler* sched_ = nullptr;
152   BOOST_COROSIO_MSVC_WARNING_PUSH 152   BOOST_COROSIO_MSVC_WARNING_PUSH
153   BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // std:: members, dll-interface 153   BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // std:: members, dll-interface
154   mutable std::mutex mutex_; 154   mutable std::mutex mutex_;
155   std::vector<heap_entry> heap_; 155   std::vector<heap_entry> heap_;
156   timer_object_pool pool_; 156   timer_object_pool pool_;
157   callback on_earliest_changed_; 157   callback on_earliest_changed_;
158   bool shutting_down_ = false; 158   bool shutting_down_ = false;
159   159  
160   // Cancellers that claimed a waiter and have yet to post it; 160   // Cancellers that claimed a waiter and have yet to post it;
161   // shutdown waits them out so the scheduler's drain finds the post. 161   // shutdown waits them out so the scheduler's drain finds the post.
162   std::size_t cancels_posting_ = 0; // guarded by mutex_ 162   std::size_t cancels_posting_ = 0; // guarded by mutex_
163   // Avoids mutex in nearest_expiry() and empty() 163   // Avoids mutex in nearest_expiry() and empty()
164   mutable std::atomic<std::int64_t> cached_nearest_ns_{ 164   mutable std::atomic<std::int64_t> cached_nearest_ns_{
165   (std::numeric_limits<std::int64_t>::max)()}; 165   (std::numeric_limits<std::int64_t>::max)()};
166   BOOST_COROSIO_MSVC_WARNING_POP 166   BOOST_COROSIO_MSVC_WARNING_POP
167   167  
168   public: 168   public:
169   /// Construct the timer service bound to a scheduler. 169   /// Construct the timer service bound to a scheduler.
HITCBC 170   4413 inline timer_service(capy::execution_context&, scheduler& sched) 170   4415 inline timer_service(capy::execution_context&, scheduler& sched)
HITCBC 171   4413 : sched_(&sched) 171   4415 : sched_(&sched)
172   { 172   {
HITCBC 173   4413 } 173   4415 }
174   174  
175   /// Return the associated scheduler. 175   /// Return the associated scheduler.
HITCBC 176   34250 inline scheduler& get_scheduler() noexcept 176   34829 inline scheduler& get_scheduler() noexcept
177   { 177   {
HITCBC 178   34250 return *sched_; 178   34829 return *sched_;
179   } 179   }
180   180  
181   /// Destroy the timer service. 181   /// Destroy the timer service.
HITCBC 182   8826 ~timer_service() override = default; 182   8830 ~timer_service() override = default;
183   183  
184   timer_service(timer_service const&) = delete; 184   timer_service(timer_service const&) = delete;
185   timer_service& operator=(timer_service const&) = delete; 185   timer_service& operator=(timer_service const&) = delete;
186   186  
187   /// Register a callback invoked when the earliest expiry changes. 187   /// Register a callback invoked when the earliest expiry changes.
HITCBC 188   4413 inline void set_on_earliest_changed(callback cb) 188   4415 inline void set_on_earliest_changed(callback cb)
189   { 189   {
HITCBC 190   4413 on_earliest_changed_ = cb; 190   4415 on_earliest_changed_ = cb;
HITCBC 191   4413 } 191   4415 }
192   192  
193   /// Return true if no timers are in the heap. 193   /// Return true if no timers are in the heap.
194   inline bool empty() const noexcept 194   inline bool empty() const noexcept
195   { 195   {
196   return cached_nearest_ns_.load(std::memory_order_acquire) == 196   return cached_nearest_ns_.load(std::memory_order_acquire) ==
197   (std::numeric_limits<std::int64_t>::max)(); 197   (std::numeric_limits<std::int64_t>::max)();
198   } 198   }
199   199  
200   /// Return the nearest timer expiry without acquiring the mutex. 200   /// Return the nearest timer expiry without acquiring the mutex.
HITCBC 201   27154 inline time_point nearest_expiry() const noexcept 201   24840 inline time_point nearest_expiry() const noexcept
202   { 202   {
HITCBC 203   27154 auto ns = cached_nearest_ns_.load(std::memory_order_acquire); 203   24840 auto ns = cached_nearest_ns_.load(std::memory_order_acquire);
HITCBC 204   27154 return time_point(time_point::duration(ns)); 204   24840 return time_point(time_point::duration(ns));
205   } 205   }
206   206  
207   /// Cancel all pending timers and free cached resources. 207   /// Cancel all pending timers and free cached resources.
208   inline void shutdown() override; 208   inline void shutdown() override;
209   209  
210   /// Construct a new timer implementation. 210   /// Construct a new timer implementation.
211   inline io_object::implementation* construct() override; 211   inline io_object::implementation* construct() override;
212   212  
213   /// Destroy a timer implementation, cancelling pending waiters. 213   /// Destroy a timer implementation, cancelling pending waiters.
214   inline void destroy(io_object::implementation* p) override; 214   inline void destroy(io_object::implementation* p) override;
215   215  
216   /// Cancel and recycle a timer implementation. 216   /// Cancel and recycle a timer implementation.
217   inline void destroy_impl(timer::implementation& impl); 217   inline void destroy_impl(timer::implementation& impl);
218   218  
219   /// Publish the timer's waiter and insert the timer into the heap. 219   /// Publish the timer's waiter and insert the timer into the heap.
220   inline void insert_waiter(timer::implementation& impl, waiter_node* w); 220   inline void insert_waiter(timer::implementation& impl, waiter_node* w);
221   221  
222   /// Cancel the timer's published waiter, if any. 222   /// Cancel the timer's published waiter, if any.
223   inline void cancel_timer(timer::implementation& impl); 223   inline void cancel_timer(timer::implementation& impl);
224   224  
225   /// Cancel one specific waiter ( stop_token callback path ). 225   /// Cancel one specific waiter ( stop_token callback path ).
226   inline void cancel_waiter(waiter_node* w); 226   inline void cancel_waiter(waiter_node* w);
227   227  
228   /// Complete all waiters whose timers have expired. 228   /// Complete all waiters whose timers have expired.
229   inline std::size_t process_expired(); 229   inline std::size_t process_expired();
230   230  
231   private: 231   private:
HITCBC 232   179550 inline void refresh_cached_nearest() noexcept 232   174412 inline void refresh_cached_nearest() noexcept
233   { 233   {
HITCBC 234   179550 auto ns = heap_.empty() ? (std::numeric_limits<std::int64_t>::max)() 234   174412 auto ns = heap_.empty() ? (std::numeric_limits<std::int64_t>::max)()
HITCBC 235   62273 : heap_[0].time_.time_since_epoch().count(); 235   59951 : heap_[0].time_.time_since_epoch().count();
HITCBC 236   179550 cached_nearest_ns_.store(ns, std::memory_order_release); 236   174412 cached_nearest_ns_.store(ns, std::memory_order_release);
HITCBC 237   179550 } 237   174412 }
238   238  
239   inline void remove_timer_impl(timer::implementation& impl); 239   inline void remove_timer_impl(timer::implementation& impl);
240   inline void up_heap(std::size_t index); 240   inline void up_heap(std::size_t index);
241   inline void down_heap(std::size_t index); 241   inline void down_heap(std::size_t index);
242   inline void swap_heap(std::size_t i1, std::size_t i2); 242   inline void swap_heap(std::size_t i1, std::size_t i2);
243   }; 243   };
244   244  
245   // Thread-local single-slot impl cache, validated by comparing svc_. 245   // Thread-local single-slot impl cache, validated by comparing svc_.
246   // Keeps a hot impl out of the shared free list; both sides still 246   // Keeps a hot impl out of the shared free list; both sides still
247   // take the pool lock to move it in and out of live tracking. 247   // take the pool lock to move it in and out of live tracking.
248   // Cleared by timer_service_invalidate_cache() during shutdown. 248   // Cleared by timer_service_invalidate_cache() during shutdown.
249   249  
250   inline thread_local_ptr<timer::implementation> tl_cached_impl; 250   inline thread_local_ptr<timer::implementation> tl_cached_impl;
251   251  
252   // The POD TLS slot above never runs destructors, so a short-lived 252   // The POD TLS slot above never runs destructors, so a short-lived
253   // run() thread would leak its cached impl. Each push arms this 253   // run() thread would leak its cached impl. Each push arms this
254   // owner, whose destructor frees the slot at thread exit. A cached 254   // owner, whose destructor frees the slot at thread exit. A cached
255   // entry is a quiescent heap object (nothing in the heap or free 255   // entry is a quiescent heap object (nothing in the heap or free
256   // list) and deletion touches no service state, so it is safe after 256   // list) and deletion touches no service state, so it is safe after
257   // the owning service is gone (the stale-entry path in 257   // the owning service is gone (the stale-entry path in
258   // try_pop_tl_cache deletes the same way). 258   // try_pop_tl_cache deletes the same way).
259   struct tl_cache_owner 259   struct tl_cache_owner
260   { 260   {
HITCBC 261   51 ~tl_cache_owner() 261   56 ~tl_cache_owner()
262   { 262   {
HITCBC 263   51 delete tl_cached_impl.get(); 263   56 delete tl_cached_impl.get();
HITCBC 264   51 tl_cached_impl.set(nullptr); 264   56 tl_cached_impl.set(nullptr);
HITCBC 265   51 } 265   56 }
266   }; 266   };
267   267  
268   inline void 268   inline void
HITCBC 269   15123 arm_tl_cache_cleanup() noexcept 269   15187 arm_tl_cache_cleanup() noexcept
270   { 270   {
HITCBC 271   15123 [[maybe_unused]] thread_local tl_cache_owner owner; 271   15187 [[maybe_unused]] thread_local tl_cache_owner owner;
HITCBC 272   15123 } 272   15187 }
273   273  
274   inline timer::implementation* 274   inline timer::implementation*
HITCBC 275   19668 try_pop_tl_cache(timer_service* svc) noexcept 275   19959 try_pop_tl_cache(timer_service* svc) noexcept
276   { 276   {
HITCBC 277   19668 auto* impl = tl_cached_impl.get(); 277   19959 auto* impl = tl_cached_impl.get();
HITCBC 278   19668 if (impl) 278   19959 if (impl)
279   { 279   {
HITCBC 280   14629 tl_cached_impl.set(nullptr); 280   14688 tl_cached_impl.set(nullptr);
HITCBC 281   14629 if (impl->svc_ == svc) 281   14688 if (impl->svc_ == svc)
HITCBC 282   14629 return impl; 282   14688 return impl;
283   // Stale impl from a destroyed service 283   // Stale impl from a destroyed service
MISUBC 284   ✗ delete impl; 284   ✗ delete impl;
285   } 285   }
HITCBC 286   5039 return nullptr; 286   5271 return nullptr;
287   } 287   }
288   288  
289   inline bool 289   inline bool
HITCBC 290   16421 try_push_tl_cache(timer::implementation* impl) noexcept 290   16712 try_push_tl_cache(timer::implementation* impl) noexcept
291   { 291   {
HITCBC 292   16421 if (!tl_cached_impl.get()) 292   16712 if (!tl_cached_impl.get())
293   { 293   {
HITCBC 294   15123 arm_tl_cache_cleanup(); 294   15187 arm_tl_cache_cleanup();
HITCBC 295   15123 tl_cached_impl.set(impl); 295   15187 tl_cached_impl.set(impl);
HITCBC 296   15123 return true; 296   15187 return true;
297   } 297   }
HITCBC 298   1298 return false; 298   1525 return false;
299   } 299   }
300   300  
301   inline void 301   inline void
HITCBC 302   8826 timer_service_invalidate_cache() noexcept 302   8830 timer_service_invalidate_cache() noexcept
303   { 303   {
HITCBC 304   8826 delete tl_cached_impl.get(); 304   8830 delete tl_cached_impl.get();
HITCBC 305   8826 tl_cached_impl.set(nullptr); 305   8830 tl_cached_impl.set(nullptr);
HITCBC 306   8826 } 306   8830 }
307   307  
308   // timer_service out-of-class member function definitions 308   // timer_service out-of-class member function definitions
309   309  
310   inline void 310   inline void
HITCBC 311   8826 timer_service::shutdown() 311   8830 timer_service::shutdown()
312   { 312   {
HITCBC 313   8826 timer_service_invalidate_cache(); 313   8830 timer_service_invalidate_cache();
HITCBC 314   8826 shutting_down_ = true; 314   8830 shutting_down_ = true;
315   // Flag only; the real drain walks the expiry heap below, not 315   // Flag only; the real drain walks the expiry heap below, not
316   // object_pool's live_ list, so the visit callback is a no-op. 316   // object_pool's live_ list, so the visit callback is a no-op.
HITCBC 317   8826 pool_.shutdown([](timer::implementation*) {}); 317   8830 pool_.shutdown([](timer::implementation*) {});
318   318  
319   // Snapshot impls and detach them from the heap so that 319   // Snapshot impls and detach them from the heap so that
320   // coroutine-owned timer destructors (run by the unwinds the 320   // coroutine-owned timer destructors (run by the unwinds the
321   // abandons below start) cannot re-enter remove_timer_impl() and 321   // abandons below start) cannot re-enter remove_timer_impl() and
322   // mutate the vector during iteration. A stop requested on another thread 322   // mutate the vector during iteration. A stop requested on another thread
323   // reaches cancel_waiter() under the same lock: whichever side 323   // reaches cancel_waiter() under the same lock: whichever side
324   // claims a waiter first owns it, and a waiter its canceller 324   // claims a waiter first owns it, and a waiter its canceller
325   // claimed is posted, so the scheduler's drain abandons it. 325   // claimed is posted, so the scheduler's drain abandons it.
HITCBC 326   8826 std::vector<timer::implementation*> impls; 326   8830 std::vector<timer::implementation*> impls;
HITCBC 327   8826 std::vector<waiter_node*> waiters; 327   8830 std::vector<waiter_node*> waiters;
328   { 328   {
HITCBC 329   8826 std::lock_guard lock(mutex_); 329   8830 std::lock_guard lock(mutex_);
HITCBC 330   8826 impls.reserve(heap_.size()); 330   8830 impls.reserve(heap_.size());
HITCBC 331   8826 waiters.reserve(heap_.size()); 331   8830 waiters.reserve(heap_.size());
HITCBC 332   12058 for (auto& entry : heap_) 332   12065 for (auto& entry : heap_)
333   { 333   {
HITCBC 334   3232 entry.timer_->heap_index_.store( 334   3235 entry.timer_->heap_index_.store(
335   (std::numeric_limits<std::size_t>::max)(), 335   (std::numeric_limits<std::size_t>::max)(),
336   std::memory_order_relaxed); 336   std::memory_order_relaxed);
HITCBC 337   3232 impls.push_back(entry.timer_); 337   3235 impls.push_back(entry.timer_);
HITCBC 338   3232 if (auto* w = std::exchange(entry.timer_->waiter_, nullptr)) 338   3235 if (auto* w = std::exchange(entry.timer_->waiter_, nullptr))
339   { 339   {
HITCBC 340   3232 w->impl_ = nullptr; 340   3235 w->impl_ = nullptr;
HITCBC 341   3232 waiters.push_back(w); 341   3235 waiters.push_back(w);
342   } 342   }
343   } 343   }
HITCBC 344   8826 heap_.clear(); 344   8830 heap_.clear();
HITCBC 345   8826 cached_nearest_ns_.store( 345   8830 cached_nearest_ns_.store(
346   (std::numeric_limits<std::int64_t>::max)(), 346   (std::numeric_limits<std::int64_t>::max)(),
347   std::memory_order_release); 347   std::memory_order_release);
HITCBC 348   8826 } 348   8830 }
349   349  
350   // A stop on another thread may have claimed its waiter before the 350   // A stop on another thread may have claimed its waiter before the
351   // pass above and still be posting it; the scheduler drains next, 351   // pass above and still be posting it; the scheduler drains next,
352   // and must find that post. 352   // and must find that post.
353   for (;;) 353   for (;;)
354   { 354   {
355   { 355   {
HITCBC 356   10554 std::lock_guard lock(mutex_); 356   8830 std::lock_guard lock(mutex_);
HITCBC 357   10554 if (cancels_posting_ == 0) 357   8830 if (cancels_posting_ == 0)
HITCBC 358   8826 break; 358   8830 break;
HITCBC 359   10554 } 359   8830 }
MISLBC 360   1728 std::this_thread::yield(); 360   ✗ std::this_thread::yield();
MISLBC 361   1728 } 361   ✗ }
362   362  
363   // Cancel waiting timers. Each waiter called work_started() 363   // Cancel waiting timers. Each waiter called work_started()
364   // in implementation::wait(); a waiter given up here never 364   // in implementation::wait(); a waiter given up here never
365   // completes, so its work is finished here to keep the count 365   // completes, so its work is finished here to keep the count
366   // balanced. Resetting the stop callback outside the lock 366   // balanced. Resetting the stop callback outside the lock
367   // waits out a canceller already running, which finds the waiter 367   // waits out a canceller already running, which finds the waiter
368   // claimed. 368   // claimed.
HITCBC 369   12058 for (auto* w : waiters) 369   12065 for (auto* w : waiters)
370   { 370   {
HITCBC 371   3232 w->reset_stop_cb(); 371   3235 w->reset_stop_cb();
HITCBC 372   3232 auto c = std::exchange(w->cont_, {}); 372   3235 auto c = std::exchange(w->cont_, {});
HITCBC 373   3232 sched_->work_finished(); 373   3235 sched_->work_finished();
374   // Unwinding the frame also ends the node's storage 374   // Unwinding the frame also ends the node's storage
HITCBC 375   3232 if (c.h) 375   3235 if (c.h)
HITCBC 376   3232 capy::abandon(c); 376   3235 capy::abandon(c);
377   } 377   }
378   378  
HITCBC 379   12058 for (auto* impl : impls) 379   12065 for (auto* impl : impls)
380   { 380   {
381   // Unlink from the pool's live_ list before the direct delete 381   // Unlink from the pool's live_ list before the direct delete
382   // below, or ~object_pool()'s unconditional sweep would delete 382   // below, or ~object_pool()'s unconditional sweep would delete
383   // this impl a second time. 383   // this impl a second time.
HITCBC 384   3232 [[maybe_unused]] bool const removed = pool_.remove(impl); 384   3235 [[maybe_unused]] bool const removed = pool_.remove(impl);
HITCBC 385   3232 BOOST_COROSIO_ASSERT(removed); 385   3235 BOOST_COROSIO_ASSERT(removed);
HITCBC 386   3232 delete impl; 386   3235 delete impl;
387   } 387   }
388   388  
389   // Anything still parked in pool_ (free-listed or live with no 389   // Anything still parked in pool_ (free-listed or live with no
390   // pending wait at shutdown) is freed by object_pool's own destructor. 390   // pending wait at shutdown) is freed by object_pool's own destructor.
HITCBC 391   8826 } 391   8830 }
392   392  
393   inline io_object::implementation* 393   inline io_object::implementation*
HITCBC 394   19668 timer_service::construct() 394   19959 timer_service::construct()
395   { 395   {
HITCBC 396   19668 timer::implementation* impl = try_pop_tl_cache(this); 396   19959 timer::implementation* impl = try_pop_tl_cache(this);
HITCBC 397   19668 if (impl) 397   19959 if (impl)
398   { 398   {
HITCBC 399   14629 impl->svc_ = this; 399   14688 impl->svc_ = this;
HITCBC 400   14629 impl->reuse(); 400   14688 impl->reuse();
HITCBC 401   14629 impl->refs_.store(1, std::memory_order_relaxed); 401   14688 impl->refs_.store(1, std::memory_order_relaxed);
402   // The TL push that parked impl here also removed it from 402   // The TL push that parked impl here also removed it from
403   // live_ (see destroy_impl) — re-adopt it now that it is live 403   // live_ (see destroy_impl) — re-adopt it now that it is live
404   // again. 404   // again.
HITCBC 405   14629 pool_.adopt(impl); 405   14688 pool_.adopt(impl);
HITCBC 406   14629 return impl; 406   14688 return impl;
407   } 407   }
408   408  
HITCBC 409   5039 return pool_.acquire(*this); 409   5271 return pool_.acquire(*this);
410   } 410   }
411   411  
412   inline void 412   inline void
HITCBC 413   19668 timer_service::destroy(io_object::implementation* p) 413   19959 timer_service::destroy(io_object::implementation* p)
414   { 414   {
415   // During shutdown the drain loop owns every impl and deletes 415   // During shutdown the drain loop owns every impl and deletes
416   // them directly. A frame destroyed by that loop can unwind a 416   // them directly. A frame destroyed by that loop can unwind a
417   // handle whose impl was freed in an earlier iteration (a 417   // handle whose impl was freed in an earlier iteration (a
418   // timeout's parent frame owns the timeout timer while 418   // timeout's parent frame owns the timeout timer while
419   // suspended on the inner delay's timer), so bail out before 419   // suspended on the inner delay's timer), so bail out before
420   // even downcasting the pointer. 420   // even downcasting the pointer.
HITCBC 421   19668 if (shutting_down_) 421   19959 if (shutting_down_)
HITCBC 422   3247 return; 422   3247 return;
HITCBC 423   16421 destroy_impl(static_cast<timer::implementation&>(*p)); 423   16712 destroy_impl(static_cast<timer::implementation&>(*p));
424   } 424   }
425   425  
426   inline void 426   inline void
HITCBC 427   16421 timer_service::destroy_impl(timer::implementation& impl) 427   16712 timer_service::destroy_impl(timer::implementation& impl)
428   { 428   {
429   // During shutdown the impl is owned by the shutdown loop. 429   // During shutdown the impl is owned by the shutdown loop.
430   // Re-entering here (from a coroutine-owned timer destructor 430   // Re-entering here (from a coroutine-owned timer destructor
431   // triggered by h.destroy()) must not modify the heap or 431   // triggered by h.destroy()) must not modify the heap or
432   // recycle the impl — shutdown deletes it directly. 432   // recycle the impl — shutdown deletes it directly.
HITCBC 433   16421 if (shutting_down_) 433   16712 if (shutting_down_)
MISUBC 434   ✗ return; 434   ✗ return;
435   435  
HITCBC 436   16421 cancel_timer(impl); 436   16712 cancel_timer(impl);
437   437  
HITCBC 438   32842 if (impl.heap_index_.load(std::memory_order_relaxed) != 438   33424 if (impl.heap_index_.load(std::memory_order_relaxed) !=
HITCBC 439   16421 (std::numeric_limits<std::size_t>::max)()) 439   16712 (std::numeric_limits<std::size_t>::max)())
440   { 440   {
MISUBC 441   ✗ std::lock_guard lock(mutex_); 441   ✗ std::lock_guard lock(mutex_);
MISUBC 442   ✗ remove_timer_impl(impl); 442   ✗ remove_timer_impl(impl);
MISUBC 443   ✗ refresh_cached_nearest(); 443   ✗ refresh_cached_nearest();
MISUBC 444   ✗ } 444   ✗ }
445   445  
HITCBC 446   16421 if (try_push_tl_cache(&impl)) 446   16712 if (try_push_tl_cache(&impl))
447   { 447   {
448   // The TL slot takes sole ownership: it is deleted directly by 448   // The TL slot takes sole ownership: it is deleted directly by
449   // tl_cache_owner / the stale-service path, never through this 449   // tl_cache_owner / the stale-service path, never through this
450   // pool, so it must leave live_ now or ~object_pool() would see 450   // pool, so it must leave live_ now or ~object_pool() would see
451   // it as abandoned (and, at the wrong moment, double-delete it 451   // it as abandoned (and, at the wrong moment, double-delete it
452   // after a later construct() re-adopts it). 452   // after a later construct() re-adopts it).
HITCBC 453   15123 [[maybe_unused]] bool const removed = pool_.remove(&impl); 453   15187 [[maybe_unused]] bool const removed = pool_.remove(&impl);
HITCBC 454   15123 BOOST_COROSIO_ASSERT(removed); 454   15187 BOOST_COROSIO_ASSERT(removed);
HITCBC 455   15123 return; 455   15187 return;
456   } 456   }
457   457  
458   // No op keepalives on a timer impl: this is the only reference, 458   // No op keepalives on a timer impl: this is the only reference,
459   // so release() always recycles immediately via retire(). 459   // so release() always recycles immediately via retire().
HITCBC 460   1298 release(&impl); 460   1525 release(&impl);
461   } 461   }
462   462  
463   inline void 463   inline void
HITCBC 464   23382 timer_service::insert_waiter(timer::implementation& impl, waiter_node* w) 464   23746 timer_service::insert_waiter(timer::implementation& impl, waiter_node* w)
465   { 465   {
HITCBC 466   23382 bool notify = false; 466   23746 bool notify = false;
HITCBC 467   23382 bool lost_cancel = false; 467   23746 bool lost_cancel = false;
468   { 468   {
HITCBC 469   23382 std::lock_guard lock(mutex_); 469   23746 std::lock_guard lock(mutex_);
470   // Grow before publishing anything, so the push_back below 470   // Grow before publishing anything, so the push_back below
471   // cannot throw: a failure here leaves the waiter untouched, 471   // cannot throw: a failure here leaves the waiter untouched,
472   // the strong guarantee rearm_wait's recovery relies on. 472   // the strong guarantee rearm_wait's recovery relies on.
HITCBC 473   23382 if (impl.heap_index_.load(std::memory_order_relaxed) == 473   23746 if (impl.heap_index_.load(std::memory_order_relaxed) ==
HITCBC 474   46764 (std::numeric_limits<std::size_t>::max)() && 474   47492 (std::numeric_limits<std::size_t>::max)() &&
HITCBC 475   23382 heap_.size() == heap_.capacity()) 475   23746 heap_.size() == heap_.capacity())
HITCBC 476   922 heap_.reserve(heap_.capacity() == 0 ? 16 : 2 * heap_.capacity()); 476   930 heap_.reserve(heap_.capacity() == 0 ? 16 : 2 * heap_.capacity());
477   // Publish: from here the waiter is visible to the fire path and 477   // Publish: from here the waiter is visible to the fire path and
478   // to its own stop callback (impl_ non-null enables cancel_waiter). 478   // to its own stop callback (impl_ non-null enables cancel_waiter).
HITCBC 479   23382 w->impl_ = &impl; 479   23746 w->impl_ = &impl;
HITCBC 480   46764 if (impl.heap_index_.load(std::memory_order_relaxed) == 480   47492 if (impl.heap_index_.load(std::memory_order_relaxed) ==
HITCBC 481   23382 (std::numeric_limits<std::size_t>::max)()) 481   23746 (std::numeric_limits<std::size_t>::max)())
482   { 482   {
HITCBC 483   23382 impl.heap_index_.store(heap_.size(), std::memory_order_relaxed); 483   23746 impl.heap_index_.store(heap_.size(), std::memory_order_relaxed);
HITCBC 484   23382 heap_.push_back({impl.expiry_, &impl}); 484   23746 heap_.push_back({impl.expiry_, &impl});
HITCBC 485   23382 up_heap(heap_.size() - 1); 485   23746 up_heap(heap_.size() - 1);
HITCBC 486   23382 notify = (impl.heap_index_.load(std::memory_order_relaxed) == 0); 486   23746 notify = (impl.heap_index_.load(std::memory_order_relaxed) == 0);
HITCBC 487   23382 refresh_cached_nearest(); 487   23746 refresh_cached_nearest();
488   } 488   }
HITCBC 489   23382 BOOST_COROSIO_ASSERT(impl.waiter_ == nullptr); 489   23746 BOOST_COROSIO_ASSERT(impl.waiter_ == nullptr);
HITCBC 490   23382 impl.waiter_ = w; 490   23746 impl.waiter_ = w;
491   491  
492   // Lost-cancel re-check: a stop requested after the canceller was 492   // Lost-cancel re-check: a stop requested after the canceller was
493   // armed in wait() but before this publication found impl_ null 493   // armed in wait() but before this publication found impl_ null
494   // and returned a no-op. Observe it now and undo the insertion. 494   // and returned a no-op. Observe it now and undo the insertion.
HITCBC 495   23382 if (w->token_->stop_requested()) 495   23746 if (w->token_->stop_requested())
496   { 496   {
HITCBC 497   2 w->impl_ = nullptr; 497   83 w->impl_ = nullptr;
HITCBC 498   2 impl.waiter_ = nullptr; 498   83 impl.waiter_ = nullptr;
HITCBC 499   2 remove_timer_impl(impl); 499   83 remove_timer_impl(impl);
HITCBC 500   2 impl.might_have_pending_waits_.store( 500   83 impl.might_have_pending_waits_.store(
501   false, std::memory_order_release); 501   false, std::memory_order_release);
HITCBC 502   2 refresh_cached_nearest(); 502   83 refresh_cached_nearest();
HITCBC 503   2 lost_cancel = true; 503   83 lost_cancel = true;
HITCBC 504   2 notify = false; // insertion undone; nearest unchanged 504   83 notify = false; // insertion undone; nearest unchanged
505   } 505   }
HITCBC 506   23382 } 506   23746 }
HITCBC 507   23382 if (notify) 507   23746 if (notify)
HITCBC 508   8036 on_earliest_changed_(); 508   7920 on_earliest_changed_();
HITCBC 509   23382 if (lost_cancel) 509   23746 if (lost_cancel)
510   { 510   {
HITCBC 511   2 w->ec_ = make_error_code(capy::error::canceled); 511   83 w->ec_ = make_error_code(capy::error::canceled);
HITCBC 512   2 sched_->post(&w->op_); 512   83 sched_->post(&w->op_);
513   } 513   }
HITCBC 514   23382 } 514   23746 }
515   515  
516   inline void 516   inline void
HITCBC 517   16421 timer_service::cancel_timer(timer::implementation& impl) 517   16712 timer_service::cancel_timer(timer::implementation& impl)
518   { 518   {
519   // Acquire pairs with the drains' release: a caller that skips the 519   // Acquire pairs with the drains' release: a caller that skips the
520   // lock may recycle the impl next, so the drain's writes to it must 520   // lock may recycle the impl next, so the drain's writes to it must
521   // happen before. 521   // happen before.
HITCBC 522   16421 if (!impl.might_have_pending_waits_.load(std::memory_order_acquire)) 522   16712 if (!impl.might_have_pending_waits_.load(std::memory_order_acquire))
HITCBC 523   16419 return; 523   16710 return;
524   524  
525   // No unlocked already-done fast-out here: it would need the 525   // No unlocked already-done fast-out here: it would need the
526   // non-atomic waiter_ (a race with concurrent drains), and an 526   // non-atomic waiter_ (a race with concurrent drains), and an
527   // index-only check is lifetime-unsafe because npos is stored 527   // index-only check is lifetime-unsafe because npos is stored
528   // before the drain finishes touching the impl. A stale-true 528   // before the drain finishes touching the impl. A stale-true
529   // flag is rare with the stateless API; the locked path below 529   // flag is rare with the stateless API; the locked path below
530   // re-validates. 530   // re-validates.
531   531  
HITCBC 532   2 waiter_node* canceled = nullptr; 532   2 waiter_node* canceled = nullptr;
533   533  
534   { 534   {
HITCBC 535   2 std::lock_guard lock(mutex_); 535   2 std::lock_guard lock(mutex_);
HITCBC 536   2 remove_timer_impl(impl); 536   2 remove_timer_impl(impl);
HITCBC 537   2 canceled = std::exchange(impl.waiter_, nullptr); 537   2 canceled = std::exchange(impl.waiter_, nullptr);
HITCBC 538   2 if (canceled) 538   2 if (canceled)
HITCBC 539   2 canceled->impl_ = nullptr; 539   2 canceled->impl_ = nullptr;
540   // Store false as the final touch of the impl under the lock so 540   // Store false as the final touch of the impl under the lock so
541   // a pre-lock false-flag check trusts it unqualified. 541   // a pre-lock false-flag check trusts it unqualified.
HITCBC 542   2 impl.might_have_pending_waits_.store(false, std::memory_order_release); 542   2 impl.might_have_pending_waits_.store(false, std::memory_order_release);
HITCBC 543   2 refresh_cached_nearest(); 543   2 refresh_cached_nearest();
HITCBC 544   2 } 544   2 }
545   545  
HITCBC 546   2 if (canceled) 546   2 if (canceled)
547   { 547   {
HITCBC 548   2 canceled->ec_ = make_error_code(capy::error::canceled); 548   2 canceled->ec_ = make_error_code(capy::error::canceled);
HITCBC 549   2 sched_->post(&canceled->op_); 549   2 sched_->post(&canceled->op_);
550   } 550   }
551   } 551   }
552   552  
553   inline void 553   inline void
HITCBC 554   1995 timer_service::cancel_waiter(waiter_node* w) 554   2325 timer_service::cancel_waiter(waiter_node* w)
555   { 555   {
556   { 556   {
HITCBC 557   1995 std::lock_guard lock(mutex_); 557   2325 std::lock_guard lock(mutex_);
558   // Already removed by another drain: cancel_timer, 558   // Already removed by another drain: cancel_timer,
559   // process_expired, or insert_waiter's lost-cancel recheck 559   // process_expired, or insert_waiter's lost-cancel recheck
HITCBC 560   1995 if (!w->impl_) 560   2325 if (!w->impl_)
HITCBC 561   273 return; 561   422 return;
HITCBC 562   1722 auto* impl = w->impl_; 562   1903 auto* impl = w->impl_;
HITCBC 563   1722 w->impl_ = nullptr; 563   1903 w->impl_ = nullptr;
HITCBC 564   1722 impl->waiter_ = nullptr; 564   1903 impl->waiter_ = nullptr;
HITCBC 565   1722 remove_timer_impl(*impl); 565   1903 remove_timer_impl(*impl);
HITCBC 566   1722 impl->might_have_pending_waits_.store(false, std::memory_order_release); 566   1903 impl->might_have_pending_waits_.store(false, std::memory_order_release);
HITCBC 567   1722 refresh_cached_nearest(); 567   1903 refresh_cached_nearest();
HITCBC 568   1722 ++cancels_posting_; 568   1903 ++cancels_posting_;
HITCBC 569   1995 } 569   2325 }
570   570  
HITCBC 571   1722 w->ec_ = make_error_code(capy::error::canceled); 571   1903 w->ec_ = make_error_code(capy::error::canceled);
HITCBC 572   1722 sched_->post(&w->op_); 572   1903 sched_->post(&w->op_);
573   573  
HITCBC 574   1722 std::lock_guard lock(mutex_); 574   1903 std::lock_guard lock(mutex_);
HITCBC 575   1722 --cancels_posting_; 575   1903 --cancels_posting_;
HITCBC 576   1722 } 576   1903 }
577   577  
578   inline std::size_t 578   inline std::size_t
HITCBC 579   154442 timer_service::process_expired() 579   148678 timer_service::process_expired()
580   { 580   {
HITCBC 581   154442 intrusive_list<waiter_node> expired; 581   148678 intrusive_list<waiter_node> expired;
582   582  
583   { 583   {
HITCBC 584   154442 std::lock_guard lock(mutex_); 584   148678 std::lock_guard lock(mutex_);
HITCBC 585   154442 auto now = clock_type::now(); 585   148678 auto now = clock_type::now();
586   586  
HITCBC 587   172866 while (!heap_.empty() && heap_[0].time_ <= now) 587   167201 while (!heap_.empty() && heap_[0].time_ <= now)
588   { 588   {
HITCBC 589   18424 timer::implementation* t = heap_[0].timer_; 589   18523 timer::implementation* t = heap_[0].timer_;
HITCBC 590   18424 remove_timer_impl(*t); 590   18523 remove_timer_impl(*t);
HITCBC 591   18424 if (auto* w = std::exchange(t->waiter_, nullptr)) 591   18523 if (auto* w = std::exchange(t->waiter_, nullptr))
592   { 592   {
HITCBC 593   18424 w->impl_ = nullptr; 593   18523 w->impl_ = nullptr;
HITCBC 594   18424 w->ec_ = {}; 594   18523 w->ec_ = {};
HITCBC 595   18424 expired.push_back(w); 595   18523 expired.push_back(w);
596   } 596   }
HITCBC 597   18424 t->might_have_pending_waits_.store( 597   18523 t->might_have_pending_waits_.store(
598   false, std::memory_order_release); 598   false, std::memory_order_release);
599   } 599   }
600   600  
HITCBC 601   154442 refresh_cached_nearest(); 601   148678 refresh_cached_nearest();
HITCBC 602   154442 } 602   148678 }
603   603  
HITCBC 604   154442 std::size_t count = 0; 604   148678 std::size_t count = 0;
HITCBC 605   172866 while (auto* w = expired.pop_front()) 605   167201 while (auto* w = expired.pop_front())
606   { 606   {
HITCBC 607   18424 sched_->post(&w->op_); 607   18523 sched_->post(&w->op_);
HITCBC 608   18424 ++count; 608   18523 ++count;
HITCBC 609   18424 } 609   18523 }
610   610  
HITCBC 611   154442 return count; 611   148678 return count;
612   } 612   }
613   613  
614   inline void 614   inline void
HITCBC 615   20150 timer_service::remove_timer_impl(timer::implementation& impl) 615   20511 timer_service::remove_timer_impl(timer::implementation& impl)
616   { 616   {
HITCBC 617   20150 std::size_t index = impl.heap_index_.load(std::memory_order_relaxed); 617   20511 std::size_t index = impl.heap_index_.load(std::memory_order_relaxed);
HITCBC 618   20150 if (index >= heap_.size()) 618   20511 if (index >= heap_.size())
MISUBC 619   ✗ return; // Not in heap 619   ✗ return; // Not in heap
620   620  
HITCBC 621   20150 if (index == heap_.size() - 1) 621   20511 if (index == heap_.size() - 1)
622   { 622   {
623   // Last element, just pop 623   // Last element, just pop
HITCBC 624   2813 impl.heap_index_.store( 624   2971 impl.heap_index_.store(
625   (std::numeric_limits<std::size_t>::max)(), 625   (std::numeric_limits<std::size_t>::max)(),
626   std::memory_order_relaxed); 626   std::memory_order_relaxed);
HITCBC 627   2813 heap_.pop_back(); 627   2971 heap_.pop_back();
628   } 628   }
629   else 629   else
630   { 630   {
631   // Swap with last and reheapify 631   // Swap with last and reheapify
HITCBC 632   17337 swap_heap(index, heap_.size() - 1); 632   17540 swap_heap(index, heap_.size() - 1);
HITCBC 633   17337 impl.heap_index_.store( 633   17540 impl.heap_index_.store(
634   (std::numeric_limits<std::size_t>::max)(), 634   (std::numeric_limits<std::size_t>::max)(),
635   std::memory_order_relaxed); 635   std::memory_order_relaxed);
HITCBC 636   17337 heap_.pop_back(); 636   17540 heap_.pop_back();
637   637  
HITCBC 638   17337 if (index > 0 && heap_[index].time_ < heap_[(index - 1) / 2].time_) 638   17540 if (index > 0 && heap_[index].time_ < heap_[(index - 1) / 2].time_)
HITGBC 639   ✗ up_heap(index); 639   4 up_heap(index);
640   else 640   else
HITCBC 641   17337 down_heap(index); 641   17536 down_heap(index);
642   } 642   }
643   } 643   }
644   644  
645   inline void 645   inline void
HITCBC 646   23382 timer_service::up_heap(std::size_t index) 646   23750 timer_service::up_heap(std::size_t index)
647   { 647   {
HITCBC 648   31135 while (index > 0) 648   31456 while (index > 0)
649   { 649   {
HITCBC 650   23097 std::size_t parent = (index - 1) / 2; 650   23455 std::size_t parent = (index - 1) / 2;
HITCBC 651   23097 if (!(heap_[index].time_ < heap_[parent].time_)) 651   23455 if (!(heap_[index].time_ < heap_[parent].time_))
HITCBC 652   15344 break; 652   15749 break;
HITCBC 653   7753 swap_heap(index, parent); 653   7706 swap_heap(index, parent);
HITCBC 654   7753 index = parent; 654   7706 index = parent;
655   } 655   }
HITCBC 656   23382 } 656   23750 }
657   657  
658   inline void 658   inline void
HITCBC 659   17337 timer_service::down_heap(std::size_t index) 659   17536 timer_service::down_heap(std::size_t index)
660   { 660   {
HITCBC 661   17337 std::size_t child = index * 2 + 1; 661   17536 std::size_t child = index * 2 + 1;
HITCBC 662   36739 while (child < heap_.size()) 662   38263 while (child < heap_.size())
663   { 663   {
HITCBC 664   22014 std::size_t min_child = (child + 1 == heap_.size() || 664   22919 std::size_t min_child = (child + 1 == heap_.size() ||
HITCBC 665   17770 heap_[child].time_ < heap_[child + 1].time_) 665   18800 heap_[child].time_ < heap_[child + 1].time_)
HITCBC 666   39784 ? child 666   41719 ? child
HITCBC 667   22014 : child + 1; 667   22919 : child + 1;
668   668  
HITCBC 669   22014 if (heap_[index].time_ < heap_[min_child].time_) 669   22919 if (heap_[index].time_ < heap_[min_child].time_)
HITCBC 670   2612 break; 670   2192 break;
671   671  
HITCBC 672   19402 swap_heap(index, min_child); 672   20727 swap_heap(index, min_child);
HITCBC 673   19402 index = min_child; 673   20727 index = min_child;
HITCBC 674   19402 child = index * 2 + 1; 674   20727 child = index * 2 + 1;
675   } 675   }
HITCBC 676   17337 } 676   17536 }
677   677  
678   inline void 678   inline void
HITCBC 679   44492 timer_service::swap_heap(std::size_t i1, std::size_t i2) 679   45973 timer_service::swap_heap(std::size_t i1, std::size_t i2)
680   { 680   {
HITCBC 681   44492 heap_entry tmp = heap_[i1]; 681   45973 heap_entry tmp = heap_[i1];
HITCBC 682   44492 heap_[i1] = heap_[i2]; 682   45973 heap_[i1] = heap_[i2];
HITCBC 683   44492 heap_[i2] = tmp; 683   45973 heap_[i2] = tmp;
HITCBC 684   44492 heap_[i1].timer_->heap_index_.store(i1, std::memory_order_relaxed); 684   45973 heap_[i1].timer_->heap_index_.store(i1, std::memory_order_relaxed);
HITCBC 685   44492 heap_[i2].timer_->heap_index_.store(i2, std::memory_order_relaxed); 685   45973 heap_[i2].timer_->heap_index_.store(i2, std::memory_order_relaxed);
HITCBC 686   44492 } 686   45973 }
687   687  
688   // waiter_node's completion_op and canceller members are defined in 688   // waiter_node's completion_op and canceller members are defined in
689   // timer.cpp alongside implementation::wait(), for the same reason 689   // timer.cpp alongside implementation::wait(), for the same reason
690   // wait() lives there (see below). 690   // wait() lives there (see below).
691   691  
692   // timer::implementation::wait() is defined in timer.cpp, not here. 692   // timer::implementation::wait() is defined in timer.cpp, not here.
693   // It must be a non-inline definition in a translation unit that is 693   // It must be a non-inline definition in a translation unit that is
694   // always pulled into the link whenever detail::timer is used (every 694   // always pulled into the link whenever detail::timer is used (every
695   // consumer needs timer's constructors from that same object file). 695   // consumer needs timer's constructors from that same object file).
696   // An inline definition in this header would only be emitted in 696   // An inline definition in this header would only be emitted in
697   // translation units that happen to also include this header, which 697   // translation units that happen to also include this header, which
698   // is not guaranteed for every caller of wait_awaitable::await_suspend 698   // is not guaranteed for every caller of wait_awaitable::await_suspend
699   // in timer.hpp (e.g. code that only reaches timer.hpp through 699   // in timer.hpp (e.g. code that only reaches timer.hpp through
700   // delay.hpp, without transitively including a scheduler header). 700   // delay.hpp, without transitively including a scheduler header).
701   701  
702   // Free functions 702   // Free functions
703   703  
704   inline timer_service& 704   inline timer_service&
HITCBC 705   4413 get_timer_service(capy::execution_context& ctx, scheduler& sched) 705   4415 get_timer_service(capy::execution_context& ctx, scheduler& sched)
706   { 706   {
HITCBC 707   4413 return ctx.make_service<timer_service>(sched); 707   4415 return ctx.make_service<timer_service>(sched);
708   } 708   }
709   709  
710   } // namespace boost::corosio::detail 710   } // namespace boost::corosio::detail
711   711  
712   #endif 712   #endif