src/internal/input_binding_gate.hpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | #ifndef DETOURMODKIT_INTERNAL_INPUT_BINDING_GATE_HPP | ||
| 2 | #define DETOURMODKIT_INTERNAL_INPUT_BINDING_GATE_HPP | ||
| 3 | |||
| 4 | /** | ||
| 5 | * @file input_binding_gate.hpp | ||
| 6 | * @brief Provides per-binding teardown gates for press and hold callbacks. | ||
| 7 | * @details Each gate serializes callback delivery against teardown. Consumer callbacks and capture destructors run | ||
| 8 | * outside the gate mutex. | ||
| 9 | * | ||
| 10 | * A control-plane release waits for every delivery and teardown span. Retirement uses a deadline and refuses | ||
| 11 | * if it cannot prove quiescence. A caller must hold no lock that consumer code can acquire. It must own no | ||
| 12 | * join that consumer code can await. | ||
| 13 | * | ||
| 14 | * A callback-side release never waits. It defers the final Hold edge to delivery unwind or emits that edge | ||
| 15 | * inline when no delivery exists. | ||
| 16 | * | ||
| 17 | * docs/design/input.md owns the delivery-scope and ABBA rationale. This private header keeps the | ||
| 18 | * synchronization unit-testable. | ||
| 19 | */ | ||
| 20 | |||
| 21 | #include "internal/input_binding_lifecycle.hpp" | ||
| 22 | #include "internal/input_delivery_scope.hpp" | ||
| 23 | |||
| 24 | #include <atomic> | ||
| 25 | #include <chrono> | ||
| 26 | #include <condition_variable> | ||
| 27 | #include <cstddef> | ||
| 28 | #include <exception> | ||
| 29 | #include <functional> | ||
| 30 | #include <memory> | ||
| 31 | #include <mutex> | ||
| 32 | #include <utility> | ||
| 33 | |||
| 34 | namespace DetourModKit | ||
| 35 | { | ||
| 36 | namespace detail | ||
| 37 | { | ||
| 38 | /** | ||
| 39 | * @class GateCallbackSlot | ||
| 40 | * @brief Owns one gate callback behind a stable heap address. | ||
| 41 | * @details A std::function move can destroy an inline source target. Retirement moves only the pointer, so the | ||
| 42 | * mutex never covers consumer destruction. | ||
| 43 | */ | ||
| 44 | template <typename Signature> class GateCallbackSlot; | ||
| 45 | |||
| 46 | template <typename Return, typename... Args> class GateCallbackSlot<Return(Args...)> | ||
| 47 | { | ||
| 48 | public: | ||
| 49 | using Function = std::function<Return(Args...)>; | ||
| 50 | using Owner = std::unique_ptr<Function>; | ||
| 51 | |||
| 52 | 1954 | GateCallbackSlot() noexcept = default; | |
| 53 | 1936 | ~GateCallbackSlot() noexcept = default; | |
| 54 | |||
| 55 | GateCallbackSlot(const GateCallbackSlot &) = delete; | ||
| 56 | GateCallbackSlot &operator=(const GateCallbackSlot &) = delete; | ||
| 57 | GateCallbackSlot(GateCallbackSlot &&) = delete; | ||
| 58 | GateCallbackSlot &operator=(GateCallbackSlot &&) = delete; | ||
| 59 | |||
| 60 | /// Installs a callback during unpublished gate setup and stores no owner for an empty callback. | ||
| 61 | 1953 | GateCallbackSlot &operator=(Function callback) | |
| 62 | { | ||
| 63 | 1953 | Owner replacement; | |
| 64 | 1953 | if (callback) | |
| 65 | { | ||
| 66 | 1751 | replacement = std::make_unique<Function>(std::move(callback)); | |
| 67 | } | ||
| 68 | 1950 | m_callback = std::move(replacement); | |
| 69 | 1950 | return *this; | |
| 70 | 1951 | } | |
| 71 | |||
| 72 | /// Clears a callback during unpublished gate setup. | ||
| 73 | 2 | GateCallbackSlot &operator=(std::nullptr_t) noexcept | |
| 74 | { | ||
| 75 | 2 | m_callback.reset(); | |
| 76 | 2 | return *this; | |
| 77 | } | ||
| 78 | |||
| 79 | /// Returns true when this slot owns a callback. | ||
| 80 | 2944 | [[nodiscard]] explicit operator bool() const noexcept { return m_callback != nullptr; } | |
| 81 | |||
| 82 | /// Invokes the owned callback. | ||
| 83 | 5031 | Return operator()(Args... args) const { return (*m_callback)(std::forward<Args>(args)...); } | |
| 84 | |||
| 85 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 86 | /// Installs a test-only probe that runs immediately after ownership transfer. | ||
| 87 | 2 | void set_after_take_probe_for_test(void (*probe)(void *) noexcept, void *context) noexcept | |
| 88 | { | ||
| 89 | 2 | m_after_take_probe_for_test = probe; | |
| 90 | 2 | m_after_take_context_for_test = context; | |
| 91 | 2 | } | |
| 92 | #endif | ||
| 93 | |||
| 94 | /// Transfers callback ownership by pointer while the std::function target stays at one address. | ||
| 95 | 355 | [[nodiscard]] Owner take() noexcept | |
| 96 | { | ||
| 97 | 710 | Owner callback = std::move(m_callback); | |
| 98 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 99 | 356 | if (m_after_take_probe_for_test != nullptr) | |
| 100 | { | ||
| 101 | 2 | m_after_take_probe_for_test(m_after_take_context_for_test); | |
| 102 | } | ||
| 103 | #endif | ||
| 104 | 356 | return callback; | |
| 105 | } | ||
| 106 | |||
| 107 | private: | ||
| 108 | Owner m_callback; | ||
| 109 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 110 | void (*m_after_take_probe_for_test)(void *) noexcept = nullptr; | ||
| 111 | void *m_after_take_context_for_test = nullptr; | ||
| 112 | #endif | ||
| 113 | }; | ||
| 114 | |||
| 115 | /** | ||
| 116 | * @struct BindingGate | ||
| 117 | * @brief Type-erased handle a binding's engine entry keeps on the gate its wrappers dispatch through. | ||
| 118 | * @details A gate has two strong owners, the poller's exploded engine entries and the BindingGuard's one-shot | ||
| 119 | * release closure, and it owns the consumer callback. Dropping the engine entries therefore retires | ||
| 120 | * the binding without destroying the callback, which is safe for an ordinary reshape and unsafe | ||
| 121 | * before a Logic DLL is unmapped. This base gives the unload drain a way to reach the gate itself. | ||
| 122 | */ | ||
| 123 | struct BindingGate | ||
| 124 | { | ||
| 125 | 1940 | virtual ~BindingGate() = default; | |
| 126 | |||
| 127 | /** | ||
| 128 | * @brief Ends delivery, emits a still-held hold's balancing edge, and destroys the consumer callback. | ||
| 129 | * @param deadline Bound on the wait for an in-flight delivery to unwind. | ||
| 130 | * @return False when the deadline expired with a delivery still running; the callback is then retained, | ||
| 131 | * because destroying a callable a poll thread is executing would free code out from under it. | ||
| 132 | * @details Idempotent. Unlike @c release(), which leaves the callback in place for the binding's guard, | ||
| 133 | * this hands ownership of the callback to the calling thread and destroys it there, so no later | ||
| 134 | * guard release can reach it. Control-plane only: the wait would deadlock a caller that is | ||
| 135 | * itself inside a delivery. | ||
| 136 | */ | ||
| 137 | [[nodiscard]] virtual bool retire(std::chrono::steady_clock::time_point deadline) = 0; | ||
| 138 | |||
| 139 | /// Keeps the delivery marker's TLS index published while this gate can open a frame. | ||
| 140 | DeliveryTlsOwner delivery_tls; | ||
| 141 | }; | ||
| 142 | |||
| 143 | /** | ||
| 144 | * @struct HoldGate | ||
| 145 | * @brief Per-binding teardown gate shared between a hold binding's callback wrapper and its guard. | ||
| 146 | * @details A hold callback carries lingering state: the consumer is told true (held) until told false | ||
| 147 | * (released), so cancelling mid-hold must deliver exactly one balancing false and never let a stale | ||
| 148 | * true land after it. A multi-combo hold ("X = combo A | combo B") explodes into N engine entries that | ||
| 149 | * all share ONE gate; the gate reference-counts the held entries in @ref active_entries and forwards | ||
| 150 | * only the 0->1 and 1->0 crossings, so the consumer sees "held" for the whole span any combo is down. | ||
| 151 | * | ||
| 152 | * Deliveries arrive from the poll thread (edges) and from control threads (a reshape's synchronized | ||
| 153 | * released(false)); @ref lifecycle guards against a late true resurrecting a torn-down binding. | ||
| 154 | */ | ||
| 155 | struct HoldGate : BindingGate | ||
| 156 | { | ||
| 157 | std::mutex mutex; | ||
| 158 | std::condition_variable idle_cv; | ||
| 159 | std::shared_ptr<std::atomic<bool>> enabled; | ||
| 160 | // Shared with the binding's engine entries. Its tombstone is the resurrection guard below; may be null for | ||
| 161 | // a gate not tied to an engine entry (never happens in the facade, tolerated for direct unit use). | ||
| 162 | std::shared_ptr<BindingLifecycle> lifecycle; | ||
| 163 | GateCallbackSlot<void(bool)> on_state_change; | ||
| 164 | |||
| 165 | // Exploded entries sharing this gate that are currently held; consumer-visible held state is (count > 0). | ||
| 166 | int active_entries = 0; | ||
| 167 | // A true edge was delivered to the user and not yet balanced by a false edge. | ||
| 168 | bool forwarded_active = false; | ||
| 169 | // The guard has torn the binding down; further edges are swallowed. | ||
| 170 | bool released = false; | ||
| 171 | // Callback invocations running outside the mutex, including a guard-release balancing edge. | ||
| 172 | int in_flight = 0; | ||
| 173 | // A release arrived while a delivery was in flight and could not block; the delivery emits the balancing | ||
| 174 | // false on its unwind. | ||
| 175 | bool deferred_final = false; | ||
| 176 | // A control-plane teardown (a guard release or the unload drain's retire) has marked the gate released but | ||
| 177 | // has not finished its consumer-code span: the release's balancing edge, or retirement's edge plus callable | ||
| 178 | // disposal. `in_flight` cannot express this on its own: the claimant drops the mutex to wait for deliveries | ||
| 179 | // to drain, so without a claim taken at the same moment as `released` a second teardown could observe | ||
| 180 | // `in_flight == 0` inside that window and report a quiesced gate while the first is about to enter the | ||
| 181 | // callback. | ||
| 182 | bool teardown_active = false; | ||
| 183 | // Native thread running that span, set and cleared with `teardown_active`. The same-gate recursion guard: | ||
| 184 | // it names the one claim a waiter must not wait for, its own. Win32 thread identity is allocation-free, | ||
| 185 | // unlike std::this_thread::get_id() on a foreign MinGW/winpthreads thread. | ||
| 186 | std::uint32_t teardown_owner = 0; | ||
| 187 | |||
| 188 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 189 | // Set these test pause points before worker start. Clear them after every worker joins. | ||
| 190 | void (*before_idle_wait_for_test)(void *) noexcept = nullptr; | ||
| 191 | void *before_idle_wait_context_for_test = nullptr; | ||
| 192 | void (*after_bookkeeping_for_test)(void *) noexcept = nullptr; | ||
| 193 | void *after_bookkeeping_context_for_test = nullptr; | ||
| 194 | #endif | ||
| 195 | |||
| 196 | /** | ||
| 197 | * @brief Holds one @ref in_flight slot for an unlocked callback. | ||
| 198 | * @details deliver() owns the primary callback slot through its critical sections. | ||
| 199 | * This holder covers a deferred false edge after that callback returns. | ||
| 200 | * release() and retire() wait for in_flight == 0 before they destroy consumer state. | ||
| 201 | * The holder releases its slot on every exit, even after an exception. | ||
| 202 | * TeardownScope covers consumer code that teardown invokes. | ||
| 203 | */ | ||
| 204 | struct InFlightSlot | ||
| 205 | { | ||
| 206 | 6 | explicit InFlightSlot(HoldGate *owner) noexcept : m_gate(owner) {} | |
| 207 | 6 | ~InFlightSlot() noexcept | |
| 208 | { | ||
| 209 | 6 | std::lock_guard<std::mutex> exit_lock(m_gate->mutex); | |
| 210 | 6 | --m_gate->in_flight; | |
| 211 | 6 | m_gate->idle_cv.notify_all(); | |
| 212 | 6 | } | |
| 213 | |||
| 214 | // One slot decrements exactly once, like the DeliveryScope it brackets. A copy would decrement twice | ||
| 215 | // and drive in_flight negative, at which point the == 0 predicate release() and retire() wait on is | ||
| 216 | // either already true while a callback runs or never true again. | ||
| 217 | InFlightSlot(const InFlightSlot &) = delete; | ||
| 218 | InFlightSlot &operator=(const InFlightSlot &) = delete; | ||
| 219 | InFlightSlot(InFlightSlot &&) = delete; | ||
| 220 | InFlightSlot &operator=(InFlightSlot &&) = delete; | ||
| 221 | |||
| 222 | private: | ||
| 223 | HoldGate *m_gate; | ||
| 224 | }; | ||
| 225 | |||
| 226 | /** | ||
| 227 | * @brief RAII holder for a teardown path's claim on @ref teardown_active. | ||
| 228 | * @details Claimed under the mutex at the same moment the path sets @ref released, and cleared once its | ||
| 229 | * consumer-code span is over. A release spans its balancing edge; retirement also spans callable | ||
| 230 | * disposal. A concurrent teardown therefore waits for the whole span rather than for the callback | ||
| 231 | * window alone. | ||
| 232 | */ | ||
| 233 | struct TeardownScope | ||
| 234 | { | ||
| 235 | 236 | explicit TeardownScope(HoldGate *owner) noexcept : m_gate(owner) {} | |
| 236 | 237 | ~TeardownScope() noexcept | |
| 237 | { | ||
| 238 | 237 | std::lock_guard<std::mutex> exit_lock(m_gate->mutex); | |
| 239 | 237 | m_gate->teardown_active = false; | |
| 240 | 237 | m_gate->teardown_owner = 0; | |
| 241 | 237 | m_gate->idle_cv.notify_all(); | |
| 242 | 237 | } | |
| 243 | |||
| 244 | // One claim, cleared exactly once, for the same reason InFlightSlot is neither copyable nor movable. | ||
| 245 | TeardownScope(const TeardownScope &) = delete; | ||
| 246 | TeardownScope &operator=(const TeardownScope &) = delete; | ||
| 247 | TeardownScope(TeardownScope &&) = delete; | ||
| 248 | TeardownScope &operator=(TeardownScope &&) = delete; | ||
| 249 | |||
| 250 | private: | ||
| 251 | HoldGate *m_gate; | ||
| 252 | }; | ||
| 253 | |||
| 254 | /** | ||
| 255 | * @brief Wrapper the poller (and a reshape's release path) invoke on each hold edge; forwards only the | ||
| 256 | * aggregate held/released transition to the user callback, run outside the mutex. | ||
| 257 | */ | ||
| 258 | 9706 | void deliver(bool active) | |
| 259 | { | ||
| 260 | 9706 | std::unique_lock<std::mutex> lock(mutex); | |
| 261 | 9742 | if (released) | |
| 262 | { | ||
| 263 | 3514 | return; | |
| 264 | } | ||
| 265 | 6228 | if (enabled && !enabled->load(std::memory_order_acquire)) | |
| 266 | { | ||
| 267 | ✗ | return; | |
| 268 | } | ||
| 269 | // Serialize this delivery's callback against any other in-flight delivery for this gate, so a true and | ||
| 270 | // a false forwarded from two threads reach the consumer in order. A delivery reached from inside a | ||
| 271 | // callback (depth > 0) skips the wait, so no wait chain runs through user code; the concurrent-delivery | ||
| 272 | // case that skip would otherwise expose is handled by the defer below. | ||
| 273 | 6228 | const bool in_callback = current_thread_in_delivery(); | |
| 274 | 6228 | if (!in_callback) | |
| 275 | { | ||
| 276 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 277 | 6226 | if (before_idle_wait_for_test != nullptr) | |
| 278 | { | ||
| 279 | 2 | before_idle_wait_for_test(before_idle_wait_context_for_test); | |
| 280 | } | ||
| 281 | #endif | ||
| 282 | 12577 | idle_cv.wait(lock, [this] { return in_flight == 0; }); | |
| 283 | 6226 | if (released) | |
| 284 | { | ||
| 285 | ✗ | return; | |
| 286 | } | ||
| 287 | // Recheck every predicate after the wait. A concurrent disable must not wake this delivery into a | ||
| 288 | // stale callback (proof: GateRaceProbe.HoldGateDisableDuringParkedDeliveryEmitsNoStaleCallback). | ||
| 289 | 6226 | if (enabled && !enabled->load(std::memory_order_acquire)) | |
| 290 | { | ||
| 291 | 1 | return; | |
| 292 | } | ||
| 293 | } | ||
| 294 | // Resurrection guard: once the binding is tombstoned, refuse a new held(true) edge; a balancing | ||
| 295 | // released(false) still passes so a removed-while-held binding ends not-held regardless of the order a | ||
| 296 | // late poll-thread true and the reshape's false arrive in. | ||
| 297 | 6227 | if (active && lifecycle && lifecycle->tombstoned()) | |
| 298 | { | ||
| 299 | ✗ | return; | |
| 300 | } | ||
| 301 | |||
| 302 | 6227 | bool crosses_boundary = false; | |
| 303 | 6227 | if (active) | |
| 304 | { | ||
| 305 | 3225 | crosses_boundary = (active_entries == 0); | |
| 306 | 3225 | ++active_entries; | |
| 307 | } | ||
| 308 | else | ||
| 309 | { | ||
| 310 | 3002 | if (active_entries == 0) | |
| 311 | { | ||
| 312 | 5 | return; | |
| 313 | } | ||
| 314 | 2997 | --active_entries; | |
| 315 | 2997 | crosses_boundary = (active_entries == 0); | |
| 316 | } | ||
| 317 | 6222 | if (!crosses_boundary) | |
| 318 | { | ||
| 319 | 4349 | return; | |
| 320 | } | ||
| 321 | |||
| 322 | // A delivery reached from inside a callback could not wait for an in-flight delivery to drain (that | ||
| 323 | // would risk the cross-binding deadlock the skip above avoids). If one is in flight on another thread, | ||
| 324 | // running this callback now would deliver two edges for one gate concurrently and out of decision order | ||
| 325 | // (a teardown false racing the poll thread's held true), which can strand the consumer observing the | ||
| 326 | // stale held. Defer this crossing to the in-flight delivery's unwind instead, so the consumer sees held | ||
| 327 | // then released in order with no concurrent callback. At depth > 0 the crossing edge is always a | ||
| 328 | // teardown false (only the poll cycle raises a held true, and it never runs inside a callback), so | ||
| 329 | // exactly one balancing false is owed and the in-flight delivery emits it. | ||
| 330 | 1873 | if (in_callback && in_flight > 0 && !active) | |
| 331 | { | ||
| 332 | 2 | deferred_final = true; | |
| 333 | 2 | return; | |
| 334 | } | ||
| 335 | |||
| 336 | // Take this thread's delivery identity before the crossing commits to a callback, and give the edge up | ||
| 337 | // if it cannot be taken. Running the callback anyway would leave a control-plane release on another | ||
| 338 | // thread unable to see that this thread is inside consumer code, and that release promises its caller | ||
| 339 | // the opposite. Nothing consumer-visible has happened yet, so refusing here costs one edge; the entry | ||
| 340 | // count taken above is the only state to undo. Constructing under the gate mutex is what makes that | ||
| 341 | // undo a single decrement rather than a re-lock and a three-field rollback. | ||
| 342 | 1871 | DeliveryScope scope; | |
| 343 | 1871 | if (!scope.admitted()) | |
| 344 | { | ||
| 345 | 3 | if (active) | |
| 346 | { | ||
| 347 | 2 | --active_entries; | |
| 348 | } | ||
| 349 | else | ||
| 350 | { | ||
| 351 | 1 | ++active_entries; | |
| 352 | } | ||
| 353 | 3 | return; | |
| 354 | } | ||
| 355 | |||
| 356 | 1868 | forwarded_active = active; | |
| 357 | 1868 | ++in_flight; | |
| 358 | 1868 | lock.unlock(); | |
| 359 | |||
| 360 | 1868 | std::exception_ptr err; | |
| 361 | try | ||
| 362 | { | ||
| 363 | 1868 | if (on_state_change) | |
| 364 | { | ||
| 365 | 1867 | on_state_change(active); | |
| 366 | } | ||
| 367 | } | ||
| 368 | 2 | catch (...) | |
| 369 | { | ||
| 370 | 2 | err = std::current_exception(); | |
| 371 | 2 | } | |
| 372 | |||
| 373 | // Claim deferred_final in the final slot-release critical section. | ||
| 374 | // Separate decisions can let two exits miss the deferred false edge. | ||
| 375 | // Keep the claimed slot through the deferred callback. | ||
| 376 | // A release() or retire() waiter must not observe false quiescence | ||
| 377 | // (proof: GateRaceProbe.HoldGateDeferredFinalClaimSharesLastSlotRelease). | ||
| 378 | 1867 | bool emit_deferred = false; | |
| 379 | { | ||
| 380 | 1867 | std::lock_guard<std::mutex> bookkeeping(mutex); | |
| 381 | 1866 | --in_flight; | |
| 382 | 1866 | if (in_flight == 0 && deferred_final) | |
| 383 | { | ||
| 384 | 387 | deferred_final = false; | |
| 385 | 387 | emit_deferred = forwarded_active; | |
| 386 | 387 | forwarded_active = false; | |
| 387 | 387 | if (emit_deferred) | |
| 388 | { | ||
| 389 | 6 | ++in_flight; | |
| 390 | } | ||
| 391 | } | ||
| 392 | 1866 | if (!emit_deferred) | |
| 393 | { | ||
| 394 | 1859 | idle_cv.notify_all(); | |
| 395 | } | ||
| 396 | 1867 | } | |
| 397 | |||
| 398 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 399 | 1865 | if (after_bookkeeping_for_test != nullptr) | |
| 400 | { | ||
| 401 | 1 | after_bookkeeping_for_test(after_bookkeeping_context_for_test); | |
| 402 | } | ||
| 403 | #endif | ||
| 404 | |||
| 405 | // A release() that could not block (self-release, or cross-binding release from inside another | ||
| 406 | // callback) deferred its balancing false to here; emit it now the callback has unwound. When the | ||
| 407 | // primary callback threw, swallow any secondary throw so the original exception is the one that | ||
| 408 | // surfaces to the poller; otherwise let it propagate to the poller's dispatch handler. | ||
| 409 | 1865 | if (emit_deferred) | |
| 410 | { | ||
| 411 | 6 | InFlightSlot balancing_slot{this}; | |
| 412 | 6 | if (on_state_change) | |
| 413 | { | ||
| 414 | 6 | if (err) | |
| 415 | { | ||
| 416 | try | ||
| 417 | { | ||
| 418 | 1 | on_state_change(false); | |
| 419 | } | ||
| 420 | ✗ | catch (...) | |
| 421 | { | ||
| 422 | ✗ | } | |
| 423 | } | ||
| 424 | else | ||
| 425 | { | ||
| 426 | 5 | on_state_change(false); | |
| 427 | } | ||
| 428 | } | ||
| 429 | 6 | } | |
| 430 | 1865 | if (err) | |
| 431 | { | ||
| 432 | 4 | std::rethrow_exception(err); | |
| 433 | } | ||
| 434 | 9749 | } | |
| 435 | |||
| 436 | /** | ||
| 437 | * @brief Guard teardown: stops further delivery and synthesizes one balancing false if still held. | ||
| 438 | * @details A control-plane release blocks until any in-flight delivery drains, then emits the balancing | ||
| 439 | * false unlocked. A release reached from inside a callback cannot block (it would deadlock), so it | ||
| 440 | * marks the gate released. If this gate has a delivery in flight, the balancing false is deferred | ||
| 441 | * to that delivery's unwind; otherwise it may run inline. A depth-zero return means the gate is | ||
| 442 | * quiesced, so a release that finds another teardown already claiming the gate waits for that one | ||
| 443 | * to finish instead of returning on the strength of its own no-op. | ||
| 444 | */ | ||
| 445 | 651 | void release() | |
| 446 | { | ||
| 447 | 651 | std::unique_lock<std::mutex> lock(mutex); | |
| 448 | 709 | if (released) | |
| 449 | { | ||
| 450 | // The unload drain's retire(), or a repeated direct release, already owns this gate's consumer-code | ||
| 451 | // span and may be inside the consumer's balancing edge right now. Returning here would tell this | ||
| 452 | // caller it may destroy the state that callback captured while the callback is still reading it. | ||
| 453 | // Wait the claimant out. A release reached from inside a delivery must not wait (that is the | ||
| 454 | // ordering discipline at the top of this file) and need not: it is not a boundary where a caller | ||
| 455 | // destroys captured state, and the delivery it is nested in is the very thing a waiter would await. | ||
| 456 | // The claim's own thread is excused for the same reason and without needing the marker, which is | ||
| 457 | // what keeps a refused DeliveryScope from turning a self-release into a wait on this very frame. | ||
| 458 | 23 | if (!current_thread_in_delivery() && teardown_owner != current_native_thread_id()) | |
| 459 | { | ||
| 460 | 29 | idle_cv.wait(lock, [this] { return !teardown_active && in_flight == 0; }); | |
| 461 | } | ||
| 462 | 23 | return; | |
| 463 | } | ||
| 464 | 686 | released = true; | |
| 465 | // Claimed before the wait below drops the mutex. Setting it together with `released` is what closes the | ||
| 466 | // window: any later release() now sees a claim rather than a merely-released gate. | ||
| 467 | 686 | teardown_active = true; | |
| 468 | 686 | teardown_owner = current_native_thread_id(); | |
| 469 | 646 | if (in_flight > 0) | |
| 470 | { | ||
| 471 | 340 | if (current_thread_in_delivery()) | |
| 472 | { | ||
| 473 | // The in-flight delivery emits the balancing false on its own unwind, inside its own in-flight | ||
| 474 | // slot, so the claim ends here and a waiter tracks that slot instead. | ||
| 475 | 325 | deferred_final = true; | |
| 476 | 325 | teardown_active = false; | |
| 477 | 325 | teardown_owner = 0; | |
| 478 | 325 | idle_cv.notify_all(); | |
| 479 | 328 | return; | |
| 480 | } | ||
| 481 | 42 | idle_cv.wait(lock, [this] { return in_flight == 0; }); | |
| 482 | } | ||
| 483 | 321 | const bool emit_false = forwarded_active; | |
| 484 | 321 | forwarded_active = false; | |
| 485 | 321 | if (!emit_false || !on_state_change) | |
| 486 | { | ||
| 487 | 103 | teardown_active = false; | |
| 488 | 103 | teardown_owner = 0; | |
| 489 | 103 | idle_cv.notify_all(); | |
| 490 | 103 | return; | |
| 491 | } | ||
| 492 | 217 | lock.unlock(); | |
| 493 | // Emit the balancing false UNWRAPPED: forwarded_active is already cleared, so a throw here cannot | ||
| 494 | // strand a stale true, and BindingGuard's composed teardown relies on the throw propagating (it runs | ||
| 495 | // its consume-suppression clear even when the balancing edge throws). The noexcept facade release | ||
| 496 | // catches it. The balancing edge is mandatory consumer code, so a MandatoryDeliveryScope brackets it: | ||
| 497 | // a nested release from this callback still defers, and it does so even when the TLS depth store is | ||
| 498 | // refused, which is the composition where two threads each running a cross-gate teardown callback | ||
| 499 | // would otherwise both read as control plane and wait on each other's claim. | ||
| 500 | // The claim outlives the call, including a throw out of it, so retire() cannot move and destroy the | ||
| 501 | // callable mid-invocation and a concurrent release() cannot report the gate quiesced. | ||
| 502 | 217 | TeardownScope teardown_slot{this}; | |
| 503 | 217 | MandatoryDeliveryScope scope; | |
| 504 | 218 | on_state_change(false); | |
| 505 | 674 | } | |
| 506 | |||
| 507 | /** | ||
| 508 | * @copydoc BindingGate::retire | ||
| 509 | * @details Takes the owner under the mutex. The false edge and destruction run on this thread after unlock. | ||
| 510 | */ | ||
| 511 | 24 | [[nodiscard]] bool retire(std::chrono::steady_clock::time_point deadline) override | |
| 512 | { | ||
| 513 | 24 | GateCallbackSlot<void(bool)>::Owner callback; | |
| 514 | 24 | bool emit_false = false; | |
| 515 | { | ||
| 516 | 24 | std::unique_lock<std::mutex> lock(mutex); | |
| 517 | // A concurrent release() runs its balancing edge unlocked under a teardown claim. Retiring through | ||
| 518 | // that claim would move and destroy the callable the other thread is still invoking, so the claim | ||
| 519 | // is part of the quiescence this deadline is waiting for. A claim this thread holds is the one | ||
| 520 | // exception: waiting for it would be waiting for the frame doing the waiting. | ||
| 521 | 24 | if (teardown_owner == current_native_thread_id()) | |
| 522 | { | ||
| 523 | ✗ | return false; | |
| 524 | } | ||
| 525 | 50 | if (!idle_cv.wait_until(lock, deadline, [this] { return in_flight == 0 && !teardown_active; })) | |
| 526 | { | ||
| 527 | 2 | return false; | |
| 528 | } | ||
| 529 | 22 | released = true; | |
| 530 | // Publish inactivity before callback transfer. A retained guard must fail closed during disposal. | ||
| 531 | 22 | if (enabled) | |
| 532 | { | ||
| 533 | 19 | enabled->store(false, std::memory_order_release); | |
| 534 | } | ||
| 535 | 22 | emit_false = forwarded_active; | |
| 536 | 22 | forwarded_active = false; | |
| 537 | 22 | callback = on_state_change.take(); | |
| 538 | 22 | if (callback) | |
| 539 | { | ||
| 540 | 19 | teardown_active = true; | |
| 541 | 19 | teardown_owner = current_native_thread_id(); | |
| 542 | } | ||
| 543 | 24 | } | |
| 544 | |||
| 545 | // Keep invocation and destruction inside both teardown scopes. A capture destructor can release this | ||
| 546 | // gate or another gate. | ||
| 547 | 22 | if (callback) | |
| 548 | { | ||
| 549 | 19 | TeardownScope teardown_slot{this}; | |
| 550 | 19 | MandatoryDeliveryScope scope; | |
| 551 | 19 | GateCallbackSlot<void(bool)>::Owner owned = std::move(callback); | |
| 552 | 19 | if (emit_false) | |
| 553 | { | ||
| 554 | 9 | (*owned)(false); | |
| 555 | } | ||
| 556 | 19 | } | |
| 557 | 22 | return true; | |
| 558 | 24 | } | |
| 559 | }; | ||
| 560 | |||
| 561 | /** | ||
| 562 | * @struct PressGate | ||
| 563 | * @brief Per-binding teardown gate that runs down an in-flight press callback before the guard releases. | ||
| 564 | * @details A press has no lingering state to balance, but shares the hold's teardown hazard: the poll thread | ||
| 565 | * can be executing on_press the instant the guard is released. A control-plane release blocks until | ||
| 566 | * on_press has finished, so the caller may destroy state the callback captured the moment release() | ||
| 567 | * returns. A one-shot press that destroys its own guard, or a callback that releases a second | ||
| 568 | * binding's guard, releases from inside the delivery and so cannot block; it marks the gate released | ||
| 569 | * and returns, and the in-flight callback observes released on its own. | ||
| 570 | */ | ||
| 571 | struct PressGate : BindingGate | ||
| 572 | { | ||
| 573 | std::mutex mutex; | ||
| 574 | std::condition_variable idle_cv; | ||
| 575 | std::shared_ptr<std::atomic<bool>> enabled; | ||
| 576 | std::shared_ptr<BindingLifecycle> lifecycle; | ||
| 577 | GateCallbackSlot<void()> on_press; | ||
| 578 | bool released = false; | ||
| 579 | int in_flight = 0; | ||
| 580 | // Native thread inside retire()'s callable disposal, which is counted in `in_flight` because it runs | ||
| 581 | // consumer destructors. Serves the same purpose as HoldGate::teardown_owner: the same-gate recursion | ||
| 582 | // guard, naming the one claim a waiter must not wait for. | ||
| 583 | std::uint32_t teardown_owner = 0; | ||
| 584 | |||
| 585 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 586 | // See HoldGate::before_idle_wait_for_test for the test-seam lifetime rule. | ||
| 587 | void (*before_idle_wait_for_test)(void *) noexcept = nullptr; | ||
| 588 | void *before_idle_wait_context_for_test = nullptr; | ||
| 589 | #endif | ||
| 590 | |||
| 591 | /** | ||
| 592 | * @brief Wrapper the poller invokes on each press edge; forwards to the user callback outside the mutex. | ||
| 593 | */ | ||
| 594 | 1112 | void deliver() | |
| 595 | { | ||
| 596 | 1112 | std::unique_lock<std::mutex> lock(mutex); | |
| 597 | 1117 | if (released) | |
| 598 | { | ||
| 599 | 257 | return; | |
| 600 | } | ||
| 601 | 860 | if (enabled && !enabled->load(std::memory_order_acquire)) | |
| 602 | { | ||
| 603 | 1 | return; | |
| 604 | } | ||
| 605 | 859 | if (lifecycle && lifecycle->tombstoned()) | |
| 606 | { | ||
| 607 | ✗ | return; | |
| 608 | } | ||
| 609 | 859 | if (!current_thread_in_delivery()) | |
| 610 | { | ||
| 611 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 612 | 859 | if (before_idle_wait_for_test != nullptr) | |
| 613 | { | ||
| 614 | 2 | before_idle_wait_for_test(before_idle_wait_context_for_test); | |
| 615 | } | ||
| 616 | #endif | ||
| 617 | 2111 | idle_cv.wait(lock, [this] { return in_flight == 0; }); | |
| 618 | 859 | if (released) | |
| 619 | { | ||
| 620 | 7 | return; | |
| 621 | } | ||
| 622 | // Recheck enabled after the wait | ||
| 623 | // (proof: GateRaceProbe.PressGateDisableDuringParkedDeliveryEmitsNoStaleCallback). | ||
| 624 | 852 | if (enabled && !enabled->load(std::memory_order_acquire)) | |
| 625 | { | ||
| 626 | 1 | return; | |
| 627 | } | ||
| 628 | 851 | if (lifecycle && lifecycle->tombstoned()) | |
| 629 | { | ||
| 630 | ✗ | return; | |
| 631 | } | ||
| 632 | } | ||
| 633 | // Same admission rule as HoldGate::deliver, and cheaper to undo: a press commits no gate state before | ||
| 634 | // this point, so a refused frame simply drops the edge. | ||
| 635 | 851 | DeliveryScope scope; | |
| 636 | 851 | if (!scope.admitted()) | |
| 637 | { | ||
| 638 | 2 | return; | |
| 639 | } | ||
| 640 | 849 | ++in_flight; | |
| 641 | 849 | lock.unlock(); | |
| 642 | |||
| 643 | 849 | std::exception_ptr err; | |
| 644 | try | ||
| 645 | { | ||
| 646 | 849 | if (on_press) | |
| 647 | { | ||
| 648 | 849 | on_press(); | |
| 649 | } | ||
| 650 | } | ||
| 651 | ✗ | catch (...) | |
| 652 | { | ||
| 653 | ✗ | err = std::current_exception(); | |
| 654 | ✗ | } | |
| 655 | |||
| 656 | 849 | lock.lock(); | |
| 657 | 849 | --in_flight; | |
| 658 | 849 | idle_cv.notify_all(); | |
| 659 | 849 | lock.unlock(); | |
| 660 | 847 | if (err) | |
| 661 | { | ||
| 662 | ✗ | std::rethrow_exception(err); | |
| 663 | } | ||
| 664 | 1118 | } | |
| 665 | |||
| 666 | /** | ||
| 667 | * @brief Guard teardown: marks the gate released and, off any callback, waits out any in-flight on_press. | ||
| 668 | */ | ||
| 669 | 1200 | void release() | |
| 670 | { | ||
| 671 | 1200 | std::unique_lock<std::mutex> lock(mutex); | |
| 672 | 1201 | released = true; | |
| 673 | 1201 | if (in_flight > 0 && !current_thread_in_delivery() && teardown_owner != current_native_thread_id()) | |
| 674 | { | ||
| 675 | 9 | idle_cv.wait(lock, [this] { return in_flight == 0; }); | |
| 676 | } | ||
| 677 | 1201 | } | |
| 678 | |||
| 679 | /** | ||
| 680 | * @copydoc BindingGate::retire | ||
| 681 | * @details A press has no lingering edge to balance, so retirement is the run-down plus handing the | ||
| 682 | * callback to this thread to destroy. Destruction happens unlocked: it runs the consumer's | ||
| 683 | * captured destructors, which must not observe the gate mutex held. It is still counted, because | ||
| 684 | * a concurrent release() promises its caller a quiesced gate and those destructors are consumer | ||
| 685 | * (Logic-DLL) code. | ||
| 686 | */ | ||
| 687 | 339 | [[nodiscard]] bool retire(std::chrono::steady_clock::time_point deadline) override | |
| 688 | { | ||
| 689 | 339 | GateCallbackSlot<void()>::Owner callback; | |
| 690 | { | ||
| 691 | 339 | std::unique_lock<std::mutex> lock(mutex); | |
| 692 | // Retiring from inside this gate's own disposal would wait on the count that disposal holds. | ||
| 693 | 339 | if (teardown_owner == current_native_thread_id()) | |
| 694 | { | ||
| 695 | ✗ | return false; | |
| 696 | } | ||
| 697 | 686 | if (!idle_cv.wait_until(lock, deadline, [this] { return in_flight == 0; })) | |
| 698 | { | ||
| 699 | 5 | return false; | |
| 700 | } | ||
| 701 | 334 | released = true; | |
| 702 | // Publish inactivity before callback transfer, as HoldGate::retire does. | ||
| 703 | 334 | if (enabled) | |
| 704 | { | ||
| 705 | 329 | enabled->store(false, std::memory_order_release); | |
| 706 | } | ||
| 707 | 333 | callback = on_press.take(); | |
| 708 | 334 | if (callback) | |
| 709 | { | ||
| 710 | 310 | ++in_flight; | |
| 711 | 310 | teardown_owner = current_native_thread_id(); | |
| 712 | } | ||
| 713 | 338 | } | |
| 714 | |||
| 715 | 334 | if (callback) | |
| 716 | { | ||
| 717 | // A capture destructor can release a gate, so keep disposal inside mandatory delivery identity. | ||
| 718 | 311 | MandatoryDeliveryScope scope; | |
| 719 | 311 | callback = nullptr; | |
| 720 | 311 | std::lock_guard<std::mutex> exit_lock(mutex); | |
| 721 | 311 | --in_flight; | |
| 722 | 311 | teardown_owner = 0; | |
| 723 | 311 | idle_cv.notify_all(); | |
| 724 | 311 | } | |
| 725 | 334 | return true; | |
| 726 | 339 | } | |
| 727 | }; | ||
| 728 | } // namespace detail | ||
| 729 | } // namespace DetourModKit | ||
| 730 | |||
| 731 | #endif // DETOURMODKIT_INTERNAL_INPUT_BINDING_GATE_HPP | ||
| 732 |