GCC Code Coverage Report

./
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
685 / 752
91.1%
Branches:
375 / 524
71.6%
Functions:
66 / 67
98.5%
src/memory_cache.cpp
Line Branch Exec Source
1 /**
2 * @file memory_cache.cpp
3 * @brief This TU implements the protection-region cache and its readability predicates.
4 *
5 * The sharded cache uses SRW locks. It provides FIFO eviction and exact invalidation without SEH.
6 * Shutdown closes reader admission and drains admitted readers before a deadline. A timeout retains the storage.
7 *
8 * This TU controls the MinGW guarded-engine lifecycle.
9 */
10
11 #include "DetourModKit/memory.hpp"
12 #include "DetourModKit/diagnostics.hpp"
13 #include "DetourModKit/logger.hpp"
14 #include "internal/diagnostics_population.hpp"
15 #include "internal/drain_backoff.hpp"
16 #include "internal/lifecycle_context.hpp"
17 #include "internal/srw_shared_mutex.hpp"
18 #include "platform.hpp"
19 #include "internal/memory_guarded.hpp"
20
21 #include <windows.h>
22
23 #include <algorithm>
24 #include <array>
25 #include <atomic>
26 #include <cassert>
27 #include <chrono>
28 #include <condition_variable>
29 #include <cstddef>
30 #include <cstdint>
31 #include <cstdlib>
32 #include <deque>
33 #include <iomanip>
34 #include <map>
35 #include <memory>
36 #include <new>
37 #include <mutex>
38 #include <shared_mutex>
39 #include <sstream>
40 #include <string>
41 #include <thread>
42 #include <type_traits>
43 #include <unordered_map>
44
45 #if defined(DMK_ENABLE_TEST_SEAMS)
46 namespace DetourModKit::detail
47 {
48 void (*g_memory_cache_before_lifecycle_lock_test_hook)() = nullptr;
49 void (*g_memory_cache_before_running_publish_test_hook)() = nullptr;
50 void (*g_memory_cache_reopen_window_test_hook)() = nullptr;
51 void (*g_memory_cache_shutdown_window_test_hook)() = nullptr;
52 void (*g_memory_cache_leader_publish_window_test_hook)() = nullptr;
53 HMODULE (*g_memory_cache_keepalive_ref_override)() noexcept = nullptr;
54 } // namespace DetourModKit::detail
55 #endif
56
57 namespace DetourModKit
58 {
59 namespace memory
60 {
61 using DetourModKit::detail::acquire_module_ref;
62 using DetourModKit::detail::release_module_ref;
63 using DetourModKit::detail::SrwSharedMutex;
64
65 namespace
66 {
67 // CachePermissions groups page-protection flags for cache permission checks. A struct preserves internal
68 // linkage through the outer anonymous namespace without a named namespace.
69 struct CachePermissions
70 {
71 static constexpr DWORD READ_PERMISSION_FLAGS = PAGE_READONLY | PAGE_READWRITE | PAGE_WRITECOPY |
72 PAGE_EXECUTE_READ | PAGE_EXECUTE_READWRITE |
73 PAGE_EXECUTE_WRITECOPY;
74 static constexpr DWORD WRITE_PERMISSION_FLAGS =
75 PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY;
76 static constexpr DWORD NOACCESS_GUARD_FLAGS = PAGE_NOACCESS | PAGE_GUARD;
77 };
78
79 /**
80 * @struct CachedMemoryRegionInfo
81 * @brief Stores a cached protection snapshot for one VirtualQuery region.
82 * @details A hit requires content_gen to equal the shard's current generation. A clear or contended
83 * invalidation therefore invalidates this entry in O(1), regardless of physical eviction.
84 */
85 struct CachedMemoryRegionInfo
86 {
87 std::uintptr_t base_address;
88 std::size_t region_size;
89 DWORD protection;
90 DWORD state;
91 std::uint64_t timestamp_ns;
92 std::uint64_t fifo_key;
93 std::uint64_t content_gen;
94 bool valid;
95
96 441 CachedMemoryRegionInfo()
97 441 : base_address(0), region_size(0), protection(0), state(0), timestamp_ns(0), fifo_key(0),
98 441 content_gen(0), valid(false)
99 {
100 441 }
101 };
102
103 /**
104 * @struct CacheShard
105 * @brief Stores one shard with a hit map, FIFO map, sorted range index, and content generation.
106 * @details The unordered_map is keyed by region base, so its O(1) hit covers only a query in a region's
107 * FIRST page. Deeper queries use the sorted range index. The std::map keyed by a
108 * monotonic counter gives oldest-first eviction. alignas(64) aligns each shard object to a 64-byte
109 * boundary. The inline mutex makes the shard non-movable, so shards use a fixed-size array that
110 * never relocates.
111 */
112 #if defined(_MSC_VER)
113 #pragma warning(push)
114 // C4324: CacheShard is intentionally padded to a full cache line by alignas(64) for cache-line hygiene.
115 #pragma warning(disable : 4324)
116 #endif
117 struct alignas(64) CacheShard
118 {
119 std::unordered_map<std::uintptr_t, CachedMemoryRegionInfo> entries;
120 std::map<std::uint64_t, std::uintptr_t> fifo_index;
121 // The shard SRW lock serializes this base-sorted O(log n) containment index. The deque prevents full
122 // buffer relocation during growth. Each entry stores [base, base + size).
123 std::deque<std::pair<std::uintptr_t, std::uintptr_t>> sorted_ranges;
124 SrwSharedMutex mtx;
125 // The first thread to CAS this 0 -> 1 becomes the VirtualQuery leader. The rest coalesce onto its
126 // result.
127 std::atomic<char> in_flight{0};
128 // Content generation provides exact invalidation. clear_cache advances it under the exclusive lock.
129 // A contended invalidate_range advances it through fetch_add. A hit requires a current entry stamp.
130 std::atomic<std::uint64_t> content_gen{0};
131 // The query thread already touches this per-shard hit and miss counter line. A busy workload therefore
132 // does not move one global counter line across every core.
133 std::atomic<std::uint64_t> hits{0};
134 std::atomic<std::uint64_t> misses{0};
135 std::uint64_t entry_counter{0};
136 std::size_t capacity;
137 std::size_t max_capacity;
138
139
2/4
✓ Branch 4 → 5 taken 11415 times.
✗ Branch 4 → 14 not taken.
✓ Branch 10 → 11 taken 11415 times.
✗ Branch 10 → 12 not taken.
 11415 CacheShard() : capacity(0), max_capacity(0) { entries.reserve(64); }
140 };
141 #if defined(_MSC_VER)
142 #pragma warning(pop)
143 #endif
144
145 static_assert(
146 std::is_same_v<
147 decltype(CacheShard::sorted_ranges),
148 std::deque<std::pair<std::uintptr_t, std::uintptr_t>>>,
149 "CacheShard::sorted_ranges is pinned to std::deque so mutation never relocates the buffer."
150 );
151
152 120702 inline std::uint64_t current_time_ns() noexcept
153 {
154 126214 return std::chrono::duration_cast<std::chrono::nanoseconds>(
155 246812 std::chrono::steady_clock::now().time_since_epoch()
156 )
157 126155 .count();
158 }
159
160 /**
161 * @brief Computes the shard index for an address.
162 * @note Uses a golden-ratio hash to spread adjacent addresses across shards.
163 */
164 117827 constexpr inline std::size_t compute_shard_index(std::uintptr_t address, std::size_t shard_count) noexcept
165 {
166 117827 return (static_cast<std::size_t>((address * 0x9E3779B97F4A7C15ULL) >> 48)) % shard_count;
167 }
168
169 // The fixed-size shard array never resizes because CacheShard is non-movable. It is null before init.
170 // A drained shutdown clears it. Abandonment or a timeout retains it until a later drain.
171 std::unique_ptr<CacheShard[]> s_cache_shards;
172 std::atomic<std::size_t> s_shard_count{0};
173 std::atomic<std::size_t> s_max_entries_per_shard{0};
174 std::atomic<unsigned int> s_configured_expiry_ms{0};
175
176 /**
177 * @enum LifecycleState
178 * @brief Defines the cache lifecycle authority: Stopped -> Starting -> Running -> Stopping -> Stopped.
179 * @details Running is published last by init_cache and cleared first by shutdown_cache. The per-stripe
180 * closed-bit words decide reader admission. This state directs control paths. Normal transitions
181 * use s_lifecycle_mutex. Loader-lock abandonment uses compare-exchange because it cannot wait for
182 * that mutex.
183 */
184 enum class LifecycleState : std::uint8_t
185 {
186 Stopped,
187 Starting,
188 Running,
189 Stopping
190 };
191 std::atomic<LifecycleState> s_lifecycle_state{LifecycleState::Stopped};
192
193 /// Reports whether the cache lifecycle is in its Running state.
194 152966 [[nodiscard]] inline bool cache_is_running() noexcept
195 {
196 152966 return s_lifecycle_state.load(std::memory_order_seq_cst) == LifecycleState::Running;
197 }
198
199 /// Returns the configured cache-entry expiry in nanoseconds.
200 122348 [[nodiscard]] inline std::uint64_t configured_expiry_ns() noexcept
201 {
202 122762 return static_cast<std::uint64_t>(s_configured_expiry_ms.load(std::memory_order_acquire)) *
203 122762 1'000'000ULL;
204 }
205
206 // Serializes shard-array mutation against cleanup. It nests inside s_lifecycle_mutex on init/shutdown.
207 SrwSharedMutex s_cache_state_mutex;
208
209 // This is the outermost lifecycle lock. Cleanup and readers never take it, and loader-lock teardown never
210 // waits on it. Lock order is lifecycle -> state -> shard. The join lock nests only inside lifecycle.
211 SrwSharedMutex s_lifecycle_mutex;
212
213 // Advances on every admitted start. The cleanup thread exits once its captured value no longer matches,
214 // so a worker cannot outlive its session and touch a later generation's shards.
215 std::atomic<std::uint64_t> s_lifecycle_generation{0};
216
217 // This sticky counter records unexpected joinable handles recovered before a new cleanup worker appears.
218 std::atomic<std::uint64_t> s_lifecycle_violations{0};
219
220 // Reader admission uses the [B-73] closed-drain pattern. Each stripe word holds a high closed bit and an
221 // admitted-reader count. One compare-exchange checks the closed bit and increments the count. Teardown sets
222 // the closed bit on every stripe before it reads the counts, so it drains a closed population.
223 // Cache-line-padded stripes keep concurrent readers off one shared line (measured in the phase 9 warm-hit
224 // contention benchmark).
225 constexpr std::size_t READER_STRIPE_COUNT = 64;
226 constexpr std::uint64_t READER_ADMISSION_CLOSED = std::uint64_t{1} << 63;
227 constexpr std::uint64_t READER_ADMISSION_COUNT_MASK = ~READER_ADMISSION_CLOSED;
228
229 #if defined(_MSC_VER)
230 #pragma warning(push)
231 // C4324: ReaderStripe is intentionally padded to a full cache line by alignas(64) so stripes never share a line.
232 #pragma warning(disable : 4324)
233 #endif
234 struct alignas(64) ReaderStripe
235 {
236 // Starts closed: admission opens only after init_cache publishes a running cache.
237 std::atomic<std::uint64_t> word{READER_ADMISSION_CLOSED};
238 };
239 #if defined(_MSC_VER)
240 #pragma warning(pop)
241 #endif
242 static_assert(std::atomic<std::uint64_t>::is_always_lock_free);
243
244 std::array<ReaderStripe, READER_STRIPE_COUNT> s_reader_stripes{};
245
246 /**
247 * @brief Returns this thread's admission stripe, derived from its Win32 thread id.
248 * @details A hash of GetCurrentThreadId allocates nothing and takes no lock, so loader lock permits it.
249 * In contrast, MinGW lowers first access to a thread_local counter into __emutls_get_address,
250 * which can allocate. The id is stable for the thread's life, so the same stripe carries the
251 * admission and its paired release. A collision adds contention but never a drain miscount.
252 */
253 33444603 [[nodiscard]] inline std::size_t reader_stripe_index() noexcept
254 {
255 33444603 const std::uint64_t mixed = static_cast<std::uint64_t>(GetCurrentThreadId()) * 0x9E3779B97F4A7C15ULL;
256 34081482 return static_cast<std::size_t>(mixed >> 48) % READER_STRIPE_COUNT;
257 }
258
259 /// Returns the admitted-reader count summed across all stripes.
260 1618 [[nodiscard]] inline std::uint64_t admitted_reader_count() noexcept
261 {
262 1618 std::uint64_t total = 0;
263
2/2
✓ Branch 11 → 3 taken 103552 times.
✓ Branch 11 → 12 taken 1618 times.
 105170 for (const ReaderStripe &stripe : s_reader_stripes)
264 {
265 207104 total += stripe.word.load(std::memory_order_seq_cst) & READER_ADMISSION_COUNT_MASK;
266 }
267 1618 return total;
268 }
269
270 /**
271 * @brief Closes reader admission on every stripe.
272 * @details After the last fetch_or returns, no stripe admits a reader, so a later drain waits on a closed
273 * population. Each fetch_or is wait-free, so loader-lock abandonment can call this.
274 */
275 1495 void close_reader_admission() noexcept
276 {
277
2/2
✓ Branch 6 → 3 taken 95680 times.
✓ Branch 6 → 7 taken 1495 times.
 97175 for (ReaderStripe &stripe : s_reader_stripes)
278 {
279 95680 stripe.word.fetch_or(READER_ADMISSION_CLOSED, std::memory_order_seq_cst);
280 }
281 1495 }
282
283 /**
284 * @brief Reopens reader admission after a drained start from each exact closed and zero-reader value.
285 * @details While a stripe is closed, its count only decreases. The caller drains every count under
286 * s_cache_state_mutex. Init opens all stripes before it publishes Running. Concurrent abandonment
287 * closes them and changes Starting to Stopped. The publication check then rolls init back.
288 */
289 740 void reopen_reader_admission() noexcept
290 {
291
2/2
✓ Branch 15 → 3 taken 47360 times.
✓ Branch 15 → 16 taken 740 times.
 96200 for (std::size_t i = 0; i < s_reader_stripes.size(); ++i)
292 {
293 47360 ReaderStripe &stripe = s_reader_stripes[i];
294 47360 std::uint64_t expected = READER_ADMISSION_CLOSED;
295 (void)stripe.word
296 47360 .compare_exchange_strong(expected, 0, std::memory_order_seq_cst, std::memory_order_seq_cst);
297 #if defined(DMK_ENABLE_TEST_SEAMS)
298
2/2
✓ Branch 9 → 10 taken 740 times.
✓ Branch 9 → 12 taken 46620 times.
 47360 if (i == 0)
299 {
300
2/2
✓ Branch 10 → 11 taken 1 time.
✓ Branch 10 → 12 taken 739 times.
 740 if (auto *const hook = DetourModKit::detail::g_memory_cache_reopen_window_test_hook)
301 1 hook();
302 }
303 #endif
304 }
305 740 }
306
307 /// Bounds every reader drain. Expiry retains the reader-visible storage instead of a free ([B-73]).
308 constexpr auto READER_DRAIN_TIMEOUT = std::chrono::seconds{1};
309
310 /// Waits for the admitted-reader count to reach zero within READER_DRAIN_TIMEOUT.
311 1485 [[nodiscard]] bool drain_admitted_readers() noexcept
312 {
313 1485 return DetourModKit::detail::drain_until_zero(
314 1613 []() noexcept { return admitted_reader_count(); },
315 2970 std::chrono::steady_clock::now() + READER_DRAIN_TIMEOUT
316 1485 );
317 }
318
319 // A cache session takes this reference before admission opens. A timeout retains it with the shard array.
320 // Guarded by s_cache_state_mutex.
321 HMODULE s_cache_self_ref{nullptr};
322
323 /// Takes the keepalive that makes a later bounded-drain timeout safe.
324 745 [[nodiscard]] HMODULE acquire_cache_keepalive_ref() noexcept
325 {
326 #if defined(DMK_ENABLE_TEST_SEAMS)
327
2/2
✓ Branch 2 → 3 taken 1 time.
✓ Branch 2 → 4 taken 744 times.
 745 if (auto *const override_fn = DetourModKit::detail::g_memory_cache_keepalive_ref_override)
328 1 return override_fn();
329 #endif
330 744 return acquire_module_ref(diagnostics::ModulePinReason::MemoryCache);
331 }
332
333 /**
334 * @brief Releases the cache keepalive after the admitted reader count reaches zero.
335 * @note
336 * Must be called with s_cache_state_mutex held.
337 */
338 743 void release_cache_keepalive_after_drain() noexcept
339 {
340
1/2
✓ Branch 2 → 3 taken 743 times.
✗ Branch 2 → 5 not taken.
 743 if (s_cache_self_ref != nullptr)
341 {
342 743 release_module_ref(s_cache_self_ref, diagnostics::ModulePinReason::MemoryCache);
343 743 s_cache_self_ref = nullptr;
344 }
345 743 }
346
347 /**
348 * @brief Records one timeout that retains the cache keepalive and reader-visible state.
349 *
350 * @note Must be called with s_cache_state_mutex held.
351 */
352 1 void record_reader_drain_timeout_retention() noexcept
353 {
354
1/2
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 1 time.
 1 assert(s_cache_self_ref != nullptr);
355 1 DetourModKit::diagnostics::record_intentional_leak(
356 DetourModKit::diagnostics::LeakSubsystem::MemoryCache
357 );
358 1 }
359
360 /**
361 * @class ReaderAdmission
362 * @brief Admits one reader through its stripe's closed-bit word and releases it on every exit path.
363 * @details A rejected admission leaves the count untouched, so the caller never joins the teardown drain
364 * population and must take the uncached route.
365 */
366 class ReaderAdmission
367 {
368 public:
369 33646240 ReaderAdmission() noexcept : m_stripe(reader_stripe_index())
370 {
371 33705581 std::atomic<std::uint64_t> &word = s_reader_stripes[m_stripe].word;
372 33148589 std::uint64_t value = word.load(std::memory_order_seq_cst);
373
2/2
✓ Branch 19 → 12 taken 142338 times.
✓ Branch 19 → 20 taken 32976988 times.
 33119326 while ((value & READER_ADMISSION_CLOSED) == 0)
374 {
375
2/2
✓ Branch 17 → 18 taken 153866 times.
✓ Branch 17 → 19 taken 1763 times.
 297967 if (word.compare_exchange_weak(
376 value,
377 value + 1,
378 std::memory_order_seq_cst,
379 std::memory_order_seq_cst
380 ))
381 {
382 153866 m_admitted = true;
383 153866 break;
384 }
385 }
386 33130854 }
387
388 33407497 ~ReaderAdmission() noexcept
389 {
390
2/2
✓ Branch 2 → 3 taken 150641 times.
✓ Branch 2 → 7 taken 33256856 times.
 33407497 if (m_admitted)
391 {
392 150641 s_reader_stripes[m_stripe].word.fetch_sub(1, std::memory_order_release);
393 }
394 33405723 }
395
396 32646298 [[nodiscard]] bool admitted() const noexcept { return m_admitted; }
397
398 ReaderAdmission(const ReaderAdmission &) = delete;
399 ReaderAdmission &operator=(const ReaderAdmission &) = delete;
400
401 private:
402 const std::size_t m_stripe;
403 bool m_admitted{false};
404 };
405
406 std::atomic<bool> s_cleanup_thread_running{false};
407
408 /**
409 * @brief The cleanup thread handle in never-destroyed storage (`[B-47]`).
410 * @details A process exit during Starting or Stopping leaves the handle joinable, and a static destructor
411 * over a joinable std::thread terminates the exiting process. shutdown_cache joins by hand.
412 * Lifecycle.CacheShutdownRacesProcessExitFromSecondThread pins the exit.
413 */
414 4459 [[nodiscard]] std::thread &cleanup_thread() noexcept
415 {
416 alignas(std::thread) static unsigned char storage[sizeof(std::thread)];
417
4/6
✓ Branch 2 → 3 taken 456 times.
✓ Branch 2 → 10 taken 4003 times.
✓ Branch 4 → 5 taken 456 times.
✗ Branch 4 → 10 not taken.
✗ Branch 8 → 9 not taken.
✓ Branch 8 → 10 taken 456 times.
 4459 static std::thread *const thread = ::new (static_cast<void *>(storage)) std::thread();
418 4459 return *thread;
419 }
420
421 // s_cleanup_self_ref holds a counted module reference acquired before thread creation. A clean join
422 // releases it. The loader-lock detach path leaks it so the detached thread's code stays mapped.
423 HMODULE s_cleanup_self_ref{nullptr};
424 std::mutex s_cleanup_mutex;
425 std::condition_variable s_cleanup_cv;
426 std::atomic<bool> s_cleanup_requested{false};
427 // Serializes the cleanup handle's join/detach/assignment. Loader-lock teardown only tries this lock.
428 SrwSharedMutex s_cleanup_join_mutex;
429
430 // This timer controls on-demand cleanup when the background thread is disabled.
431 std::atomic<std::uint64_t> s_last_cleanup_time_ns{0};
432 constexpr std::uint64_t CLEANUP_INTERVAL_NS = 1'000'000'000ULL;
433
434 // These instance-wide COLD counters advance only off the read hot path. The hot hit and miss tallies live
435 // per shard. Each counter is alignas(64), so the three never share a cache line.
436 #if defined(_MSC_VER)
437 #pragma warning(push)
438 // C4324: each counter is intentionally padded to a full cache line by alignas(64) to prevent line overlap.
439 #pragma warning(disable : 4324)
440 #endif
441 struct CacheStats
442 {
443 alignas(64) std::atomic<std::uint64_t> invalidations{0};
444 alignas(64) std::atomic<std::uint64_t> coalesced_queries{0};
445 alignas(64) std::atomic<std::uint64_t> on_demand_cleanups{0};
446 };
447 #if defined(_MSC_VER)
448 #pragma warning(pop)
449 #endif
450 CacheStats s_stats;
451
452 /**
453 * @brief Checks if a cache entry is valid, current for the shard generation, and covers [address, address +
454 * size).
455 * @param shard_content_gen The shard's current content generation. A clear or contended invalidation
456 * advances it. An older stamp marks the entry invalid, so lookup returns a miss.
457 */
458 118345 constexpr inline bool is_entry_valid_and_covers(
459 const CachedMemoryRegionInfo &entry,
460 std::uintptr_t address,
461 std::size_t size,
462 std::uint64_t current_ns,
463 std::uint64_t expiry_ns,
464 std::uint64_t shard_content_gen
465 ) noexcept
466 {
467
1/2
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 118345 times.
 118345 if (!entry.valid)
468 ✗ return false;
469
470
2/2
✓ Branch 4 → 5 taken 191 times.
✓ Branch 4 → 6 taken 118154 times.
 118345 if (entry.content_gen != shard_content_gen)
471 191 return false;
472
473 118154 const std::uint64_t entry_age = current_ns - entry.timestamp_ns;
474
2/2
✓ Branch 6 → 7 taken 12 times.
✓ Branch 6 → 8 taken 118142 times.
 118154 if (entry_age > expiry_ns)
475 12 return false;
476
477 118142 const std::uintptr_t end_address = address + size;
478
2/2
✓ Branch 8 → 9 taken 4 times.
✓ Branch 8 → 10 taken 118138 times.
 118142 if (end_address < address)
479 4 return false;
480
481 118138 const std::uintptr_t entry_end_address = entry.base_address + entry.region_size;
482
1/2
✗ Branch 10 → 11 not taken.
✓ Branch 10 → 12 taken 118138 times.
 118138 if (entry_end_address < entry.base_address)
483 ✗ return false;
484
485
2/4
✓ Branch 12 → 13 taken 120113 times.
✗ Branch 12 → 15 not taken.
✓ Branch 13 → 14 taken 120222 times.
✗ Branch 13 → 15 not taken.
 118138 return address >= entry.base_address && end_address <= entry_end_address;
486 }
487
488 /// Checks protection flags for read permission.
489 235418 constexpr inline bool check_read_permission(DWORD protection) noexcept
490 {
491
1/2
✓ Branch 2 → 3 taken 236843 times.
✗ Branch 2 → 5 not taken.
 472261 return (protection & CachePermissions::READ_PERMISSION_FLAGS) != 0 &&
492
1/2
✓ Branch 3 → 4 taken 237161 times.
✗ Branch 3 → 5 not taken.
 472261 (protection & CachePermissions::NOACCESS_GUARD_FLAGS) == 0;
493 }
494
495 /// Checks protection flags for write permission.
496 3928 constexpr inline bool check_write_permission(DWORD protection) noexcept
497 {
498
1/2
✓ Branch 2 → 3 taken 3929 times.
✗ Branch 2 → 5 not taken.
 7857 return (protection & CachePermissions::WRITE_PERMISSION_FLAGS) != 0 &&
499
2/2
✓ Branch 3 → 4 taken 3901 times.
✓ Branch 3 → 5 taken 28 times.
 7857 (protection & CachePermissions::NOACCESS_GUARD_FLAGS) == 0;
500 }
501
502 /**
503 * @brief Inserts a range into the shard's sorted auxiliary container.
504 * @note Must be called with the shard mutex held (exclusive). This helper is deliberately not noexcept
505 * because bad_alloc must reach update_shard_with_region's fail-soft catch.
506 */
507 431 void insert_sorted_range(CacheShard &shard, std::uintptr_t base_addr, std::size_t region_size)
508 {
509 431 auto range = std::make_pair(base_addr, base_addr + region_size);
510
1/2
✓ Branch 5 → 6 taken 431 times.
✗ Branch 5 → 9 not taken.
 430 auto pos = std::lower_bound(shard.sorted_ranges.begin(), shard.sorted_ranges.end(), range);
511
2/2
✓ Branch 7 → 8 taken 429 times.
✓ Branch 7 → 11 taken 2 times.
 431 shard.sorted_ranges.insert(pos, range);
512 429 }
513
514 /**
515 * @brief Removes a range from the shard's sorted auxiliary container.
516 * @note Must be called with the shard mutex held (exclusive).
517 */
518 58 void remove_sorted_range(CacheShard &shard, std::uintptr_t base_addr) noexcept
519 {
520 auto it = std::lower_bound(
521 58 shard.sorted_ranges.begin(),
522 58 shard.sorted_ranges.end(),
523 58 std::make_pair(base_addr, std::uintptr_t{0})
524 58 );
525
3/6
✓ Branch 8 → 9 taken 58 times.
✗ Branch 8 → 12 not taken.
✓ Branch 10 → 11 taken 58 times.
✗ Branch 10 → 12 not taken.
✓ Branch 13 → 14 taken 58 times.
✗ Branch 13 → 17 not taken.
 58 if (it != shard.sorted_ranges.end() && it->first == base_addr)
526 58 shard.sorted_ranges.erase(it);
527 58 }
528
529 /**
530 * @brief Finds and validates a cache entry in a shard that covers [address, address + size).
531 * @note Must be called with the shard mutex held (shared or exclusive).
532 * @note The shard-local direct probe is a first-page fast path because the map key is the region base.
533 * Deeper queries use the O(log n) search over sorted_ranges. A miss returns nullptr. The caller then
534 * queries through VirtualQuery and inserts the result. There is deliberately no
535 * per-page index. The per-shard entry count is small and bounded.
536 */
537 125839 CachedMemoryRegionInfo *find_in_shard(
538 CacheShard &shard,
539 std::uintptr_t address,
540 std::size_t size,
541 std::uint64_t current_ns,
542 std::uint64_t expiry_ns
543 ) noexcept
544 {
545 // One acquire load reads the shard generation for both tiers. An entry with an older stamp is skipped.
546 // Thus a lost physical eviction or a clear that races a leader cannot serve a stale hit.
547 125839 const std::uint64_t shard_content_gen = shard.content_gen.load(std::memory_order_acquire);
548
549 125801 const std::uintptr_t base_addr = address & ~static_cast<std::uintptr_t>(0xFFF);
550 125801 auto it = shard.entries.find(base_addr);
551
2/2
✓ Branch 12 → 13 taken 119814 times.
✓ Branch 12 → 17 taken 101 times.
 121060 if (it != shard.entries.end())
552 {
553 119814 CachedMemoryRegionInfo &entry = it->second;
554
1/2
✓ Branch 15 → 16 taken 119871 times.
✗ Branch 15 → 17 not taken.
 119102 if (is_entry_valid_and_covers(entry, address, size, current_ns, expiry_ns, shard_content_gen))
555 {
556 119871 return &entry;
557 }
558 }
559
560 auto range_it = std::upper_bound(
561 681 shard.sorted_ranges.begin(),
562 682 shard.sorted_ranges.end(),
563 ✗ std::make_pair(address, UINTPTR_MAX)
564 682 );
565
2/2
✓ Branch 23 → 24 taken 297 times.
✓ Branch 23 → 42 taken 377 times.
 674 if (range_it != shard.sorted_ranges.begin())
566 {
567 297 --range_it;
568
6/6
✓ Branch 26 → 27 taken 296 times.
✓ Branch 26 → 30 taken 4 times.
✓ Branch 28 → 29 taken 229 times.
✓ Branch 28 → 30 taken 66 times.
✓ Branch 31 → 32 taken 229 times.
✓ Branch 31 → 42 taken 70 times.
 296 if (address >= range_it->first && address < range_it->second)
569 {
570 229 auto entry_it = shard.entries.find(range_it->first);
571
1/2
✓ Branch 36 → 37 taken 231 times.
✗ Branch 36 → 41 not taken.
 227 if (entry_it != shard.entries.end())
572 {
573 231 CachedMemoryRegionInfo &entry = entry_it->second;
574
2/2
✓ Branch 39 → 40 taken 125 times.
✓ Branch 39 → 41 taken 107 times.
 228 if (is_entry_valid_and_covers(
575 entry,
576 address,
577 size,
578 current_ns,
579 expiry_ns,
580 shard_content_gen
581 ))
582 {
583 125 return &entry;
584 }
585 }
586 }
587 }
588
589 552 return nullptr;
590 }
591
592 /**
593 * @brief Evicts the oldest shard entry through an O(log n) FIFO lookup.
594 * @note Must be called with the shard mutex held (exclusive).
595 * @return true if an entry was evicted, false if the shard is empty.
596 */
597 45 bool evict_oldest_entry(CacheShard &shard) noexcept
598 {
599
1/2
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 5 taken 45 times.
 45 if (shard.fifo_index.empty())
600 ✗ return false;
601
602 45 const auto fifo_it = shard.fifo_index.begin();
603 45 const std::uintptr_t oldest_base = fifo_it->second;
604
605 45 shard.fifo_index.erase(fifo_it);
606
607 45 const auto entry_it = shard.entries.find(oldest_base);
608
1/2
✓ Branch 11 → 12 taken 45 times.
✗ Branch 11 → 15 not taken.
 45 if (entry_it != shard.entries.end())
609 {
610 45 shard.entries.erase(entry_it);
611 45 remove_sorted_range(shard, oldest_base);
612 45 return true;
613 }
614 ✗ return false;
615 }
616
617 /**
618 * @brief Force-evicts entries until the shard is at or below max_capacity.
619 * @note Must be called with the shard mutex held (exclusive).
620 */
621 23945 void trim_to_max_capacity(CacheShard &shard) noexcept
622 {
623
2/6
✗ Branch 5 → 6 not taken.
✓ Branch 5 → 9 taken 23945 times.
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 9 not taken.
✗ Branch 10 → 3 not taken.
✓ Branch 10 → 11 taken 23945 times.
 23945 while (shard.entries.size() > shard.max_capacity && !shard.fifo_index.empty())
624 {
625 ✗ evict_oldest_entry(shard);
626 }
627 23945 }
628
629 /**
630 * @brief Updates or inserts a cache entry in a specific shard and can throw.
631 * @param content_gen Generation captured before the leader's VirtualQuery. An entry published across a
632 * clear or invalidation starts stale, so the next hit queries again.
633 * @details If a call throws, the function erases each index change. An update without its old FIFO record
634 * drops its entry. Proof: MemoryTest.CacheInsertFaultRollbackStaysConsistent and
635 * MemoryCacheLifecycleProof.ContendedInvalidationAdvancesContentGeneration.
636 * @note Must be called with the shard mutex held (exclusive). It can throw. The noexcept
637 * @ref update_shard_with_region wrapper fails soft on that.
638 */
639 512 void update_shard_with_region_impl(
640 CacheShard &shard,
641 const MEMORY_BASIC_INFORMATION &mbi,
642 std::uint64_t current_ns,
643 std::uint64_t content_gen
644 )
645 {
646 512 const std::uintptr_t base_addr = reinterpret_cast<std::uintptr_t>(mbi.BaseAddress);
647
648
1/2
✓ Branch 2 → 3 taken 512 times.
✗ Branch 2 → 56 not taken.
 512 auto it = shard.entries.find(base_addr);
649
2/2
✓ Branch 5 → 6 taken 71 times.
✓ Branch 5 → 22 taken 441 times.
 512 if (it != shard.entries.end())
650 {
651 71 CachedMemoryRegionInfo &old_entry = it->second;
652
1/2
✓ Branch 7 → 8 taken 71 times.
✗ Branch 7 → 43 not taken.
 71 const auto fifo_it = shard.fifo_index.find(old_entry.fifo_key);
653
3/6
✓ Branch 10 → 11 taken 71 times.
✗ Branch 10 → 14 not taken.
✓ Branch 12 → 13 taken 71 times.
✗ Branch 12 → 14 not taken.
✓ Branch 15 → 16 taken 71 times.
✗ Branch 15 → 17 not taken.
 71 if (fifo_it != shard.fifo_index.end() && fifo_it->second == base_addr)
654 {
655
1/2
✓ Branch 16 → 17 taken 71 times.
✗ Branch 16 → 43 not taken.
 71 shard.fifo_index.erase(fifo_it);
656 }
657
658 71 const std::uint64_t new_fifo_key = shard.entry_counter++;
659
660 // A later failure leaves no FIFO record for the old entry. Erase all entry indexes before the
661 // exception escapes.
662 try
663 {
664
1/2
✗ Branch 17 → 18 not taken.
✓ Branch 17 → 20 taken 71 times.
 71 if (old_entry.region_size != mbi.RegionSize)
665 {
666 ✗ remove_sorted_range(shard, base_addr);
667
0/2
✗ Branch 19 → 20 not taken.
✗ Branch 19 → 35 not taken.
 ✗ insert_sorted_range(shard, base_addr, mbi.RegionSize);
668 }
669
2/2
✓ Branch 20 → 21 taken 70 times.
✓ Branch 20 → 34 taken 1 time.
 71 shard.fifo_index.emplace(new_fifo_key, base_addr);
670 }
671 1 catch (...)
672 {
673
1/2
✓ Branch 37 → 38 taken 1 time.
✗ Branch 37 → 41 not taken.
 1 shard.fifo_index.erase(new_fifo_key);
674 1 remove_sorted_range(shard, base_addr);
675
1/2
✓ Branch 39 → 40 taken 1 time.
✗ Branch 39 → 41 not taken.
 1 shard.entries.erase(base_addr);
676 1 throw;
677 1 }
678
679 70 old_entry.base_address = base_addr;
680 70 old_entry.region_size = mbi.RegionSize;
681 70 old_entry.protection = mbi.Protect;
682 70 old_entry.state = mbi.State;
683 70 old_entry.timestamp_ns = current_ns;
684 70 old_entry.fifo_key = new_fifo_key;
685 70 old_entry.content_gen = content_gen;
686 70 old_entry.valid = true;
687 }
688 else
689 {
690
2/2
✓ Branch 23 → 24 taken 45 times.
✓ Branch 23 → 25 taken 396 times.
 441 if (shard.entries.size() >= shard.capacity)
691 {
692 45 evict_oldest_entry(shard);
693 }
694
695
1/2
✗ Branch 26 → 27 not taken.
✓ Branch 26 → 28 taken 441 times.
 441 if (shard.entries.size() >= shard.max_capacity)
696 {
697 ✗ trim_to_max_capacity(shard);
698 }
699
700 441 const std::uint64_t new_fifo_key = shard.entry_counter++;
701
702 441 CachedMemoryRegionInfo new_entry;
703 441 new_entry.base_address = base_addr;
704 441 new_entry.region_size = mbi.RegionSize;
705 441 new_entry.protection = mbi.Protect;
706 441 new_entry.state = mbi.State;
707 441 new_entry.timestamp_ns = current_ns;
708 441 new_entry.fifo_key = new_fifo_key;
709 441 new_entry.content_gen = content_gen;
710 441 new_entry.valid = true;
711
712 // Treat all three index writes as one transaction. The catch erases each prior commit after a
713 // later failure.
714 441 bool map_committed = false;
715 441 bool fifo_committed = false;
716 try
717 {
718
2/2
✓ Branch 29 → 30 taken 436 times.
✓ Branch 29 → 44 taken 5 times.
 441 shard.entries.insert_or_assign(base_addr, new_entry);
719 436 map_committed = true;
720
2/2
✓ Branch 30 → 31 taken 431 times.
✓ Branch 30 → 45 taken 5 times.
 436 shard.fifo_index.emplace(new_fifo_key, base_addr);
721 431 fifo_committed = true;
722
2/2
✓ Branch 31 → 32 taken 429 times.
✓ Branch 31 → 46 taken 2 times.
 431 insert_sorted_range(shard, base_addr, mbi.RegionSize);
723 }
724 12 catch (...)
725 {
726
2/2
✓ Branch 48 → 49 taken 2 times.
✓ Branch 48 → 50 taken 10 times.
 12 if (fifo_committed)
727 {
728
1/2
✓ Branch 49 → 50 taken 2 times.
✗ Branch 49 → 53 not taken.
 2 shard.fifo_index.erase(new_fifo_key);
729 }
730
2/2
✓ Branch 50 → 51 taken 7 times.
✓ Branch 50 → 52 taken 5 times.
 12 if (map_committed)
731 {
732
1/2
✓ Branch 51 → 52 taken 7 times.
✗ Branch 51 → 53 not taken.
 7 shard.entries.erase(base_addr);
733 }
734 12 throw;
735 12 }
736 }
737 499 }
738
739 /**
740 * @brief Updates or inserts a cache entry in a specific shard with soft allocation failure.
741 * @details The cache is a hint over the authoritative VirtualQuery result. The implementation erases its
742 * partial work after a failure, so the three shard indexes remain equal.
743 * @note Must be called with the shard mutex held (exclusive).
744 */
745 512 void update_shard_with_region(
746 CacheShard &shard,
747 const MEMORY_BASIC_INFORMATION &mbi,
748 std::uint64_t current_ns,
749 std::uint64_t content_gen
750 ) noexcept
751 {
752 try
753 {
754
2/2
✓ Branch 2 → 3 taken 499 times.
✓ Branch 2 → 4 taken 13 times.
 512 update_shard_with_region_impl(shard, mbi, current_ns, content_gen);
755 }
756 13 catch (...)
757 {
758 13 }
759 512 }
760
761 /**
762 * @brief Removes expired entries from a shard.
763 * @note Must be called with the shard mutex held (exclusive).
764 * @return Number of entries removed from this shard.
765 */
766 23945 std::size_t cleanup_expired_entries_in_shard(
767 CacheShard &shard,
768 std::uint64_t current_ns,
769 std::uint64_t expiry_ns
770 ) noexcept
771 {
772 23945 std::size_t removed = 0;
773 23945 auto it = shard.entries.begin();
774
2/2
✓ Branch 25 → 4 taken 26 times.
✓ Branch 25 → 26 taken 23945 times.
 23971 while (it != shard.entries.end())
775 {
776 26 const CachedMemoryRegionInfo &entry = it->second;
777 26 const std::uint64_t entry_age = current_ns - entry.timestamp_ns;
778
779
3/4
✓ Branch 5 → 6 taken 26 times.
✗ Branch 5 → 7 not taken.
✓ Branch 6 → 7 taken 1 time.
✓ Branch 6 → 21 taken 25 times.
 26 if (!entry.valid || entry_age > expiry_ns)
780 {
781 1 const auto fifo_it = shard.fifo_index.find(entry.fifo_key);
782
3/6
✓ Branch 10 → 11 taken 1 time.
✗ Branch 10 → 15 not taken.
✓ Branch 13 → 14 taken 1 time.
✗ Branch 13 → 15 not taken.
✓ Branch 16 → 17 taken 1 time.
✗ Branch 16 → 18 not taken.
 1 if (fifo_it != shard.fifo_index.end() && fifo_it->second == it->first)
783 {
784 1 shard.fifo_index.erase(fifo_it);
785 }
786
787 1 remove_sorted_range(shard, entry.base_address);
788 1 it = shard.entries.erase(it);
789 1 ++removed;
790 1 }
791 else
792 {
793 25 ++it;
794 }
795 }
796 23945 return removed;
797 }
798
799 /**
800 * @brief Performs cleanup of expired cache entries across all shards.
801 * @param force Force cleanup regardless of state-mutex contention.
802 */
803 1595 void cleanup_expired_entries(bool force) noexcept
804 {
805 // The state mutex stays locked while this function iterates the shards. It excludes shard-array release
806 // by shutdown_cache. On-demand cleanup uses try_lock, so the hot path never blocks. Forced cleanup
807 // waits for the lock.
808 1595 std::unique_lock lock(s_cache_state_mutex, std::defer_lock);
809
1/2
✓ Branch 3 → 4 taken 1595 times.
✗ Branch 3 → 5 not taken.
 1595 if (force)
810 {
811 1595 lock.lock();
812 }
813
0/2
✗ Branch 6 → 7 not taken.
✗ Branch 6 → 8 not taken.
 ✗ else if (!lock.try_lock())
814 {
815 ✗ return;
816 }
817
818
1/2
✗ Branch 9 → 10 not taken.
✓ Branch 9 → 11 taken 1595 times.
 1595 if (!s_cache_shards)
819 ✗ return;
820
821 1595 const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire);
822
1/2
✗ Branch 18 → 19 not taken.
✓ Branch 18 → 20 taken 1595 times.
 1595 if (shard_count == 0)
823 ✗ return;
824
825 1595 const std::uint64_t current_ts = current_time_ns();
826 1595 const std::uint64_t expiry_ns = configured_expiry_ns();
827
828
2/2
✓ Branch 33 → 23 taken 24275 times.
✓ Branch 33 → 34 taken 1595 times.
 25870 for (std::size_t i = 0; i < shard_count; ++i)
829 {
830 24275 std::unique_lock<SrwSharedMutex> shard_lock(s_cache_shards[i].mtx, std::try_to_lock);
831
2/2
✓ Branch 26 → 27 taken 23945 times.
✓ Branch 26 → 31 taken 330 times.
 24275 if (shard_lock.owns_lock())
832 {
833 23945 cleanup_expired_entries_in_shard(s_cache_shards[i], current_ts, expiry_ns);
834 23945 trim_to_max_capacity(s_cache_shards[i]);
835 }
836 24275 }
837
1/2
✓ Branch 36 → 37 taken 1595 times.
✗ Branch 36 → 39 not taken.
 1595 }
838
839 /**
840 * @brief Checks whether elapsed time permits on-demand cleanup.
841 * @return true if this caller claims the cleanup trigger. Contended shards can remain unprocessed.
842 */
843 ✗ bool try_trigger_on_demand_cleanup() noexcept
844 {
845
0/2
✗ Branch 3 → 4 not taken.
✗ Branch 3 → 5 not taken.
 ✗ if (!cache_is_running())
846 ✗ return false;
847
848 ✗ const std::uint64_t now_ns = current_time_ns();
849 ✗ const std::uint64_t last_cleanup = s_last_cleanup_time_ns.load(std::memory_order_acquire);
850 ✗ const std::uint64_t elapsed_ns = now_ns - last_cleanup;
851
852
0/2
✗ Branch 13 → 14 not taken.
✗ Branch 13 → 28 not taken.
 ✗ if (elapsed_ns >= CLEANUP_INTERVAL_NS)
853 {
854 ✗ std::uint64_t expected = last_cleanup;
855
0/2
✗ Branch 22 → 23 not taken.
✗ Branch 22 → 27 not taken.
 ✗ if (s_last_cleanup_time_ns.compare_exchange_strong(expected, now_ns, std::memory_order_acq_rel))
856 {
857 ✗ cleanup_expired_entries(false);
858 s_stats.on_demand_cleanups.fetch_add(1, std::memory_order_relaxed);
859 ✗ return true;
860 }
861 }
862 ✗ return false;
863 }
864
865 /**
866 * @brief Runs the background cleanup thread for one lifecycle generation.
867 * @param generation Lifecycle generation captured at thread creation.
868 * @note Exits after the live generation no longer matches @p generation.
869 */
870 740 void cleanup_thread_func(std::uint64_t generation) noexcept
871 {
872
4/4
✓ Branch 24 → 25 taken 1705 times.
✓ Branch 24 → 34 taken 630 times.
✓ Branch 35 → 3 taken 1705 times.
✓ Branch 35 → 36 taken 630 times.
 4040 while (s_cleanup_thread_running.load(std::memory_order_acquire) &&
873
1/2
✓ Branch 32 → 33 taken 1705 times.
✗ Branch 32 → 34 not taken.
 1705 s_lifecycle_generation.load(std::memory_order_acquire) == generation)
874 {
875 {
876 1705 std::unique_lock<std::mutex> lock(s_cleanup_mutex);
877 1705 s_cleanup_cv.wait_for(
878 lock,
879 1705 std::chrono::seconds(1),
880 3328 [&]()
881 {
882 3328 return s_cleanup_requested.load(std::memory_order_acquire) ||
883
4/4
✓ Branch 3 → 4 taken 1732 times.
✓ Branch 3 → 14 taken 1596 times.
✓ Branch 5 → 6 taken 1625 times.
✓ Branch 5 → 14 taken 107 times.
 4953 !s_cleanup_thread_running.load(std::memory_order_acquire) ||
884
1/2
✗ Branch 13 → 14 not taken.
✓ Branch 13 → 15 taken 1625 times.
 4953 s_lifecycle_generation.load(std::memory_order_acquire) != generation;
885 }
886 );
887 1704 }
888
889
4/4
✓ Branch 8 → 9 taken 1595 times.
✓ Branch 8 → 17 taken 109 times.
✓ Branch 19 → 20 taken 109 times.
✓ Branch 19 → 21 taken 1595 times.
 3299 if (!s_cleanup_thread_running.load(std::memory_order_acquire) ||
890
1/2
✗ Branch 16 → 17 not taken.
✓ Branch 16 → 18 taken 1595 times.
 1595 s_lifecycle_generation.load(std::memory_order_acquire) != generation)
891 109 break;
892
893 1595 cleanup_expired_entries(true);
894 1595 s_cleanup_requested.store(false, std::memory_order_relaxed);
895 }
896 739 }
897
898 /**
899 * @brief Requests cleanup through the worker or the on-demand path.
900 * @details Signals the worker when it is active. Otherwise, attempts on-demand cleanup.
901 */
902 2983 void request_cleanup() noexcept
903 {
904
1/2
✓ Branch 3 → 4 taken 2983 times.
✗ Branch 3 → 6 not taken.
 2983 if (s_cleanup_thread_running.load(std::memory_order_acquire))
905 {
906 2983 s_cleanup_requested.store(true, std::memory_order_relaxed);
907 2983 s_cleanup_cv.notify_one();
908 }
909 else
910 {
911 ✗ try_trigger_on_demand_cleanup();
912 }
913 2983 }
914
915 /**
916 * @brief Detaches the cleanup thread and retains its counted module reference.
917 * @return true if no joinable thread exists or detachment succeeds. Returns false if std::thread::detach()
918 * throws.
919 */
920 9 bool detach_cleanup_thread_retained() noexcept
921 {
922
2/2
✓ Branch 4 → 5 taken 6 times.
✓ Branch 4 → 6 taken 3 times.
 9 if (!cleanup_thread().joinable())
923 {
924 6 return true;
925 }
926
927 try
928 {
929 // The retained module reference keeps the detached worker's code mapped.
930
1/2
✓ Branch 7 → 8 taken 3 times.
✗ Branch 7 → 11 not taken.
 3 cleanup_thread().detach();
931 3 DetourModKit::diagnostics::record_intentional_leak(
932 DetourModKit::diagnostics::LeakSubsystem::MemoryCache
933 );
934 3 return true;
935 }
936 ✗ catch (...)
937 {
938 s_lifecycle_violations.fetch_add(1, std::memory_order_relaxed);
939 ✗ return false;
940 ✗ }
941 }
942
943 /**
944 * @brief Tries to claim and detach the cleanup thread without a wait for an unauthorized teardown.
945 * @return true if this caller claims the mutex and finds no joinable thread or detaches it. Returns false
946 * on contention or if std::thread::detach() throws.
947 */
948 9 bool try_detach_cleanup_thread_unauthorized() noexcept
949 {
950 9 std::unique_lock join_lock(s_cleanup_join_mutex, std::try_to_lock);
951
1/2
✗ Branch 4 → 5 not taken.
✓ Branch 4 → 6 taken 9 times.
 9 if (!join_lock.owns_lock())
952 {
953 // Do not wait because the owner already controls the join or detach decision.
954 ✗ return false;
955 }
956
957 9 return detach_cleanup_thread_retained();
958 9 }
959
960 /**
961 * @brief Claims and joins the cleanup thread after the caller authorizes a teardown wait.
962 * @return true if a joinable handle was joined (init uses this to count a reaped leftover as a lifecycle
963 * violation). Returns false when nothing needs a reap or retention contains a failed join.
964 * @note The joinable() check and mutation both run under s_cleanup_join_mutex. Callers must not check
965 * joinable() first outside that mutex because this races the abandon-path detach.
966 */
967 1485 bool join_cleanup_thread() noexcept
968 {
969 1485 std::lock_guard join_lock(s_cleanup_join_mutex);
970
2/2
✓ Branch 5 → 6 taken 749 times.
✓ Branch 5 → 7 taken 736 times.
 1485 if (!cleanup_thread().joinable())
971 {
972 749 return false;
973 }
974
975 try
976 {
977
1/2
✓ Branch 8 → 9 taken 736 times.
✗ Branch 8 → 16 not taken.
 736 cleanup_thread().join();
978 }
979 ✗ catch (...)
980 {
981 s_lifecycle_violations.fetch_add(1, std::memory_order_relaxed);
982 ✗ (void)detach_cleanup_thread_retained();
983 ✗ return false;
984 ✗ }
985 // After a join outside the loader lock, drop the reference from creation. The caller retains another
986 // module reference, so this is never the terminal release.
987
1/2
✓ Branch 9 → 10 taken 736 times.
✗ Branch 9 → 12 not taken.
 736 if (s_cleanup_self_ref != nullptr)
988 {
989 736 release_module_ref(s_cleanup_self_ref, diagnostics::ModulePinReason::MemoryCache);
990 736 s_cleanup_self_ref = nullptr;
991 }
992 736 return true;
993 1485 }
994
995 #if !defined(_MSC_VER) && defined(_WIN64)
996 /** @brief Applies the Session retirement policy to a cache teardown. */
997 14571 void shutdown_guarded_engine() noexcept
998 {
999
2/2
✓ Branch 4 → 5 taken 1637 times.
✓ Branch 4 → 6 taken 12934 times.
 14571 if (DetourModKit::detail::lifecycle().state() == DetourModKit::detail::LifecycleState::Stopping)
1000 1637 detail::retire_session_guarded_engine();
1001 else
1002 12934 detail::release_guarded_engine();
1003 14571 }
1004 #endif
1005
1006 /**
1007 * @brief Abandons the cache when teardown lacks block authority.
1008 * @details A lifecycle-mutex wait can deadlock against thread creation under the loader lock.
1009 * The shutdown_cache contract owns the guarded-handler release exception.
1010 */
1011 9 void abandon_cache_unauthorized() noexcept
1012 {
1013 #if !defined(_MSC_VER) && defined(_WIN64)
1014 9 shutdown_guarded_engine();
1015 #endif
1016 9 s_cleanup_thread_running.store(false, std::memory_order_release);
1017 9 s_cleanup_cv.notify_one();
1018 // If another thread owns the handle lock, it completes the join/detach decision after loader unlock.
1019 9 (void)try_detach_cleanup_thread_unauthorized();
1020
1021 // Close admission before another reader enters the retained shard array. Each fetch_or is wait-free
1022 // and loader-lock safe. The next authorized init or shutdown drains admitted readers before any free.
1023 9 close_reader_admission();
1024
1025 9 LifecycleState state = s_lifecycle_state.load(std::memory_order_seq_cst);
1026
3/4
✓ Branch 13 → 9 taken 3 times.
✓ Branch 13 → 14 taken 6 times.
✗ Branch 14 → 9 not taken.
✓ Branch 14 → 15 taken 6 times.
 9 while (state == LifecycleState::Starting || state == LifecycleState::Running)
1027 {
1028
1/2
✓ Branch 10 → 11 taken 3 times.
✗ Branch 10 → 12 not taken.
 3 if (s_lifecycle_state.compare_exchange_weak(
1029 state,
1030 LifecycleState::Stopped,
1031 std::memory_order_seq_cst,
1032 std::memory_order_seq_cst
1033 ))
1034 {
1035 3 break;
1036 }
1037 }
1038 9 }
1039
1040 /**
1041 * @brief Evicts every entry in a shard whose region overlaps [address, end_address).
1042 * @note Must be called with the shard mutex held (exclusive).
1043 * @note Scans the whole shard: one region can be cached in several shards under the same base key (the
1044 * shard is chosen from the query address). The shard remains bounded by max_capacity, and this scan
1045 * never runs on a read hot path.
1046 */
1047 38937 std::size_t evict_overlapping_entries_in_shard(
1048 CacheShard &shard,
1049 std::uintptr_t address,
1050 std::uintptr_t end_address
1051 ) noexcept
1052 {
1053 38937 std::size_t evicted = 0;
1054 38937 auto it = shard.entries.begin();
1055
2/2
✓ Branch 34 → 4 taken 136 times.
✓ Branch 34 → 35 taken 38938 times.
 39075 while (it != shard.entries.end())
1056 {
1057 136 const CachedMemoryRegionInfo &entry = it->second;
1058 136 const std::uintptr_t entry_end_address = entry.base_address + entry.region_size;
1059 // A VirtualQuery region cannot extend past the address space, but a corrupt cached size can.
1060 // Treat a wrapped end as the top of the address space so a poisoned entry is still evicted.
1061 136 const std::uintptr_t clamped_entry_end =
1062
1/2
✓ Branch 5 → 6 taken 136 times.
✗ Branch 5 → 7 not taken.
 136 (entry_end_address < entry.base_address) ? UINTPTR_MAX : entry_end_address;
1063 136 const bool overlaps =
1064
4/6
✓ Branch 8 → 9 taken 136 times.
✗ Branch 8 → 12 not taken.
✓ Branch 9 → 10 taken 136 times.
✗ Branch 9 → 12 not taken.
✓ Branch 10 → 11 taken 11 times.
✓ Branch 10 → 12 taken 125 times.
 136 entry.valid && address < clamped_entry_end && end_address > entry.base_address;
1065
2/2
✓ Branch 13 → 14 taken 11 times.
✓ Branch 13 → 30 taken 125 times.
 136 if (overlaps)
1066 {
1067 11 const auto fifo_it = shard.fifo_index.find(entry.fifo_key);
1068
3/6
✓ Branch 17 → 18 taken 11 times.
✗ Branch 17 → 22 not taken.
✓ Branch 20 → 21 taken 11 times.
✗ Branch 20 → 22 not taken.
✓ Branch 23 → 24 taken 11 times.
✗ Branch 23 → 25 not taken.
 11 if (fifo_it != shard.fifo_index.end() && fifo_it->second == it->first)
1069 {
1070 11 shard.fifo_index.erase(fifo_it);
1071 }
1072 11 remove_sorted_range(shard, entry.base_address);
1073 11 it = shard.entries.erase(it);
1074 s_stats.invalidations.fetch_add(1, std::memory_order_relaxed);
1075 11 ++evicted;
1076 }
1077 else
1078 {
1079 125 ++it;
1080 }
1081 }
1082 38938 return evicted;
1083 }
1084
1085 /**
1086 * @brief Invalidates cache entries that overlap [address, address + size) across all shards.
1087 * @details Uses one try-lock per shard. On success, entries that overlap are physically evicted. Under
1088 * contention, the content generation advances instead. This invalidates every entry in O(1), so
1089 * correctness never depends on lock success. The fallback over-invalidates the contended
1090 * shard, an accepted trade on the rare, off-hot-path protection-change caller.
1091 */
1092 2982 void invalidate_range_internal(std::uintptr_t address, std::size_t size) noexcept
1093 {
1094
3/6
✓ Branch 3 → 4 taken 2982 times.
✗ Branch 3 → 5 not taken.
✗ Branch 4 → 5 not taken.
✓ Branch 4 → 6 taken 2982 times.
✗ Branch 7 → 8 not taken.
✓ Branch 7 → 9 taken 2982 times.
 2982 if (!s_cache_shards || size == 0)
1095 ✗ return;
1096
1097
2/2
✓ Branch 9 → 10 taken 2979 times.
✓ Branch 9 → 11 taken 3 times.
 2982 const std::uintptr_t end_address = (address + size < address) ? UINTPTR_MAX : address + size;
1098 2982 const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire);
1099
1100
2/2
✓ Branch 43 → 20 taken 40116 times.
✓ Branch 43 → 44 taken 2981 times.
 43097 for (std::size_t shard_idx = 0; shard_idx < shard_count; ++shard_idx)
1101 {
1102 40116 CacheShard &shard = s_cache_shards[shard_idx];
1103 40111 std::unique_lock<SrwSharedMutex> lock(shard.mtx, std::try_to_lock);
1104
2/2
✓ Branch 23 → 24 taken 38940 times.
✓ Branch 23 → 36 taken 1172 times.
 40114 if (lock.owns_lock())
1105 {
1106 38940 evict_overlapping_entries_in_shard(shard, address, end_address);
1107 // An earlier leader keeps in_flight set until publication under this lock. A generation advance
1108 // makes its entry stale. A later leader queries after this eviction and the caller's completed
1109 // protection change, so physical eviction suffices without a shard-wide generation change.
1110
2/2
✓ Branch 32 → 33 taken 91 times.
✓ Branch 32 → 41 taken 38843 times.
 77871 if (shard.in_flight.load(std::memory_order_acquire) != 0)
1111 {
1112 91 shard.content_gen.fetch_add(1, std::memory_order_acq_rel);
1113 }
1114 }
1115 else
1116 {
1117 // Contention advances the generation, so every entry becomes invalid. A leader that captured
1118 // the old generation republishes a stale entry.
1119 1172 shard.content_gen.fetch_add(1, std::memory_order_acq_rel);
1120 s_stats.invalidations.fetch_add(1, std::memory_order_relaxed);
1121 }
1122 40106 }
1123 }
1124
1125 /**
1126 * @brief Performs one-time cache initialization (allocates the shard array, configures bounds).
1127 */
1128 745 bool perform_cache_initialization(
1129 std::size_t cache_size,
1130 unsigned int expiry_ms,
1131 std::size_t shard_count
1132 ) noexcept
1133 {
1134
1/2
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 745 times.
 745 if (cache_size == 0)
1135 ✗ cache_size = MIN_CACHE_SIZE;
1136
2/2
✓ Branch 4 → 5 taken 1 time.
✓ Branch 4 → 6 taken 744 times.
 745 if (shard_count == 0)
1137 1 shard_count = 1;
1138
1139 // Quotient plus remainder computes ceiling division without an addition that can wrap.
1140 745 const std::size_t entries_per_shard =
1141
2/2
✓ Branch 6 → 7 taken 3 times.
✓ Branch 6 → 8 taken 742 times.
 745 cache_size / shard_count + ((cache_size % shard_count != 0) ? 1 : 0);
1142
1143 // Reject invalid bounds before publication. The explicit max_size check also contains length_error.
1144 745 const bool multiplier_overflows = entries_per_shard > SIZE_MAX / DEFAULT_MAX_CACHE_SIZE_MULTIPLIER;
1145
2/2
✓ Branch 9 → 10 taken 3 times.
✓ Branch 9 → 12 taken 742 times.
 745 if (multiplier_overflows)
1146 {
1147 3 s_cache_shards.reset();
1148 3 return false;
1149 }
1150 742 const std::size_t hard_max_per_shard = entries_per_shard * DEFAULT_MAX_CACHE_SIZE_MULTIPLIER;
1151
1152 try
1153 {
1154
2/2
✓ Branch 15 → 16 taken 2 times.
✓ Branch 15 → 18 taken 740 times.
 742 if (hard_max_per_shard > decltype(CacheShard::entries){}.max_size())
1155 {
1156 2 s_cache_shards.reset();
1157 2 return false;
1158 }
1159
1/2
✓ Branch 18 → 19 taken 740 times.
✗ Branch 18 → 63 not taken.
 740 s_cache_shards = std::make_unique<CacheShard[]>(shard_count);
1160
2/2
✓ Branch 27 → 22 taken 11415 times.
✓ Branch 27 → 28 taken 740 times.
 12155 for (std::size_t i = 0; i < shard_count; ++i)
1161 {
1162
1/2
✓ Branch 23 → 24 taken 11415 times.
✗ Branch 23 → 64 not taken.
 11415 s_cache_shards[i].entries.reserve(hard_max_per_shard);
1163 11415 s_cache_shards[i].capacity = entries_per_shard;
1164 11415 s_cache_shards[i].max_capacity = hard_max_per_shard;
1165 }
1166 }
1167 ✗ catch (...)
1168 {
1169 ✗ s_cache_shards.reset();
1170 ✗ return false;
1171 ✗ }
1172
1173 s_max_entries_per_shard.store(entries_per_shard, std::memory_order_release);
1174 s_configured_expiry_ms.store(expiry_ms, std::memory_order_release);
1175 740 s_last_cleanup_time_ns.store(current_time_ns(), std::memory_order_release);
1176 // Publish the shard count LAST so a reader that observes Running also sees the shard array and
1177 // config fields, never a torn half-initialized snapshot.
1178 s_shard_count.store(shard_count, std::memory_order_release);
1179
1180 740 return true;
1181 }
1182
1183 /**
1184 * @brief Performs VirtualQuery and updates the cache with stampede coalescence.
1185 * @return true if VirtualQuery (or a coalesced follower read) succeeded.
1186 */
1187 bool
1188 533 query_and_update_cache(std::size_t shard_idx, LPCVOID address, MEMORY_BASIC_INFORMATION &mbi_out) noexcept
1189 {
1190 533 CacheShard &shard = s_cache_shards[shard_idx];
1191
1192 532 char expected = 0;
1193
2/2
✓ Branch 11 → 12 taken 488 times.
✓ Branch 11 → 38 taken 46 times.
 1066 if (shard.in_flight.compare_exchange_strong(expected, 1, std::memory_order_acq_rel))
1194 {
1195 // Capture the generation before the query. A clear or invalidation before publication stamps this
1196 // entry stale.
1197 488 const std::uint64_t gen_at_query = shard.content_gen.load(std::memory_order_acquire);
1198 488 const bool result = VirtualQuery(address, &mbi_out, sizeof(mbi_out)) != 0;
1199 488 const std::uint64_t now_ns = current_time_ns();
1200
1201 #if defined(DMK_ENABLE_TEST_SEAMS)
1202 // Inject a clear/protection change into the post-query, pre-publish window.
1203
2/2
✓ Branch 21 → 22 taken 2 times.
✓ Branch 21 → 23 taken 486 times.
 488 if (auto *const hook = DetourModKit::detail::g_memory_cache_leader_publish_window_test_hook)
1204 2 hook();
1205 #endif
1206
1207
1/2
✓ Branch 23 → 24 taken 488 times.
✗ Branch 23 → 29 not taken.
 488 if (result)
1208 {
1209 488 std::unique_lock<SrwSharedMutex> lock(s_cache_shards[shard_idx].mtx);
1210 488 update_shard_with_region(shard, mbi_out, now_ns, gen_at_query);
1211 488 }
1212
1213 488 shard.in_flight.store(0, std::memory_order_release);
1214 488 return result;
1215 }
1216 else
1217 {
1218 46 const std::uint64_t expiry_ns = configured_expiry_ns();
1219 48 constexpr std::size_t MAX_FOLLOWER_YIELDS = 8;
1220
1221
2/2
✓ Branch 63 → 40 taken 339 times.
✓ Branch 63 → 64 taken 20 times.
 359 for (std::size_t yield_count = 0; yield_count < MAX_FOLLOWER_YIELDS; ++yield_count)
1222 {
1223
2/2
✓ Branch 47 → 48 taken 28 times.
✓ Branch 47 → 61 taken 310 times.
 677 if (shard.in_flight.load(std::memory_order_acquire) == 0)
1224 {
1225 28 const std::uintptr_t addr_val = reinterpret_cast<std::uintptr_t>(address);
1226 28 std::shared_lock<SrwSharedMutex> lock(s_cache_shards[shard_idx].mtx);
1227 CachedMemoryRegionInfo *cached =
1228 28 find_in_shard(shard, addr_val, 1, current_time_ns(), expiry_ns);
1229
2/2
✓ Branch 52 → 53 taken 12 times.
✓ Branch 52 → 56 taken 16 times.
 28 if (cached)
1230 {
1231 s_stats.coalesced_queries.fetch_add(1, std::memory_order_relaxed);
1232 12 mbi_out.BaseAddress = reinterpret_cast<PVOID>(cached->base_address);
1233 12 mbi_out.RegionSize = cached->region_size;
1234 12 mbi_out.Protect = cached->protection;
1235 12 mbi_out.State = cached->state;
1236 12 return true;
1237 }
1238 16 break;
1239
2/2
✓ Branch 58 → 59 taken 12 times.
✓ Branch 58 → 60 taken 16 times.
 28 }
1240
1241 310 std::this_thread::yield();
1242 }
1243
1244 36 expected = 0;
1245
2/2
✓ Branch 72 → 73 taken 24 times.
✓ Branch 72 → 97 taken 12 times.
 72 if (shard.in_flight.compare_exchange_strong(expected, 1, std::memory_order_acq_rel))
1246 {
1247 24 const std::uint64_t gen_at_query = shard.content_gen.load(std::memory_order_acquire);
1248 24 const bool result = VirtualQuery(address, &mbi_out, sizeof(mbi_out)) != 0;
1249
1/2
✓ Branch 81 → 82 taken 24 times.
✗ Branch 81 → 88 not taken.
 24 if (result)
1250 {
1251 24 std::unique_lock<SrwSharedMutex> lock(s_cache_shards[shard_idx].mtx);
1252 24 const std::uint64_t now_ns = current_time_ns();
1253 24 update_shard_with_region(shard, mbi_out, now_ns, gen_at_query);
1254 24 }
1255 24 shard.in_flight.store(0, std::memory_order_release);
1256 24 return result;
1257 }
1258
1259 12 return VirtualQuery(address, &mbi_out, sizeof(mbi_out)) != 0;
1260 }
1261 }
1262
1263 /**
1264 * @brief Walks a range through VirtualQuery without cache involvement.
1265 * @param address Start of the range. Callers screen a zero address before the call.
1266 * @param size Byte length of the range. Callers screen a zero size before the call.
1267 * @param check_permission Predicate over one region's protection flags.
1268 * @return true only when every region the range touches is committed and satisfies @p check_permission,
1269 * with no unmapped gap between them.
1270 * @details The loop repeats only when the range crosses different protections. VirtualQuery already
1271 * coalesces neighbors with the same protection, so the common single-region case uses one query.
1272 * Every failure closes the result. A wrap, invalid sub-region, disallowed protection, failed
1273 * query, or no cursor advance returns false. The returned region contains `cursor`, which
1274 * guarantees progress.
1275 */
1276 97399 bool range_permission_uncached(
1277 std::uintptr_t address,
1278 std::size_t size,
1279 bool (*check_permission)(DWORD) noexcept
1280 ) noexcept
1281 {
1282 97399 const std::uintptr_t query_end = address + size;
1283
2/2
✓ Branch 2 → 3 taken 3 times.
✓ Branch 2 → 4 taken 97396 times.
 97399 if (query_end < address)
1284 3 return false;
1285
1286 97396 std::uintptr_t cursor = address;
1287
2/2
✓ Branch 17 → 5 taken 96872 times.
✓ Branch 17 → 18 taken 114656 times.
 211528 while (cursor < query_end)
1288 {
1289 96872 MEMORY_BASIC_INFORMATION mbi{};
1290
1/2
✗ Branch 6 → 7 not taken.
✓ Branch 6 → 8 taken 117320 times.
 96872 if (VirtualQuery(reinterpret_cast<LPCVOID>(cursor), &mbi, sizeof(mbi)) == 0)
1291 14 return false;
1292
2/2
✓ Branch 8 → 9 taken 4 times.
✓ Branch 8 → 10 taken 117316 times.
 117320 if (mbi.State != MEM_COMMIT)
1293 4 return false;
1294
2/2
✓ Branch 11 → 12 taken 10 times.
✓ Branch 11 → 13 taken 114132 times.
 117316 if (!check_permission(mbi.Protect))
1295 10 return false;
1296
1297 114132 const std::uintptr_t region_end =
1298 114132 reinterpret_cast<std::uintptr_t>(mbi.BaseAddress) + mbi.RegionSize;
1299 // A zero-size or wrapped region fails closed instead of a spin.
1300
1/2
✗ Branch 13 → 14 not taken.
✓ Branch 13 → 15 taken 114132 times.
 114132 if (region_end <= cursor)
1301 ✗ return false;
1302 114132 cursor = region_end;
1303 }
1304 114656 return true;
1305 }
1306
1307 /**
1308 * @brief Checks permissions for is_readable and is_writable.
1309 * @param address Start address of the query (0 fails closed).
1310 * @param size Number of bytes to check (0 fails closed).
1311 * @param check_permission Predicate that validates the protection flags.
1312 */
1313 219436 bool check_memory_permission(
1314 std::uintptr_t address,
1315 std::size_t size,
1316 bool (*check_permission)(DWORD) noexcept
1317 ) noexcept
1318 {
1319
2/4
✓ Branch 2 → 3 taken 221159 times.
✗ Branch 2 → 4 not taken.
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 5 taken 221626 times.
 219436 if (address == 0 || size == 0)
1320 ✗ return false;
1321
1322 // One compare-exchange combines admission with the closed check. A query after closure never joins the
1323 // drain population ([B-73]). While admitted, teardown cannot free the shard array.
1324 221626 ReaderAdmission admission;
1325
1326 // Without an admitted running cache, walk the range directly. The walk spans protection boundaries,
1327 // so a re-protected interior page is answered correctly.
1328
5/6
✓ Branch 7 → 8 taken 123989 times.
✓ Branch 7 → 10 taken 98777 times.
✗ Branch 9 → 10 not taken.
✓ Branch 9 → 11 taken 122703 times.
✓ Branch 12 → 13 taken 98431 times.
✓ Branch 12 → 14 taken 122408 times.
 224993 if (!admission.admitted() || !cache_is_running())
1329 {
1330 98431 return range_permission_uncached(address, size, check_permission);
1331 }
1332 118433 const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire);
1333
1/2
✗ Branch 21 → 22 not taken.
✓ Branch 21 → 23 taken 118433 times.
 118433 if (shard_count == 0)
1334 {
1335 ✗ return range_permission_uncached(address, size, check_permission);
1336 }
1337
1338 118433 const std::size_t shard_idx = compute_shard_index(address, shard_count);
1339 118349 const std::uint64_t now_ns = current_time_ns();
1340 121713 const std::uint64_t expiry_ns = configured_expiry_ns();
1341
1342 {
1343 119425 std::shared_lock<SrwSharedMutex> lock(s_cache_shards[shard_idx].mtx);
1344 CachedMemoryRegionInfo *cached_info =
1345 131368 find_in_shard(s_cache_shards[shard_idx], address, size, now_ns, expiry_ns);
1346
2/2
✓ Branch 30 → 31 taken 119596 times.
✓ Branch 30 → 40 taken 533 times.
 120129 if (cached_info)
1347 {
1348 119596 s_cache_shards[shard_idx].hits.fetch_add(1, std::memory_order_relaxed);
1349 // Require MEM_COMMIT exactly as the miss and uncached paths do. Non-committed regions report
1350 // Protect == 0, so check_permission already rejects them. The explicit state check keeps every
1351 // path symmetric.
1352
2/4
✓ Branch 34 → 35 taken 130120 times.
✗ Branch 34 → 38 not taken.
✓ Branch 36 → 37 taken 124030 times.
✗ Branch 36 → 38 not taken.
 119956 return cached_info->state == MEM_COMMIT && check_permission(cached_info->protection);
1353 }
1354
2/2
✓ Branch 42 → 43 taken 532 times.
✓ Branch 42 → 49 taken 125425 times.
 113937 }
1355
1356 532 s_cache_shards[shard_idx].misses.fetch_add(1, std::memory_order_relaxed);
1357
1358 534 MEMORY_BASIC_INFORMATION mbi{};
1359
1/2
✗ Branch 48 → 50 not taken.
✓ Branch 48 → 51 taken 536 times.
 534 if (!query_and_update_cache(shard_idx, reinterpret_cast<LPCVOID>(address), mbi))
1360 ✗ return false;
1361
1362
2/2
✓ Branch 51 → 52 taken 4 times.
✓ Branch 51 → 53 taken 532 times.
 536 if (mbi.State != MEM_COMMIT)
1363 4 return false;
1364
1365
2/2
✓ Branch 54 → 55 taken 16 times.
✓ Branch 54 → 56 taken 516 times.
 532 if (!check_permission(mbi.Protect))
1366 16 return false;
1367
1368 516 const std::uintptr_t region_end_addr =
1369 516 reinterpret_cast<std::uintptr_t>(mbi.BaseAddress) + mbi.RegionSize;
1370 516 const std::uintptr_t query_end_addr = address + size;
1371
1372
2/2
✓ Branch 56 → 57 taken 4 times.
✓ Branch 56 → 58 taken 512 times.
 516 if (query_end_addr < address)
1373 4 return false;
1374
1/2
✗ Branch 58 → 59 not taken.
✓ Branch 58 → 60 taken 512 times.
 512 if (region_end_addr <= address)
1375 ✗ return false;
1376
1377 // The common case fits within the cached region, so answer directly. A range past it crosses into an
1378 // adjacent protection region that one cache entry cannot cover. Walk the remainder without the cache.
1379 // Fail closed on any uncommitted, disallowed, or gapped sub-region.
1380
2/2
✓ Branch 60 → 61 taken 506 times.
✓ Branch 60 → 62 taken 6 times.
 512 if (query_end_addr <= region_end_addr)
1381 506 return true;
1382
1383 6 return range_permission_uncached(region_end_addr, query_end_addr - region_end_addr, check_permission);
1384 236650 }
1385
1386 #if defined(DMK_ENABLE_TEST_SEAMS)
1387 /**
1388 * @brief Holds the selected shard's shared lock and invokes a deterministic test callback.
1389 */
1390 2 void hold_shard_shared_lock_for_test(Address address, void (*callback)() noexcept) noexcept
1391 {
1392
1/2
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 2 times.
 2 if (callback == nullptr)
1393 ✗ return;
1394
1395 2 ReaderAdmission admission;
1396
3/6
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 9 not taken.
✗ Branch 8 → 9 not taken.
✓ Branch 8 → 10 taken 2 times.
✗ Branch 11 → 12 not taken.
✓ Branch 11 → 13 taken 2 times.
 2 if (!admission.admitted() || !cache_is_running())
1397 ✗ return;
1398
1399 2 const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire);
1400
1/2
✗ Branch 20 → 21 not taken.
✓ Branch 20 → 22 taken 2 times.
 2 if (shard_count == 0)
1401 ✗ return;
1402
1403 2 const std::size_t shard_idx = compute_shard_index(address.raw(), shard_count);
1404 2 std::shared_lock<SrwSharedMutex> lock(s_cache_shards[shard_idx].mtx);
1405 2 callback();
1406
1/2
✓ Branch 30 → 31 taken 2 times.
✗ Branch 30 → 33 not taken.
 2 }
1407
1408 /**
1409 * @brief Reports all index sizes for the shard that owns @p address under one shared lock.
1410 *
1411 * @details This test seam exposes FIFO and range index state that MemoryStats omits.
1412 */
1413 26 void shard_index_sizes_for_test(
1414 Address address,
1415 std::size_t &entries,
1416 std::size_t &fifo,
1417 std::size_t &ranges
1418 ) noexcept
1419 {
1420 26 entries = 0;
1421 26 fifo = 0;
1422 26 ranges = 0;
1423
1424 26 ReaderAdmission admission;
1425
3/6
✓ Branch 4 → 5 taken 26 times.
✗ Branch 4 → 7 not taken.
✗ Branch 6 → 7 not taken.
✓ Branch 6 → 8 taken 26 times.
✗ Branch 9 → 10 not taken.
✓ Branch 9 → 11 taken 26 times.
 26 if (!admission.admitted() || !cache_is_running())
1426 ✗ return;
1427
1428 26 const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire);
1429
1/2
✗ Branch 18 → 19 not taken.
✓ Branch 18 → 20 taken 26 times.
 26 if (shard_count == 0)
1430 ✗ return;
1431
1432 26 CacheShard &shard = s_cache_shards[compute_shard_index(address.raw(), shard_count)];
1433 26 std::shared_lock<SrwSharedMutex> lock(shard.mtx);
1434 26 entries = shard.entries.size();
1435 26 fifo = shard.fifo_index.size();
1436 26 ranges = shard.sorted_ranges.size();
1437
1/2
✓ Branch 30 → 31 taken 26 times.
✗ Branch 30 → 33 not taken.
 26 }
1438 #endif
1439
1440 /**
1441 * @brief Drains the closed reader population, takes the cache keepalive, and builds the shard array.
1442 * @details The caller holds s_lifecycle_mutex. Each failure publishes Stopped before it returns false.
1443 */
1444 746 [[nodiscard]] bool drain_readers_and_build_shards(
1445 std::size_t cache_size,
1446 unsigned int expiry_ms,
1447 std::size_t shard_count
1448 ) noexcept
1449 {
1450 746 std::lock_guard state_lock(s_cache_state_mutex);
1451 // Close every stripe before the shard array changes. Then drain the closed reader population before
1452 // the deadline. Abandonment or a prior timeout can leave reader-visible state retained. Free that state
1453 // only after this drain reaches zero. On expiry the start fails and retains the state under [B-73].
1454 746 close_reader_admission();
1455 s_shard_count.store(0, std::memory_order_release);
1456
1/2
✗ Branch 13 → 14 not taken.
✓ Branch 13 → 17 taken 746 times.
 746 if (!drain_admitted_readers())
1457 {
1458 ✗ record_reader_drain_timeout_retention();
1459 ✗ s_lifecycle_state.store(LifecycleState::Stopped, std::memory_order_seq_cst);
1460 ✗ return false;
1461 }
1462
1463
2/2
✓ Branch 17 → 18 taken 745 times.
✓ Branch 17 → 39 taken 1 time.
 746 if (s_cache_self_ref == nullptr)
1464 {
1465 745 s_cache_self_ref = acquire_cache_keepalive_ref();
1466
2/2
✓ Branch 19 → 20 taken 1 time.
✓ Branch 19 → 39 taken 744 times.
 745 if (s_cache_self_ref == nullptr)
1467 {
1468 1 s_cache_shards.reset();
1469 s_configured_expiry_ms.store(0, std::memory_order_relaxed);
1470 s_max_entries_per_shard.store(0, std::memory_order_relaxed);
1471 1 s_lifecycle_state.store(LifecycleState::Stopped, std::memory_order_seq_cst);
1472 1 return false;
1473 }
1474 }
1475
1476
2/2
✓ Branch 40 → 41 taken 5 times.
✓ Branch 40 → 60 taken 740 times.
 745 if (!perform_cache_initialization(cache_size, expiry_ms, shard_count))
1477 {
1478 5 release_cache_keepalive_after_drain();
1479 s_configured_expiry_ms.store(0, std::memory_order_relaxed);
1480 s_max_entries_per_shard.store(0, std::memory_order_relaxed);
1481 5 s_lifecycle_state.store(LifecycleState::Stopped, std::memory_order_seq_cst);
1482 5 return false;
1483 }
1484 740 return true;
1485 746 }
1486
1487 /**
1488 * @brief Starts the cleanup thread for the next generation, or falls back to on-demand cleanup.
1489 * @details The caller holds s_lifecycle_mutex and calls this only after drain_readers_and_build_shards
1490 * succeeds.
1491 */
1492 740 void start_cleanup_thread() noexcept
1493 {
1494 740 const std::uint64_t generation = s_lifecycle_generation.fetch_add(1, std::memory_order_acq_rel) + 1;
1495
1496 740 s_cleanup_thread_running.store(true, std::memory_order_release);
1497 // Hold a counted reference before cleanup thread creation. A creation failure releases it below.
1498 740 s_cleanup_self_ref = acquire_module_ref(diagnostics::ModulePinReason::MemoryCache);
1499
1/2
✗ Branch 6 → 7 not taken.
✓ Branch 6 → 11 taken 740 times.
 740 if (s_cleanup_self_ref == nullptr)
1500 {
1501 ✗ s_cleanup_thread_running.store(false, std::memory_order_release);
1502 ✗ (void)log().try_log(
1503 LogLevel::Debug,
1504 "MemoryCache: Module reference unavailable, using on-demand cleanup instead of "
1505 "background cleanup."
1506 );
1507 }
1508 else
1509 {
1510 try
1511 {
1512 // Publish the handle under the join mutex so this never races the loader-lock detach path. A
1513 // concurrent detach tries the mutex, fails, and returns without access to the handle. This lock
1514 // can remain held across thread creation without a deadlock.
1515 740 std::lock_guard join_lock(s_cleanup_join_mutex);
1516
1/2
✗ Branch 14 → 15 not taken.
✓ Branch 14 → 16 taken 740 times.
 740 assert(!cleanup_thread().joinable());
1517
1/2
✓ Branch 17 → 18 taken 740 times.
✗ Branch 17 → 24 not taken.
 740 cleanup_thread() = std::thread(cleanup_thread_func, generation);
1518 740 }
1519 ✗ catch (...)
1520 {
1521 ✗ release_module_ref(s_cleanup_self_ref, diagnostics::ModulePinReason::MemoryCache);
1522 ✗ s_cleanup_self_ref = nullptr;
1523 ✗ s_cleanup_thread_running.store(false, std::memory_order_release);
1524 ✗ (void)log().try_log(
1525 LogLevel::Debug,
1526 "MemoryCache: Background cleanup thread unavailable, using on-demand "
1527 "cleanup."
1528 );
1529 ✗ }
1530 }
1531 740 }
1532
1533 /**
1534 * @brief Joins the cleanup thread, then frees or retains the shard array of a canceled start.
1535 * @details The caller holds s_lifecycle_mutex.
1536 */
1537 3 void roll_back_cancelled_start() noexcept
1538 {
1539 3 s_cleanup_thread_running.store(false, std::memory_order_release);
1540 3 s_cleanup_cv.notify_one();
1541 3 (void)join_cleanup_thread();
1542
1543 3 std::lock_guard<SrwSharedMutex> state_lock(s_cache_state_mutex);
1544 3 close_reader_admission();
1545 s_shard_count.store(0, std::memory_order_release);
1546
1/2
✓ Branch 16 → 17 taken 3 times.
✗ Branch 16 → 35 not taken.
 3 if (drain_admitted_readers())
1547 {
1548 3 s_cache_shards.reset();
1549 s_configured_expiry_ms.store(0, std::memory_order_relaxed);
1550 s_max_entries_per_shard.store(0, std::memory_order_relaxed);
1551 3 release_cache_keepalive_after_drain();
1552 }
1553 else
1554 {
1555 ✗ record_reader_drain_timeout_retention();
1556 }
1557 #if !defined(_MSC_VER) && defined(_WIN64)
1558 3 detail::release_guarded_engine();
1559 #endif
1560 3 }
1561
1562 } // namespace
1563
1564 4748 bool init_cache(std::size_t cache_size, unsigned int expiry_ms, std::size_t shard_count)
1565 {
1566 // A live cache needs no new start. Report it before the loader-lock veto.
1567
2/2
✓ Branch 3 → 4 taken 4000 times.
✓ Branch 3 → 5 taken 748 times.
 4748 if (cache_is_running())
1568 {
1569 4000 return true;
1570 }
1571
1572 // Refuse whenever the lifecycle gate does not authorize a wait. Cleanup-thread creation under loader lock
1573 // deadlocks. Readers use uncached VirtualQuery until an authorized init succeeds.
1574
2/2
✓ Branch 6 → 7 taken 1 time.
✓ Branch 6 → 8 taken 747 times.
 748 if (!DetourModKit::detail::blocking_teardown_permitted())
1575 {
1576 1 return false;
1577 }
1578
1579 #if defined(DMK_ENABLE_TEST_SEAMS)
1580
2/2
✓ Branch 8 → 9 taken 1 time.
✓ Branch 8 → 10 taken 746 times.
 747 if (auto *const hook = DetourModKit::detail::g_memory_cache_before_lifecycle_lock_test_hook)
1581
1/2
✓ Branch 9 → 10 taken 1 time.
✗ Branch 9 → 55 not taken.
 1 hook();
1582 #endif
1583
1584 // Serialize the whole start against shutdown_cache across the cleanup-thread handle.
1585 747 std::lock_guard lifecycle_lock(s_lifecycle_mutex);
1586
1587
2/2
✓ Branch 12 → 13 taken 1 time.
✓ Branch 12 → 14 taken 746 times.
 747 if (s_lifecycle_state.load(std::memory_order_seq_cst) == LifecycleState::Running)
1588 1 return true;
1589
1590 // Recover any unexpected joinable handle before assignment of the next worker.
1591 746 const bool reaped_leftover = join_cleanup_thread();
1592
1/2
✗ Branch 15 → 16 not taken.
✓ Branch 15 → 19 taken 746 times.
 746 if (reaped_leftover)
1593 {
1594 s_lifecycle_violations.fetch_add(1, std::memory_order_relaxed);
1595 }
1596 {
1597 746 std::lock_guard join_lock(s_cleanup_join_mutex);
1598
1/2
✗ Branch 22 → 23 not taken.
✓ Branch 22 → 26 taken 746 times.
 746 if (cleanup_thread().joinable())
1599 {
1600 s_lifecycle_violations.fetch_add(1, std::memory_order_relaxed);
1601 ✗ return false;
1602 }
1603
1/2
✓ Branch 28 → 29 taken 746 times.
✗ Branch 28 → 33 not taken.
 746 }
1604
1605 746 s_lifecycle_state.store(LifecycleState::Starting, std::memory_order_seq_cst);
1606
1607
2/2
✓ Branch 32 → 34 taken 6 times.
✓ Branch 32 → 35 taken 740 times.
 746 if (!drain_readers_and_build_shards(cache_size, expiry_ms, shard_count))
1608 6 return false;
1609
1610 740 start_cleanup_thread();
1611
1612 // The atexit handler is a last-resort safety net when the consumer omits shutdown_cache.
1613 static bool atexit_registered = false;
1614
2/2
✓ Branch 36 → 37 taken 451 times.
✓ Branch 36 → 40 taken 289 times.
 740 if (!atexit_registered)
1615 {
1616 451 std::atexit(
1617 902 []()
1618 {
1619
2/2
✓ Branch 3 → 4 taken 450 times.
✓ Branch 3 → 5 taken 1 time.
 451 if (s_lifecycle_state.load(std::memory_order_seq_cst) != LifecycleState::Running)
1620 450 return;
1621 1 shutdown_cache();
1622 }
1623 );
1624 451 atexit_registered = true;
1625 }
1626
1627 // Open every stripe before Running publishes. A concurrent abandonment closes the stripes and changes
1628 // Starting to Stopped. The publication check below then rolls the start back.
1629 740 reopen_reader_admission();
1630
1631 #if defined(DMK_ENABLE_TEST_SEAMS)
1632
2/2
✓ Branch 41 → 42 taken 2 times.
✓ Branch 41 → 43 taken 738 times.
 740 if (auto *const hook = DetourModKit::detail::g_memory_cache_before_running_publish_test_hook)
1633
1/2
✓ Branch 42 → 43 taken 2 times.
✗ Branch 42 → 53 not taken.
 2 hook();
1634 #endif
1635
1636 // Publish only if unauthorized abandonment did not cancel this Starting generation.
1637 740 LifecycleState expected_state = LifecycleState::Starting;
1638
2/2
✓ Branch 44 → 45 taken 3 times.
✓ Branch 44 → 47 taken 737 times.
 740 if (!s_lifecycle_state.compare_exchange_strong(
1639 expected_state,
1640 LifecycleState::Running,
1641 std::memory_order_seq_cst,
1642 std::memory_order_seq_cst
1643 ))
1644 {
1645 3 roll_back_cancelled_start();
1646 3 return false;
1647 }
1648
1649 #if !defined(_MSC_VER) && defined(_WIN64)
1650 // The reopen follows the Running publication, so a rolled-back start keeps a retired epoch closed
1651 // (SessionTeardown.RolledBackCacheStartKeepsTheRetiredHandlerEpoch). The cache keepalive reference pins
1652 // this image through a concurrent abandonment.
1653 737 detail::reopen_guarded_engine();
1654 737 detail::ensure_guarded_engine_installed();
1655 #endif
1656 737 return true;
1657 747 }
1658
1659 35 void clear_cache() noexcept
1660 {
1661 {
1662 35 std::lock_guard state_lock(s_cache_state_mutex);
1663
1664
2/2
✓ Branch 4 → 5 taken 1 time.
✓ Branch 4 → 6 taken 34 times.
 35 if (!cache_is_running())
1665 1 return;
1666
1667 34 const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire);
1668
1/2
✗ Branch 13 → 14 not taken.
✓ Branch 13 → 15 taken 34 times.
 34 if (shard_count == 0)
1669 ✗ return;
1670
1671 // Acquire an exclusive lock for each shard and wait if needed. The cleanup thread uses try_lock, so it
1672 // skips held shards.
1673
2/2
✓ Branch 47 → 16 taken 505 times.
✓ Branch 47 → 48 taken 34 times.
 539 for (std::size_t i = 0; i < shard_count; ++i)
1674 {
1675 505 std::unique_lock<SrwSharedMutex> shard_lock(s_cache_shards[i].mtx);
1676 505 s_cache_shards[i].entries.clear();
1677 505 s_cache_shards[i].fifo_index.clear();
1678 505 s_cache_shards[i].sorted_ranges.clear();
1679 505 s_cache_shards[i].hits.store(0, std::memory_order_relaxed);
1680 505 s_cache_shards[i].misses.store(0, std::memory_order_relaxed);
1681 // Advance the generation so an in-flight leader cannot republish a pre-clear result.
1682 505 s_cache_shards[i].content_gen.fetch_add(1, std::memory_order_release);
1683 505 }
1684
1685 s_stats.invalidations.store(0, std::memory_order_relaxed);
1686 s_stats.coalesced_queries.store(0, std::memory_order_relaxed);
1687 s_stats.on_demand_cleanups.store(0, std::memory_order_relaxed);
1688
1689 34 s_last_cleanup_time_ns.store(current_time_ns(), std::memory_order_relaxed);
1690
2/2
✓ Branch 83 → 84 taken 34 times.
✓ Branch 83 → 88 taken 1 time.
 35 }
1691
1692 // Deferred-log order: the optional diagnostic tail runs after every cache mutex is released. A sink or
1693 // format failure drops the line.
1694 34 (void)log().try_log(LogLevel::Debug, "MemoryCache: All entries cleared.");
1695 }
1696
1697 14569 void shutdown_cache() noexcept
1698 {
1699 // Decide the block policy once for the whole teardown. A second query lets a concurrent publication split
1700 // one teardown across both policies.
1701 14569 const bool may_block = DetourModKit::detail::blocking_teardown_permitted();
1702 // Session teardown releases the diagnostics owners as its own leaf, so only a module without an active
1703 // Session releases them here.
1704
2/2
✓ Branch 5 → 6 taken 12927 times.
✓ Branch 5 → 7 taken 1641 times.
 14569 if (DetourModKit::detail::lifecycle().state() == DetourModKit::detail::LifecycleState::Stopped)
1705 {
1706 12927 DetourModKit::detail::release_diagnostics_emit_owners(
1707 DetourModKit::detail::DiagnosticsTeardown::CacheShutdown,
1708 may_block
1709 );
1710 }
1711
2/2
✓ Branch 7 → 8 taken 6 times.
✓ Branch 7 → 10 taken 14563 times.
 14569 if (!may_block)
1712 {
1713 6 abandon_cache_unauthorized();
1714 13832 return;
1715 }
1716
1717 // Serialize the whole stop against init_cache across the cleanup-thread handle.
1718 14563 std::lock_guard lifecycle_lock(s_lifecycle_mutex);
1719
1720 14563 const LifecycleState state = s_lifecycle_state.load(std::memory_order_seq_cst);
1721
6/8
✓ Branch 12 → 13 taken 13826 times.
✓ Branch 12 → 17 taken 737 times.
✓ Branch 13 → 14 taken 13826 times.
✗ Branch 13 → 16 not taken.
✓ Branch 15 → 16 taken 13826 times.
✗ Branch 15 → 17 not taken.
✓ Branch 18 → 19 taken 13826 times.
✓ Branch 18 → 21 taken 737 times.
 14563 if (state != LifecycleState::Running && !(state == LifecycleState::Stopped && s_cache_shards))
1722 {
1723 #if !defined(_MSC_VER) && defined(_WIN64)
1724 13826 shutdown_guarded_engine();
1725 #endif
1726 13826 return;
1727 }
1728
1729 // Stopped with a live shard array is a prior loader-lock abandonment or drain timeout, safe to finish here
1730 // off the loader lock. Stopping also prevents a concurrent loader-lock callback from another Stopped
1731 // publication.
1732 737 s_lifecycle_state.store(LifecycleState::Stopping, std::memory_order_seq_cst);
1733
1734 // Close every admission stripe before the drain reads the counts. The drain then sees a closed population.
1735 // A query after this point takes the uncached route ([B-73]).
1736 737 close_reader_admission();
1737
1738 #if defined(DMK_ENABLE_TEST_SEAMS)
1739 // Force an initializer to attempt this lifecycle lock before teardown continues.
1740
2/2
✓ Branch 23 → 24 taken 3 times.
✓ Branch 23 → 25 taken 734 times.
 737 if (auto *const hook = DetourModKit::detail::g_memory_cache_shutdown_window_test_hook)
1741 3 hook();
1742 #endif
1743
1744 // Join the cleanup thread before acquisition of the state mutex. The thread takes s_cache_state_mutex in
1745 // forced cleanup. A join under that mutex can deadlock. The joined population is closed: the flag store
1746 // above makes the one worker exit its wait promptly.
1747 736 s_cleanup_thread_running.store(false, std::memory_order_release);
1748 736 s_cleanup_cv.notify_one();
1749 736 (void)join_cleanup_thread();
1750
1751 736 bool drained = false;
1752 {
1753 736 std::lock_guard state_lock(s_cache_state_mutex);
1754
1755 // Capture the shard count before its reset to zero because the destroy loop needs the length.
1756 736 const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire);
1757 s_shard_count.store(0, std::memory_order_release);
1758
1759 736 drained = drain_admitted_readers();
1760
2/2
✓ Branch 45 → 46 taken 735 times.
✓ Branch 45 → 108 taken 1 time.
 736 if (drained)
1761 {
1762
2/2
✓ Branch 57 → 47 taken 11335 times.
✓ Branch 57 → 58 taken 735 times.
 12070 for (std::size_t i = 0; i < shard_count; ++i)
1763 {
1764 11335 std::unique_lock<SrwSharedMutex> shard_lock(s_cache_shards[i].mtx);
1765 11335 s_cache_shards[i].entries.clear();
1766 11335 s_cache_shards[i].fifo_index.clear();
1767 11335 s_cache_shards[i].sorted_ranges.clear();
1768 11335 }
1769
1770 735 s_cache_shards.reset();
1771
1772 // Only the global cold counters need an explicit reset. s_lifecycle_violations is intentionally
1773 // NOT reset: a sticky diagnostic that must survive a restart.
1774 s_stats.invalidations.store(0, std::memory_order_relaxed);
1775 s_stats.coalesced_queries.store(0, std::memory_order_relaxed);
1776 s_stats.on_demand_cleanups.store(0, std::memory_order_relaxed);
1777 s_last_cleanup_time_ns.store(0, std::memory_order_relaxed);
1778 s_configured_expiry_ms.store(0, std::memory_order_relaxed);
1779 s_max_entries_per_shard.store(0, std::memory_order_relaxed);
1780 735 release_cache_keepalive_after_drain();
1781 }
1782 else
1783 {
1784 // A stalled admitted reader can still hold shard pointers. Retain the shard array, configuration,
1785 // and precommitted module reference. A later init_cache or shutdown_cache call retries the drain.
1786 1 record_reader_drain_timeout_retention();
1787 }
1788 736 s_cleanup_requested.store(false, std::memory_order_relaxed);
1789
1790 #if !defined(_MSC_VER) && defined(_WIN64)
1791 736 shutdown_guarded_engine();
1792 #endif
1793
1794 // Publish Stopped last, under the lifecycle mutex, so the next init_cache admits a fresh start (which
1795 // reopens admission only from the exact closed and zero-reader state).
1796 736 s_lifecycle_state.store(LifecycleState::Stopped, std::memory_order_seq_cst);
1797 736 }
1798
1799 // Deferred-log order: the optional diagnostic tail runs after every cache mutex is released.
1800
2/2
✓ Branch 113 → 114 taken 735 times.
✓ Branch 113 → 117 taken 1 time.
 736 if (drained)
1801 {
1802 735 (void)log().try_log(LogLevel::Debug, "MemoryCache: Shutdown complete.");
1803 }
1804 else
1805 {
1806 1 (void)log().try_log(LogLevel::Debug, "MemoryCache: Shutdown drain timed out, cache storage retained.");
1807 }
1808
2/2
✓ Branch 122 → 123 taken 736 times.
✓ Branch 122 → 125 taken 13826 times.
 14562 }
1809
1810 35165928 MemoryStats get_memory_stats() noexcept
1811 {
1812 35165928 MemoryStats stats{};
1813 // The COLD counters are independent of the shard-array lifetime, so a relaxed load outside the reader
1814 // guard is safe. The hot per-shard tallies are summed under the guard below.
1815 33529170 stats.invalidations = s_stats.invalidations.load(std::memory_order_relaxed);
1816 34475553 stats.coalesced_queries = s_stats.coalesced_queries.load(std::memory_order_relaxed);
1817 35206669 stats.on_demand_cleanups = s_stats.on_demand_cleanups.load(std::memory_order_relaxed);
1818 33752055 stats.lifecycle_violations = s_lifecycle_violations.load(std::memory_order_relaxed);
1819
1820 // Capture the config fields and entry totals under the same reader admission that permission readers use.
1821 // A plain acquire of s_shard_count alone lets a concurrent shutdown free the array between a stale
1822 // non-zero count and the loop. Every field stays at its zero default while the cache is down.
1823 {
1824 33752055 ReaderAdmission admission;
1825
6/6
✓ Branch 32 → 33 taken 22069 times.
✓ Branch 32 → 36 taken 33080699 times.
✓ Branch 34 → 35 taken 20439 times.
✓ Branch 34 → 36 taken 1595 times.
✓ Branch 37 → 38 taken 20438 times.
✓ Branch 37 → 86 taken 33082295 times.
 32694760 if (admission.admitted() && cache_is_running())
1826 {
1827 20387 const std::size_t active_shard_count = s_shard_count.load(std::memory_order_acquire);
1828
1/2
✓ Branch 45 → 46 taken 20399 times.
✗ Branch 45 → 86 not taken.
 20387 if (active_shard_count > 0)
1829 {
1830 20399 stats.shard_count = active_shard_count;
1831 20384 stats.max_entries_per_shard = s_max_entries_per_shard.load(std::memory_order_acquire);
1832 20365 stats.expiry_ms = s_configured_expiry_ms.load(std::memory_order_acquire);
1833
1834 20365 std::size_t total_hard_max = 0;
1835
2/2
✓ Branch 84 → 61 taken 310546 times.
✓ Branch 84 → 85 taken 18029 times.
 328575 for (std::size_t i = 0; i < active_shard_count; ++i)
1836 {
1837 310546 std::shared_lock<SrwSharedMutex> shard_lock(s_cache_shards[i].mtx);
1838 305487 stats.total_entries += s_cache_shards[i].entries.size();
1839 299529 total_hard_max += s_cache_shards[i].max_capacity;
1840 295225 stats.hits += s_cache_shards[i].hits.load(std::memory_order_relaxed);
1841 297003 stats.misses += s_cache_shards[i].misses.load(std::memory_order_relaxed);
1842 297785 }
1843 18029 stats.hard_max_per_shard = total_hard_max / active_shard_count;
1844 }
1845 }
1846 33100312 }
1847
1848 33092486 const std::uint64_t total_queries = stats.hits + stats.misses;
1849 33092486 stats.hit_rate_percent =
1850
2/2
✓ Branch 87 → 88 taken 19960 times.
✓ Branch 87 → 89 taken 33072526 times.
 33092486 (total_queries > 0) ? (static_cast<double>(stats.hits) / static_cast<double>(total_queries)) * 100.0
1851 : -1.0;
1852 33092486 return stats;
1853 }
1854
1855 25 std::string get_cache_stats()
1856 {
1857 25 const MemoryStats s = get_memory_stats();
1858
1859
1/2
✓ Branch 3 → 4 taken 25 times.
✗ Branch 3 → 42 not taken.
 25 std::ostringstream oss;
1860
4/8
✓ Branch 4 → 5 taken 25 times.
✗ Branch 4 → 40 not taken.
✓ Branch 5 → 6 taken 25 times.
✗ Branch 5 → 40 not taken.
✓ Branch 6 → 7 taken 25 times.
✗ Branch 6 → 40 not taken.
✓ Branch 7 → 8 taken 25 times.
✗ Branch 7 → 40 not taken.
 25 oss << "MemoryCache Stats (Shards: " << s.shard_count << ", Entries/Shard: " << s.max_entries_per_shard
1861
4/8
✓ Branch 8 → 9 taken 25 times.
✗ Branch 8 → 40 not taken.
✓ Branch 9 → 10 taken 25 times.
✗ Branch 9 → 40 not taken.
✓ Branch 10 → 11 taken 25 times.
✗ Branch 10 → 40 not taken.
✓ Branch 11 → 12 taken 25 times.
✗ Branch 11 → 40 not taken.
 25 << ", HardMax/Shard: " << s.hard_max_per_shard << ", Expiry: " << s.expiry_ms << "ms) - "
1862
7/14
✓ Branch 12 → 13 taken 25 times.
✗ Branch 12 → 40 not taken.
✓ Branch 13 → 14 taken 25 times.
✗ Branch 13 → 40 not taken.
✓ Branch 14 → 15 taken 25 times.
✗ Branch 14 → 40 not taken.
✓ Branch 15 → 16 taken 25 times.
✗ Branch 15 → 40 not taken.
✓ Branch 16 → 17 taken 25 times.
✗ Branch 16 → 40 not taken.
✓ Branch 17 → 18 taken 25 times.
✗ Branch 17 → 40 not taken.
✓ Branch 18 → 19 taken 25 times.
✗ Branch 18 → 40 not taken.
 25 << "Hits: " << s.hits << ", Misses: " << s.misses << ", Invalidations: " << s.invalidations
1863
4/8
✓ Branch 19 → 20 taken 25 times.
✗ Branch 19 → 40 not taken.
✓ Branch 20 → 21 taken 25 times.
✗ Branch 20 → 40 not taken.
✓ Branch 21 → 22 taken 25 times.
✗ Branch 21 → 40 not taken.
✓ Branch 22 → 23 taken 25 times.
✗ Branch 22 → 40 not taken.
 25 << ", Coalesced: " << s.coalesced_queries << ", OnDemandCleanups: " << s.on_demand_cleanups
1864
4/8
✓ Branch 23 → 24 taken 25 times.
✗ Branch 23 → 40 not taken.
✓ Branch 24 → 25 taken 25 times.
✗ Branch 24 → 40 not taken.
✓ Branch 25 → 26 taken 25 times.
✗ Branch 25 → 40 not taken.
✓ Branch 26 → 27 taken 25 times.
✗ Branch 26 → 40 not taken.
 25 << ", TotalEntries: " << s.total_entries << ", LifecycleViolations: " << s.lifecycle_violations;
1865
1866
2/2
✓ Branch 27 → 28 taken 16 times.
✓ Branch 27 → 34 taken 9 times.
 25 if (s.hit_rate_percent >= 0.0)
1867 {
1868
4/8
✓ Branch 28 → 29 taken 16 times.
✗ Branch 28 → 40 not taken.
✓ Branch 29 → 30 taken 16 times.
✗ Branch 29 → 40 not taken.
✓ Branch 32 → 33 taken 16 times.
✗ Branch 32 → 40 not taken.
✓ Branch 33 → 35 taken 16 times.
✗ Branch 33 → 40 not taken.
 16 oss << ", Hit Rate: " << std::fixed << std::setprecision(2) << s.hit_rate_percent << "%";
1869 }
1870 else
1871 {
1872
1/2
✓ Branch 34 → 35 taken 9 times.
✗ Branch 34 → 40 not taken.
 9 oss << ", Hit Rate: N/A (no queries tracked)";
1873 }
1874
1/2
✓ Branch 35 → 36 taken 25 times.
✗ Branch 35 → 40 not taken.
 50 return oss.str();
1875 25 }
1876
1877 2996 void invalidate_range(Region range) noexcept
1878 {
1879
6/6
✓ Branch 3 → 4 taken 2995 times.
✓ Branch 3 → 5 taken 1 time.
✓ Branch 4 → 5 taken 1 time.
✓ Branch 4 → 6 taken 2994 times.
✓ Branch 7 → 8 taken 2 times.
✓ Branch 7 → 9 taken 2994 times.
 2996 if (!range.base || range.size == 0)
1880 13 return;
1881
1882 // Admission keeps shutdown from a shard-array free during the sweep. A rejected caller has no live cache.
1883 2994 ReaderAdmission admission;
1884
5/6
✓ Branch 11 → 12 taken 2982 times.
✓ Branch 11 → 14 taken 11 times.
✗ Branch 13 → 14 not taken.
✓ Branch 13 → 15 taken 2982 times.
✓ Branch 16 → 17 taken 11 times.
✓ Branch 16 → 18 taken 2982 times.
 2993 if (!admission.admitted() || !cache_is_running())
1885 11 return;
1886
1887 2982 const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire);
1888
1/2
✗ Branch 25 → 26 not taken.
✓ Branch 25 → 27 taken 2982 times.
 2982 if (shard_count == 0)
1889 ✗ return;
1890
1891 2982 invalidate_range_internal(range.base.raw(), range.size);
1892
1893 // The on-demand cleanup fallback holds s_cache_state_mutex while it iterates the shards.
1894 2983 request_cleanup();
1895
2/2
✓ Branch 32 → 33 taken 2983 times.
✓ Branch 32 → 35 taken 11 times.
 2994 }
1896
1897 218760 bool is_readable(Region range) noexcept
1898 {
1899 218760 return check_memory_permission(range.base.raw(), range.size, check_read_permission);
1900 }
1901
1902 3865 bool is_writable(Region range) noexcept
1903 {
1904 3865 return check_memory_permission(range.base.raw(), range.size, check_write_permission);
1905 }
1906
1907 16 ReadableStatus is_readable_nonblocking(Region range) noexcept
1908 {
1909 16 const std::uintptr_t address = range.base.raw();
1910 16 const std::size_t size = range.size;
1911
4/4
✓ Branch 3 → 4 taken 15 times.
✓ Branch 3 → 5 taken 1 time.
✓ Branch 4 → 5 taken 1 time.
✓ Branch 4 → 6 taken 14 times.
 16 if (address == 0 || size == 0)
1912 2 return ReadableStatus::NotReadable;
1913
1914 14 ReaderAdmission admission;
1915
5/6
✓ Branch 8 → 9 taken 10 times.
✓ Branch 8 → 11 taken 4 times.
✗ Branch 10 → 11 not taken.
✓ Branch 10 → 12 taken 10 times.
✓ Branch 13 → 14 taken 4 times.
✓ Branch 13 → 19 taken 10 times.
 14 if (!admission.admitted() || !cache_is_running())
1916 {
1917 // No cache is available. Use a range walk that can wait and return a definite answer. The
1918 // cache-present path below never issues a VirtualQuery and returns Unknown on a miss.
1919
2/2
✓ Branch 15 → 16 taken 2 times.
✓ Branch 15 → 17 taken 2 times.
 4 return range_permission_uncached(address, size, check_read_permission) ? ReadableStatus::Readable
1920 4 : ReadableStatus::NotReadable;
1921 }
1922
1923 10 const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire);
1924
1/2
✗ Branch 26 → 27 not taken.
✓ Branch 26 → 28 taken 10 times.
 10 if (shard_count == 0)
1925 ✗ return ReadableStatus::Unknown;
1926
1927 10 const std::size_t shard_idx = compute_shard_index(address, shard_count);
1928 10 const std::uint64_t now_ns = current_time_ns();
1929 10 const std::uint64_t expiry_ns = configured_expiry_ns();
1930
1931 // The shared try-lock prevents a wait by a latency-sensitive thread on a contended shard.
1932 10 std::shared_lock<SrwSharedMutex> lock(s_cache_shards[shard_idx].mtx, std::try_to_lock);
1933
1/2
✗ Branch 34 → 35 not taken.
✓ Branch 34 → 36 taken 10 times.
 10 if (!lock.owns_lock())
1934 ✗ return ReadableStatus::Unknown;
1935
1936 CachedMemoryRegionInfo *cached_info =
1937 10 find_in_shard(s_cache_shards[shard_idx], address, size, now_ns, expiry_ns);
1938
2/2
✓ Branch 38 → 39 taken 6 times.
✓ Branch 38 → 48 taken 4 times.
 10 if (cached_info)
1939 {
1940 6 s_cache_shards[shard_idx].hits.fetch_add(1, std::memory_order_relaxed);
1941 // Require MEM_COMMIT alongside the read permission, symmetric with the blocking hit path.
1942 5 return (cached_info->state == MEM_COMMIT && check_read_permission(cached_info->protection))
1943
4/4
✓ Branch 42 → 43 taken 5 times.
✓ Branch 42 → 45 taken 1 time.
✓ Branch 44 → 45 taken 2 times.
✓ Branch 44 → 46 taken 3 times.
 11 ? ReadableStatus::Readable
1944 6 : ReadableStatus::NotReadable;
1945 }
1946
1947 // Under non-blocking semantics, return Unknown on a cache miss instead of a VirtualQuery call.
1948 4 return ReadableStatus::Unknown;
1949 14 }
1950 } // namespace memory
1951 } // namespace DetourModKit
1952
1953 #if defined(DMK_ENABLE_TEST_SEAMS)
1954 namespace DetourModKit::detail
1955 {
1956 3 void memory_cache_abandon_for_test() noexcept
1957 {
1958 3 memory::abandon_cache_unauthorized();
1959 3 }
1960
1961 5 std::uint64_t memory_cache_admitted_reader_count_for_test() noexcept
1962 {
1963 5 return memory::admitted_reader_count();
1964 }
1965
1966 1 bool memory_cache_has_retained_shards_for_test() noexcept
1967 {
1968 1 return memory::s_cache_shards != nullptr;
1969 }
1970
1971 2 bool memory_cache_reader_would_be_admitted_for_test() noexcept
1972 {
1973 2 memory::ReaderAdmission admission;
1974 2 return admission.admitted();
1975 2 }
1976
1977 2 void memory_cache_hold_shared_shard_lock_for_test(Address address, void (*callback)() noexcept) noexcept
1978 {
1979 2 memory::hold_shard_shared_lock_for_test(address, callback);
1980 2 }
1981
1982 26 void memory_cache_shard_index_sizes_for_test(
1983 Address address,
1984 std::size_t &entries,
1985 std::size_t &fifo,
1986 std::size_t &ranges
1987 ) noexcept
1988 {
1989 26 memory::shard_index_sizes_for_test(address, entries, fifo, ranges);
1990 26 }
1991 } // namespace DetourModKit::detail
1992 #endif
1993