src/memory_cache.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /** | ||
| 2 | * @file memory_cache.cpp | ||
| 3 | * @brief This TU implements the protection-region cache and its readability predicates. | ||
| 4 | * | ||
| 5 | * The sharded cache uses SRW locks. It provides FIFO eviction and exact invalidation without SEH. | ||
| 6 | * Shutdown closes reader admission and drains admitted readers before a deadline. A timeout retains the storage. | ||
| 7 | * | ||
| 8 | * This TU controls the MinGW guarded-engine lifecycle. | ||
| 9 | */ | ||
| 10 | |||
| 11 | #include "DetourModKit/memory.hpp" | ||
| 12 | #include "DetourModKit/diagnostics.hpp" | ||
| 13 | #include "DetourModKit/logger.hpp" | ||
| 14 | #include "internal/drain_backoff.hpp" | ||
| 15 | #include "internal/lifecycle_context.hpp" | ||
| 16 | #include "internal/srw_shared_mutex.hpp" | ||
| 17 | #include "platform.hpp" | ||
| 18 | #include "internal/memory_guarded.hpp" | ||
| 19 | |||
| 20 | #include <windows.h> | ||
| 21 | |||
| 22 | #include <algorithm> | ||
| 23 | #include <array> | ||
| 24 | #include <atomic> | ||
| 25 | #include <cassert> | ||
| 26 | #include <chrono> | ||
| 27 | #include <condition_variable> | ||
| 28 | #include <cstddef> | ||
| 29 | #include <cstdint> | ||
| 30 | #include <cstdlib> | ||
| 31 | #include <deque> | ||
| 32 | #include <iomanip> | ||
| 33 | #include <map> | ||
| 34 | #include <memory> | ||
| 35 | #include <mutex> | ||
| 36 | #include <shared_mutex> | ||
| 37 | #include <sstream> | ||
| 38 | #include <string> | ||
| 39 | #include <thread> | ||
| 40 | #include <type_traits> | ||
| 41 | #include <unordered_map> | ||
| 42 | |||
| 43 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 44 | namespace DetourModKit::detail | ||
| 45 | { | ||
| 46 | void (*g_memory_cache_before_lifecycle_lock_test_hook)() = nullptr; | ||
| 47 | void (*g_memory_cache_before_running_publish_test_hook)() = nullptr; | ||
| 48 | void (*g_memory_cache_reopen_window_test_hook)() = nullptr; | ||
| 49 | void (*g_memory_cache_shutdown_window_test_hook)() = nullptr; | ||
| 50 | void (*g_memory_cache_leader_publish_window_test_hook)() = nullptr; | ||
| 51 | HMODULE (*g_memory_cache_keepalive_ref_override)() noexcept = nullptr; | ||
| 52 | } // namespace DetourModKit::detail | ||
| 53 | #endif | ||
| 54 | |||
| 55 | namespace DetourModKit | ||
| 56 | { | ||
| 57 | namespace memory | ||
| 58 | { | ||
| 59 | using DetourModKit::detail::acquire_module_ref; | ||
| 60 | using DetourModKit::detail::release_module_ref; | ||
| 61 | using DetourModKit::detail::SrwSharedMutex; | ||
| 62 | |||
| 63 | namespace | ||
| 64 | { | ||
| 65 | // CachePermissions groups page-protection flags for cache permission checks. A struct preserves internal | ||
| 66 | // linkage through the outer anonymous namespace without a named namespace. | ||
| 67 | struct CachePermissions | ||
| 68 | { | ||
| 69 | static constexpr DWORD READ_PERMISSION_FLAGS = PAGE_READONLY | PAGE_READWRITE | PAGE_WRITECOPY | | ||
| 70 | PAGE_EXECUTE_READ | PAGE_EXECUTE_READWRITE | | ||
| 71 | PAGE_EXECUTE_WRITECOPY; | ||
| 72 | static constexpr DWORD WRITE_PERMISSION_FLAGS = | ||
| 73 | PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY; | ||
| 74 | static constexpr DWORD NOACCESS_GUARD_FLAGS = PAGE_NOACCESS | PAGE_GUARD; | ||
| 75 | }; | ||
| 76 | |||
| 77 | /** | ||
| 78 | * @struct CachedMemoryRegionInfo | ||
| 79 | * @brief Stores a cached protection snapshot for one VirtualQuery region. | ||
| 80 | * @details A hit requires content_gen to equal the shard's current generation. A clear or contended | ||
| 81 | * invalidation therefore invalidates this entry in O(1), regardless of physical eviction. | ||
| 82 | */ | ||
| 83 | struct CachedMemoryRegionInfo | ||
| 84 | { | ||
| 85 | std::uintptr_t base_address; | ||
| 86 | std::size_t region_size; | ||
| 87 | DWORD protection; | ||
| 88 | DWORD state; | ||
| 89 | std::uint64_t timestamp_ns; | ||
| 90 | std::uint64_t fifo_key; | ||
| 91 | std::uint64_t content_gen; | ||
| 92 | bool valid; | ||
| 93 | |||
| 94 | 440 | CachedMemoryRegionInfo() | |
| 95 | 440 | : base_address(0), region_size(0), protection(0), state(0), timestamp_ns(0), fifo_key(0), | |
| 96 | 440 | content_gen(0), valid(false) | |
| 97 | { | ||
| 98 | 440 | } | |
| 99 | }; | ||
| 100 | |||
| 101 | /** | ||
| 102 | * @struct CacheShard | ||
| 103 | * @brief Stores one shard with a hit map, FIFO map, sorted range index, and content generation. | ||
| 104 | * @details The unordered_map is keyed by region base, so its O(1) hit covers only a query in a region's | ||
| 105 | * FIRST page. Deeper queries use the sorted range index. The std::map keyed by a | ||
| 106 | * monotonic counter gives oldest-first eviction. alignas(64) aligns each shard object to a 64-byte | ||
| 107 | * boundary. The inline mutex makes the shard non-movable, so shards use a fixed-size array that | ||
| 108 | * never relocates. | ||
| 109 | */ | ||
| 110 | #if defined(_MSC_VER) | ||
| 111 | #pragma warning(push) | ||
| 112 | // C4324: CacheShard is intentionally padded to a full cache line by alignas(64) for cache-line hygiene. | ||
| 113 | #pragma warning(disable : 4324) | ||
| 114 | #endif | ||
| 115 | struct alignas(64) CacheShard | ||
| 116 | { | ||
| 117 | std::unordered_map<std::uintptr_t, CachedMemoryRegionInfo> entries; | ||
| 118 | std::map<std::uint64_t, std::uintptr_t> fifo_index; | ||
| 119 | // The shard SRW lock serializes this base-sorted O(log n) containment index. The deque prevents full | ||
| 120 | // buffer relocation during growth. Each entry stores [base, base + size). | ||
| 121 | std::deque<std::pair<std::uintptr_t, std::uintptr_t>> sorted_ranges; | ||
| 122 | SrwSharedMutex mtx; | ||
| 123 | // The first thread to CAS this 0 -> 1 becomes the VirtualQuery leader. The rest coalesce onto its | ||
| 124 | // result. | ||
| 125 | std::atomic<char> in_flight{0}; | ||
| 126 | // Content generation provides exact invalidation. clear_cache advances it under the exclusive lock. | ||
| 127 | // A contended invalidate_range advances it through fetch_add. A hit requires a current entry stamp. | ||
| 128 | std::atomic<std::uint64_t> content_gen{0}; | ||
| 129 | // The query thread already touches this per-shard hit and miss counter line. A busy workload therefore | ||
| 130 | // does not move one global counter line across every core. | ||
| 131 | std::atomic<std::uint64_t> hits{0}; | ||
| 132 | std::atomic<std::uint64_t> misses{0}; | ||
| 133 | std::uint64_t entry_counter{0}; | ||
| 134 | std::size_t capacity; | ||
| 135 | std::size_t max_capacity; | ||
| 136 | |||
| 137 |
2/4✓ Branch 4 → 5 taken 11479 times.
✗ Branch 4 → 14 not taken.
✓ Branch 10 → 11 taken 11479 times.
✗ Branch 10 → 12 not taken.
|
11479 | CacheShard() : capacity(0), max_capacity(0) { entries.reserve(64); } |
| 138 | }; | ||
| 139 | #if defined(_MSC_VER) | ||
| 140 | #pragma warning(pop) | ||
| 141 | #endif | ||
| 142 | |||
| 143 | static_assert( | ||
| 144 | std::is_same_v< | ||
| 145 | decltype(CacheShard::sorted_ranges), | ||
| 146 | std::deque<std::pair<std::uintptr_t, std::uintptr_t>>>, | ||
| 147 | "CacheShard::sorted_ranges is pinned to std::deque so mutation never relocates the buffer." | ||
| 148 | ); | ||
| 149 | |||
| 150 | 281514 | inline std::uint64_t current_time_ns() noexcept | |
| 151 | { | ||
| 152 | 287402 | return std::chrono::duration_cast<std::chrono::nanoseconds>( | |
| 153 | 566668 | std::chrono::steady_clock::now().time_since_epoch() | |
| 154 | ) | ||
| 155 | 287206 | .count(); | |
| 156 | } | ||
| 157 | |||
| 158 | /** | ||
| 159 | * @brief Computes the shard index for an address. | ||
| 160 | * @note Uses a golden-ratio hash to spread adjacent addresses across shards. | ||
| 161 | */ | ||
| 162 | 279237 | constexpr inline std::size_t compute_shard_index(std::uintptr_t address, std::size_t shard_count) noexcept | |
| 163 | { | ||
| 164 | 279237 | return (static_cast<std::size_t>((address * 0x9E3779B97F4A7C15ULL) >> 48)) % shard_count; | |
| 165 | } | ||
| 166 | |||
| 167 | // The fixed-size shard array never resizes because CacheShard is non-movable. It is null before init. | ||
| 168 | // A drained shutdown clears it. Abandonment or a timeout retains it until a later drain. | ||
| 169 | std::unique_ptr<CacheShard[]> s_cache_shards; | ||
| 170 | std::atomic<std::size_t> s_shard_count{0}; | ||
| 171 | std::atomic<std::size_t> s_max_entries_per_shard{0}; | ||
| 172 | std::atomic<unsigned int> s_configured_expiry_ms{0}; | ||
| 173 | |||
| 174 | /** | ||
| 175 | * @enum LifecycleState | ||
| 176 | * @brief Defines the cache lifecycle authority: Stopped -> Starting -> Running -> Stopping -> Stopped. | ||
| 177 | * @details Running is published last by init_cache and cleared first by shutdown_cache. The per-stripe | ||
| 178 | * closed-bit words decide reader admission. This state directs control paths. Normal transitions | ||
| 179 | * use s_lifecycle_mutex. Loader-lock abandonment uses compare-exchange because it cannot wait for | ||
| 180 | * that mutex. | ||
| 181 | */ | ||
| 182 | enum class LifecycleState : std::uint8_t | ||
| 183 | { | ||
| 184 | Stopped, | ||
| 185 | Starting, | ||
| 186 | Running, | ||
| 187 | Stopping | ||
| 188 | }; | ||
| 189 | std::atomic<LifecycleState> s_lifecycle_state{LifecycleState::Stopped}; | ||
| 190 | |||
| 191 | /// Reports whether the cache lifecycle is in its Running state. | ||
| 192 | 319374 | [[nodiscard]] inline bool cache_is_running() noexcept | |
| 193 | { | ||
| 194 | 319374 | return s_lifecycle_state.load(std::memory_order_seq_cst) == LifecycleState::Running; | |
| 195 | } | ||
| 196 | |||
| 197 | /// Returns the configured cache-entry expiry in nanoseconds. | ||
| 198 | 280097 | [[nodiscard]] inline std::uint64_t configured_expiry_ns() noexcept | |
| 199 | { | ||
| 200 | 282989 | return static_cast<std::uint64_t>(s_configured_expiry_ms.load(std::memory_order_acquire)) * | |
| 201 | 282989 | 1'000'000ULL; | |
| 202 | } | ||
| 203 | |||
| 204 | // Serializes shard-array mutation against cleanup. It nests inside s_lifecycle_mutex on init/shutdown. | ||
| 205 | SrwSharedMutex s_cache_state_mutex; | ||
| 206 | |||
| 207 | // This is the outermost lifecycle lock. Cleanup and readers never take it, and loader-lock teardown never | ||
| 208 | // waits on it. Lock order is lifecycle -> state -> shard. The join lock nests only inside lifecycle. | ||
| 209 | SrwSharedMutex s_lifecycle_mutex; | ||
| 210 | |||
| 211 | // Advances on every admitted start. The cleanup thread exits once its captured value no longer matches, | ||
| 212 | // so a worker cannot outlive its session and touch a later generation's shards. | ||
| 213 | std::atomic<std::uint64_t> s_lifecycle_generation{0}; | ||
| 214 | |||
| 215 | // This sticky counter records unexpected joinable handles recovered before a new cleanup worker appears. | ||
| 216 | std::atomic<std::uint64_t> s_lifecycle_violations{0}; | ||
| 217 | |||
| 218 | // Reader admission uses the [B-73] closed-drain pattern. Each stripe word holds a high closed bit and an | ||
| 219 | // admitted-reader count. One compare-exchange checks the closed bit and increments the count. Teardown sets | ||
| 220 | // the closed bit on every stripe before it reads the counts, so it drains a closed population. | ||
| 221 | // Cache-line-padded stripes keep concurrent readers off one shared line (measured in the phase 9 warm-hit | ||
| 222 | // contention benchmark). | ||
| 223 | constexpr std::size_t READER_STRIPE_COUNT = 64; | ||
| 224 | constexpr std::uint64_t READER_ADMISSION_CLOSED = std::uint64_t{1} << 63; | ||
| 225 | constexpr std::uint64_t READER_ADMISSION_COUNT_MASK = ~READER_ADMISSION_CLOSED; | ||
| 226 | |||
| 227 | #if defined(_MSC_VER) | ||
| 228 | #pragma warning(push) | ||
| 229 | // C4324: ReaderStripe is intentionally padded to a full cache line by alignas(64) so stripes never share a line. | ||
| 230 | #pragma warning(disable : 4324) | ||
| 231 | #endif | ||
| 232 | struct alignas(64) ReaderStripe | ||
| 233 | { | ||
| 234 | // Starts closed: admission opens only after init_cache publishes a running cache. | ||
| 235 | std::atomic<std::uint64_t> word{READER_ADMISSION_CLOSED}; | ||
| 236 | }; | ||
| 237 | #if defined(_MSC_VER) | ||
| 238 | #pragma warning(pop) | ||
| 239 | #endif | ||
| 240 | static_assert(std::atomic<std::uint64_t>::is_always_lock_free); | ||
| 241 | |||
| 242 | std::array<ReaderStripe, READER_STRIPE_COUNT> s_reader_stripes{}; | ||
| 243 | |||
| 244 | /** | ||
| 245 | * @brief Returns this thread's admission stripe, derived from its Win32 thread id. | ||
| 246 | * @details A hash of GetCurrentThreadId allocates nothing and takes no lock, so loader lock permits it. | ||
| 247 | * In contrast, MinGW lowers first access to a thread_local counter into __emutls_get_address, | ||
| 248 | * which can allocate. The id is stable for the thread's life, so the same stripe carries the | ||
| 249 | * admission and its paired release. A collision adds contention but never a drain miscount. | ||
| 250 | */ | ||
| 251 | 61601070 | [[nodiscard]] inline std::size_t reader_stripe_index() noexcept | |
| 252 | { | ||
| 253 | 61601070 | const std::uint64_t mixed = static_cast<std::uint64_t>(GetCurrentThreadId()) * 0x9E3779B97F4A7C15ULL; | |
| 254 | 63506040 | return static_cast<std::size_t>(mixed >> 48) % READER_STRIPE_COUNT; | |
| 255 | } | ||
| 256 | |||
| 257 | /// Returns the admitted-reader count summed across all stripes. | ||
| 258 | 1628 | [[nodiscard]] inline std::uint64_t admitted_reader_count() noexcept | |
| 259 | { | ||
| 260 | 1628 | std::uint64_t total = 0; | |
| 261 |
2/2✓ Branch 11 → 3 taken 104192 times.
✓ Branch 11 → 12 taken 1628 times.
|
105820 | for (const ReaderStripe &stripe : s_reader_stripes) |
| 262 | { | ||
| 263 | 208384 | total += stripe.word.load(std::memory_order_seq_cst) & READER_ADMISSION_COUNT_MASK; | |
| 264 | } | ||
| 265 | 1628 | return total; | |
| 266 | } | ||
| 267 | |||
| 268 | /** | ||
| 269 | * @brief Closes reader admission on every stripe. | ||
| 270 | * @details After the last fetch_or returns, no stripe admits a reader, so a later drain waits on a closed | ||
| 271 | * population. Each fetch_or is wait-free, so loader-lock abandonment can call this. | ||
| 272 | */ | ||
| 273 | 1496 | void close_reader_admission() noexcept | |
| 274 | { | ||
| 275 |
2/2✓ Branch 6 → 3 taken 95744 times.
✓ Branch 6 → 7 taken 1496 times.
|
97240 | for (ReaderStripe &stripe : s_reader_stripes) |
| 276 | { | ||
| 277 | 95744 | stripe.word.fetch_or(READER_ADMISSION_CLOSED, std::memory_order_seq_cst); | |
| 278 | } | ||
| 279 | 1496 | } | |
| 280 | |||
| 281 | /** | ||
| 282 | * @brief Reopens reader admission after a drained start from each exact closed and zero-reader value. | ||
| 283 | * @details While a stripe is closed, its count only decreases. The caller drains every count under | ||
| 284 | * s_cache_state_mutex. Init opens all stripes before it publishes Running. Concurrent abandonment | ||
| 285 | * closes them and changes Starting to Stopped. The publication check then rolls init back. | ||
| 286 | */ | ||
| 287 | 744 | void reopen_reader_admission() noexcept | |
| 288 | { | ||
| 289 |
2/2✓ Branch 15 → 3 taken 47616 times.
✓ Branch 15 → 16 taken 744 times.
|
96720 | for (std::size_t i = 0; i < s_reader_stripes.size(); ++i) |
| 290 | { | ||
| 291 | 47616 | ReaderStripe &stripe = s_reader_stripes[i]; | |
| 292 | 47616 | std::uint64_t expected = READER_ADMISSION_CLOSED; | |
| 293 | (void)stripe.word | ||
| 294 | 47616 | .compare_exchange_strong(expected, 0, std::memory_order_seq_cst, std::memory_order_seq_cst); | |
| 295 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 296 |
2/2✓ Branch 9 → 10 taken 744 times.
✓ Branch 9 → 12 taken 46872 times.
|
47616 | if (i == 0) |
| 297 | { | ||
| 298 |
2/2✓ Branch 10 → 11 taken 1 time.
✓ Branch 10 → 12 taken 743 times.
|
744 | if (auto *const hook = DetourModKit::detail::g_memory_cache_reopen_window_test_hook) |
| 299 | 1 | hook(); | |
| 300 | } | ||
| 301 | #endif | ||
| 302 | } | ||
| 303 | 744 | } | |
| 304 | |||
| 305 | /// Bounds every reader drain. Expiry retains the reader-visible storage instead of a free ([B-73]). | ||
| 306 | constexpr auto READER_DRAIN_TIMEOUT = std::chrono::seconds{1}; | ||
| 307 | |||
| 308 | /// Waits for the admitted-reader count to reach zero within READER_DRAIN_TIMEOUT. | ||
| 309 | 1494 | [[nodiscard]] bool drain_admitted_readers() noexcept | |
| 310 | { | ||
| 311 | 1494 | return DetourModKit::detail::drain_until_zero( | |
| 312 | 1623 | []() noexcept { return admitted_reader_count(); }, | |
| 313 | 2988 | std::chrono::steady_clock::now() + READER_DRAIN_TIMEOUT | |
| 314 | 1494 | ); | |
| 315 | } | ||
| 316 | |||
| 317 | // A cache session takes this reference before admission opens. A timeout retains it with the shard array. | ||
| 318 | // Guarded by s_cache_state_mutex. | ||
| 319 | HMODULE s_cache_self_ref{nullptr}; | ||
| 320 | |||
| 321 | /// Takes the keepalive that makes a later bounded-drain timeout safe. | ||
| 322 | 749 | [[nodiscard]] HMODULE acquire_cache_keepalive_ref() noexcept | |
| 323 | { | ||
| 324 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 325 |
2/2✓ Branch 2 → 3 taken 1 time.
✓ Branch 2 → 4 taken 748 times.
|
749 | if (auto *const override_fn = DetourModKit::detail::g_memory_cache_keepalive_ref_override) |
| 326 | 1 | return override_fn(); | |
| 327 | #endif | ||
| 328 | 748 | return acquire_module_ref(diagnostics::ModulePinReason::MemoryCache); | |
| 329 | } | ||
| 330 | |||
| 331 | /** | ||
| 332 | * @brief Releases the cache keepalive after the admitted reader count reaches zero. | ||
| 333 | * @note | ||
| 334 | * Must be called with s_cache_state_mutex held. | ||
| 335 | */ | ||
| 336 | 748 | void release_cache_keepalive_after_drain() noexcept | |
| 337 | { | ||
| 338 |
1/2✓ Branch 2 → 3 taken 748 times.
✗ Branch 2 → 5 not taken.
|
748 | if (s_cache_self_ref != nullptr) |
| 339 | { | ||
| 340 | 748 | release_module_ref(s_cache_self_ref, diagnostics::ModulePinReason::MemoryCache); | |
| 341 | 748 | s_cache_self_ref = nullptr; | |
| 342 | } | ||
| 343 | 748 | } | |
| 344 | |||
| 345 | /** | ||
| 346 | * @brief Records one timeout that retains the cache keepalive and reader-visible state. | ||
| 347 | * | ||
| 348 | * @note Must be called with s_cache_state_mutex held. | ||
| 349 | */ | ||
| 350 | 1 | void record_reader_drain_timeout_retention() noexcept | |
| 351 | { | ||
| 352 |
1/2✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 1 time.
|
1 | assert(s_cache_self_ref != nullptr); |
| 353 | 1 | DetourModKit::diagnostics::record_intentional_leak( | |
| 354 | DetourModKit::diagnostics::LeakSubsystem::MemoryCache | ||
| 355 | ); | ||
| 356 | 1 | } | |
| 357 | |||
| 358 | /** | ||
| 359 | * @class ReaderAdmission | ||
| 360 | * @brief Admits one reader through its stripe's closed-bit word and releases it on every exit path. | ||
| 361 | * @details A rejected admission leaves the count untouched, so the caller never joins the teardown drain | ||
| 362 | * population and must take the uncached route. | ||
| 363 | */ | ||
| 364 | class ReaderAdmission | ||
| 365 | { | ||
| 366 | public: | ||
| 367 | 62183401 | ReaderAdmission() noexcept : m_stripe(reader_stripe_index()) | |
| 368 | { | ||
| 369 | 62838881 | std::atomic<std::uint64_t> &word = s_reader_stripes[m_stripe].word; | |
| 370 | 63303546 | std::uint64_t value = word.load(std::memory_order_seq_cst); | |
| 371 |
2/2✓ Branch 19 → 12 taken 320010 times.
✓ Branch 19 → 20 taken 63120795 times.
|
63440805 | while ((value & READER_ADMISSION_CLOSED) == 0) |
| 372 | { | ||
| 373 |
2/2✓ Branch 17 → 18 taken 324662 times.
✓ Branch 17 → 19 taken 496 times.
|
645168 | if (word.compare_exchange_weak( |
| 374 | value, | ||
| 375 | value + 1, | ||
| 376 | std::memory_order_seq_cst, | ||
| 377 | std::memory_order_seq_cst | ||
| 378 | )) | ||
| 379 | { | ||
| 380 | 324662 | m_admitted = true; | |
| 381 | 324662 | break; | |
| 382 | } | ||
| 383 | } | ||
| 384 | 63445457 | } | |
| 385 | |||
| 386 | 62267721 | ~ReaderAdmission() noexcept | |
| 387 | { | ||
| 388 |
2/2✓ Branch 2 → 3 taken 322454 times.
✓ Branch 2 → 7 taken 61945267 times.
|
62267721 | if (m_admitted) |
| 389 | { | ||
| 390 | 322454 | s_reader_stripes[m_stripe].word.fetch_sub(1, std::memory_order_release); | |
| 391 | } | ||
| 392 | 62264571 | } | |
| 393 | |||
| 394 | 62085741 | [[nodiscard]] bool admitted() const noexcept { return m_admitted; } | |
| 395 | |||
| 396 | ReaderAdmission(const ReaderAdmission &) = delete; | ||
| 397 | ReaderAdmission &operator=(const ReaderAdmission &) = delete; | ||
| 398 | |||
| 399 | private: | ||
| 400 | const std::size_t m_stripe; | ||
| 401 | bool m_admitted{false}; | ||
| 402 | }; | ||
| 403 | |||
| 404 | // Use std::thread, not jthread. The jthread auto-join destructor runs after s_cleanup_cv and | ||
| 405 | // s_cleanup_mutex are destroyed in reverse declaration order. Manual join in shutdown_cache avoids this. | ||
| 406 | std::atomic<bool> s_cleanup_thread_running{false}; | ||
| 407 | std::thread s_cleanup_thread; | ||
| 408 | // s_cleanup_self_ref holds a counted module reference acquired before thread creation. A clean join | ||
| 409 | // releases it. The loader-lock detach path leaks it so the detached thread's code stays mapped. | ||
| 410 | HMODULE s_cleanup_self_ref{nullptr}; | ||
| 411 | std::mutex s_cleanup_mutex; | ||
| 412 | std::condition_variable s_cleanup_cv; | ||
| 413 | std::atomic<bool> s_cleanup_requested{false}; | ||
| 414 | // Serializes the cleanup handle's join/detach/assignment. Loader-lock teardown only tries this lock. | ||
| 415 | SrwSharedMutex s_cleanup_join_mutex; | ||
| 416 | |||
| 417 | // This timer controls on-demand cleanup when the background thread is disabled. | ||
| 418 | std::atomic<std::uint64_t> s_last_cleanup_time_ns{0}; | ||
| 419 | constexpr std::uint64_t CLEANUP_INTERVAL_NS = 1'000'000'000ULL; | ||
| 420 | |||
| 421 | // These instance-wide COLD counters advance only off the read hot path. The hot hit and miss tallies live | ||
| 422 | // per shard. Each counter is alignas(64), so the three never share a cache line. | ||
| 423 | #if defined(_MSC_VER) | ||
| 424 | #pragma warning(push) | ||
| 425 | // C4324: each counter is intentionally padded to a full cache line by alignas(64) to prevent line overlap. | ||
| 426 | #pragma warning(disable : 4324) | ||
| 427 | #endif | ||
| 428 | struct CacheStats | ||
| 429 | { | ||
| 430 | alignas(64) std::atomic<std::uint64_t> invalidations{0}; | ||
| 431 | alignas(64) std::atomic<std::uint64_t> coalesced_queries{0}; | ||
| 432 | alignas(64) std::atomic<std::uint64_t> on_demand_cleanups{0}; | ||
| 433 | }; | ||
| 434 | #if defined(_MSC_VER) | ||
| 435 | #pragma warning(pop) | ||
| 436 | #endif | ||
| 437 | CacheStats s_stats; | ||
| 438 | |||
| 439 | /** | ||
| 440 | * @brief Checks if a cache entry is valid, current for the shard generation, and covers [address, address + | ||
| 441 | * size). | ||
| 442 | * @param shard_content_gen The shard's current content generation. A clear or contended invalidation | ||
| 443 | * advances it. An older stamp marks the entry invalid, so lookup returns a miss. | ||
| 444 | */ | ||
| 445 | 282247 | constexpr inline bool is_entry_valid_and_covers( | |
| 446 | const CachedMemoryRegionInfo &entry, | ||
| 447 | std::uintptr_t address, | ||
| 448 | std::size_t size, | ||
| 449 | std::uint64_t current_ns, | ||
| 450 | std::uint64_t expiry_ns, | ||
| 451 | std::uint64_t shard_content_gen | ||
| 452 | ) noexcept | ||
| 453 | { | ||
| 454 |
1/2✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 282247 times.
|
282247 | if (!entry.valid) |
| 455 | ✗ | return false; | |
| 456 | |||
| 457 |
2/2✓ Branch 4 → 5 taken 78 times.
✓ Branch 4 → 6 taken 282169 times.
|
282247 | if (entry.content_gen != shard_content_gen) |
| 458 | 78 | return false; | |
| 459 | |||
| 460 | 282169 | const std::uint64_t entry_age = current_ns - entry.timestamp_ns; | |
| 461 |
2/2✓ Branch 6 → 7 taken 4 times.
✓ Branch 6 → 8 taken 282165 times.
|
282169 | if (entry_age > expiry_ns) |
| 462 | 4 | return false; | |
| 463 | |||
| 464 | 282165 | const std::uintptr_t end_address = address + size; | |
| 465 |
2/2✓ Branch 8 → 9 taken 4 times.
✓ Branch 8 → 10 taken 282161 times.
|
282165 | if (end_address < address) |
| 466 | 4 | return false; | |
| 467 | |||
| 468 | 282161 | const std::uintptr_t entry_end_address = entry.base_address + entry.region_size; | |
| 469 |
1/2✗ Branch 10 → 11 not taken.
✓ Branch 10 → 12 taken 282161 times.
|
282161 | if (entry_end_address < entry.base_address) |
| 470 | ✗ | return false; | |
| 471 | |||
| 472 |
3/4✓ Branch 12 → 13 taken 281753 times.
✓ Branch 12 → 15 taken 408 times.
✓ Branch 13 → 14 taken 283436 times.
✗ Branch 13 → 15 not taken.
|
282161 | return address >= entry.base_address && end_address <= entry_end_address; |
| 473 | } | ||
| 474 | |||
| 475 | /// Checks protection flags for read permission. | ||
| 476 | 295994 | constexpr inline bool check_read_permission(DWORD protection) noexcept | |
| 477 | { | ||
| 478 |
1/2✓ Branch 2 → 3 taken 297006 times.
✗ Branch 2 → 5 not taken.
|
593000 | return (protection & CachePermissions::READ_PERMISSION_FLAGS) != 0 && |
| 479 |
2/2✓ Branch 3 → 4 taken 295256 times.
✓ Branch 3 → 5 taken 1750 times.
|
593000 | (protection & CachePermissions::NOACCESS_GUARD_FLAGS) == 0; |
| 480 | } | ||
| 481 | |||
| 482 | /// Checks protection flags for write permission. | ||
| 483 | 3954 | constexpr inline bool check_write_permission(DWORD protection) noexcept | |
| 484 | { | ||
| 485 |
2/2✓ Branch 2 → 3 taken 3951 times.
✓ Branch 2 → 5 taken 3 times.
|
7905 | return (protection & CachePermissions::WRITE_PERMISSION_FLAGS) != 0 && |
| 486 |
1/2✓ Branch 3 → 4 taken 3952 times.
✗ Branch 3 → 5 not taken.
|
7905 | (protection & CachePermissions::NOACCESS_GUARD_FLAGS) == 0; |
| 487 | } | ||
| 488 | |||
| 489 | /** | ||
| 490 | * @brief Inserts a range into the shard's sorted auxiliary container. | ||
| 491 | * @note Must be called with the shard mutex held (exclusive). This helper is deliberately not noexcept | ||
| 492 | * because bad_alloc must reach update_shard_with_region's fail-soft catch. | ||
| 493 | */ | ||
| 494 | 430 | void insert_sorted_range(CacheShard &shard, std::uintptr_t base_addr, std::size_t region_size) | |
| 495 | { | ||
| 496 | 430 | auto range = std::make_pair(base_addr, base_addr + region_size); | |
| 497 |
1/2✓ Branch 5 → 6 taken 430 times.
✗ Branch 5 → 9 not taken.
|
430 | auto pos = std::lower_bound(shard.sorted_ranges.begin(), shard.sorted_ranges.end(), range); |
| 498 |
2/2✓ Branch 7 → 8 taken 428 times.
✓ Branch 7 → 11 taken 2 times.
|
430 | shard.sorted_ranges.insert(pos, range); |
| 499 | 428 | } | |
| 500 | |||
| 501 | /** | ||
| 502 | * @brief Removes a range from the shard's sorted auxiliary container. | ||
| 503 | * @note Must be called with the shard mutex held (exclusive). | ||
| 504 | */ | ||
| 505 | 58 | void remove_sorted_range(CacheShard &shard, std::uintptr_t base_addr) noexcept | |
| 506 | { | ||
| 507 | auto it = std::lower_bound( | ||
| 508 | 58 | shard.sorted_ranges.begin(), | |
| 509 | 58 | shard.sorted_ranges.end(), | |
| 510 | 58 | std::make_pair(base_addr, std::uintptr_t{0}) | |
| 511 | 58 | ); | |
| 512 |
3/6✓ Branch 8 → 9 taken 58 times.
✗ Branch 8 → 12 not taken.
✓ Branch 10 → 11 taken 58 times.
✗ Branch 10 → 12 not taken.
✓ Branch 13 → 14 taken 58 times.
✗ Branch 13 → 17 not taken.
|
58 | if (it != shard.sorted_ranges.end() && it->first == base_addr) |
| 513 | 58 | shard.sorted_ranges.erase(it); | |
| 514 | 58 | } | |
| 515 | |||
| 516 | /** | ||
| 517 | * @brief Finds and validates a cache entry in a shard that covers [address, address + size). | ||
| 518 | * @note Must be called with the shard mutex held (shared or exclusive). | ||
| 519 | * @note The shard-local direct probe is a first-page fast path because the map key is the region base. | ||
| 520 | * Deeper queries use the O(log n) search over sorted_ranges. A miss returns nullptr. The caller then | ||
| 521 | * queries through VirtualQuery and inserts the result. There is deliberately no | ||
| 522 | * per-page index. The per-shard entry count is small and bounded. | ||
| 523 | */ | ||
| 524 | 289014 | CachedMemoryRegionInfo *find_in_shard( | |
| 525 | CacheShard &shard, | ||
| 526 | std::uintptr_t address, | ||
| 527 | std::size_t size, | ||
| 528 | std::uint64_t current_ns, | ||
| 529 | std::uint64_t expiry_ns | ||
| 530 | ) noexcept | ||
| 531 | { | ||
| 532 | // One acquire load reads the shard generation for both tiers. An entry with an older stamp is skipped. | ||
| 533 | // Thus a lost physical eviction or a clear that races a leader cannot serve a stale hit. | ||
| 534 | 289014 | const std::uint64_t shard_content_gen = shard.content_gen.load(std::memory_order_acquire); | |
| 535 | |||
| 536 | 289263 | const std::uintptr_t base_addr = address & ~static_cast<std::uintptr_t>(0xFFF); | |
| 537 | 289263 | auto it = shard.entries.find(base_addr); | |
| 538 |
1/2✓ Branch 12 → 13 taken 285725 times.
✗ Branch 12 → 17 not taken.
|
284337 | if (it != shard.entries.end()) |
| 539 | { | ||
| 540 | 285725 | CachedMemoryRegionInfo &entry = it->second; | |
| 541 |
1/2✓ Branch 15 → 16 taken 283617 times.
✗ Branch 15 → 17 not taken.
|
285135 | if (is_entry_valid_and_covers(entry, address, size, current_ns, expiry_ns, shard_content_gen)) |
| 542 | { | ||
| 543 | 283617 | return &entry; | |
| 544 | } | ||
| 545 | } | ||
| 546 | |||
| 547 | auto range_it = std::upper_bound( | ||
| 548 | 632 | shard.sorted_ranges.begin(), | |
| 549 | 632 | shard.sorted_ranges.end(), | |
| 550 | ✗ | std::make_pair(address, UINTPTR_MAX) | |
| 551 | 633 | ); | |
| 552 |
2/2✓ Branch 23 → 24 taken 254 times.
✓ Branch 23 → 42 taken 379 times.
|
633 | if (range_it != shard.sorted_ranges.begin()) |
| 553 | { | ||
| 554 | 254 | --range_it; | |
| 555 |
5/6✓ Branch 26 → 27 taken 254 times.
✗ Branch 26 → 30 not taken.
✓ Branch 28 → 29 taken 190 times.
✓ Branch 28 → 30 taken 64 times.
✓ Branch 31 → 32 taken 190 times.
✓ Branch 31 → 42 taken 64 times.
|
254 | if (address >= range_it->first && address < range_it->second) |
| 556 | { | ||
| 557 | 190 | auto entry_it = shard.entries.find(range_it->first); | |
| 558 |
1/2✓ Branch 36 → 37 taken 190 times.
✗ Branch 36 → 41 not taken.
|
190 | if (entry_it != shard.entries.end()) |
| 559 | { | ||
| 560 | 190 | CachedMemoryRegionInfo &entry = entry_it->second; | |
| 561 |
2/2✓ Branch 39 → 40 taken 144 times.
✓ Branch 39 → 41 taken 46 times.
|
190 | if (is_entry_valid_and_covers( |
| 562 | entry, | ||
| 563 | address, | ||
| 564 | size, | ||
| 565 | current_ns, | ||
| 566 | expiry_ns, | ||
| 567 | shard_content_gen | ||
| 568 | )) | ||
| 569 | { | ||
| 570 | 144 | return &entry; | |
| 571 | } | ||
| 572 | } | ||
| 573 | } | ||
| 574 | } | ||
| 575 | |||
| 576 | 489 | return nullptr; | |
| 577 | } | ||
| 578 | |||
| 579 | /** | ||
| 580 | * @brief Evicts the oldest shard entry through an O(log n) FIFO lookup. | ||
| 581 | * @note Must be called with the shard mutex held (exclusive). | ||
| 582 | * @return true if an entry was evicted, false if the shard is empty. | ||
| 583 | */ | ||
| 584 | 45 | bool evict_oldest_entry(CacheShard &shard) noexcept | |
| 585 | { | ||
| 586 |
1/2✗ Branch 3 → 4 not taken.
✓ Branch 3 → 5 taken 45 times.
|
45 | if (shard.fifo_index.empty()) |
| 587 | ✗ | return false; | |
| 588 | |||
| 589 | 45 | const auto fifo_it = shard.fifo_index.begin(); | |
| 590 | 45 | const std::uintptr_t oldest_base = fifo_it->second; | |
| 591 | |||
| 592 | 45 | shard.fifo_index.erase(fifo_it); | |
| 593 | |||
| 594 | 45 | const auto entry_it = shard.entries.find(oldest_base); | |
| 595 |
1/2✓ Branch 11 → 12 taken 45 times.
✗ Branch 11 → 15 not taken.
|
45 | if (entry_it != shard.entries.end()) |
| 596 | { | ||
| 597 | 45 | shard.entries.erase(entry_it); | |
| 598 | 45 | remove_sorted_range(shard, oldest_base); | |
| 599 | 45 | return true; | |
| 600 | } | ||
| 601 | ✗ | return false; | |
| 602 | } | ||
| 603 | |||
| 604 | /** | ||
| 605 | * @brief Force-evicts entries until the shard is at or below max_capacity. | ||
| 606 | * @note Must be called with the shard mutex held (exclusive). | ||
| 607 | */ | ||
| 608 | 14431 | void trim_to_max_capacity(CacheShard &shard) noexcept | |
| 609 | { | ||
| 610 |
2/6✗ Branch 5 → 6 not taken.
✓ Branch 5 → 9 taken 14431 times.
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 9 not taken.
✗ Branch 10 → 3 not taken.
✓ Branch 10 → 11 taken 14431 times.
|
14431 | while (shard.entries.size() > shard.max_capacity && !shard.fifo_index.empty()) |
| 611 | { | ||
| 612 | ✗ | evict_oldest_entry(shard); | |
| 613 | } | ||
| 614 | 14431 | } | |
| 615 | |||
| 616 | /** | ||
| 617 | * @brief Updates or inserts a cache entry in a specific shard and can throw. | ||
| 618 | * @param content_gen Generation captured before the leader's VirtualQuery. An entry published across a | ||
| 619 | * clear or invalidation starts stale, so the next hit queries again. | ||
| 620 | * @details If a call throws, the function erases each index change. An update without its old FIFO record | ||
| 621 | * drops its entry. Proof: MemoryTest.CacheInsertFaultRollbackStaysConsistent and | ||
| 622 | * MemoryCacheLifecycleProof.ContendedInvalidationAdvancesContentGeneration. | ||
| 623 | * @note Must be called with the shard mutex held (exclusive). It can throw. The noexcept | ||
| 624 | * @ref update_shard_with_region wrapper fails soft on that. | ||
| 625 | */ | ||
| 626 | 478 | void update_shard_with_region_impl( | |
| 627 | CacheShard &shard, | ||
| 628 | const MEMORY_BASIC_INFORMATION &mbi, | ||
| 629 | std::uint64_t current_ns, | ||
| 630 | std::uint64_t content_gen | ||
| 631 | ) | ||
| 632 | { | ||
| 633 | 478 | const std::uintptr_t base_addr = reinterpret_cast<std::uintptr_t>(mbi.BaseAddress); | |
| 634 | |||
| 635 |
1/2✓ Branch 2 → 3 taken 478 times.
✗ Branch 2 → 56 not taken.
|
478 | auto it = shard.entries.find(base_addr); |
| 636 |
2/2✓ Branch 5 → 6 taken 38 times.
✓ Branch 5 → 22 taken 440 times.
|
478 | if (it != shard.entries.end()) |
| 637 | { | ||
| 638 | 38 | CachedMemoryRegionInfo &old_entry = it->second; | |
| 639 |
1/2✓ Branch 7 → 8 taken 38 times.
✗ Branch 7 → 43 not taken.
|
38 | const auto fifo_it = shard.fifo_index.find(old_entry.fifo_key); |
| 640 |
3/6✓ Branch 10 → 11 taken 38 times.
✗ Branch 10 → 14 not taken.
✓ Branch 12 → 13 taken 38 times.
✗ Branch 12 → 14 not taken.
✓ Branch 15 → 16 taken 38 times.
✗ Branch 15 → 17 not taken.
|
38 | if (fifo_it != shard.fifo_index.end() && fifo_it->second == base_addr) |
| 641 | { | ||
| 642 |
1/2✓ Branch 16 → 17 taken 38 times.
✗ Branch 16 → 43 not taken.
|
38 | shard.fifo_index.erase(fifo_it); |
| 643 | } | ||
| 644 | |||
| 645 | 38 | const std::uint64_t new_fifo_key = shard.entry_counter++; | |
| 646 | |||
| 647 | // A later failure leaves no FIFO record for the old entry. Erase all entry indexes before the | ||
| 648 | // exception escapes. | ||
| 649 | try | ||
| 650 | { | ||
| 651 |
1/2✗ Branch 17 → 18 not taken.
✓ Branch 17 → 20 taken 38 times.
|
38 | if (old_entry.region_size != mbi.RegionSize) |
| 652 | { | ||
| 653 | ✗ | remove_sorted_range(shard, base_addr); | |
| 654 | ✗ | insert_sorted_range(shard, base_addr, mbi.RegionSize); | |
| 655 | } | ||
| 656 |
2/2✓ Branch 20 → 21 taken 37 times.
✓ Branch 20 → 34 taken 1 time.
|
38 | shard.fifo_index.emplace(new_fifo_key, base_addr); |
| 657 | } | ||
| 658 | 1 | catch (...) | |
| 659 | { | ||
| 660 |
1/2✓ Branch 37 → 38 taken 1 time.
✗ Branch 37 → 41 not taken.
|
1 | shard.fifo_index.erase(new_fifo_key); |
| 661 | 1 | remove_sorted_range(shard, base_addr); | |
| 662 |
1/2✓ Branch 39 → 40 taken 1 time.
✗ Branch 39 → 41 not taken.
|
1 | shard.entries.erase(base_addr); |
| 663 | 1 | throw; | |
| 664 | 1 | } | |
| 665 | |||
| 666 | 37 | old_entry.base_address = base_addr; | |
| 667 | 37 | old_entry.region_size = mbi.RegionSize; | |
| 668 | 37 | old_entry.protection = mbi.Protect; | |
| 669 | 37 | old_entry.state = mbi.State; | |
| 670 | 37 | old_entry.timestamp_ns = current_ns; | |
| 671 | 37 | old_entry.fifo_key = new_fifo_key; | |
| 672 | 37 | old_entry.content_gen = content_gen; | |
| 673 | 37 | old_entry.valid = true; | |
| 674 | } | ||
| 675 | else | ||
| 676 | { | ||
| 677 |
2/2✓ Branch 23 → 24 taken 45 times.
✓ Branch 23 → 25 taken 395 times.
|
440 | if (shard.entries.size() >= shard.capacity) |
| 678 | { | ||
| 679 | 45 | evict_oldest_entry(shard); | |
| 680 | } | ||
| 681 | |||
| 682 |
1/2✗ Branch 26 → 27 not taken.
✓ Branch 26 → 28 taken 440 times.
|
440 | if (shard.entries.size() >= shard.max_capacity) |
| 683 | { | ||
| 684 | ✗ | trim_to_max_capacity(shard); | |
| 685 | } | ||
| 686 | |||
| 687 | 440 | const std::uint64_t new_fifo_key = shard.entry_counter++; | |
| 688 | |||
| 689 | 440 | CachedMemoryRegionInfo new_entry; | |
| 690 | 440 | new_entry.base_address = base_addr; | |
| 691 | 440 | new_entry.region_size = mbi.RegionSize; | |
| 692 | 440 | new_entry.protection = mbi.Protect; | |
| 693 | 440 | new_entry.state = mbi.State; | |
| 694 | 440 | new_entry.timestamp_ns = current_ns; | |
| 695 | 440 | new_entry.fifo_key = new_fifo_key; | |
| 696 | 440 | new_entry.content_gen = content_gen; | |
| 697 | 440 | new_entry.valid = true; | |
| 698 | |||
| 699 | // Treat all three index writes as one transaction. The catch erases each prior commit after a | ||
| 700 | // later failure. | ||
| 701 | 440 | bool map_committed = false; | |
| 702 | 440 | bool fifo_committed = false; | |
| 703 | try | ||
| 704 | { | ||
| 705 |
2/2✓ Branch 29 → 30 taken 435 times.
✓ Branch 29 → 44 taken 5 times.
|
440 | shard.entries.insert_or_assign(base_addr, new_entry); |
| 706 | 435 | map_committed = true; | |
| 707 |
2/2✓ Branch 30 → 31 taken 430 times.
✓ Branch 30 → 45 taken 5 times.
|
435 | shard.fifo_index.emplace(new_fifo_key, base_addr); |
| 708 | 430 | fifo_committed = true; | |
| 709 |
2/2✓ Branch 31 → 32 taken 428 times.
✓ Branch 31 → 46 taken 2 times.
|
430 | insert_sorted_range(shard, base_addr, mbi.RegionSize); |
| 710 | } | ||
| 711 | 12 | catch (...) | |
| 712 | { | ||
| 713 |
2/2✓ Branch 48 → 49 taken 2 times.
✓ Branch 48 → 50 taken 10 times.
|
12 | if (fifo_committed) |
| 714 | { | ||
| 715 |
1/2✓ Branch 49 → 50 taken 2 times.
✗ Branch 49 → 53 not taken.
|
2 | shard.fifo_index.erase(new_fifo_key); |
| 716 | } | ||
| 717 |
2/2✓ Branch 50 → 51 taken 7 times.
✓ Branch 50 → 52 taken 5 times.
|
12 | if (map_committed) |
| 718 | { | ||
| 719 |
1/2✓ Branch 51 → 52 taken 7 times.
✗ Branch 51 → 53 not taken.
|
7 | shard.entries.erase(base_addr); |
| 720 | } | ||
| 721 | 12 | throw; | |
| 722 | 12 | } | |
| 723 | } | ||
| 724 | 465 | } | |
| 725 | |||
| 726 | /** | ||
| 727 | * @brief Updates or inserts a cache entry in a specific shard with soft allocation failure. | ||
| 728 | * @details The cache is a hint over the authoritative VirtualQuery result. The implementation erases its | ||
| 729 | * partial work after a failure, so the three shard indexes remain equal. | ||
| 730 | * @note Must be called with the shard mutex held (exclusive). | ||
| 731 | */ | ||
| 732 | 478 | void update_shard_with_region( | |
| 733 | CacheShard &shard, | ||
| 734 | const MEMORY_BASIC_INFORMATION &mbi, | ||
| 735 | std::uint64_t current_ns, | ||
| 736 | std::uint64_t content_gen | ||
| 737 | ) noexcept | ||
| 738 | { | ||
| 739 | try | ||
| 740 | { | ||
| 741 |
2/2✓ Branch 2 → 3 taken 465 times.
✓ Branch 2 → 4 taken 13 times.
|
478 | update_shard_with_region_impl(shard, mbi, current_ns, content_gen); |
| 742 | } | ||
| 743 | 13 | catch (...) | |
| 744 | { | ||
| 745 | 13 | } | |
| 746 | 478 | } | |
| 747 | |||
| 748 | /** | ||
| 749 | * @brief Removes expired entries from a shard. | ||
| 750 | * @note Must be called with the shard mutex held (exclusive). | ||
| 751 | * @return Number of entries removed from this shard. | ||
| 752 | */ | ||
| 753 | 14431 | std::size_t cleanup_expired_entries_in_shard( | |
| 754 | CacheShard &shard, | ||
| 755 | std::uint64_t current_ns, | ||
| 756 | std::uint64_t expiry_ns | ||
| 757 | ) noexcept | ||
| 758 | { | ||
| 759 | 14431 | std::size_t removed = 0; | |
| 760 | 14431 | auto it = shard.entries.begin(); | |
| 761 |
2/2✓ Branch 25 → 4 taken 20 times.
✓ Branch 25 → 26 taken 14431 times.
|
14451 | while (it != shard.entries.end()) |
| 762 | { | ||
| 763 | 20 | const CachedMemoryRegionInfo &entry = it->second; | |
| 764 | 20 | const std::uint64_t entry_age = current_ns - entry.timestamp_ns; | |
| 765 | |||
| 766 |
3/4✓ Branch 5 → 6 taken 20 times.
✗ Branch 5 → 7 not taken.
✓ Branch 6 → 7 taken 1 time.
✓ Branch 6 → 21 taken 19 times.
|
20 | if (!entry.valid || entry_age > expiry_ns) |
| 767 | { | ||
| 768 | 1 | const auto fifo_it = shard.fifo_index.find(entry.fifo_key); | |
| 769 |
3/6✓ Branch 10 → 11 taken 1 time.
✗ Branch 10 → 15 not taken.
✓ Branch 13 → 14 taken 1 time.
✗ Branch 13 → 15 not taken.
✓ Branch 16 → 17 taken 1 time.
✗ Branch 16 → 18 not taken.
|
1 | if (fifo_it != shard.fifo_index.end() && fifo_it->second == it->first) |
| 770 | { | ||
| 771 | 1 | shard.fifo_index.erase(fifo_it); | |
| 772 | } | ||
| 773 | |||
| 774 | 1 | remove_sorted_range(shard, entry.base_address); | |
| 775 | 1 | it = shard.entries.erase(it); | |
| 776 | 1 | ++removed; | |
| 777 | 1 | } | |
| 778 | else | ||
| 779 | { | ||
| 780 | 19 | ++it; | |
| 781 | } | ||
| 782 | } | ||
| 783 | 14431 | return removed; | |
| 784 | } | ||
| 785 | |||
| 786 | /** | ||
| 787 | * @brief Performs cleanup of expired cache entries across all shards. | ||
| 788 | * @param force Force cleanup regardless of state-mutex contention. | ||
| 789 | */ | ||
| 790 | 975 | void cleanup_expired_entries(bool force) noexcept | |
| 791 | { | ||
| 792 | // The state mutex stays locked while this function iterates the shards. It excludes shard-array release | ||
| 793 | // by shutdown_cache. On-demand cleanup uses try_lock, so the hot path never blocks. Forced cleanup | ||
| 794 | // waits for the lock. | ||
| 795 | 975 | std::unique_lock lock(s_cache_state_mutex, std::defer_lock); | |
| 796 |
1/2✓ Branch 3 → 4 taken 975 times.
✗ Branch 3 → 5 not taken.
|
975 | if (force) |
| 797 | { | ||
| 798 | 975 | lock.lock(); | |
| 799 | } | ||
| 800 | ✗ | else if (!lock.try_lock()) | |
| 801 | { | ||
| 802 | ✗ | return; | |
| 803 | } | ||
| 804 | |||
| 805 |
1/2✗ Branch 9 → 10 not taken.
✓ Branch 9 → 11 taken 975 times.
|
975 | if (!s_cache_shards) |
| 806 | ✗ | return; | |
| 807 | |||
| 808 | 975 | const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire); | |
| 809 |
1/2✗ Branch 18 → 19 not taken.
✓ Branch 18 → 20 taken 975 times.
|
975 | if (shard_count == 0) |
| 810 | ✗ | return; | |
| 811 | |||
| 812 | 975 | const std::uint64_t current_ts = current_time_ns(); | |
| 813 | 975 | const std::uint64_t expiry_ns = configured_expiry_ns(); | |
| 814 | |||
| 815 |
2/2✓ Branch 33 → 23 taken 14805 times.
✓ Branch 33 → 34 taken 975 times.
|
15780 | for (std::size_t i = 0; i < shard_count; ++i) |
| 816 | { | ||
| 817 | 14805 | std::unique_lock<SrwSharedMutex> shard_lock(s_cache_shards[i].mtx, std::try_to_lock); | |
| 818 |
2/2✓ Branch 26 → 27 taken 14431 times.
✓ Branch 26 → 31 taken 374 times.
|
14805 | if (shard_lock.owns_lock()) |
| 819 | { | ||
| 820 | 14431 | cleanup_expired_entries_in_shard(s_cache_shards[i], current_ts, expiry_ns); | |
| 821 | 14431 | trim_to_max_capacity(s_cache_shards[i]); | |
| 822 | } | ||
| 823 | 14805 | } | |
| 824 |
1/2✓ Branch 36 → 37 taken 975 times.
✗ Branch 36 → 39 not taken.
|
975 | } |
| 825 | |||
| 826 | /** | ||
| 827 | * @brief Checks whether elapsed time permits on-demand cleanup. | ||
| 828 | * @return true if this caller claims the cleanup trigger. Contended shards can remain unprocessed. | ||
| 829 | */ | ||
| 830 | ✗ | bool try_trigger_on_demand_cleanup() noexcept | |
| 831 | { | ||
| 832 | ✗ | if (!cache_is_running()) | |
| 833 | ✗ | return false; | |
| 834 | |||
| 835 | ✗ | const std::uint64_t now_ns = current_time_ns(); | |
| 836 | ✗ | const std::uint64_t last_cleanup = s_last_cleanup_time_ns.load(std::memory_order_acquire); | |
| 837 | ✗ | const std::uint64_t elapsed_ns = now_ns - last_cleanup; | |
| 838 | |||
| 839 | ✗ | if (elapsed_ns >= CLEANUP_INTERVAL_NS) | |
| 840 | { | ||
| 841 | ✗ | std::uint64_t expected = last_cleanup; | |
| 842 | ✗ | if (s_last_cleanup_time_ns.compare_exchange_strong(expected, now_ns, std::memory_order_acq_rel)) | |
| 843 | { | ||
| 844 | ✗ | cleanup_expired_entries(false); | |
| 845 | s_stats.on_demand_cleanups.fetch_add(1, std::memory_order_relaxed); | ||
| 846 | ✗ | return true; | |
| 847 | } | ||
| 848 | } | ||
| 849 | ✗ | return false; | |
| 850 | } | ||
| 851 | |||
| 852 | /** | ||
| 853 | * @brief Runs the background cleanup thread for one lifecycle generation. | ||
| 854 | * @param generation Lifecycle generation captured at thread creation. | ||
| 855 | * @note Exits after the live generation no longer matches @p generation. | ||
| 856 | */ | ||
| 857 | 744 | void cleanup_thread_func(std::uint64_t generation) noexcept | |
| 858 | { | ||
| 859 |
4/4✓ Branch 24 → 25 taken 1087 times.
✓ Branch 24 → 34 taken 632 times.
✓ Branch 35 → 3 taken 1087 times.
✓ Branch 35 → 36 taken 632 times.
|
2806 | while (s_cleanup_thread_running.load(std::memory_order_acquire) && |
| 860 |
1/2✓ Branch 32 → 33 taken 1087 times.
✗ Branch 32 → 34 not taken.
|
1087 | s_lifecycle_generation.load(std::memory_order_acquire) == generation) |
| 861 | { | ||
| 862 | { | ||
| 863 | 1087 | std::unique_lock<std::mutex> lock(s_cleanup_mutex); | |
| 864 | 1087 | s_cleanup_cv.wait_for( | |
| 865 | lock, | ||
| 866 | 1087 | std::chrono::seconds(1), | |
| 867 | 2091 | [&]() | |
| 868 | { | ||
| 869 | 2091 | return s_cleanup_requested.load(std::memory_order_acquire) || | |
| 870 |
4/4✓ Branch 3 → 4 taken 1117 times.
✓ Branch 3 → 14 taken 974 times.
✓ Branch 5 → 6 taken 1005 times.
✓ Branch 5 → 14 taken 112 times.
|
3096 | !s_cleanup_thread_running.load(std::memory_order_acquire) || |
| 871 |
1/2✗ Branch 13 → 14 not taken.
✓ Branch 13 → 15 taken 1005 times.
|
3096 | s_lifecycle_generation.load(std::memory_order_acquire) != generation; |
| 872 | } | ||
| 873 | ); | ||
| 874 | 1087 | } | |
| 875 | |||
| 876 |
4/4✓ Branch 8 → 9 taken 975 times.
✓ Branch 8 → 17 taken 112 times.
✓ Branch 19 → 20 taken 112 times.
✓ Branch 19 → 21 taken 975 times.
|
2062 | if (!s_cleanup_thread_running.load(std::memory_order_acquire) || |
| 877 |
1/2✗ Branch 16 → 17 not taken.
✓ Branch 16 → 18 taken 975 times.
|
975 | s_lifecycle_generation.load(std::memory_order_acquire) != generation) |
| 878 | 112 | break; | |
| 879 | |||
| 880 | 975 | cleanup_expired_entries(true); | |
| 881 | 975 | s_cleanup_requested.store(false, std::memory_order_relaxed); | |
| 882 | } | ||
| 883 | 744 | } | |
| 884 | |||
| 885 | /** | ||
| 886 | * @brief Requests cleanup through the worker or the on-demand path. | ||
| 887 | * @details Signals the worker when it is active. Otherwise, attempts on-demand cleanup. | ||
| 888 | */ | ||
| 889 | 2980 | void request_cleanup() noexcept | |
| 890 | { | ||
| 891 |
1/2✓ Branch 3 → 4 taken 2976 times.
✗ Branch 3 → 6 not taken.
|
2980 | if (s_cleanup_thread_running.load(std::memory_order_acquire)) |
| 892 | { | ||
| 893 | 2976 | s_cleanup_requested.store(true, std::memory_order_relaxed); | |
| 894 | 2981 | s_cleanup_cv.notify_one(); | |
| 895 | } | ||
| 896 | else | ||
| 897 | { | ||
| 898 | ✗ | try_trigger_on_demand_cleanup(); | |
| 899 | } | ||
| 900 | 2981 | } | |
| 901 | |||
| 902 | /** | ||
| 903 | * @brief Detaches the cleanup thread and retains its counted module reference. | ||
| 904 | * @return true if no joinable thread exists or detachment succeeds. Returns false if std::thread::detach() | ||
| 905 | * throws. | ||
| 906 | */ | ||
| 907 | 2 | bool detach_cleanup_thread_retained() noexcept | |
| 908 | { | ||
| 909 |
1/2✗ Branch 3 → 4 not taken.
✓ Branch 3 → 5 taken 2 times.
|
2 | if (!s_cleanup_thread.joinable()) |
| 910 | { | ||
| 911 | ✗ | return true; | |
| 912 | } | ||
| 913 | |||
| 914 | try | ||
| 915 | { | ||
| 916 | // The retained module reference keeps the detached worker's code mapped. | ||
| 917 |
1/2✓ Branch 5 → 6 taken 2 times.
✗ Branch 5 → 9 not taken.
|
2 | s_cleanup_thread.detach(); |
| 918 | 2 | DetourModKit::diagnostics::record_intentional_leak( | |
| 919 | DetourModKit::diagnostics::LeakSubsystem::MemoryCache | ||
| 920 | ); | ||
| 921 | 2 | return true; | |
| 922 | } | ||
| 923 | ✗ | catch (...) | |
| 924 | { | ||
| 925 | s_lifecycle_violations.fetch_add(1, std::memory_order_relaxed); | ||
| 926 | ✗ | return false; | |
| 927 | ✗ | } | |
| 928 | } | ||
| 929 | |||
| 930 | /** | ||
| 931 | * @brief Tries to claim and detach the cleanup thread without a wait for an unauthorized teardown. | ||
| 932 | * @return true if this caller claims the mutex and finds no joinable thread or detaches it. Returns false | ||
| 933 | * on contention or if std::thread::detach() throws. | ||
| 934 | */ | ||
| 935 | 2 | bool try_detach_cleanup_thread_unauthorized() noexcept | |
| 936 | { | ||
| 937 | 2 | std::unique_lock join_lock(s_cleanup_join_mutex, std::try_to_lock); | |
| 938 |
1/2✗ Branch 4 → 5 not taken.
✓ Branch 4 → 6 taken 2 times.
|
2 | if (!join_lock.owns_lock()) |
| 939 | { | ||
| 940 | // Do not wait because the owner already controls the join or detach decision. | ||
| 941 | ✗ | return false; | |
| 942 | } | ||
| 943 | |||
| 944 | 2 | return detach_cleanup_thread_retained(); | |
| 945 | 2 | } | |
| 946 | |||
| 947 | /** | ||
| 948 | * @brief Claims and joins the cleanup thread after the caller authorizes a teardown wait. | ||
| 949 | * @return true if a joinable handle was joined (init uses this to count a reaped leftover as a lifecycle | ||
| 950 | * violation). Returns false when nothing needs a reap or retention contains a failed join. | ||
| 951 | * @note The joinable() check and mutation both run under s_cleanup_join_mutex. Callers must not check | ||
| 952 | * joinable() first outside that mutex because this races the abandon-path detach. | ||
| 953 | */ | ||
| 954 | 1494 | bool join_cleanup_thread() noexcept | |
| 955 | { | ||
| 956 | 1494 | std::lock_guard join_lock(s_cleanup_join_mutex); | |
| 957 |
2/2✓ Branch 4 → 5 taken 752 times.
✓ Branch 4 → 6 taken 742 times.
|
1494 | if (!s_cleanup_thread.joinable()) |
| 958 | { | ||
| 959 | 752 | return false; | |
| 960 | } | ||
| 961 | |||
| 962 | try | ||
| 963 | { | ||
| 964 |
1/2✓ Branch 6 → 7 taken 742 times.
✗ Branch 6 → 14 not taken.
|
742 | s_cleanup_thread.join(); |
| 965 | } | ||
| 966 | ✗ | catch (...) | |
| 967 | { | ||
| 968 | s_lifecycle_violations.fetch_add(1, std::memory_order_relaxed); | ||
| 969 | ✗ | (void)detach_cleanup_thread_retained(); | |
| 970 | ✗ | return false; | |
| 971 | ✗ | } | |
| 972 | // After a join outside the loader lock, drop the reference from creation. The caller retains another | ||
| 973 | // module reference, so this is never the terminal release. | ||
| 974 |
1/2✓ Branch 7 → 8 taken 742 times.
✗ Branch 7 → 10 not taken.
|
742 | if (s_cleanup_self_ref != nullptr) |
| 975 | { | ||
| 976 | 742 | release_module_ref(s_cleanup_self_ref, diagnostics::ModulePinReason::MemoryCache); | |
| 977 | 742 | s_cleanup_self_ref = nullptr; | |
| 978 | } | ||
| 979 | 742 | return true; | |
| 980 | 1494 | } | |
| 981 | |||
| 982 | /** | ||
| 983 | * @brief Abandons the cache without a wait when teardown lacks block authority. | ||
| 984 | * @details A wait on s_lifecycle_mutex under the loader lock can deadlock against an initializer that | ||
| 985 | * creates the cleanup thread because thread creation takes the loader lock. This path stops and | ||
| 986 | * attempts to detach the cleanup thread, drops the guarded engine, and unpublishes a Starting or | ||
| 987 | * Running generation. It drains no readers and frees no shards. | ||
| 988 | */ | ||
| 989 | 2 | void abandon_cache_unauthorized() noexcept | |
| 990 | { | ||
| 991 | #if !defined(_MSC_VER) && defined(_WIN64) | ||
| 992 | // Remove the vectored fault handler before module unload. This operation takes the VEH mutex and drains | ||
| 993 | // in-flight guarded accesses, so it is not wait-free. A handler in unmapped code faults the host. | ||
| 994 | // Therefore, the wait is safer than omission of handler removal. | ||
| 995 | 2 | detail::release_guarded_engine(); | |
| 996 | #endif | ||
| 997 | 2 | s_cleanup_thread_running.store(false, std::memory_order_release); | |
| 998 | 2 | s_cleanup_cv.notify_one(); | |
| 999 | // If another thread owns the handle lock, it completes the join/detach decision after loader unlock. | ||
| 1000 | 2 | (void)try_detach_cleanup_thread_unauthorized(); | |
| 1001 | |||
| 1002 | // Close admission before another reader enters the retained shard array. Each fetch_or is wait-free | ||
| 1003 | // and loader-lock safe. The next authorized init or shutdown drains admitted readers before any free. | ||
| 1004 | 2 | close_reader_admission(); | |
| 1005 | |||
| 1006 | 2 | LifecycleState state = s_lifecycle_state.load(std::memory_order_seq_cst); | |
| 1007 |
1/4✓ Branch 13 → 9 taken 2 times.
✗ Branch 13 → 14 not taken.
✗ Branch 14 → 9 not taken.
✗ Branch 14 → 15 not taken.
|
2 | while (state == LifecycleState::Starting || state == LifecycleState::Running) |
| 1008 | { | ||
| 1009 |
1/2✓ Branch 10 → 11 taken 2 times.
✗ Branch 10 → 12 not taken.
|
2 | if (s_lifecycle_state.compare_exchange_weak( |
| 1010 | state, | ||
| 1011 | LifecycleState::Stopped, | ||
| 1012 | std::memory_order_seq_cst, | ||
| 1013 | std::memory_order_seq_cst | ||
| 1014 | )) | ||
| 1015 | { | ||
| 1016 | 2 | break; | |
| 1017 | } | ||
| 1018 | } | ||
| 1019 | 2 | } | |
| 1020 | |||
| 1021 | /** | ||
| 1022 | * @brief Evicts every entry in a shard whose region overlaps [address, end_address). | ||
| 1023 | * @note Must be called with the shard mutex held (exclusive). | ||
| 1024 | * @note Scans the whole shard: one region can be cached in several shards under the same base key (the | ||
| 1025 | * shard is chosen from the query address). The shard remains bounded by max_capacity, and this scan | ||
| 1026 | * never runs on a read hot path. | ||
| 1027 | */ | ||
| 1028 | 38034 | std::size_t evict_overlapping_entries_in_shard( | |
| 1029 | CacheShard &shard, | ||
| 1030 | std::uintptr_t address, | ||
| 1031 | std::uintptr_t end_address | ||
| 1032 | ) noexcept | ||
| 1033 | { | ||
| 1034 | 38034 | std::size_t evicted = 0; | |
| 1035 | 38034 | auto it = shard.entries.begin(); | |
| 1036 |
2/2✓ Branch 34 → 4 taken 204 times.
✓ Branch 34 → 35 taken 38072 times.
|
38239 | while (it != shard.entries.end()) |
| 1037 | { | ||
| 1038 | 204 | const CachedMemoryRegionInfo &entry = it->second; | |
| 1039 | 204 | const std::uintptr_t entry_end_address = entry.base_address + entry.region_size; | |
| 1040 | // A VirtualQuery region cannot extend past the address space, but a corrupt cached size can. | ||
| 1041 | // Treat a wrapped end as the top of the address space so a poisoned entry is still evicted. | ||
| 1042 | 204 | const std::uintptr_t clamped_entry_end = | |
| 1043 |
1/2✓ Branch 5 → 6 taken 204 times.
✗ Branch 5 → 7 not taken.
|
204 | (entry_end_address < entry.base_address) ? UINTPTR_MAX : entry_end_address; |
| 1044 | 204 | const bool overlaps = | |
| 1045 |
4/6✓ Branch 8 → 9 taken 204 times.
✗ Branch 8 → 12 not taken.
✓ Branch 9 → 10 taken 204 times.
✗ Branch 9 → 12 not taken.
✓ Branch 10 → 11 taken 11 times.
✓ Branch 10 → 12 taken 193 times.
|
204 | entry.valid && address < clamped_entry_end && end_address > entry.base_address; |
| 1046 |
2/2✓ Branch 13 → 14 taken 11 times.
✓ Branch 13 → 30 taken 193 times.
|
204 | if (overlaps) |
| 1047 | { | ||
| 1048 | 11 | const auto fifo_it = shard.fifo_index.find(entry.fifo_key); | |
| 1049 |
3/6✓ Branch 17 → 18 taken 11 times.
✗ Branch 17 → 22 not taken.
✓ Branch 20 → 21 taken 11 times.
✗ Branch 20 → 22 not taken.
✓ Branch 23 → 24 taken 11 times.
✗ Branch 23 → 25 not taken.
|
11 | if (fifo_it != shard.fifo_index.end() && fifo_it->second == it->first) |
| 1050 | { | ||
| 1051 | 11 | shard.fifo_index.erase(fifo_it); | |
| 1052 | } | ||
| 1053 | 11 | remove_sorted_range(shard, entry.base_address); | |
| 1054 | 11 | it = shard.entries.erase(it); | |
| 1055 | s_stats.invalidations.fetch_add(1, std::memory_order_relaxed); | ||
| 1056 | 11 | ++evicted; | |
| 1057 | } | ||
| 1058 | else | ||
| 1059 | { | ||
| 1060 | 193 | ++it; | |
| 1061 | } | ||
| 1062 | } | ||
| 1063 | 38072 | return evicted; | |
| 1064 | } | ||
| 1065 | |||
| 1066 | /** | ||
| 1067 | * @brief Invalidates cache entries that overlap [address, address + size) across all shards. | ||
| 1068 | * @details Uses one try-lock per shard. On success, entries that overlap are physically evicted. Under | ||
| 1069 | * contention, the content generation advances instead. This invalidates every entry in O(1), so | ||
| 1070 | * correctness never depends on lock success. The fallback over-invalidates the contended | ||
| 1071 | * shard, an accepted trade on the rare, off-hot-path protection-change caller. | ||
| 1072 | */ | ||
| 1073 | 2982 | void invalidate_range_internal(std::uintptr_t address, std::size_t size) noexcept | |
| 1074 | { | ||
| 1075 |
4/6✓ Branch 3 → 4 taken 2983 times.
✗ Branch 3 → 5 not taken.
✓ Branch 4 → 5 taken 2 times.
✓ Branch 4 → 6 taken 2981 times.
✗ Branch 7 → 8 not taken.
✓ Branch 7 → 9 taken 2983 times.
|
2982 | if (!s_cache_shards || size == 0) |
| 1076 | ✗ | return; | |
| 1077 | |||
| 1078 |
2/2✓ Branch 9 → 10 taken 2978 times.
✓ Branch 9 → 11 taken 5 times.
|
2983 | const std::uintptr_t end_address = (address + size < address) ? UINTPTR_MAX : address + size; |
| 1079 | 2982 | const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire); | |
| 1080 | |||
| 1081 |
2/2✓ Branch 43 → 20 taken 39487 times.
✓ Branch 43 → 44 taken 2835 times.
|
42322 | for (std::size_t shard_idx = 0; shard_idx < shard_count; ++shard_idx) |
| 1082 | { | ||
| 1083 | 39487 | CacheShard &shard = s_cache_shards[shard_idx]; | |
| 1084 | 39484 | std::unique_lock<SrwSharedMutex> lock(shard.mtx, std::try_to_lock); | |
| 1085 |
2/2✓ Branch 23 → 24 taken 38137 times.
✓ Branch 23 → 36 taken 1275 times.
|
39368 | if (lock.owns_lock()) |
| 1086 | { | ||
| 1087 | 38137 | evict_overlapping_entries_in_shard(shard, address, end_address); | |
| 1088 | // An earlier leader keeps in_flight set until publication under this lock. A generation advance | ||
| 1089 | // makes its entry stale. A later leader queries after this eviction and the caller's completed | ||
| 1090 | // protection change, so physical eviction suffices without a shard-wide generation change. | ||
| 1091 |
2/2✓ Branch 32 → 33 taken 156 times.
✓ Branch 32 → 41 taken 37889 times.
|
76033 | if (shard.in_flight.load(std::memory_order_acquire) != 0) |
| 1092 | { | ||
| 1093 | 156 | shard.content_gen.fetch_add(1, std::memory_order_acq_rel); | |
| 1094 | } | ||
| 1095 | } | ||
| 1096 | else | ||
| 1097 | { | ||
| 1098 | // Contention advances the generation, so every entry becomes invalid. A leader that captured | ||
| 1099 | // the old generation republishes a stale entry. | ||
| 1100 | 1275 | shard.content_gen.fetch_add(1, std::memory_order_acq_rel); | |
| 1101 | s_stats.invalidations.fetch_add(1, std::memory_order_relaxed); | ||
| 1102 | } | ||
| 1103 | 39320 | } | |
| 1104 | } | ||
| 1105 | |||
| 1106 | /** | ||
| 1107 | * @brief Performs one-time cache initialization (allocates the shard array, configures bounds). | ||
| 1108 | */ | ||
| 1109 | 749 | bool perform_cache_initialization( | |
| 1110 | std::size_t cache_size, | ||
| 1111 | unsigned int expiry_ms, | ||
| 1112 | std::size_t shard_count | ||
| 1113 | ) noexcept | ||
| 1114 | { | ||
| 1115 |
1/2✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 749 times.
|
749 | if (cache_size == 0) |
| 1116 | ✗ | cache_size = MIN_CACHE_SIZE; | |
| 1117 |
2/2✓ Branch 4 → 5 taken 1 time.
✓ Branch 4 → 6 taken 748 times.
|
749 | if (shard_count == 0) |
| 1118 | 1 | shard_count = 1; | |
| 1119 | |||
| 1120 | // Quotient plus remainder computes ceiling division without an addition that can wrap. | ||
| 1121 | 749 | const std::size_t entries_per_shard = | |
| 1122 |
2/2✓ Branch 6 → 7 taken 3 times.
✓ Branch 6 → 8 taken 746 times.
|
749 | cache_size / shard_count + ((cache_size % shard_count != 0) ? 1 : 0); |
| 1123 | |||
| 1124 | // Reject invalid bounds before publication. The explicit max_size check also contains length_error. | ||
| 1125 | 749 | const bool multiplier_overflows = entries_per_shard > SIZE_MAX / DEFAULT_MAX_CACHE_SIZE_MULTIPLIER; | |
| 1126 |
2/2✓ Branch 9 → 10 taken 3 times.
✓ Branch 9 → 12 taken 746 times.
|
749 | if (multiplier_overflows) |
| 1127 | { | ||
| 1128 | 3 | s_cache_shards.reset(); | |
| 1129 | 3 | return false; | |
| 1130 | } | ||
| 1131 | 746 | const std::size_t hard_max_per_shard = entries_per_shard * DEFAULT_MAX_CACHE_SIZE_MULTIPLIER; | |
| 1132 | |||
| 1133 | try | ||
| 1134 | { | ||
| 1135 |
2/2✓ Branch 15 → 16 taken 2 times.
✓ Branch 15 → 18 taken 744 times.
|
746 | if (hard_max_per_shard > decltype(CacheShard::entries){}.max_size()) |
| 1136 | { | ||
| 1137 | 2 | s_cache_shards.reset(); | |
| 1138 | 2 | return false; | |
| 1139 | } | ||
| 1140 |
1/2✓ Branch 18 → 19 taken 744 times.
✗ Branch 18 → 63 not taken.
|
744 | s_cache_shards = std::make_unique<CacheShard[]>(shard_count); |
| 1141 |
2/2✓ Branch 27 → 22 taken 11479 times.
✓ Branch 27 → 28 taken 744 times.
|
12223 | for (std::size_t i = 0; i < shard_count; ++i) |
| 1142 | { | ||
| 1143 |
1/2✓ Branch 23 → 24 taken 11479 times.
✗ Branch 23 → 64 not taken.
|
11479 | s_cache_shards[i].entries.reserve(hard_max_per_shard); |
| 1144 | 11479 | s_cache_shards[i].capacity = entries_per_shard; | |
| 1145 | 11479 | s_cache_shards[i].max_capacity = hard_max_per_shard; | |
| 1146 | } | ||
| 1147 | } | ||
| 1148 | ✗ | catch (...) | |
| 1149 | { | ||
| 1150 | ✗ | s_cache_shards.reset(); | |
| 1151 | ✗ | return false; | |
| 1152 | ✗ | } | |
| 1153 | |||
| 1154 | s_max_entries_per_shard.store(entries_per_shard, std::memory_order_release); | ||
| 1155 | s_configured_expiry_ms.store(expiry_ms, std::memory_order_release); | ||
| 1156 | 744 | s_last_cleanup_time_ns.store(current_time_ns(), std::memory_order_release); | |
| 1157 | // Publish the shard count LAST so a reader that observes Running also sees the shard array and | ||
| 1158 | // config fields, never a torn half-initialized snapshot. | ||
| 1159 | s_shard_count.store(shard_count, std::memory_order_release); | ||
| 1160 | |||
| 1161 | 744 | return true; | |
| 1162 | } | ||
| 1163 | |||
| 1164 | /** | ||
| 1165 | * @brief Performs VirtualQuery and updates the cache with stampede coalescence. | ||
| 1166 | * @return true if VirtualQuery (or a coalesced follower read) succeeded. | ||
| 1167 | */ | ||
| 1168 | bool | ||
| 1169 | 483 | query_and_update_cache(std::size_t shard_idx, LPCVOID address, MEMORY_BASIC_INFORMATION &mbi_out) noexcept | |
| 1170 | { | ||
| 1171 | 483 | CacheShard &shard = s_cache_shards[shard_idx]; | |
| 1172 | |||
| 1173 | 483 | char expected = 0; | |
| 1174 |
2/2✓ Branch 11 → 12 taken 476 times.
✓ Branch 11 → 38 taken 7 times.
|
966 | if (shard.in_flight.compare_exchange_strong(expected, 1, std::memory_order_acq_rel)) |
| 1175 | { | ||
| 1176 | // Capture the generation before the query. A clear or invalidation before publication stamps this | ||
| 1177 | // entry stale. | ||
| 1178 | 476 | const std::uint64_t gen_at_query = shard.content_gen.load(std::memory_order_acquire); | |
| 1179 | 476 | const bool result = VirtualQuery(address, &mbi_out, sizeof(mbi_out)) != 0; | |
| 1180 | 476 | const std::uint64_t now_ns = current_time_ns(); | |
| 1181 | |||
| 1182 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 1183 | // Inject a clear/protection change into the post-query, pre-publish window. | ||
| 1184 |
2/2✓ Branch 21 → 22 taken 2 times.
✓ Branch 21 → 23 taken 474 times.
|
476 | if (auto *const hook = DetourModKit::detail::g_memory_cache_leader_publish_window_test_hook) |
| 1185 | 2 | hook(); | |
| 1186 | #endif | ||
| 1187 | |||
| 1188 |
1/2✓ Branch 23 → 24 taken 476 times.
✗ Branch 23 → 29 not taken.
|
476 | if (result) |
| 1189 | { | ||
| 1190 | 476 | std::unique_lock<SrwSharedMutex> lock(s_cache_shards[shard_idx].mtx); | |
| 1191 | 476 | update_shard_with_region(shard, mbi_out, now_ns, gen_at_query); | |
| 1192 | 476 | } | |
| 1193 | |||
| 1194 | 476 | shard.in_flight.store(0, std::memory_order_release); | |
| 1195 | 476 | return result; | |
| 1196 | } | ||
| 1197 | else | ||
| 1198 | { | ||
| 1199 | 7 | const std::uint64_t expiry_ns = configured_expiry_ns(); | |
| 1200 | 7 | constexpr std::size_t MAX_FOLLOWER_YIELDS = 8; | |
| 1201 | |||
| 1202 |
2/2✓ Branch 63 → 40 taken 42 times.
✓ Branch 63 → 64 taken 4 times.
|
46 | for (std::size_t yield_count = 0; yield_count < MAX_FOLLOWER_YIELDS; ++yield_count) |
| 1203 | { | ||
| 1204 |
2/2✓ Branch 47 → 48 taken 3 times.
✓ Branch 47 → 61 taken 39 times.
|
84 | if (shard.in_flight.load(std::memory_order_acquire) == 0) |
| 1205 | { | ||
| 1206 | 3 | const std::uintptr_t addr_val = reinterpret_cast<std::uintptr_t>(address); | |
| 1207 | 3 | std::shared_lock<SrwSharedMutex> lock(s_cache_shards[shard_idx].mtx); | |
| 1208 | CachedMemoryRegionInfo *cached = | ||
| 1209 | 3 | find_in_shard(shard, addr_val, 1, current_time_ns(), expiry_ns); | |
| 1210 |
2/2✓ Branch 52 → 53 taken 1 time.
✓ Branch 52 → 56 taken 2 times.
|
3 | if (cached) |
| 1211 | { | ||
| 1212 | s_stats.coalesced_queries.fetch_add(1, std::memory_order_relaxed); | ||
| 1213 | 1 | mbi_out.BaseAddress = reinterpret_cast<PVOID>(cached->base_address); | |
| 1214 | 1 | mbi_out.RegionSize = cached->region_size; | |
| 1215 | 1 | mbi_out.Protect = cached->protection; | |
| 1216 | 1 | mbi_out.State = cached->state; | |
| 1217 | 1 | return true; | |
| 1218 | } | ||
| 1219 | 2 | break; | |
| 1220 |
2/2✓ Branch 58 → 59 taken 1 time.
✓ Branch 58 → 60 taken 2 times.
|
3 | } |
| 1221 | |||
| 1222 | 39 | std::this_thread::yield(); | |
| 1223 | } | ||
| 1224 | |||
| 1225 | 6 | expected = 0; | |
| 1226 |
2/2✓ Branch 72 → 73 taken 2 times.
✓ Branch 72 → 97 taken 4 times.
|
12 | if (shard.in_flight.compare_exchange_strong(expected, 1, std::memory_order_acq_rel)) |
| 1227 | { | ||
| 1228 | 2 | const std::uint64_t gen_at_query = shard.content_gen.load(std::memory_order_acquire); | |
| 1229 | 2 | const bool result = VirtualQuery(address, &mbi_out, sizeof(mbi_out)) != 0; | |
| 1230 |
1/2✓ Branch 81 → 82 taken 2 times.
✗ Branch 81 → 88 not taken.
|
2 | if (result) |
| 1231 | { | ||
| 1232 | 2 | std::unique_lock<SrwSharedMutex> lock(s_cache_shards[shard_idx].mtx); | |
| 1233 | 2 | const std::uint64_t now_ns = current_time_ns(); | |
| 1234 | 2 | update_shard_with_region(shard, mbi_out, now_ns, gen_at_query); | |
| 1235 | 2 | } | |
| 1236 | 2 | shard.in_flight.store(0, std::memory_order_release); | |
| 1237 | 2 | return result; | |
| 1238 | } | ||
| 1239 | |||
| 1240 | 4 | return VirtualQuery(address, &mbi_out, sizeof(mbi_out)) != 0; | |
| 1241 | } | ||
| 1242 | } | ||
| 1243 | |||
| 1244 | /** | ||
| 1245 | * @brief Walks a range through VirtualQuery without cache involvement. | ||
| 1246 | * @param address Start of the range. Callers screen a zero address before the call. | ||
| 1247 | * @param size Byte length of the range. Callers screen a zero size before the call. | ||
| 1248 | * @param check_permission Predicate over one region's protection flags. | ||
| 1249 | * @return true only when every region the range touches is committed and satisfies @p check_permission, | ||
| 1250 | * with no unmapped gap between them. | ||
| 1251 | * @details The loop repeats only when the range crosses different protections. VirtualQuery already | ||
| 1252 | * coalesces neighbors with the same protection, so the common single-region case uses one query. | ||
| 1253 | * Every failure closes the result. A wrap, invalid sub-region, disallowed protection, failed | ||
| 1254 | * query, or no cursor advance returns false. The returned region contains `cursor`, which | ||
| 1255 | * guarantees progress. | ||
| 1256 | */ | ||
| 1257 | 13784 | bool range_permission_uncached( | |
| 1258 | std::uintptr_t address, | ||
| 1259 | std::size_t size, | ||
| 1260 | bool (*check_permission)(DWORD) noexcept | ||
| 1261 | ) noexcept | ||
| 1262 | { | ||
| 1263 | 13784 | const std::uintptr_t query_end = address + size; | |
| 1264 |
2/2✓ Branch 2 → 3 taken 3 times.
✓ Branch 2 → 4 taken 13781 times.
|
13784 | if (query_end < address) |
| 1265 | 3 | return false; | |
| 1266 | |||
| 1267 | 13781 | std::uintptr_t cursor = address; | |
| 1268 |
2/2✓ Branch 17 → 5 taken 13793 times.
✓ Branch 17 → 18 taken 13869 times.
|
27662 | while (cursor < query_end) |
| 1269 | { | ||
| 1270 | 13793 | MEMORY_BASIC_INFORMATION mbi{}; | |
| 1271 |
1/2✗ Branch 6 → 7 not taken.
✓ Branch 6 → 8 taken 13906 times.
|
13793 | if (VirtualQuery(reinterpret_cast<LPCVOID>(cursor), &mbi, sizeof(mbi)) == 0) |
| 1272 | 14 | return false; | |
| 1273 |
2/2✓ Branch 8 → 9 taken 4 times.
✓ Branch 8 → 10 taken 13902 times.
|
13906 | if (mbi.State != MEM_COMMIT) |
| 1274 | 4 | return false; | |
| 1275 |
2/2✓ Branch 11 → 12 taken 10 times.
✓ Branch 11 → 13 taken 13881 times.
|
13902 | if (!check_permission(mbi.Protect)) |
| 1276 | 10 | return false; | |
| 1277 | |||
| 1278 | 13881 | const std::uintptr_t region_end = | |
| 1279 | 13881 | reinterpret_cast<std::uintptr_t>(mbi.BaseAddress) + mbi.RegionSize; | |
| 1280 | // A zero-size or wrapped region fails closed instead of a spin. | ||
| 1281 |
1/2✗ Branch 13 → 14 not taken.
✓ Branch 13 → 15 taken 13881 times.
|
13881 | if (region_end <= cursor) |
| 1282 | ✗ | return false; | |
| 1283 | 13881 | cursor = region_end; | |
| 1284 | } | ||
| 1285 | 13869 | return true; | |
| 1286 | } | ||
| 1287 | |||
| 1288 | /** | ||
| 1289 | * @brief Checks permissions for is_readable and is_writable. | ||
| 1290 | * @param address Start address of the query (0 fails closed). | ||
| 1291 | * @param size Number of bytes to check (0 fails closed). | ||
| 1292 | * @param check_permission Predicate that validates the protection flags. | ||
| 1293 | */ | ||
| 1294 | 285920 | bool check_memory_permission( | |
| 1295 | std::uintptr_t address, | ||
| 1296 | std::size_t size, | ||
| 1297 | bool (*check_permission)(DWORD) noexcept | ||
| 1298 | ) noexcept | ||
| 1299 | { | ||
| 1300 |
2/4✓ Branch 2 → 3 taken 296233 times.
✗ Branch 2 → 4 not taken.
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 5 taken 297054 times.
|
285920 | if (address == 0 || size == 0) |
| 1301 | ✗ | return false; | |
| 1302 | |||
| 1303 | // One compare-exchange combines admission with the closed check. A query after closure never joins the | ||
| 1304 | // drain population ([B-73]). While admitted, teardown cannot free the shard array. | ||
| 1305 | 297054 | ReaderAdmission admission; | |
| 1306 | |||
| 1307 | // Without an admitted running cache, walk the range directly. The walk spans protection boundaries, | ||
| 1308 | // so a re-protected interior page is answered correctly. | ||
| 1309 |
5/6✓ Branch 7 → 8 taken 283219 times.
✓ Branch 7 → 10 taken 14550 times.
✗ Branch 9 → 10 not taken.
✓ Branch 9 → 11 taken 281709 times.
✓ Branch 12 → 13 taken 13775 times.
✓ Branch 12 → 14 taken 281730 times.
|
297952 | if (!admission.admitted() || !cache_is_running()) |
| 1310 | { | ||
| 1311 | 13775 | return range_permission_uncached(address, size, check_permission); | |
| 1312 | } | ||
| 1313 | 281730 | const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire); | |
| 1314 |
1/2✗ Branch 21 → 22 not taken.
✓ Branch 21 → 23 taken 281730 times.
|
281730 | if (shard_count == 0) |
| 1315 | { | ||
| 1316 | ✗ | return range_permission_uncached(address, size, check_permission); | |
| 1317 | } | ||
| 1318 | |||
| 1319 | 281730 | const std::size_t shard_idx = compute_shard_index(address, shard_count); | |
| 1320 | 279007 | const std::uint64_t now_ns = current_time_ns(); | |
| 1321 | 280072 | const std::uint64_t expiry_ns = configured_expiry_ns(); | |
| 1322 | |||
| 1323 | { | ||
| 1324 | 279772 | std::shared_lock<SrwSharedMutex> lock(s_cache_shards[shard_idx].mtx); | |
| 1325 | CachedMemoryRegionInfo *cached_info = | ||
| 1326 | 291406 | find_in_shard(s_cache_shards[shard_idx], address, size, now_ns, expiry_ns); | |
| 1327 |
2/2✓ Branch 30 → 31 taken 283532 times.
✓ Branch 30 → 40 taken 483 times.
|
284015 | if (cached_info) |
| 1328 | { | ||
| 1329 | 283532 | s_cache_shards[shard_idx].hits.fetch_add(1, std::memory_order_relaxed); | |
| 1330 | // Require MEM_COMMIT exactly as the miss and uncached paths do. Non-committed regions report | ||
| 1331 | // Protect == 0, so check_permission already rejects them. The explicit state check keeps every | ||
| 1332 | // path symmetric. | ||
| 1333 |
2/4✓ Branch 34 → 35 taken 290202 times.
✗ Branch 34 → 38 not taken.
✓ Branch 36 → 37 taken 283848 times.
✗ Branch 36 → 38 not taken.
|
286857 | return cached_info->state == MEM_COMMIT && check_permission(cached_info->protection); |
| 1334 | } | ||
| 1335 |
2/2✓ Branch 42 → 43 taken 483 times.
✓ Branch 42 → 49 taken 289534 times.
|
280364 | } |
| 1336 | |||
| 1337 | 483 | s_cache_shards[shard_idx].misses.fetch_add(1, std::memory_order_relaxed); | |
| 1338 | |||
| 1339 | 483 | MEMORY_BASIC_INFORMATION mbi{}; | |
| 1340 |
1/2✗ Branch 48 → 50 not taken.
✓ Branch 48 → 51 taken 483 times.
|
483 | if (!query_and_update_cache(shard_idx, reinterpret_cast<LPCVOID>(address), mbi)) |
| 1341 | ✗ | return false; | |
| 1342 | |||
| 1343 |
2/2✓ Branch 51 → 52 taken 4 times.
✓ Branch 51 → 53 taken 479 times.
|
483 | if (mbi.State != MEM_COMMIT) |
| 1344 | 4 | return false; | |
| 1345 | |||
| 1346 |
2/2✓ Branch 54 → 55 taken 16 times.
✓ Branch 54 → 56 taken 463 times.
|
479 | if (!check_permission(mbi.Protect)) |
| 1347 | 16 | return false; | |
| 1348 | |||
| 1349 | 463 | const std::uintptr_t region_end_addr = | |
| 1350 | 463 | reinterpret_cast<std::uintptr_t>(mbi.BaseAddress) + mbi.RegionSize; | |
| 1351 | 463 | const std::uintptr_t query_end_addr = address + size; | |
| 1352 | |||
| 1353 |
2/2✓ Branch 56 → 57 taken 4 times.
✓ Branch 56 → 58 taken 459 times.
|
463 | if (query_end_addr < address) |
| 1354 | 4 | return false; | |
| 1355 |
1/2✗ Branch 58 → 59 not taken.
✓ Branch 58 → 60 taken 459 times.
|
459 | if (region_end_addr <= address) |
| 1356 | ✗ | return false; | |
| 1357 | |||
| 1358 | // The common case fits within the cached region, so answer directly. A range past it crosses into an | ||
| 1359 | // adjacent protection region that one cache entry cannot cover. Walk the remainder without the cache. | ||
| 1360 | // Fail closed on any uncommitted, disallowed, or gapped sub-region. | ||
| 1361 |
2/2✓ Branch 60 → 61 taken 453 times.
✓ Branch 60 → 62 taken 6 times.
|
459 | if (query_end_addr <= region_end_addr) |
| 1362 | 453 | return true; | |
| 1363 | |||
| 1364 | 6 | return range_permission_uncached(region_end_addr, query_end_addr - region_end_addr, check_permission); | |
| 1365 | 303905 | } | |
| 1366 | |||
| 1367 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 1368 | /** | ||
| 1369 | * @brief Holds the selected shard's shared lock and invokes a deterministic test callback. | ||
| 1370 | */ | ||
| 1371 | 2 | void hold_shard_shared_lock_for_test(Address address, void (*callback)() noexcept) noexcept | |
| 1372 | { | ||
| 1373 |
1/2✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 2 times.
|
2 | if (callback == nullptr) |
| 1374 | ✗ | return; | |
| 1375 | |||
| 1376 | 2 | ReaderAdmission admission; | |
| 1377 |
3/6✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 9 not taken.
✗ Branch 8 → 9 not taken.
✓ Branch 8 → 10 taken 2 times.
✗ Branch 11 → 12 not taken.
✓ Branch 11 → 13 taken 2 times.
|
2 | if (!admission.admitted() || !cache_is_running()) |
| 1378 | ✗ | return; | |
| 1379 | |||
| 1380 | 2 | const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire); | |
| 1381 |
1/2✗ Branch 20 → 21 not taken.
✓ Branch 20 → 22 taken 2 times.
|
2 | if (shard_count == 0) |
| 1382 | ✗ | return; | |
| 1383 | |||
| 1384 | 2 | const std::size_t shard_idx = compute_shard_index(address.raw(), shard_count); | |
| 1385 | 2 | std::shared_lock<SrwSharedMutex> lock(s_cache_shards[shard_idx].mtx); | |
| 1386 | 2 | callback(); | |
| 1387 |
1/2✓ Branch 30 → 31 taken 2 times.
✗ Branch 30 → 33 not taken.
|
2 | } |
| 1388 | |||
| 1389 | /** | ||
| 1390 | * @brief Reports all index sizes for the shard that owns @p address under one shared lock. | ||
| 1391 | * | ||
| 1392 | * @details This test seam exposes FIFO and range index state that MemoryStats omits. | ||
| 1393 | */ | ||
| 1394 | 26 | void shard_index_sizes_for_test( | |
| 1395 | Address address, | ||
| 1396 | std::size_t &entries, | ||
| 1397 | std::size_t &fifo, | ||
| 1398 | std::size_t &ranges | ||
| 1399 | ) noexcept | ||
| 1400 | { | ||
| 1401 | 26 | entries = 0; | |
| 1402 | 26 | fifo = 0; | |
| 1403 | 26 | ranges = 0; | |
| 1404 | |||
| 1405 | 26 | ReaderAdmission admission; | |
| 1406 |
3/6✓ Branch 4 → 5 taken 26 times.
✗ Branch 4 → 7 not taken.
✗ Branch 6 → 7 not taken.
✓ Branch 6 → 8 taken 26 times.
✗ Branch 9 → 10 not taken.
✓ Branch 9 → 11 taken 26 times.
|
26 | if (!admission.admitted() || !cache_is_running()) |
| 1407 | ✗ | return; | |
| 1408 | |||
| 1409 | 26 | const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire); | |
| 1410 |
1/2✗ Branch 18 → 19 not taken.
✓ Branch 18 → 20 taken 26 times.
|
26 | if (shard_count == 0) |
| 1411 | ✗ | return; | |
| 1412 | |||
| 1413 | 26 | CacheShard &shard = s_cache_shards[compute_shard_index(address.raw(), shard_count)]; | |
| 1414 | 26 | std::shared_lock<SrwSharedMutex> lock(shard.mtx); | |
| 1415 | 26 | entries = shard.entries.size(); | |
| 1416 | 26 | fifo = shard.fifo_index.size(); | |
| 1417 | 26 | ranges = shard.sorted_ranges.size(); | |
| 1418 |
1/2✓ Branch 30 → 31 taken 26 times.
✗ Branch 30 → 33 not taken.
|
26 | } |
| 1419 | #endif | ||
| 1420 | |||
| 1421 | } // namespace | ||
| 1422 | |||
| 1423 | 4742 | bool init_cache(std::size_t cache_size, unsigned int expiry_ms, std::size_t shard_count) | |
| 1424 | { | ||
| 1425 | // A live cache needs no new start. Report it before the loader-lock veto. | ||
| 1426 |
2/2✓ Branch 3 → 4 taken 3991 times.
✓ Branch 3 → 5 taken 751 times.
|
4742 | if (cache_is_running()) |
| 1427 | { | ||
| 1428 | 3991 | return true; | |
| 1429 | } | ||
| 1430 | |||
| 1431 | // Refuse whenever the lifecycle gate does not authorize a wait. Cleanup-thread creation under loader lock | ||
| 1432 | // deadlocks. Readers use uncached VirtualQuery until an authorized init succeeds. | ||
| 1433 |
2/2✓ Branch 6 → 7 taken 1 time.
✓ Branch 6 → 8 taken 750 times.
|
751 | if (!DetourModKit::detail::blocking_teardown_permitted()) |
| 1434 | { | ||
| 1435 | 1 | return false; | |
| 1436 | } | ||
| 1437 | |||
| 1438 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 1439 |
2/2✓ Branch 8 → 9 taken 1 time.
✓ Branch 8 → 10 taken 749 times.
|
750 | if (auto *const hook = DetourModKit::detail::g_memory_cache_before_lifecycle_lock_test_hook) |
| 1440 |
1/2✓ Branch 9 → 10 taken 1 time.
✗ Branch 9 → 177 not taken.
|
1 | hook(); |
| 1441 | #endif | ||
| 1442 | |||
| 1443 | // Serialize the whole start against shutdown_cache across the cleanup-thread handle. | ||
| 1444 | 750 | std::lock_guard lifecycle_lock(s_lifecycle_mutex); | |
| 1445 | |||
| 1446 |
1/2✗ Branch 12 → 13 not taken.
✓ Branch 12 → 14 taken 750 times.
|
750 | if (s_lifecycle_state.load(std::memory_order_seq_cst) == LifecycleState::Running) |
| 1447 | ✗ | return true; | |
| 1448 | |||
| 1449 | // Recover any unexpected joinable handle before assignment of the next worker. | ||
| 1450 | 750 | const bool reaped_leftover = join_cleanup_thread(); | |
| 1451 |
1/2✗ Branch 15 → 16 not taken.
✓ Branch 15 → 19 taken 750 times.
|
750 | if (reaped_leftover) |
| 1452 | { | ||
| 1453 | s_lifecycle_violations.fetch_add(1, std::memory_order_relaxed); | ||
| 1454 | } | ||
| 1455 | { | ||
| 1456 | 750 | std::lock_guard join_lock(s_cleanup_join_mutex); | |
| 1457 |
1/2✗ Branch 21 → 22 not taken.
✓ Branch 21 → 25 taken 750 times.
|
750 | if (s_cleanup_thread.joinable()) |
| 1458 | { | ||
| 1459 | s_lifecycle_violations.fetch_add(1, std::memory_order_relaxed); | ||
| 1460 | ✗ | return false; | |
| 1461 | } | ||
| 1462 |
1/2✓ Branch 27 → 28 taken 750 times.
✗ Branch 27 → 42 not taken.
|
750 | } |
| 1463 | |||
| 1464 | 750 | s_lifecycle_state.store(LifecycleState::Starting, std::memory_order_seq_cst); | |
| 1465 | |||
| 1466 | { | ||
| 1467 | 750 | std::lock_guard state_lock(s_cache_state_mutex); | |
| 1468 | // Close every stripe before the shard array changes. Then drain the closed reader population before | ||
| 1469 | // the deadline. Abandonment or a prior timeout can leave reader-visible state retained. Free that state | ||
| 1470 | // only after this drain reaches zero. On expiry the start fails and retains the state under [B-73]. | ||
| 1471 | 750 | close_reader_admission(); | |
| 1472 | s_shard_count.store(0, std::memory_order_release); | ||
| 1473 |
1/2✗ Branch 41 → 43 not taken.
✓ Branch 41 → 46 taken 750 times.
|
750 | if (!drain_admitted_readers()) |
| 1474 | { | ||
| 1475 | ✗ | record_reader_drain_timeout_retention(); | |
| 1476 | ✗ | s_lifecycle_state.store(LifecycleState::Stopped, std::memory_order_seq_cst); | |
| 1477 | ✗ | return false; | |
| 1478 | } | ||
| 1479 | |||
| 1480 |
2/2✓ Branch 46 → 47 taken 749 times.
✓ Branch 46 → 68 taken 1 time.
|
750 | if (s_cache_self_ref == nullptr) |
| 1481 | { | ||
| 1482 | 749 | s_cache_self_ref = acquire_cache_keepalive_ref(); | |
| 1483 |
2/2✓ Branch 48 → 49 taken 1 time.
✓ Branch 48 → 68 taken 748 times.
|
749 | if (s_cache_self_ref == nullptr) |
| 1484 | { | ||
| 1485 | 1 | s_cache_shards.reset(); | |
| 1486 | s_configured_expiry_ms.store(0, std::memory_order_relaxed); | ||
| 1487 | s_max_entries_per_shard.store(0, std::memory_order_relaxed); | ||
| 1488 | 1 | s_lifecycle_state.store(LifecycleState::Stopped, std::memory_order_seq_cst); | |
| 1489 | 1 | return false; | |
| 1490 | } | ||
| 1491 | } | ||
| 1492 | |||
| 1493 |
2/2✓ Branch 69 → 70 taken 5 times.
✓ Branch 69 → 89 taken 744 times.
|
749 | if (!perform_cache_initialization(cache_size, expiry_ms, shard_count)) |
| 1494 | { | ||
| 1495 | 5 | release_cache_keepalive_after_drain(); | |
| 1496 | s_configured_expiry_ms.store(0, std::memory_order_relaxed); | ||
| 1497 | s_max_entries_per_shard.store(0, std::memory_order_relaxed); | ||
| 1498 | 5 | s_lifecycle_state.store(LifecycleState::Stopped, std::memory_order_seq_cst); | |
| 1499 | 5 | return false; | |
| 1500 | } | ||
| 1501 |
2/2✓ Branch 91 → 92 taken 744 times.
✓ Branch 91 → 99 taken 6 times.
|
750 | } |
| 1502 | |||
| 1503 | // Advance the generation this session's cleanup thread binds to, after the shards are built. | ||
| 1504 | 744 | const std::uint64_t generation = s_lifecycle_generation.fetch_add(1, std::memory_order_acq_rel) + 1; | |
| 1505 | |||
| 1506 | #if !defined(_MSC_VER) && defined(_WIN64) | ||
| 1507 | // MinGW has no frame-based SEH. A successful vectored-handler install avoids the per-call VirtualQuery | ||
| 1508 | // fallback. Installation remains best-effort and independent of cache success. | ||
| 1509 | 744 | detail::ensure_guarded_engine_installed(); | |
| 1510 | #endif | ||
| 1511 | |||
| 1512 | 744 | s_cleanup_thread_running.store(true, std::memory_order_release); | |
| 1513 | // Hold a counted reference before cleanup thread creation. A creation failure releases it below. | ||
| 1514 | 744 | s_cleanup_self_ref = acquire_module_ref(diagnostics::ModulePinReason::MemoryCache); | |
| 1515 |
1/2✗ Branch 98 → 100 not taken.
✓ Branch 98 → 104 taken 744 times.
|
744 | if (s_cleanup_self_ref == nullptr) |
| 1516 | { | ||
| 1517 | ✗ | s_cleanup_thread_running.store(false, std::memory_order_release); | |
| 1518 | ✗ | (void)log().try_log( | |
| 1519 | LogLevel::Debug, | ||
| 1520 | "MemoryCache: Module reference unavailable, using on-demand cleanup instead of " | ||
| 1521 | "background cleanup." | ||
| 1522 | ); | ||
| 1523 | } | ||
| 1524 | else | ||
| 1525 | { | ||
| 1526 | try | ||
| 1527 | { | ||
| 1528 | // Publish the handle under the join mutex so this never races the loader-lock detach path. A | ||
| 1529 | // concurrent detach tries the mutex, fails, and returns without access to the handle. This lock can | ||
| 1530 | // remain held across thread creation without a deadlock. | ||
| 1531 | 744 | std::lock_guard join_lock(s_cleanup_join_mutex); | |
| 1532 |
1/2✗ Branch 106 → 107 not taken.
✓ Branch 106 → 108 taken 744 times.
|
744 | assert(!s_cleanup_thread.joinable()); |
| 1533 |
1/2✓ Branch 109 → 110 taken 744 times.
✗ Branch 109 → 164 not taken.
|
744 | s_cleanup_thread = std::thread(cleanup_thread_func, generation); |
| 1534 | 744 | } | |
| 1535 | ✗ | catch (...) | |
| 1536 | { | ||
| 1537 | ✗ | release_module_ref(s_cleanup_self_ref, diagnostics::ModulePinReason::MemoryCache); | |
| 1538 | ✗ | s_cleanup_self_ref = nullptr; | |
| 1539 | ✗ | s_cleanup_thread_running.store(false, std::memory_order_release); | |
| 1540 | ✗ | (void)log().try_log( | |
| 1541 | LogLevel::Debug, | ||
| 1542 | "MemoryCache: Background cleanup thread unavailable, using on-demand " | ||
| 1543 | "cleanup." | ||
| 1544 | ); | ||
| 1545 | ✗ | } | |
| 1546 | } | ||
| 1547 | |||
| 1548 | // The atexit handler is a last-resort safety net when the consumer omits shutdown_cache. | ||
| 1549 | static bool atexit_registered = false; | ||
| 1550 |
2/2✓ Branch 114 → 115 taken 445 times.
✓ Branch 114 → 118 taken 299 times.
|
744 | if (!atexit_registered) |
| 1551 | { | ||
| 1552 | 445 | std::atexit( | |
| 1553 | 890 | []() | |
| 1554 | { | ||
| 1555 |
2/2✓ Branch 3 → 4 taken 444 times.
✓ Branch 3 → 5 taken 1 time.
|
445 | if (s_lifecycle_state.load(std::memory_order_seq_cst) != LifecycleState::Running) |
| 1556 | 444 | return; | |
| 1557 | 1 | shutdown_cache(); | |
| 1558 | } | ||
| 1559 | ); | ||
| 1560 | 445 | atexit_registered = true; | |
| 1561 | } | ||
| 1562 | |||
| 1563 | // Open every stripe before Running publishes. A concurrent abandonment closes the stripes and changes | ||
| 1564 | // Starting to Stopped. The publication check below then rolls the start back. | ||
| 1565 | 744 | reopen_reader_admission(); | |
| 1566 | |||
| 1567 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 1568 |
2/2✓ Branch 119 → 120 taken 1 time.
✓ Branch 119 → 121 taken 743 times.
|
744 | if (auto *const hook = DetourModKit::detail::g_memory_cache_before_running_publish_test_hook) |
| 1569 |
1/2✓ Branch 120 → 121 taken 1 time.
✗ Branch 120 → 175 not taken.
|
1 | hook(); |
| 1570 | #endif | ||
| 1571 | |||
| 1572 | // Publish only if unauthorized abandonment did not cancel this Starting generation. | ||
| 1573 | 744 | LifecycleState expected_state = LifecycleState::Starting; | |
| 1574 |
2/2✓ Branch 122 → 123 taken 2 times.
✓ Branch 122 → 160 taken 742 times.
|
744 | if (!s_lifecycle_state.compare_exchange_strong( |
| 1575 | expected_state, | ||
| 1576 | LifecycleState::Running, | ||
| 1577 | std::memory_order_seq_cst, | ||
| 1578 | std::memory_order_seq_cst | ||
| 1579 | )) | ||
| 1580 | { | ||
| 1581 | 2 | s_cleanup_thread_running.store(false, std::memory_order_release); | |
| 1582 | 2 | s_cleanup_cv.notify_one(); | |
| 1583 | 2 | (void)join_cleanup_thread(); | |
| 1584 | |||
| 1585 | 2 | std::lock_guard<SrwSharedMutex> state_lock(s_cache_state_mutex); | |
| 1586 | 2 | close_reader_admission(); | |
| 1587 | s_shard_count.store(0, std::memory_order_release); | ||
| 1588 |
1/2✓ Branch 137 → 138 taken 2 times.
✗ Branch 137 → 156 not taken.
|
2 | if (drain_admitted_readers()) |
| 1589 | { | ||
| 1590 | 2 | s_cache_shards.reset(); | |
| 1591 | s_configured_expiry_ms.store(0, std::memory_order_relaxed); | ||
| 1592 | s_max_entries_per_shard.store(0, std::memory_order_relaxed); | ||
| 1593 | 2 | release_cache_keepalive_after_drain(); | |
| 1594 | } | ||
| 1595 | else | ||
| 1596 | { | ||
| 1597 | ✗ | record_reader_drain_timeout_retention(); | |
| 1598 | } | ||
| 1599 | #if !defined(_MSC_VER) && defined(_WIN64) | ||
| 1600 | 2 | detail::release_guarded_engine(); | |
| 1601 | #endif | ||
| 1602 | 2 | return false; | |
| 1603 | 2 | } | |
| 1604 | |||
| 1605 | 742 | return true; | |
| 1606 | 750 | } | |
| 1607 | |||
| 1608 | 35 | void clear_cache() noexcept | |
| 1609 | { | ||
| 1610 | { | ||
| 1611 | 35 | std::lock_guard state_lock(s_cache_state_mutex); | |
| 1612 | |||
| 1613 |
2/2✓ Branch 4 → 5 taken 1 time.
✓ Branch 4 → 6 taken 34 times.
|
35 | if (!cache_is_running()) |
| 1614 | 1 | return; | |
| 1615 | |||
| 1616 | 34 | const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire); | |
| 1617 |
1/2✗ Branch 13 → 14 not taken.
✓ Branch 13 → 15 taken 34 times.
|
34 | if (shard_count == 0) |
| 1618 | ✗ | return; | |
| 1619 | |||
| 1620 | // Acquire an exclusive lock for each shard and wait if needed. The cleanup thread uses try_lock, so it | ||
| 1621 | // skips held shards. | ||
| 1622 |
2/2✓ Branch 47 → 16 taken 505 times.
✓ Branch 47 → 48 taken 34 times.
|
539 | for (std::size_t i = 0; i < shard_count; ++i) |
| 1623 | { | ||
| 1624 | 505 | std::unique_lock<SrwSharedMutex> shard_lock(s_cache_shards[i].mtx); | |
| 1625 | 505 | s_cache_shards[i].entries.clear(); | |
| 1626 | 505 | s_cache_shards[i].fifo_index.clear(); | |
| 1627 | 505 | s_cache_shards[i].sorted_ranges.clear(); | |
| 1628 | 505 | s_cache_shards[i].hits.store(0, std::memory_order_relaxed); | |
| 1629 | 505 | s_cache_shards[i].misses.store(0, std::memory_order_relaxed); | |
| 1630 | // Advance the generation so an in-flight leader cannot republish a pre-clear result. | ||
| 1631 | 505 | s_cache_shards[i].content_gen.fetch_add(1, std::memory_order_release); | |
| 1632 | 505 | } | |
| 1633 | |||
| 1634 | s_stats.invalidations.store(0, std::memory_order_relaxed); | ||
| 1635 | s_stats.coalesced_queries.store(0, std::memory_order_relaxed); | ||
| 1636 | s_stats.on_demand_cleanups.store(0, std::memory_order_relaxed); | ||
| 1637 | |||
| 1638 | 34 | s_last_cleanup_time_ns.store(current_time_ns(), std::memory_order_relaxed); | |
| 1639 |
2/2✓ Branch 83 → 84 taken 34 times.
✓ Branch 83 → 88 taken 1 time.
|
35 | } |
| 1640 | |||
| 1641 | // Deferred-log order: the optional diagnostic tail runs after every cache mutex is released. A sink or | ||
| 1642 | // format failure drops the line. | ||
| 1643 | 34 | (void)log().try_log(LogLevel::Debug, "MemoryCache: All entries cleared."); | |
| 1644 | } | ||
| 1645 | |||
| 1646 | 4924 | void shutdown_cache() noexcept | |
| 1647 | { | ||
| 1648 | // Decide the block policy once for the whole teardown. A second query lets a concurrent publication split | ||
| 1649 | // one teardown across both policies. | ||
| 1650 | 4924 | const bool may_block = DetourModKit::detail::blocking_teardown_permitted(); | |
| 1651 |
1/2✗ Branch 3 → 4 not taken.
✓ Branch 3 → 6 taken 4924 times.
|
4924 | if (!may_block) |
| 1652 | { | ||
| 1653 | ✗ | abandon_cache_unauthorized(); | |
| 1654 | 4182 | return; | |
| 1655 | } | ||
| 1656 | |||
| 1657 | // Serialize the whole stop against init_cache across the cleanup-thread handle. | ||
| 1658 | 4924 | std::lock_guard lifecycle_lock(s_lifecycle_mutex); | |
| 1659 | |||
| 1660 | 4924 | const LifecycleState state = s_lifecycle_state.load(std::memory_order_seq_cst); | |
| 1661 |
6/8✓ Branch 8 → 9 taken 4182 times.
✓ Branch 8 → 13 taken 742 times.
✓ Branch 9 → 10 taken 4182 times.
✗ Branch 9 → 12 not taken.
✓ Branch 11 → 12 taken 4182 times.
✗ Branch 11 → 13 not taken.
✓ Branch 14 → 15 taken 4182 times.
✓ Branch 14 → 16 taken 742 times.
|
4924 | if (state != LifecycleState::Running && !(state == LifecycleState::Stopped && s_cache_shards)) |
| 1662 | 4182 | return; | |
| 1663 | |||
| 1664 | // Stopped with a live shard array is a prior loader-lock abandonment or drain timeout, safe to finish here | ||
| 1665 | // off the loader lock. Stopping also prevents a concurrent loader-lock callback from another Stopped | ||
| 1666 | // publication. | ||
| 1667 | 742 | s_lifecycle_state.store(LifecycleState::Stopping, std::memory_order_seq_cst); | |
| 1668 | |||
| 1669 | // Close every admission stripe before the drain reads the counts. The drain then sees a closed population. | ||
| 1670 | // A query after this point takes the uncached route ([B-73]). | ||
| 1671 | 742 | close_reader_admission(); | |
| 1672 | |||
| 1673 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 1674 | // Force an initializer to attempt this lifecycle lock before teardown continues. | ||
| 1675 |
2/2✓ Branch 18 → 19 taken 2 times.
✓ Branch 18 → 20 taken 740 times.
|
742 | if (auto *const hook = DetourModKit::detail::g_memory_cache_shutdown_window_test_hook) |
| 1676 | 2 | hook(); | |
| 1677 | #endif | ||
| 1678 | |||
| 1679 | // Join the cleanup thread before acquisition of the state mutex. The thread takes s_cache_state_mutex in | ||
| 1680 | // forced cleanup. A join under that mutex can deadlock. The joined population is closed: the flag store | ||
| 1681 | // above makes the one worker exit its wait promptly. | ||
| 1682 | 742 | s_cleanup_thread_running.store(false, std::memory_order_release); | |
| 1683 | 742 | s_cleanup_cv.notify_one(); | |
| 1684 | 742 | (void)join_cleanup_thread(); | |
| 1685 | |||
| 1686 | 742 | bool drained = false; | |
| 1687 | { | ||
| 1688 | 742 | std::lock_guard state_lock(s_cache_state_mutex); | |
| 1689 | |||
| 1690 | // Capture the shard count before its reset to zero because the destroy loop needs the length. | ||
| 1691 | 742 | const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire); | |
| 1692 | s_shard_count.store(0, std::memory_order_release); | ||
| 1693 | |||
| 1694 | 742 | drained = drain_admitted_readers(); | |
| 1695 |
2/2✓ Branch 40 → 41 taken 741 times.
✓ Branch 40 → 103 taken 1 time.
|
742 | if (drained) |
| 1696 | { | ||
| 1697 |
2/2✓ Branch 52 → 42 taken 11431 times.
✓ Branch 52 → 53 taken 741 times.
|
12172 | for (std::size_t i = 0; i < shard_count; ++i) |
| 1698 | { | ||
| 1699 | 11431 | std::unique_lock<SrwSharedMutex> shard_lock(s_cache_shards[i].mtx); | |
| 1700 | 11431 | s_cache_shards[i].entries.clear(); | |
| 1701 | 11431 | s_cache_shards[i].fifo_index.clear(); | |
| 1702 | 11431 | s_cache_shards[i].sorted_ranges.clear(); | |
| 1703 | 11431 | } | |
| 1704 | |||
| 1705 | 741 | s_cache_shards.reset(); | |
| 1706 | |||
| 1707 | // Only the global cold counters need an explicit reset. s_lifecycle_violations is intentionally | ||
| 1708 | // NOT reset: a sticky diagnostic that must survive a restart. | ||
| 1709 | s_stats.invalidations.store(0, std::memory_order_relaxed); | ||
| 1710 | s_stats.coalesced_queries.store(0, std::memory_order_relaxed); | ||
| 1711 | s_stats.on_demand_cleanups.store(0, std::memory_order_relaxed); | ||
| 1712 | s_last_cleanup_time_ns.store(0, std::memory_order_relaxed); | ||
| 1713 | s_configured_expiry_ms.store(0, std::memory_order_relaxed); | ||
| 1714 | s_max_entries_per_shard.store(0, std::memory_order_relaxed); | ||
| 1715 | 741 | release_cache_keepalive_after_drain(); | |
| 1716 | } | ||
| 1717 | else | ||
| 1718 | { | ||
| 1719 | // A stalled admitted reader can still hold shard pointers. Retain the shard array, configuration, | ||
| 1720 | // and precommitted module reference. A later init_cache or shutdown_cache call retries the drain. | ||
| 1721 | 1 | record_reader_drain_timeout_retention(); | |
| 1722 | } | ||
| 1723 | 742 | s_cleanup_requested.store(false, std::memory_order_relaxed); | |
| 1724 | |||
| 1725 | #if !defined(_MSC_VER) && defined(_WIN64) | ||
| 1726 | // Remove the vectored fault handler so it cannot dangle into freed code if the DMK module is unloaded | ||
| 1727 | // after teardown. The engine drains guarded reads on the handler path before handler removal. An | ||
| 1728 | // in-flight read cannot fault into a missing handler. The operation is idempotent. A later guarded | ||
| 1729 | // read reinstalls it. | ||
| 1730 | 742 | detail::release_guarded_engine(); | |
| 1731 | #endif | ||
| 1732 | |||
| 1733 | // Publish Stopped last, under the lifecycle mutex, so the next init_cache admits a fresh start (which | ||
| 1734 | // reopens admission only from the exact closed and zero-reader state). | ||
| 1735 | 742 | s_lifecycle_state.store(LifecycleState::Stopped, std::memory_order_seq_cst); | |
| 1736 | 742 | } | |
| 1737 | |||
| 1738 | // Deferred-log order: the optional diagnostic tail runs after every cache mutex is released. | ||
| 1739 |
2/2✓ Branch 108 → 109 taken 741 times.
✓ Branch 108 → 112 taken 1 time.
|
742 | if (drained) |
| 1740 | { | ||
| 1741 | 741 | (void)log().try_log(LogLevel::Debug, "MemoryCache: Shutdown complete."); | |
| 1742 | } | ||
| 1743 | else | ||
| 1744 | { | ||
| 1745 | 1 | (void)log().try_log(LogLevel::Debug, "MemoryCache: Shutdown drain timed out, cache storage retained."); | |
| 1746 | } | ||
| 1747 |
2/2✓ Branch 117 → 118 taken 742 times.
✓ Branch 117 → 120 taken 4182 times.
|
4924 | } |
| 1748 | |||
| 1749 | 59777642 | MemoryStats get_memory_stats() noexcept | |
| 1750 | { | ||
| 1751 | 59777642 | MemoryStats stats{}; | |
| 1752 | // The COLD counters are independent of the shard-array lifetime, so a relaxed load outside the reader | ||
| 1753 | // guard is safe. The hot per-shard tallies are summed under the guard below. | ||
| 1754 | 63778573 | stats.invalidations = s_stats.invalidations.load(std::memory_order_relaxed); | |
| 1755 | 64273352 | stats.coalesced_queries = s_stats.coalesced_queries.load(std::memory_order_relaxed); | |
| 1756 | 61224657 | stats.on_demand_cleanups = s_stats.on_demand_cleanups.load(std::memory_order_relaxed); | |
| 1757 | 64197273 | stats.lifecycle_violations = s_lifecycle_violations.load(std::memory_order_relaxed); | |
| 1758 | |||
| 1759 | // Capture the config fields and entry totals under the same reader admission that permission readers use. | ||
| 1760 | // A plain acquire of s_shard_count alone lets a concurrent shutdown free the array between a stale | ||
| 1761 | // non-zero count and the loop. Every field stays at its zero default while the cache is down. | ||
| 1762 | { | ||
| 1763 | 64197273 | ReaderAdmission admission; | |
| 1764 |
6/6✓ Branch 32 → 33 taken 30924 times.
✓ Branch 32 → 36 taken 62049394 times.
✓ Branch 34 → 35 taken 29545 times.
✓ Branch 34 → 36 taken 1372 times.
✓ Branch 37 → 38 taken 29559 times.
✓ Branch 37 → 86 taken 62050752 times.
|
62670636 | if (admission.admitted() && cache_is_running()) |
| 1765 | { | ||
| 1766 | 29582 | const std::size_t active_shard_count = s_shard_count.load(std::memory_order_acquire); | |
| 1767 |
1/2✓ Branch 45 → 46 taken 29598 times.
✗ Branch 45 → 86 not taken.
|
29582 | if (active_shard_count > 0) |
| 1768 | { | ||
| 1769 | 29598 | stats.shard_count = active_shard_count; | |
| 1770 | 29547 | stats.max_entries_per_shard = s_max_entries_per_shard.load(std::memory_order_acquire); | |
| 1771 | 29590 | stats.expiry_ms = s_configured_expiry_ms.load(std::memory_order_acquire); | |
| 1772 | |||
| 1773 | 29590 | std::size_t total_hard_max = 0; | |
| 1774 |
2/2✓ Branch 84 → 61 taken 460037 times.
✓ Branch 84 → 85 taken 26353 times.
|
486390 | for (std::size_t i = 0; i < active_shard_count; ++i) |
| 1775 | { | ||
| 1776 | 460037 | std::shared_lock<SrwSharedMutex> shard_lock(s_cache_shards[i].mtx); | |
| 1777 | 454828 | stats.total_entries += s_cache_shards[i].entries.size(); | |
| 1778 | 453193 | total_hard_max += s_cache_shards[i].max_capacity; | |
| 1779 | 449826 | stats.hits += s_cache_shards[i].hits.load(std::memory_order_relaxed); | |
| 1780 | 454301 | stats.misses += s_cache_shards[i].misses.load(std::memory_order_relaxed); | |
| 1781 | 452481 | } | |
| 1782 | 26353 | stats.hard_max_per_shard = total_hard_max / active_shard_count; | |
| 1783 | } | ||
| 1784 | } | ||
| 1785 | 62077089 | } | |
| 1786 | |||
| 1787 | 61515055 | const std::uint64_t total_queries = stats.hits + stats.misses; | |
| 1788 | 61515055 | stats.hit_rate_percent = | |
| 1789 |
2/2✓ Branch 87 → 88 taken 29589 times.
✓ Branch 87 → 89 taken 61485466 times.
|
61515055 | (total_queries > 0) ? (static_cast<double>(stats.hits) / static_cast<double>(total_queries)) * 100.0 |
| 1790 | : -1.0; | ||
| 1791 | 61515055 | return stats; | |
| 1792 | } | ||
| 1793 | |||
| 1794 | 25 | std::string get_cache_stats() | |
| 1795 | { | ||
| 1796 | 25 | const MemoryStats s = get_memory_stats(); | |
| 1797 | |||
| 1798 |
1/2✓ Branch 3 → 4 taken 25 times.
✗ Branch 3 → 42 not taken.
|
25 | std::ostringstream oss; |
| 1799 |
4/8✓ Branch 4 → 5 taken 25 times.
✗ Branch 4 → 40 not taken.
✓ Branch 5 → 6 taken 25 times.
✗ Branch 5 → 40 not taken.
✓ Branch 6 → 7 taken 25 times.
✗ Branch 6 → 40 not taken.
✓ Branch 7 → 8 taken 25 times.
✗ Branch 7 → 40 not taken.
|
25 | oss << "MemoryCache Stats (Shards: " << s.shard_count << ", Entries/Shard: " << s.max_entries_per_shard |
| 1800 |
4/8✓ Branch 8 → 9 taken 25 times.
✗ Branch 8 → 40 not taken.
✓ Branch 9 → 10 taken 25 times.
✗ Branch 9 → 40 not taken.
✓ Branch 10 → 11 taken 25 times.
✗ Branch 10 → 40 not taken.
✓ Branch 11 → 12 taken 25 times.
✗ Branch 11 → 40 not taken.
|
25 | << ", HardMax/Shard: " << s.hard_max_per_shard << ", Expiry: " << s.expiry_ms << "ms) - " |
| 1801 |
7/14✓ Branch 12 → 13 taken 25 times.
✗ Branch 12 → 40 not taken.
✓ Branch 13 → 14 taken 25 times.
✗ Branch 13 → 40 not taken.
✓ Branch 14 → 15 taken 25 times.
✗ Branch 14 → 40 not taken.
✓ Branch 15 → 16 taken 25 times.
✗ Branch 15 → 40 not taken.
✓ Branch 16 → 17 taken 25 times.
✗ Branch 16 → 40 not taken.
✓ Branch 17 → 18 taken 25 times.
✗ Branch 17 → 40 not taken.
✓ Branch 18 → 19 taken 25 times.
✗ Branch 18 → 40 not taken.
|
25 | << "Hits: " << s.hits << ", Misses: " << s.misses << ", Invalidations: " << s.invalidations |
| 1802 |
4/8✓ Branch 19 → 20 taken 25 times.
✗ Branch 19 → 40 not taken.
✓ Branch 20 → 21 taken 25 times.
✗ Branch 20 → 40 not taken.
✓ Branch 21 → 22 taken 25 times.
✗ Branch 21 → 40 not taken.
✓ Branch 22 → 23 taken 25 times.
✗ Branch 22 → 40 not taken.
|
25 | << ", Coalesced: " << s.coalesced_queries << ", OnDemandCleanups: " << s.on_demand_cleanups |
| 1803 |
4/8✓ Branch 23 → 24 taken 25 times.
✗ Branch 23 → 40 not taken.
✓ Branch 24 → 25 taken 25 times.
✗ Branch 24 → 40 not taken.
✓ Branch 25 → 26 taken 25 times.
✗ Branch 25 → 40 not taken.
✓ Branch 26 → 27 taken 25 times.
✗ Branch 26 → 40 not taken.
|
25 | << ", TotalEntries: " << s.total_entries << ", LifecycleViolations: " << s.lifecycle_violations; |
| 1804 | |||
| 1805 |
2/2✓ Branch 27 → 28 taken 16 times.
✓ Branch 27 → 34 taken 9 times.
|
25 | if (s.hit_rate_percent >= 0.0) |
| 1806 | { | ||
| 1807 |
4/8✓ Branch 28 → 29 taken 16 times.
✗ Branch 28 → 40 not taken.
✓ Branch 29 → 30 taken 16 times.
✗ Branch 29 → 40 not taken.
✓ Branch 32 → 33 taken 16 times.
✗ Branch 32 → 40 not taken.
✓ Branch 33 → 35 taken 16 times.
✗ Branch 33 → 40 not taken.
|
16 | oss << ", Hit Rate: " << std::fixed << std::setprecision(2) << s.hit_rate_percent << "%"; |
| 1808 | } | ||
| 1809 | else | ||
| 1810 | { | ||
| 1811 |
1/2✓ Branch 34 → 35 taken 9 times.
✗ Branch 34 → 40 not taken.
|
9 | oss << ", Hit Rate: N/A (no queries tracked)"; |
| 1812 | } | ||
| 1813 |
1/2✓ Branch 35 → 36 taken 25 times.
✗ Branch 35 → 40 not taken.
|
50 | return oss.str(); |
| 1814 | 25 | } | |
| 1815 | |||
| 1816 | 2988 | void invalidate_range(Region range) noexcept | |
| 1817 | { | ||
| 1818 |
6/6✓ Branch 3 → 4 taken 2986 times.
✓ Branch 3 → 5 taken 1 time.
✓ Branch 4 → 5 taken 1 time.
✓ Branch 4 → 6 taken 2985 times.
✓ Branch 7 → 8 taken 2 times.
✓ Branch 7 → 9 taken 2985 times.
|
2988 | if (!range.base || range.size == 0) |
| 1819 | 5 | return; | |
| 1820 | |||
| 1821 | // Admission keeps shutdown from a shard-array free during the sweep. A rejected caller has no live cache. | ||
| 1822 | 2985 | ReaderAdmission admission; | |
| 1823 |
5/6✓ Branch 11 → 12 taken 2983 times.
✓ Branch 11 → 14 taken 4 times.
✗ Branch 13 → 14 not taken.
✓ Branch 13 → 15 taken 2982 times.
✓ Branch 16 → 17 taken 3 times.
✓ Branch 16 → 18 taken 2982 times.
|
2986 | if (!admission.admitted() || !cache_is_running()) |
| 1824 | 3 | return; | |
| 1825 | |||
| 1826 | 2982 | const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire); | |
| 1827 |
1/2✗ Branch 25 → 26 not taken.
✓ Branch 25 → 27 taken 2982 times.
|
2982 | if (shard_count == 0) |
| 1828 | ✗ | return; | |
| 1829 | |||
| 1830 | 2982 | invalidate_range_internal(range.base.raw(), range.size); | |
| 1831 | |||
| 1832 | // The on-demand cleanup fallback holds s_cache_state_mutex while it iterates the shards. | ||
| 1833 | 2982 | request_cleanup(); | |
| 1834 |
2/2✓ Branch 32 → 33 taken 2982 times.
✓ Branch 32 → 35 taken 3 times.
|
2983 | } |
| 1835 | |||
| 1836 | 292828 | bool is_readable(Region range) noexcept | |
| 1837 | { | ||
| 1838 | 292828 | return check_memory_permission(range.base.raw(), range.size, check_read_permission); | |
| 1839 | } | ||
| 1840 | |||
| 1841 | 3892 | bool is_writable(Region range) noexcept | |
| 1842 | { | ||
| 1843 | 3892 | return check_memory_permission(range.base.raw(), range.size, check_write_permission); | |
| 1844 | } | ||
| 1845 | |||
| 1846 | 16 | ReadableStatus is_readable_nonblocking(Region range) noexcept | |
| 1847 | { | ||
| 1848 | 16 | const std::uintptr_t address = range.base.raw(); | |
| 1849 | 16 | const std::size_t size = range.size; | |
| 1850 |
4/4✓ Branch 3 → 4 taken 15 times.
✓ Branch 3 → 5 taken 1 time.
✓ Branch 4 → 5 taken 1 time.
✓ Branch 4 → 6 taken 14 times.
|
16 | if (address == 0 || size == 0) |
| 1851 | 2 | return ReadableStatus::NotReadable; | |
| 1852 | |||
| 1853 | 14 | ReaderAdmission admission; | |
| 1854 |
5/6✓ Branch 8 → 9 taken 10 times.
✓ Branch 8 → 11 taken 4 times.
✗ Branch 10 → 11 not taken.
✓ Branch 10 → 12 taken 10 times.
✓ Branch 13 → 14 taken 4 times.
✓ Branch 13 → 19 taken 10 times.
|
14 | if (!admission.admitted() || !cache_is_running()) |
| 1855 | { | ||
| 1856 | // No cache is available. Use a range walk that can wait and return a definite answer. The | ||
| 1857 | // cache-present path below never issues a VirtualQuery and returns Unknown on a miss. | ||
| 1858 |
2/2✓ Branch 15 → 16 taken 2 times.
✓ Branch 15 → 17 taken 2 times.
|
4 | return range_permission_uncached(address, size, check_read_permission) ? ReadableStatus::Readable |
| 1859 | 4 | : ReadableStatus::NotReadable; | |
| 1860 | } | ||
| 1861 | |||
| 1862 | 10 | const std::size_t shard_count = s_shard_count.load(std::memory_order_acquire); | |
| 1863 |
1/2✗ Branch 26 → 27 not taken.
✓ Branch 26 → 28 taken 10 times.
|
10 | if (shard_count == 0) |
| 1864 | ✗ | return ReadableStatus::Unknown; | |
| 1865 | |||
| 1866 | 10 | const std::size_t shard_idx = compute_shard_index(address, shard_count); | |
| 1867 | 10 | const std::uint64_t now_ns = current_time_ns(); | |
| 1868 | 10 | const std::uint64_t expiry_ns = configured_expiry_ns(); | |
| 1869 | |||
| 1870 | // The shared try-lock prevents a wait by a latency-sensitive thread on a contended shard. | ||
| 1871 | 10 | std::shared_lock<SrwSharedMutex> lock(s_cache_shards[shard_idx].mtx, std::try_to_lock); | |
| 1872 |
1/2✗ Branch 34 → 35 not taken.
✓ Branch 34 → 36 taken 10 times.
|
10 | if (!lock.owns_lock()) |
| 1873 | ✗ | return ReadableStatus::Unknown; | |
| 1874 | |||
| 1875 | CachedMemoryRegionInfo *cached_info = | ||
| 1876 | 10 | find_in_shard(s_cache_shards[shard_idx], address, size, now_ns, expiry_ns); | |
| 1877 |
2/2✓ Branch 38 → 39 taken 6 times.
✓ Branch 38 → 48 taken 4 times.
|
10 | if (cached_info) |
| 1878 | { | ||
| 1879 | 6 | s_cache_shards[shard_idx].hits.fetch_add(1, std::memory_order_relaxed); | |
| 1880 | // Require MEM_COMMIT alongside the read permission, symmetric with the blocking hit path. | ||
| 1881 | 5 | return (cached_info->state == MEM_COMMIT && check_read_permission(cached_info->protection)) | |
| 1882 |
4/4✓ Branch 42 → 43 taken 5 times.
✓ Branch 42 → 45 taken 1 time.
✓ Branch 44 → 45 taken 2 times.
✓ Branch 44 → 46 taken 3 times.
|
11 | ? ReadableStatus::Readable |
| 1883 | 6 | : ReadableStatus::NotReadable; | |
| 1884 | } | ||
| 1885 | |||
| 1886 | // Under non-blocking semantics, return Unknown on a cache miss instead of a VirtualQuery call. | ||
| 1887 | 4 | return ReadableStatus::Unknown; | |
| 1888 | 14 | } | |
| 1889 | } // namespace memory | ||
| 1890 | } // namespace DetourModKit | ||
| 1891 | |||
| 1892 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 1893 | namespace DetourModKit::detail | ||
| 1894 | { | ||
| 1895 | 2 | void memory_cache_abandon_for_test() noexcept | |
| 1896 | { | ||
| 1897 | 2 | memory::abandon_cache_unauthorized(); | |
| 1898 | 2 | } | |
| 1899 | |||
| 1900 | 5 | std::uint64_t memory_cache_admitted_reader_count_for_test() noexcept | |
| 1901 | { | ||
| 1902 | 5 | return memory::admitted_reader_count(); | |
| 1903 | } | ||
| 1904 | |||
| 1905 | 1 | bool memory_cache_has_retained_shards_for_test() noexcept | |
| 1906 | { | ||
| 1907 | 1 | return memory::s_cache_shards != nullptr; | |
| 1908 | } | ||
| 1909 | |||
| 1910 | 2 | bool memory_cache_reader_would_be_admitted_for_test() noexcept | |
| 1911 | { | ||
| 1912 | 2 | memory::ReaderAdmission admission; | |
| 1913 | 2 | return admission.admitted(); | |
| 1914 | 2 | } | |
| 1915 | |||
| 1916 | 2 | void memory_cache_hold_shared_shard_lock_for_test(Address address, void (*callback)() noexcept) noexcept | |
| 1917 | { | ||
| 1918 | 2 | memory::hold_shard_shared_lock_for_test(address, callback); | |
| 1919 | 2 | } | |
| 1920 | |||
| 1921 | 26 | void memory_cache_shard_index_sizes_for_test( | |
| 1922 | Address address, | ||
| 1923 | std::size_t &entries, | ||
| 1924 | std::size_t &fifo, | ||
| 1925 | std::size_t &ranges | ||
| 1926 | ) noexcept | ||
| 1927 | { | ||
| 1928 | 26 | memory::shard_index_sizes_for_test(address, entries, fifo, ranges); | |
| 1929 | 26 | } | |
| 1930 | } // namespace DetourModKit::detail | ||
| 1931 | #endif | ||
| 1932 |