src/hook_retirement.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /** | ||
| 2 | * @file hook_retirement.cpp | ||
| 3 | * @brief This TU implements the counted thunk decode, the counted drain proof, and HookStack::retire (`[B-106]`). | ||
| 4 | */ | ||
| 5 | |||
| 6 | #include "DetourModKit/hook.hpp" | ||
| 7 | |||
| 8 | #include "internal/drain_backoff.hpp" | ||
| 9 | #include "internal/hook_backend.hpp" | ||
| 10 | #include "internal/hook_backend_visit.hpp" | ||
| 11 | #include "internal/hook_fault_boundary.hpp" | ||
| 12 | #include "internal/lifecycle_context.hpp" | ||
| 13 | #include "internal/memory_guarded.hpp" | ||
| 14 | #include "internal/mid_hook_adapter.hpp" | ||
| 15 | |||
| 16 | #include "DetourModKit/logger.hpp" | ||
| 17 | |||
| 18 | #include "x86_decode.hpp" | ||
| 19 | |||
| 20 | #include <windows.h> | ||
| 21 | |||
| 22 | #include <Zydis/Zydis.h> | ||
| 23 | |||
| 24 | #include <algorithm> | ||
| 25 | #include <array> | ||
| 26 | #include <atomic> | ||
| 27 | #include <chrono> | ||
| 28 | #include <cstddef> | ||
| 29 | #include <cstdint> | ||
| 30 | #include <expected> | ||
| 31 | #include <limits> | ||
| 32 | #include <memory> | ||
| 33 | #include <optional> | ||
| 34 | #include <span> | ||
| 35 | #include <thread> | ||
| 36 | #include <utility> | ||
| 37 | #include <vector> | ||
| 38 | |||
| 39 | namespace DetourModKit | ||
| 40 | { | ||
| 41 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 42 | void (*detail::g_counted_drain_probe)(detail::CountedDrainStage) noexcept = nullptr; | ||
| 43 | #endif | ||
| 44 | |||
| 45 | namespace | ||
| 46 | { | ||
| 47 | /// The largest thunk the decode copies. An unoptimized thunk with eight stack arguments stays far below it. | ||
| 48 | constexpr std::size_t COUNTED_THUNK_MAX_BYTES = 2048; | ||
| 49 | /// The incremental-link jumps that the decode follows from a function address to its body. | ||
| 50 | constexpr int INCREMENTAL_JUMP_LIMIT = 4; | ||
| 51 | /// The calls that one thunk can hold. The detour call and a few unoptimized-build helpers stay below it. | ||
| 52 | constexpr std::size_t COUNTED_THUNK_MAX_CALLS = 16; | ||
| 53 | /// The return sites that one list holds, and the largest offset from its base. | ||
| 54 | constexpr std::size_t RETURN_SITES_PER_LIST = 8; | ||
| 55 | constexpr std::uintptr_t RETURN_SITE_MAX_OFFSET = 0xFF; | ||
| 56 | |||
| 57 | /// Follows the jump that an incremental link puts at a function address, so the decode reads the body. | ||
| 58 | 4422 | [[nodiscard]] std::uintptr_t follow_incremental_jumps(std::uintptr_t address) noexcept | |
| 59 | { | ||
| 60 | 4422 | for (int hop = 0; hop < INCREMENTAL_JUMP_LIMIT; ++hop) | |
| 61 | { | ||
| 62 | 4422 | const std::optional<std::uintptr_t> next = DetourModKit::detail::decode_e9_rel32(address); | |
| 63 | 4422 | if (!next) | |
| 64 | { | ||
| 65 | 4422 | break; | |
| 66 | } | ||
| 67 | ✗ | address = *next; | |
| 68 | } | ||
| 69 | 4422 | return address; | |
| 70 | } | ||
| 71 | |||
| 72 | /** | ||
| 73 | * @brief Packs the returns of the helper calls, given as offsets from @p begin, into return-site lists. | ||
| 74 | * @details A stack word equal to a site keeps the drain Busy, so no site address lives in a local here. Each | ||
| 75 | * list keeps its base one byte below its first site, so no base equals a site either. | ||
| 76 | * @return False when the sites need more lists than a thunk holds. | ||
| 77 | */ | ||
| 78 | 4414 | [[nodiscard]] bool pack_helper_sites( | |
| 79 | const std::uint8_t *begin, | ||
| 80 | std::span<const std::size_t> site_offsets, | ||
| 81 | hook::CountedThunk &thunk | ||
| 82 | ) noexcept | ||
| 83 | { | ||
| 84 | 4414 | thunk.helper_site_lists = 0; | |
| 85 | 4414 | std::size_t list_base_offset = 0; | |
| 86 | 8853 | for (const std::size_t site_offset : site_offsets) | |
| 87 | { | ||
| 88 | safetyhook::ReturnSites *list = | ||
| 89 | 25 | thunk.helper_site_lists == 0 ? nullptr : &thunk.helper_sites[thunk.helper_site_lists - 1]; | |
| 90 | 25 | const bool fits = list != nullptr && list->count < RETURN_SITES_PER_LIST && | |
| 91 | ✗ | site_offset - list_base_offset <= RETURN_SITE_MAX_OFFSET; | |
| 92 | 25 | if (!fits) | |
| 93 | { | ||
| 94 | 50 | if (thunk.helper_site_lists == thunk.helper_sites.size()) | |
| 95 | { | ||
| 96 | ✗ | return false; | |
| 97 | } | ||
| 98 | 25 | list = &thunk.helper_sites[thunk.helper_site_lists++]; | |
| 99 | 25 | list_base_offset = site_offset - 1; | |
| 100 | 25 | list->code = begin + list_base_offset; | |
| 101 | 25 | list->count = 0; | |
| 102 | } | ||
| 103 | 25 | list->offsets[list->count++] = static_cast<std::uint8_t>(site_offset - list_base_offset); | |
| 104 | } | ||
| 105 | 4414 | return true; | |
| 106 | } | ||
| 107 | |||
| 108 | /// Runs one idle pass. An unusable pass reads as busy, because the caller then retries or retains. | ||
| 109 | 9728 | [[nodiscard]] bool idle_pass( | |
| 110 | std::span<const safetyhook::AddressRange> ranges, | ||
| 111 | std::span<const safetyhook::ReturnSites *const> sites | ||
| 112 | ) noexcept | ||
| 113 | { | ||
| 114 | try | ||
| 115 | { | ||
| 116 | 9728 | const std::expected<bool, safetyhook::OsError> idle = safetyhook::threads_idle_outside(ranges, sites); | |
| 117 | 9728 | return idle.has_value() && *idle; | |
| 118 | } | ||
| 119 | ✗ | catch (...) | |
| 120 | { | ||
| 121 | ✗ | return false; | |
| 122 | ✗ | } | |
| 123 | } | ||
| 124 | |||
| 125 | /// Sums one counter over every stripe of every block with seq_cst loads (`[B-43]`). | ||
| 126 | template <class Member> | ||
| 127 | [[nodiscard]] std::uint64_t | ||
| 128 | 10230 | sum_counters(std::span<hook::detail::CountedBlock *const> blocks, Member member) noexcept | |
| 129 | { | ||
| 130 | 10230 | std::uint64_t total = 0; | |
| 131 | 30808 | for (hook::detail::CountedBlock *const block : blocks) | |
| 132 | { | ||
| 133 | 175916 | for (hook::detail::CountedStripe &stripe : block->stripes) | |
| 134 | { | ||
| 135 | 165568 | total += std::atomic_ref<std::uint64_t>(stripe.*member).load(std::memory_order_seq_cst); | |
| 136 | } | ||
| 137 | } | ||
| 138 | 10230 | return total; | |
| 139 | } | ||
| 140 | |||
| 141 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 142 | 15345 | void note_counted_drain_stage(DetourModKit::detail::CountedDrainStage stage) noexcept | |
| 143 | { | ||
| 144 | 15345 | if (auto *probe = DetourModKit::detail::g_counted_drain_probe) | |
| 145 | { | ||
| 146 | 222 | probe(stage); | |
| 147 | } | ||
| 148 | 15345 | } | |
| 149 | #endif | ||
| 150 | } // namespace | ||
| 151 | |||
| 152 | namespace hook | ||
| 153 | { | ||
| 154 | 4422 | std::expected<detail::CountedThunkShape, const char *> detail::inspect_counted_thunk(void *entry) noexcept | |
| 155 | { | ||
| 156 | 4422 | const std::uintptr_t begin = follow_incremental_jumps(reinterpret_cast<std::uintptr_t>(entry)); | |
| 157 | 4422 | const std::optional<DetourModKit::detail::FunctionBound> bound = DetourModKit::detail::unwind_bound(begin); | |
| 158 | 4422 | if (!bound || bound->lo != begin) | |
| 159 | { | ||
| 160 | ✗ | return std::unexpected("no unwind record starts at the thunk"); | |
| 161 | } | ||
| 162 | 4422 | const std::size_t size = bound->hi - bound->lo; | |
| 163 | 4422 | if (size > COUNTED_THUNK_MAX_BYTES) | |
| 164 | { | ||
| 165 | ✗ | return std::unexpected("the thunk is larger than the decode bound"); | |
| 166 | } | ||
| 167 | 4422 | std::array<std::uint8_t, COUNTED_THUNK_MAX_BYTES> bytes{}; | |
| 168 | 4422 | if (!DetourModKit::detail::guarded_read_bytes(begin, bytes.data(), size)) | |
| 169 | { | ||
| 170 | ✗ | return std::unexpected("the thunk bytes are not readable"); | |
| 171 | } | ||
| 172 | 4422 | ZydisDecoder decoder{}; | |
| 173 | 4422 | if (!ZYAN_SUCCESS(ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64))) | |
| 174 | { | ||
| 175 | ✗ | return std::unexpected("the decoder did not start"); | |
| 176 | } | ||
| 177 | |||
| 178 | // Straight-line code runs in address order, so the first and last locked increments bound the counted | ||
| 179 | // window and every call outside them runs uncounted. | ||
| 180 | 4422 | std::array<std::size_t, COUNTED_THUNK_MAX_CALLS> call_starts{}; | |
| 181 | 4422 | std::array<std::size_t, COUNTED_THUNK_MAX_CALLS> call_ends{}; | |
| 182 | 4422 | std::size_t calls = 0; | |
| 183 | 4422 | std::size_t pushes = 0; | |
| 184 | 4422 | std::size_t pops = 0; | |
| 185 | 4422 | std::size_t locks = 0; | |
| 186 | 4422 | std::size_t first_lock = 0; | |
| 187 | 4422 | std::size_t last_lock_end = 0; | |
| 188 | 4422 | bool returned = false; | |
| 189 | 4422 | std::size_t offset = 0; | |
| 190 | 143246 | while (offset < size && !returned) | |
| 191 | { | ||
| 192 | 138831 | ZydisDecodedInstruction instruction{}; | |
| 193 | 277662 | if (!ZYAN_SUCCESS(ZydisDecoderDecodeInstruction( | |
| 194 | &decoder, | ||
| 195 | nullptr, | ||
| 196 | bytes.data() + offset, | ||
| 197 | size - offset, | ||
| 198 | &instruction | ||
| 199 | ))) | ||
| 200 | { | ||
| 201 | ✗ | return std::unexpected("an instruction of the thunk does not decode"); | |
| 202 | } | ||
| 203 | 138831 | const std::size_t end = offset + instruction.length; | |
| 204 | 138831 | pushes += instruction.mnemonic == ZYDIS_MNEMONIC_PUSH ? 1 : 0; | |
| 205 | 138831 | pops += instruction.mnemonic == ZYDIS_MNEMONIC_POP ? 1 : 0; | |
| 206 | 138831 | if ((instruction.attributes & ZYDIS_ATTRIB_HAS_LOCK) != 0) | |
| 207 | { | ||
| 208 | 8836 | first_lock = locks == 0 ? offset : first_lock; | |
| 209 | 8836 | last_lock_end = end; | |
| 210 | 8836 | ++locks; | |
| 211 | } | ||
| 212 | 138831 | switch (instruction.meta.category) | |
| 213 | { | ||
| 214 | 4447 | case ZYDIS_CATEGORY_CALL: | |
| 215 | 4447 | if (calls == call_ends.size()) | |
| 216 | { | ||
| 217 | ✗ | return std::unexpected("the thunk holds more calls than the decode bound"); | |
| 218 | } | ||
| 219 | 4447 | call_starts[calls] = offset; | |
| 220 | 4447 | call_ends[calls] = end; | |
| 221 | 4447 | ++calls; | |
| 222 | 4447 | break; | |
| 223 | 4415 | case ZYDIS_CATEGORY_RET: | |
| 224 | 4415 | returned = true; | |
| 225 | 4415 | break; | |
| 226 | 7 | case ZYDIS_CATEGORY_COND_BR: | |
| 227 | case ZYDIS_CATEGORY_UNCOND_BR: | ||
| 228 | case ZYDIS_CATEGORY_INTERRUPT: | ||
| 229 | case ZYDIS_CATEGORY_SYSCALL: | ||
| 230 | 7 | return std::unexpected("the thunk branches or traps, so its code is not straight-line"); | |
| 231 | 129962 | default: | |
| 232 | 129962 | break; | |
| 233 | } | ||
| 234 | 138824 | offset = end; | |
| 235 | } | ||
| 236 | 4415 | if (!returned) | |
| 237 | { | ||
| 238 | ✗ | return std::unexpected("the thunk does not return inside its unwind record"); | |
| 239 | } | ||
| 240 | 4415 | if (locks != 2) | |
| 241 | { | ||
| 242 | 1 | return std::unexpected("the thunk does not hold exactly two locked increments"); | |
| 243 | } | ||
| 244 | |||
| 245 | 4414 | std::array<std::size_t, COUNTED_THUNK_MAX_CALLS> helper_site_offsets{}; | |
| 246 | 4414 | std::size_t helpers = 0; | |
| 247 | 4414 | std::size_t counted_calls = 0; | |
| 248 | 8853 | for (std::size_t i = 0; i < calls; ++i) | |
| 249 | { | ||
| 250 | 4439 | if (call_ends[i] <= first_lock || call_starts[i] >= last_lock_end) | |
| 251 | { | ||
| 252 | 25 | helper_site_offsets[helpers++] = call_ends[i]; | |
| 253 | } | ||
| 254 | else | ||
| 255 | { | ||
| 256 | 4414 | ++counted_calls; | |
| 257 | } | ||
| 258 | } | ||
| 259 | 4414 | if (counted_calls == 0) | |
| 260 | { | ||
| 261 | ✗ | return std::unexpected("the thunk calls nothing between its counts"); | |
| 262 | } | ||
| 263 | |||
| 264 | 4414 | CountedThunkShape shape{}; | |
| 265 | 8828 | shape.thunk.range = { | |
| 266 | 4414 | reinterpret_cast<std::uint8_t *>(bound->lo), | |
| 267 | 4414 | reinterpret_cast<std::uint8_t *>(bound->hi), | |
| 268 | }; | ||
| 269 | 4414 | const std::span<const std::size_t> site_offsets{helper_site_offsets.data(), helpers}; | |
| 270 | 4414 | if (!pack_helper_sites(shape.thunk.range.begin, site_offsets, shape.thunk)) | |
| 271 | { | ||
| 272 | ✗ | return std::unexpected("the calls outside the counted window need more return sites than a hook holds"); | |
| 273 | } | ||
| 274 | 4414 | shape.enter_count_offset = first_lock; | |
| 275 | 4414 | shape.after_exit_count_offset = last_lock_end; | |
| 276 | 4414 | shape.helper_calls = helpers; | |
| 277 | 4414 | shape.counted_calls = counted_calls; | |
| 278 | 4414 | shape.pushes = pushes; | |
| 279 | 4414 | shape.pops = pops; | |
| 280 | 4414 | return shape; | |
| 281 | } | ||
| 282 | |||
| 283 | 4342 | detail::CountedDrain detail::drain_counted( | |
| 284 | std::span<const safetyhook::AddressRange> ranges, | ||
| 285 | std::span<const safetyhook::ReturnSites *const> sites, | ||
| 286 | std::span<CountedBlock *const> blocks, | ||
| 287 | std::chrono::steady_clock::time_point deadline | ||
| 288 | ) noexcept | ||
| 289 | { | ||
| 290 | 4342 | DetourModKit::detail::DrainBackoff backoff; | |
| 291 | 4342 | CountedDrain verdict{}; | |
| 292 | for (;;) | ||
| 293 | { | ||
| 294 | 5398 | if (idle_pass(ranges, sites)) | |
| 295 | { | ||
| 296 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 297 | 5115 | note_counted_drain_stage(DetourModKit::detail::CountedDrainStage::AfterFirstPass); | |
| 298 | #endif | ||
| 299 | // Exits first: equal sums then prove an instant at which no call sat between enter and exit. | ||
| 300 | 5115 | const std::uint64_t exits = sum_counters(blocks, &CountedStripe::exits); | |
| 301 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 302 | 5115 | note_counted_drain_stage(DetourModKit::detail::CountedDrainStage::BetweenCounterReads); | |
| 303 | #endif | ||
| 304 | 5115 | const std::uint64_t enters = sum_counters(blocks, &CountedStripe::enters); | |
| 305 | 5115 | verdict.in_flight = enters >= exits ? enters - exits : 0; | |
| 306 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 307 | 5115 | note_counted_drain_stage(DetourModKit::detail::CountedDrainStage::AfterCounterReads); | |
| 308 | #endif | ||
| 309 | // Pass 2 finds a thread that still runs the tail after its exit count. | ||
| 310 | 5115 | if (enters == exits && idle_pass(ranges, sites)) | |
| 311 | { | ||
| 312 | 4329 | verdict.idle = true; | |
| 313 | 4329 | return verdict; | |
| 314 | } | ||
| 315 | } | ||
| 316 | 1069 | if (std::chrono::steady_clock::now() >= deadline) | |
| 317 | { | ||
| 318 | 13 | return verdict; | |
| 319 | } | ||
| 320 | 1056 | backoff.pause(); | |
| 321 | 1056 | } | |
| 322 | } | ||
| 323 | |||
| 324 | 6350 | bool detail::current_thread_returns_into(std::span<const safetyhook::AddressRange> ranges) noexcept | |
| 325 | { | ||
| 326 | 6350 | constexpr DWORD FRAMES_PER_CAPTURE = 62; | |
| 327 | 6350 | constexpr DWORD FRAME_LIMIT = 1024; | |
| 328 | 6350 | std::array<void *, FRAMES_PER_CAPTURE> frames{}; | |
| 329 | 6350 | for (DWORD skipped = 0; skipped < FRAME_LIMIT; skipped += FRAMES_PER_CAPTURE) | |
| 330 | { | ||
| 331 | 6350 | const WORD captured = RtlCaptureStackBackTrace(skipped, FRAMES_PER_CAPTURE, frames.data(), nullptr); | |
| 332 | 124155 | for (WORD i = 0; i < captured; ++i) | |
| 333 | { | ||
| 334 | 117806 | const auto *const address = static_cast<const std::uint8_t *>(frames[i]); | |
| 335 | 324411 | for (const safetyhook::AddressRange &range : ranges) | |
| 336 | { | ||
| 337 | 88800 | if (address >= range.begin && address < range.end) | |
| 338 | { | ||
| 339 | 1 | return true; | |
| 340 | } | ||
| 341 | } | ||
| 342 | } | ||
| 343 | 6349 | if (captured < FRAMES_PER_CAPTURE) | |
| 344 | { | ||
| 345 | 6349 | break; | |
| 346 | } | ||
| 347 | } | ||
| 348 | 6349 | return false; | |
| 349 | } | ||
| 350 | |||
| 351 | namespace | ||
| 352 | { | ||
| 353 | /// The state that one retirement changed on one hook, so a Busy or Refused result can undo it. | ||
| 354 | template <class HookImpl> struct RetiringHook | ||
| 355 | { | ||
| 356 | HookImpl *impl{nullptr}; | ||
| 357 | bool relay_switched{false}; | ||
| 358 | bool route_closed{false}; | ||
| 359 | /// The commit restores this hook, so it adds no residue position. | ||
| 360 | bool restores{false}; | ||
| 361 | }; | ||
| 362 | |||
| 363 | /** | ||
| 364 | * @brief Puts every relay and route that this retirement changed back as it was. | ||
| 365 | * @details A mid route returns to LIVE only for an Active hook that no disable bypassed. Every other mid | ||
| 366 | * route stays CLOSED, because nothing reaches its stub. | ||
| 367 | */ | ||
| 368 | 13 | template <class HookImpl> void undo_retirement(std::span<RetiringHook<HookImpl>> hooks) noexcept | |
| 369 | { | ||
| 370 | 44 | for (RetiringHook<HookImpl> &hook : hooks) | |
| 371 | { | ||
| 372 | 18 | if (hook.route_closed) | |
| 373 | { | ||
| 374 | 2 | const bool reopen = hook.impl->status.load(std::memory_order_acquire) == HookState::Active && | |
| 375 | 1 | !hook.impl->bypassed; | |
| 376 | 1 | (void)DetourModKit::detail::apply_backend( | |
| 377 | 1 | hook.impl->backend, | |
| 378 | 3 | [reopen](auto &backend) noexcept | |
| 379 | 1 | { reopen ? backend.cancel_route_rundown() : backend.finish_route_rundown(); } | |
| 380 | ); | ||
| 381 | 1 | hook.route_closed = false; | |
| 382 | } | ||
| 383 | 18 | if (hook.relay_switched) | |
| 384 | { | ||
| 385 | 16 | (void)DetourModKit::detail::switch_backend(hook.impl->backend, true); | |
| 386 | 16 | hook.relay_switched = false; | |
| 387 | } | ||
| 388 | } | ||
| 389 | 13 | } | |
| 390 | |||
| 391 | /// The newest records on one target that the commit prediction reads. | ||
| 392 | constexpr std::size_t PREDICTED_RECORDS = 32; | ||
| 393 | |||
| 394 | /// Returns the coordinator key of a hook: the start of its trampoline. | ||
| 395 | 169 | template <class HookImpl> [[nodiscard]] std::uint8_t *coordinator_key(HookImpl &impl) noexcept | |
| 396 | { | ||
| 397 | 169 | return DetourModKit::detail::backend_value_or( | |
| 398 | 169 | impl.backend, | |
| 399 | static_cast<std::uint8_t *>(nullptr), | ||
| 400 | 338 | [](const auto &backend) noexcept { return backend.trampoline().data(); } | |
| 401 | 169 | ); | |
| 402 | } | ||
| 403 | |||
| 404 | /** | ||
| 405 | * @brief Marks each hook that the newest-first commit restores. | ||
| 406 | * @details The coordinator top restores when the target holds its own patch or the bytes that it saved, so | ||
| 407 | * a disabled top layer restores. A lower hook also restores when every newer record on its target | ||
| 408 | * belongs to a hook that restores first, because each restore puts the next patch back. A newer | ||
| 409 | * layer on an overlapping target with another address reads as no layer here. The budget guard of | ||
| 410 | * ~Hook still pins that case. | ||
| 411 | */ | ||
| 412 | template <class HookImpl> | ||
| 413 | 2099 | void predict_restores( | |
| 414 | std::span<RetiringHook<HookImpl>> hooks, | ||
| 415 | const safetyhook::ProcessCoordinator &coordinator | ||
| 416 | ) noexcept | ||
| 417 | { | ||
| 418 | 2115 | const auto restores = [hooks](const std::uint8_t *key) noexcept | |
| 419 | { | ||
| 420 | 32 | return std::ranges::any_of( | |
| 421 | 16 | hooks, | |
| 422 | 18 | [key](const RetiringHook<HookImpl> &hook) noexcept | |
| 423 | 18 | { return hook.restores && coordinator_key(*hook.impl) == key; } | |
| 424 | 16 | ); | |
| 425 | }; | ||
| 426 | 2099 | bool changed = true; | |
| 427 | 6277 | while (changed) | |
| 428 | { | ||
| 429 | 4178 | changed = false; | |
| 430 | 12678 | for (RetiringHook<HookImpl> &hook : hooks) | |
| 431 | { | ||
| 432 | 4322 | if (hook.restores || DetourModKit::detail::backend_claimed(hook.impl->backend)) | |
| 433 | { | ||
| 434 | 4304 | continue; | |
| 435 | } | ||
| 436 | 2165 | if (!DetourModKit::detail::backend_committed(hook.impl->backend)) | |
| 437 | { | ||
| 438 | 2001 | hook.restores = changed = true; | |
| 439 | 2001 | continue; | |
| 440 | } | ||
| 441 | 164 | std::uint8_t *const key = coordinator_key(*hook.impl); | |
| 442 | const DetourModKit::detail::PatchWitness witness = | ||
| 443 | 164 | DetourModKit::detail::witness_of(hook.impl->backend); | |
| 444 | 164 | const bool own_bytes = witness == DetourModKit::detail::PatchWitness::OwnedPatch || | |
| 445 | witness == DetourModKit::detail::PatchWitness::Original; | ||
| 446 | 164 | if (own_bytes && coordinator.top(key)) | |
| 447 | { | ||
| 448 | 146 | hook.restores = changed = true; | |
| 449 | 146 | continue; | |
| 450 | } | ||
| 451 | 18 | std::array<safetyhook::ProcessRouteRecord, PREDICTED_RECORDS> records{}; | |
| 452 | const std::size_t count = | ||
| 453 | 18 | coordinator.records_at(reinterpret_cast<std::uint8_t *>(hook.impl->target), records); | |
| 454 | 36 | if (count == 0 || count > records.size()) | |
| 455 | { | ||
| 456 | ✗ | continue; | |
| 457 | } | ||
| 458 | // The records arrive newest first. top() skips a retained record that is not enabled. | ||
| 459 | 18 | bool newer_restore = true; | |
| 460 | 18 | bool found = false; | |
| 461 | 26 | for (std::size_t i = 0; i < count && !found; ++i) | |
| 462 | { | ||
| 463 | 21 | const safetyhook::ProcessRouteRecord &record = records[i]; | |
| 464 | 21 | found = record.trampoline.begin == key; | |
| 465 | 21 | if (!found && (!record.retained || record.enabled) && !restores(record.trampoline.begin)) | |
| 466 | { | ||
| 467 | 13 | newer_restore = false; | |
| 468 | 13 | break; | |
| 469 | } | ||
| 470 | } | ||
| 471 | 18 | if (found && newer_restore && records[0].trampoline.begin != key) | |
| 472 | { | ||
| 473 | 3 | hook.restores = changed = true; | |
| 474 | } | ||
| 475 | } | ||
| 476 | } | ||
| 477 | 2099 | } | |
| 478 | |||
| 479 | /** | ||
| 480 | * @brief Measures the time-stamp counter rate while every route is still open. | ||
| 481 | * @details A CLOSING spin reads only the counter, because a clock read can be an export that the closing | ||
| 482 | * route hooks. The rate converts the drain deadline into counter ticks before any route closes. | ||
| 483 | */ | ||
| 484 | 7 | [[nodiscard]] double counter_ticks_per_nanosecond() noexcept | |
| 485 | { | ||
| 486 | 7 | const auto first_time = std::chrono::steady_clock::now(); | |
| 487 | 7 | const std::uint64_t first_ticks = __rdtsc(); | |
| 488 | 7 | std::this_thread::sleep_for(std::chrono::milliseconds{2}); | |
| 489 | 7 | const std::uint64_t last_ticks = __rdtsc(); | |
| 490 | const auto elapsed = | ||
| 491 | 7 | std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - first_time) | |
| 492 | 7 | .count(); | |
| 493 | 7 | if (elapsed <= 0 || last_ticks <= first_ticks) | |
| 494 | { | ||
| 495 | ✗ | return 1.0; | |
| 496 | } | ||
| 497 | 7 | return static_cast<double>(last_ticks - first_ticks) / static_cast<double>(elapsed); | |
| 498 | } | ||
| 499 | |||
| 500 | /** | ||
| 501 | * @brief Returns the counter ticks left before @p deadline. | ||
| 502 | * @details The result never exceeds the closing budget of ~Hook, because callers of a CLOSING route wait at | ||
| 503 | * its gateway for the whole spin. | ||
| 504 | */ | ||
| 505 | [[nodiscard]] std::uint64_t | ||
| 506 | 8 | remaining_ticks(std::chrono::steady_clock::time_point deadline, double ticks_per_nanosecond) noexcept | |
| 507 | { | ||
| 508 | 8 | const auto remaining = deadline - std::chrono::steady_clock::now(); | |
| 509 | 8 | if (remaining <= std::chrono::steady_clock::duration::zero()) | |
| 510 | { | ||
| 511 | ✗ | return 0; | |
| 512 | } | ||
| 513 | const double ticks = | ||
| 514 | 8 | static_cast<double>(std::chrono::duration_cast<std::chrono::nanoseconds>(remaining).count()) * | |
| 515 | 8 | ticks_per_nanosecond; | |
| 516 | 8 | const auto budget = static_cast<double>(DetourModKit::detail::CLOSING_DRAIN_TICK_BUDGET); | |
| 517 | 8 | return ticks >= budget ? DetourModKit::detail::CLOSING_DRAIN_TICK_BUDGET | |
| 518 | 8 | : static_cast<std::uint64_t>(ticks); | |
| 519 | } | ||
| 520 | |||
| 521 | /** | ||
| 522 | * @brief Proves each deferred mid route idle, with retries until @p deadline, and retains every route that | ||
| 523 | * stays refused. An expired deadline still leaves one attempt. | ||
| 524 | * @details No thread enters a restored route, and the threads inside it only move forward. A later attempt | ||
| 525 | * can therefore hold where an earlier one refused (`[B-108]`). | ||
| 526 | */ | ||
| 527 | 2085 | void reclaim_deferred_routes( | |
| 528 | std::span<detail::HookAccess::DeferredRestore> deferred, | ||
| 529 | std::chrono::steady_clock::time_point deadline | ||
| 530 | ) noexcept | ||
| 531 | { | ||
| 532 | 2085 | DetourModKit::detail::DrainBackoff backoff; | |
| 533 | for (;;) | ||
| 534 | { | ||
| 535 | 2290 | bool waiting = false; | |
| 536 | 4908 | for (detail::HookAccess::DeferredRestore &restore : deferred) | |
| 537 | { | ||
| 538 | 328 | if (auto *const mid_backend = std::get_if<safetyhook::MidHook>(&restore.impl->backend)) | |
| 539 | { | ||
| 540 | 210 | waiting = !mid_backend->reclaim_deferred_route() || waiting; | |
| 541 | } | ||
| 542 | } | ||
| 543 | 2290 | if (!waiting || std::chrono::steady_clock::now() >= deadline) | |
| 544 | { | ||
| 545 | 2085 | break; | |
| 546 | } | ||
| 547 | 205 | backoff.pause(); | |
| 548 | 205 | } | |
| 549 | 4293 | for (detail::HookAccess::DeferredRestore &restore : deferred) | |
| 550 | { | ||
| 551 | 123 | if (auto *const mid_backend = std::get_if<safetyhook::MidHook>(&restore.impl->backend)) | |
| 552 | { | ||
| 553 | 5 | mid_backend->retain_deferred_route(); | |
| 554 | } | ||
| 555 | } | ||
| 556 | 2085 | } | |
| 557 | |||
| 558 | /// Builds the result of a retirement that changed nothing and logs why. | ||
| 559 | [[nodiscard]] HookRetirement | ||
| 560 | 17 | unchanged(RetirementOutcome outcome, ErrorCode refusal, std::uint64_t in_flight, const char *why) noexcept | |
| 561 | { | ||
| 562 | 17 | (void)log().try_log(LogLevel::Info, "hook: HookStack::retire changed nothing: {}.", why); | |
| 563 | return HookRetirement{ | ||
| 564 | .outcome = outcome, | ||
| 565 | .in_flight = static_cast<std::uint32_t>( | ||
| 566 | 17 | std::min<std::uint64_t>(in_flight, (std::numeric_limits<std::uint32_t>::max)()) | |
| 567 | ), | ||
| 568 | .refusal = refusal, | ||
| 569 | 34 | }; | |
| 570 | } | ||
| 571 | } // namespace | ||
| 572 | |||
| 573 | 14346 | void HookStack::order_by_layer() noexcept | |
| 574 | { | ||
| 575 | 324 | const auto stamp = [](const std::unique_ptr<Hook> &hook) noexcept -> std::uint64_t | |
| 576 | { | ||
| 577 | 324 | const auto *const impl = hook ? detail::HookAccess::impl(*hook) : nullptr; | |
| 578 | 324 | return impl != nullptr ? impl->layer_stamp : 0; | |
| 579 | }; | ||
| 580 | // An insertion sort keeps equal stamps in push order and allocates nothing. | ||
| 581 | 14508 | for (std::size_t index = 1; index < m_hooks.size(); ++index) | |
| 582 | { | ||
| 583 | 163 | for (std::size_t slot = index; slot > 0 && stamp(m_hooks[slot - 1]) > stamp(m_hooks[slot]); --slot) | |
| 584 | { | ||
| 585 | 1 | std::swap(m_hooks[slot - 1], m_hooks[slot]); | |
| 586 | } | ||
| 587 | } | ||
| 588 | 14346 | } | |
| 589 | |||
| 590 | 2112 | HookRetirement HookStack::retire(std::chrono::milliseconds deadline) noexcept | |
| 591 | { | ||
| 592 | 2112 | if (m_hooks.empty()) | |
| 593 | { | ||
| 594 | 10 | return HookRetirement{}; | |
| 595 | } | ||
| 596 | 2102 | if (!DetourModKit::detail::blocking_teardown_permitted()) | |
| 597 | { | ||
| 598 | 1 | return unchanged( | |
| 599 | RetirementOutcome::Busy, | ||
| 600 | ErrorCode::LoaderLockActive, | ||
| 601 | 0, | ||
| 602 | "the caller holds the loader lock or runs in a loader phase" | ||
| 603 | 1 | ); | |
| 604 | } | ||
| 605 | 2101 | const auto now = std::chrono::steady_clock::now(); | |
| 606 | 2101 | const auto headroom = std::chrono::steady_clock::time_point::max() - now; | |
| 607 | 2101 | const auto stop = deadline <= std::chrono::milliseconds::zero() ? now | |
| 608 | 2101 | : deadline >= std::chrono::duration_cast<std::chrono::milliseconds>(headroom) | |
| 609 | 2101 | ? std::chrono::steady_clock::time_point::max() | |
| 610 | 2100 | : now + std::chrono::duration_cast<std::chrono::steady_clock::duration>(deadline); | |
| 611 | |||
| 612 | using HookImpl = std::remove_pointer_t<decltype(detail::HookAccess::impl(std::declval<const Hook &>()))>; | ||
| 613 | 2101 | order_by_layer(); | |
| 614 | 2101 | std::vector<RetiringHook<HookImpl>> hooks; | |
| 615 | 2101 | std::vector<safetyhook::AddressRange> ranges; | |
| 616 | 2101 | std::vector<const safetyhook::ReturnSites *> sites; | |
| 617 | 2101 | std::vector<detail::CountedBlock *> blocks; | |
| 618 | 2101 | std::vector<detail::HookAccess::DeferredRestore> deferred; | |
| 619 | 2101 | std::vector<safetyhook::InlineHook *> deferred_backends; | |
| 620 | try | ||
| 621 | { | ||
| 622 | 2101 | hooks.reserve(m_hooks.size()); | |
| 623 | 2101 | deferred.reserve(m_hooks.size()); | |
| 624 | 2101 | deferred_backends.reserve(m_hooks.size()); | |
| 625 | 4274 | for (auto it = m_hooks.rbegin(); it != m_hooks.rend(); ++it) | |
| 626 | { | ||
| 627 | 2173 | HookImpl *const impl = *it ? detail::HookAccess::impl(**it) : nullptr; | |
| 628 | 2173 | if (impl == nullptr) | |
| 629 | { | ||
| 630 | ✗ | continue; | |
| 631 | } | ||
| 632 | 2173 | hooks.push_back(RetiringHook<HookImpl>{.impl = impl}); | |
| 633 | 2173 | if (impl->counted) | |
| 634 | { | ||
| 635 | 156 | CountedThunk &thunk = *impl->counted; | |
| 636 | 156 | ranges.push_back(thunk.range); | |
| 637 | 312 | if (std::ranges::find(blocks, thunk.block) == blocks.end()) | |
| 638 | { | ||
| 639 | 154 | blocks.push_back(thunk.block); | |
| 640 | } | ||
| 641 | 158 | for (std::size_t i = 0; i < thunk.helper_site_lists; ++i) | |
| 642 | { | ||
| 643 | 2 | sites.push_back(&thunk.helper_sites[i]); | |
| 644 | } | ||
| 645 | } | ||
| 646 | } | ||
| 647 | } | ||
| 648 | ✗ | catch (...) | |
| 649 | { | ||
| 650 | ✗ | return unchanged(RetirementOutcome::Refused, ErrorCode::OutOfMemory, 0, "the proof storage failed"); | |
| 651 | ✗ | } | |
| 652 | |||
| 653 | // A counted detour or a mid callback of this stack that retires its own stack waits on itself. | ||
| 654 | 2101 | bool reentrant = detail::current_thread_returns_into(ranges); | |
| 655 | 6375 | for (const RetiringHook<HookImpl> &hook : hooks) | |
| 656 | { | ||
| 657 | 2173 | if (hook.impl->mid_slot < DetourModKit::detail::MID_ADAPTER_CAPACITY) | |
| 658 | { | ||
| 659 | 9 | reentrant = reentrant || | |
| 660 | 9 | DetourModKit::detail::thread_is_inside_mid_adapter( | |
| 661 | 9 | DetourModKit::detail::mid_adapter_slots()[hook.impl->mid_slot] | |
| 662 | 18 | ) || | |
| 663 | 9 | DetourModKit::detail::backend_value_or( | |
| 664 | 9 | hook.impl->backend, | |
| 665 | true, | ||
| 666 | 18 | [](const auto &backend) noexcept { return backend.caller_owns_route(); } | |
| 667 | ); | ||
| 668 | } | ||
| 669 | } | ||
| 670 | 2101 | if (reentrant) | |
| 671 | { | ||
| 672 | 1 | return unchanged( | |
| 673 | RetirementOutcome::Refused, | ||
| 674 | ErrorCode::ReentrantCallRejected, | ||
| 675 | 0, | ||
| 676 | "the caller runs inside a counted detour or a mid callback of this stack" | ||
| 677 | 1 | ); | |
| 678 | } | ||
| 679 | { | ||
| 680 | 2100 | const safetyhook::ProcessCoordinator coordinator; | |
| 681 | 2100 | if (!coordinator) | |
| 682 | { | ||
| 683 | 1 | return unchanged( | |
| 684 | RetirementOutcome::Busy, | ||
| 685 | ErrorCode::Ok, | ||
| 686 | 0, | ||
| 687 | "the process coordinator is unavailable" | ||
| 688 | 1 | ); | |
| 689 | } | ||
| 690 | 2099 | predict_restores(std::span<RetiringHook<HookImpl>>{hooks}, coordinator); | |
| 691 | 6368 | for (const RetiringHook<HookImpl> &hook : hooks) | |
| 692 | { | ||
| 693 | // A claimed hook adds no position, and a restored hook has none to leave. | ||
| 694 | 2192 | const bool fresh_residue = !hook.restores && | |
| 695 | 2185 | !DetourModKit::detail::backend_claimed(hook.impl->backend) && | |
| 696 | 14 | DetourModKit::detail::witness_of(hook.impl->backend) != | |
| 697 | 2171 | DetourModKit::detail::PatchWitness::Original; | |
| 698 | 2171 | if (fresh_residue && !DetourModKit::detail::residue_fits_budget(hook.impl->backend)) | |
| 699 | { | ||
| 700 | 1 | return unchanged( | |
| 701 | RetirementOutcome::Refused, | ||
| 702 | ErrorCode::LayerResidueLimitReached, | ||
| 703 | 0, | ||
| 704 | "a layer that cannot restore exceeds the residue budget" | ||
| 705 | 1 | ); | |
| 706 | } | ||
| 707 | } | ||
| 708 | 2100 | } | |
| 709 | |||
| 710 | // Switch every inline relay off, top layers included. No cell turns Vacant before the commit, so an undo | ||
| 711 | // never overwrites a claimed slot. | ||
| 712 | 6366 | for (RetiringHook<HookImpl> &hook : hooks) | |
| 713 | { | ||
| 714 | 2170 | if (!hook.impl->is_inline || DetourModKit::detail::backend_bypassed(hook.impl->backend)) | |
| 715 | { | ||
| 716 | 9 | continue; | |
| 717 | } | ||
| 718 | 2161 | if (!DetourModKit::detail::switch_backend(hook.impl->backend, false)) | |
| 719 | { | ||
| 720 | ✗ | undo_retirement(std::span<RetiringHook<HookImpl>>{hooks}); | |
| 721 | ✗ | return unchanged( | |
| 722 | RetirementOutcome::Refused, | ||
| 723 | ErrorCode::BackendFailed, | ||
| 724 | 0, | ||
| 725 | "a relay cannot switch off" | ||
| 726 | ✗ | ); | |
| 727 | } | ||
| 728 | 2161 | hook.relay_switched = true; | |
| 729 | } | ||
| 730 | |||
| 731 | 2098 | if (!blocks.empty()) | |
| 732 | { | ||
| 733 | 93 | const detail::CountedDrain drained = detail::drain_counted(ranges, sites, blocks, stop); | |
| 734 | 93 | if (!drained.idle) | |
| 735 | { | ||
| 736 | 12 | undo_retirement(std::span<RetiringHook<HookImpl>>{hooks}); | |
| 737 | 12 | return unchanged( | |
| 738 | RetirementOutcome::Busy, | ||
| 739 | ErrorCode::Ok, | ||
| 740 | 12 | drained.in_flight, | |
| 741 | "a counted call stayed inside its detour past the deadline" | ||
| 742 | 12 | ); | |
| 743 | } | ||
| 744 | } | ||
| 745 | |||
| 746 | // Each mid route drains alone while CLOSING, so a callback of one route can still call through another. | ||
| 747 | // The coordinator comes first, because its wait can be an export that the closing route hooks. | ||
| 748 | 2086 | const bool has_mid_route = std::ranges::any_of( | |
| 749 | hooks, | ||
| 750 | 2148 | [](const RetiringHook<HookImpl> &hook) noexcept { return !hook.impl->is_inline; } | |
| 751 | ); | ||
| 752 | 2086 | const double ticks_per_nanosecond = has_mid_route ? counter_ticks_per_nanosecond() : 1.0; | |
| 753 | 6324 | for (RetiringHook<HookImpl> &hook : hooks) | |
| 754 | { | ||
| 755 | 2153 | if (hook.impl->is_inline) | |
| 756 | { | ||
| 757 | 2145 | continue; | |
| 758 | } | ||
| 759 | 8 | const std::uint64_t budget = remaining_ticks(stop, ticks_per_nanosecond); | |
| 760 | 8 | bool drained = false; | |
| 761 | { | ||
| 762 | 8 | const safetyhook::ProcessCoordinator coordinator; | |
| 763 | 8 | (void)DetourModKit::detail::apply_backend( | |
| 764 | 8 | hook.impl->backend, | |
| 765 | 16 | [](auto &backend) noexcept { backend.begin_route_rundown(); } | |
| 766 | ); | ||
| 767 | 8 | hook.route_closed = true; | |
| 768 | 8 | drained = DetourModKit::detail::drain_closing_route(hook.impl->backend, budget); | |
| 769 | 8 | (void)DetourModKit::detail::apply_backend( | |
| 770 | 8 | hook.impl->backend, | |
| 771 | 16 | [](auto &backend) noexcept { backend.finish_route_rundown(); } | |
| 772 | ); | ||
| 773 | 8 | } | |
| 774 | 8 | if (!drained) | |
| 775 | { | ||
| 776 | 1 | undo_retirement(std::span<RetiringHook<HookImpl>>{hooks}); | |
| 777 | 1 | return unchanged( | |
| 778 | RetirementOutcome::Busy, | ||
| 779 | ErrorCode::Ok, | ||
| 780 | 0, | ||
| 781 | "a mid callback stayed inside its route past the deadline" | ||
| 782 | 1 | ); | |
| 783 | } | ||
| 784 | } | ||
| 785 | |||
| 786 | 2085 | HookRetirement result{}; | |
| 787 | 2275 | const auto count = [&result](TeardownOutcome outcome) noexcept | |
| 788 | { | ||
| 789 | 2275 | switch (outcome) | |
| 790 | { | ||
| 791 | 2134 | case TeardownOutcome::Restored: | |
| 792 | 2134 | ++result.restored; | |
| 793 | 2134 | break; | |
| 794 | 18 | case TeardownOutcome::Vacated: | |
| 795 | 18 | ++result.vacated; | |
| 796 | 18 | break; | |
| 797 | ✗ | case TeardownOutcome::Pinned: | |
| 798 | ✗ | ++result.pinned; | |
| 799 | ✗ | break; | |
| 800 | 123 | case TeardownOutcome::Empty: | |
| 801 | case TeardownOutcome::Deferred: | ||
| 802 | 123 | break; | |
| 803 | } | ||
| 804 | 4360 | }; | |
| 805 | // The drain above covered every counted detour, and every relay stays off until the commit. | ||
| 806 | 4237 | while (!m_hooks.empty()) | |
| 807 | { | ||
| 808 | 4304 | const std::unique_ptr<Hook> hook = std::move(m_hooks.back()); | |
| 809 | 2152 | m_hooks.pop_back(); | |
| 810 | 2152 | if (hook) | |
| 811 | { | ||
| 812 | 2152 | count(detail::HookAccess::teardown(*hook, detail::CountedDrainProof::ProvenByRetire, &deferred)); | |
| 813 | } | ||
| 814 | 2152 | } | |
| 815 | // One idle pass after the last restore proves every deferred trampoline. | ||
| 816 | 4293 | for (detail::HookAccess::DeferredRestore &restore : deferred) | |
| 817 | { | ||
| 818 | 123 | if (auto *const inline_backend = std::get_if<safetyhook::InlineHook>(&restore.impl->backend)) | |
| 819 | { | ||
| 820 | 118 | deferred_backends.push_back(inline_backend); | |
| 821 | } | ||
| 822 | } | ||
| 823 | 2085 | safetyhook::InlineHook::reclaim_deferred(deferred_backends); | |
| 824 | 2085 | reclaim_deferred_routes(std::span<detail::HookAccess::DeferredRestore>{deferred}, stop); | |
| 825 | 4293 | for (detail::HookAccess::DeferredRestore &restore : deferred) | |
| 826 | { | ||
| 827 | 123 | count(detail::HookAccess::finish_deferred_restore(restore)); | |
| 828 | } | ||
| 829 | 2085 | (void)log().try_log( | |
| 830 | LogLevel::Info, | ||
| 831 | "hook: HookStack::retire retired its hooks: {} restored, {} vacated, {} pinned.", | ||
| 832 | result.restored, | ||
| 833 | result.vacated, | ||
| 834 | result.pinned | ||
| 835 | ); | ||
| 836 | 2085 | return result; | |
| 837 | 2101 | } | |
| 838 | } // namespace hook | ||
| 839 | |||
| 840 | #if defined(DMK_ENABLE_TEST_SEAMS) | ||
| 841 | 5 | detail::CountedThunkProbe detail::counted_thunk_for_test(const hook::Hook &hook) noexcept | |
| 842 | { | ||
| 843 | 5 | const auto *const impl = hook::detail::HookAccess::impl(hook); | |
| 844 | 5 | if (impl == nullptr || !impl->counted) | |
| 845 | { | ||
| 846 | ✗ | return CountedThunkProbe{}; | |
| 847 | } | ||
| 848 | const std::expected<hook::detail::CountedThunkShape, const char *> shape = | ||
| 849 | 5 | hook::detail::inspect_counted_thunk(impl->counted->range.begin); | |
| 850 | 5 | if (!shape) | |
| 851 | { | ||
| 852 | ✗ | return CountedThunkProbe{}; | |
| 853 | } | ||
| 854 | return CountedThunkProbe{ | ||
| 855 | 5 | .entry = shape->thunk.range.begin, | |
| 856 | 10 | .range_end = shape->thunk.range.end, | |
| 857 | 5 | .enter_count = shape->thunk.range.begin + shape->enter_count_offset, | |
| 858 | 5 | .after_exit_count = shape->thunk.range.begin + shape->after_exit_count_offset, | |
| 859 | 10 | .helper_calls = shape->helper_calls, | |
| 860 | 10 | .counted_calls = shape->counted_calls, | |
| 861 | 10 | .pushes = shape->pushes, | |
| 862 | 10 | .pops = shape->pops, | |
| 863 | 15 | }; | |
| 864 | } | ||
| 865 | #endif | ||
| 866 | } // namespace DetourModKit | ||
| 867 |