structured-zstd 0.2.6

Pure-Rust Zstandard (zstd) compression and decompression: all levels, streaming, dictionaries, no_std and WebAssembly ready — no FFI, no cmake
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
//! Dictionary priming and CDict-equivalent snapshot lifecycle for the
//! [`MatchGeneratorDriver`].
//!
//! Holds the matcher's dictionary entry points (prime / restore / capture /
//! invalidate / resident reapply / entropy seed) as inherent `*_impl` helpers;
//! the parent module's trait surface forwards to them. Kept as a child module
//! so the snapshot path can touch the driver's private invariant-bearing
//! `primed` / `reset_shape` fields directly without widening them to the rest
//! of the encoding module.

use super::{MIN_MATCH_LEN, MatchGeneratorDriver, Matcher, MatcherStorage, PrimedKey};

impl MatchGeneratorDriver {
    pub(super) fn dictionary_is_resident_impl(&self) -> bool {
        match &self.storage {
            MatcherStorage::HashChain(hc) => hc.table.dict_resident,
            MatcherStorage::Simple(s) => s.dict_resident(),
            MatcherStorage::Dfast(d) => d.dict_resident(),
            MatcherStorage::Row(r) => r.dict_resident(),
        }
    }

    pub(super) fn reapply_resident_dictionary_impl(&mut self, offset_hist: [u32; 3]) {
        // Same offset-history head as `prime_with_dictionary`, without the dict
        // commit / re-index (resident dict bytes + cached dms already in place).
        match self.active_backend() {
            super::super::strategy::BackendTag::Simple => {
                self.simple_mut().prime_offset_history(offset_hist)
            }
            super::super::strategy::BackendTag::Dfast => {
                self.dfast_matcher_mut().offset_hist = offset_hist
            }
            super::super::strategy::BackendTag::Row => {
                self.row_matcher_mut().set_offset_hist(offset_hist)
            }
            super::super::strategy::BackendTag::HashChain => {
                let matcher = self.hc_matcher_mut();
                matcher.table.offset_hist = offset_hist;
                matcher.table.mark_dictionary_primed();
            }
        }
        // Restore the retained-dictionary budget the per-frame `reset` cleared.
        // The matcher's `reset` re-inflated `max_window_size` by the resident
        // dict region (so the dict + next input both stay in the eviction band),
        // exactly as `prime_with_dictionary` does — but the resident path skips
        // that prime, so without this the driver-level budget stays 0 and
        // `retire_dictionary_budget` never shrinks the inflated window as input
        // evicts the dict. For HashChain (whose `window_low` is measured against
        // `max_window_size`), a stuck-inflated window would let a post-eviction
        // match exceed the frame header's base window and emit an over-window
        // offset. The inflation equals `max_window_size - base`, and
        // `reported_window_size` is the base `1 << window_log` set by `reset`.
        let base = self.reported_window_size;
        let inflated = match self.active_backend() {
            super::super::strategy::BackendTag::Simple => self.simple_mut().max_window_size,
            super::super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().max_window_size,
            super::super::strategy::BackendTag::Row => self.row_matcher_mut().max_window_size,
            super::super::strategy::BackendTag::HashChain => {
                self.hc_matcher_mut().table.max_window_size
            }
        };
        self.dictionary_retained_budget = inflated.saturating_sub(base);
        // The reset kept the dictionary's bytes and index but cleared the
        // long-distance table, which a freshly primed frame would hold filled.
        #[cfg(feature = "ldm")]
        self.index_dictionary_in_place();
    }

    /// [`crate::encoding::Matcher::prime_with_dictionary`] compiled for the kernel tier `K`, the one the encoder chose for the
    /// frame.
    pub(crate) fn prime_with_dictionary_with<K: crate::encoding::fastpath::EncodeTier>(
        &mut self,
        dict_content: &[u8],
        offset_hist: [u32; 3],
    ) {
        match self.active_backend() {
            super::super::strategy::BackendTag::Simple => {
                // Routes through prime_offset_history so BOTH
                // offset_hist (wire encoder) and rep[0..2] (kernel)
                // are updated atomically. Without this, the two
                // tracks drift after dict priming — kernel emits
                // repcode matches against stale FAST_INITIAL_REP
                // while the wire encoder uses the primed history,
                // producing divergent wire encoding (Copilot review
                // #15 on #216).
                self.simple_mut().prime_offset_history(offset_hist);
            }
            super::super::strategy::BackendTag::Dfast => {
                self.dfast_matcher_mut().offset_hist = offset_hist
            }
            super::super::strategy::BackendTag::Row => {
                self.row_matcher_mut().set_offset_hist(offset_hist)
            }
            super::super::strategy::BackendTag::HashChain => {
                let matcher = self.hc_matcher_mut();
                // Clear the chain/hash tables (deferred from the dict-active
                // `reset`): prime rebuilds them from the dict, so they must start
                // empty. The reuse hot path skips prime and `clone_from`s a clean
                // snapshot instead, so only the first-prime / key-mismatch frames
                // pay the fill -- not every reused-CDict frame.
                matcher.table.clear_chain_hash_tables();
                matcher.table.offset_hist = offset_hist;
                matcher.table.mark_dictionary_primed();
            }
        }

        if dict_content.is_empty() {
            return;
        }
        #[cfg(feature = "ldm")]
        let ldm_dict_start = self.ldm_committed_end;
        // The match-finder tables index only as much of the dictionary's end as
        // the CDict's tables hold, the same for every frame of the dictionary
        // (upstream `ZSTD_loadDictionaryContent`, `zstd_compress.c:4962`); the
        // whole of it is committed all the same, so every byte stays in reach
        // of an offset.
        let reach_log = self.dict_reach_log.unwrap_or_else(|| {
            self.reset_shape
                .map_or(31, |(params, ..)| params.dictionary_reach_log())
        });
        self.storage
            .set_dictionary_index_floor(dict_content.len().saturating_sub(1usize << reach_log));
        // A dictionary attached again after frames that copied it takes back
        // the tables built for it, so a compressor alternating small and large
        // frames builds them once per dictionary, as upstream builds a CDict.
        match self.active_backend() {
            super::super::strategy::BackendTag::Dfast
                if self.dfast_attaches_dictionary(dict_content.len()) =>
            {
                self.dfast_matcher_mut()
                    .revive_dict_cache(dict_content.len());
            }
            // The row backend's plan says whether this frame attaches.
            super::super::strategy::BackendTag::Row => {
                self.row_matcher_mut().revive_dict_cache(dict_content.len());
            }
            _ => {}
        }

        // Dictionary bytes should stay addressable until produced frame output
        // itself exceeds the live window size. We bump `max_window_size`
        // by the dictionary length so the eviction band keeps the
        // primed bytes in `history`.
        //
        // Cap: `with_params`/`reset` enforce `window_log <= 30` so the
        // eviction band `2 * max_window_size` stays below `u32::MAX`
        // with headroom for one MAX_BLOCK_SIZE pending block — the
        // kernel asserts `data.len() <= u32::MAX`. A large enough
        // dictionary could otherwise push `max_window_size` past
        // that ceiling via the `saturating_add` below and silently
        // re-introduce the same overflow the `window_log` cap was
        // designed to prevent. Clamp the post-priming size so the
        // doubled-band-plus-block invariant survives.
        use super::super::match_table::storage::MAX_PRIMED_WINDOW_SIZE;

        // `requested_dict_budget` is what the caller asked for;
        // `base_max_window_size` snapshots the pre-priming cap so we
        // can compute how much window the cap actually GRANTED below.
        // The cap may clip the requested growth, in which case the
        // bookkeeping (`dictionary_retained_budget` retire path) must
        // track only the granted portion — otherwise
        // `retire_dictionary_budget()` would later reclaim more than
        // was actually added and shrink the matcher below its real
        // base window (and `cap = 2 * max_window_size` would shrink
        // with it, risking under-allocation on subsequent commits).
        // The `granted_retained_budget` calculation further below is
        // the load-bearing piece — see its block-level comment for
        // the post-clip / post-uncommitted-tail math.
        let requested_dict_budget = dict_content.len();
        let base_max_window_size = match self.active_backend() {
            super::super::strategy::BackendTag::Simple => self.simple_mut().max_window_size,
            super::super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().max_window_size,
            super::super::strategy::BackendTag::Row => self.row_matcher_mut().max_window_size,
            super::super::strategy::BackendTag::HashChain => {
                self.hc_matcher_mut().table.max_window_size
            }
        };
        match self.active_backend() {
            super::super::strategy::BackendTag::Simple => {
                let matcher = self.simple_mut();
                matcher.max_window_size = matcher
                    .max_window_size
                    .saturating_add(requested_dict_budget)
                    .min(MAX_PRIMED_WINDOW_SIZE);
                matcher.fit_table_to_window();
            }
            super::super::strategy::BackendTag::Dfast => {
                let matcher = self.dfast_matcher_mut();
                matcher.max_window_size = matcher
                    .max_window_size
                    .saturating_add(requested_dict_budget)
                    .min(MAX_PRIMED_WINDOW_SIZE);
            }
            super::super::strategy::BackendTag::Row => {
                let matcher = self.row_matcher_mut();
                matcher.max_window_size = matcher
                    .max_window_size
                    .saturating_add(requested_dict_budget)
                    .min(MAX_PRIMED_WINDOW_SIZE);
            }
            super::super::strategy::BackendTag::HashChain => {
                let matcher = self.hc_matcher_mut();
                matcher.table.max_window_size = matcher
                    .table
                    .max_window_size
                    .saturating_add(requested_dict_budget)
                    .min(MAX_PRIMED_WINDOW_SIZE);
            }
        }

        let mut start = 0usize;
        let mut committed_dict_budget = 0usize;
        // insert_position needs 4 bytes of lookahead for hashing;
        // backfill_boundary_positions re-visits tail positions once the
        // next slice extends history, but cannot hash <4 byte fragments.
        let min_primed_tail = match self.active_backend() {
            super::super::strategy::BackendTag::Simple => MIN_MATCH_LEN,
            super::super::strategy::BackendTag::Dfast
            | super::super::strategy::BackendTag::Row
            | super::super::strategy::BackendTag::HashChain => 4,
        };
        // A dictionary too short to hash at all is left out whole: no match can
        // reach it, so offsets stay right. A longer one goes in whole, since the
        // decoder places every byte of it before the frame and a byte missing
        // here shifts every offset into it. A last piece too short to hash is
        // avoided by ending the piece before it that much earlier, or, when
        // that would leave this piece too short instead, by running it to the
        // end.
        let hashable = dict_content.len() >= min_primed_tail;
        while hashable && start < dict_content.len() {
            let mut end = (start + self.slice_size).min(dict_content.len());
            if end != dict_content.len() && dict_content.len() - end < min_primed_tail {
                let shortened = dict_content.len() - min_primed_tail;
                end = if shortened - start >= min_primed_tail {
                    shortened
                } else {
                    dict_content.len()
                };
            }
            debug_assert!(end - start >= min_primed_tail);
            // The chunk goes straight into the history, as a block of input
            // does.
            let chunk = &dict_content[start..end];
            self.fill_in_place(chunk.len(), &mut |history| {
                history.extend_from_slice(chunk);
                (chunk.len(), false)
            });
            self.commit_filled(chunk.len());
            self.skip_matching_for_dictionary_priming::<K>(dict_content.len());
            committed_dict_budget += end - start;
            start = end;
        }
        // The long-distance matcher indexes the dictionary as upstream does when
        // it loads one (`ZSTD_ldm_fillHashTable` from
        // `ZSTD_loadDictionaryContent`), so a frame can reach it from as far as
        // the window allows.
        #[cfg(feature = "ldm")]
        {
            self.ldm_dict_end = self.ldm_committed_end;
            if let Some(producer) = self.ldm.as_mut() {
                let window = self.storage.committed_window();
                let window_start = self.ldm_committed_end - window.len();
                producer.fill_table(
                    window,
                    window_start,
                    ldm_dict_start.max(window_start),
                    self.ldm_committed_end,
                );
            }
        }

        // Derive `granted_retained_budget` directly from the two real
        // bounds — bytes actually committed and bytes the cap allows
        // — instead of doing a cap-clip pass followed by an
        // uncommitted-tail subtract. Previous shape double-discounted
        // when the cap clipped: clip lost `(requested - allowed)`,
        // then tail-subtract lost ANOTHER `(requested - committed)`,
        // leaving `max_window_size` shy of the dictionary that was
        // actually retained (e.g. cap=900, committed=998, uncommitted=2
        // landed at granted=898 instead of the correct 900).
        let capped_retained_budget = MAX_PRIMED_WINDOW_SIZE.saturating_sub(base_max_window_size);
        let granted_retained_budget = committed_dict_budget.min(capped_retained_budget);
        let final_max_window_size = base_max_window_size.saturating_add(granted_retained_budget);
        match self.active_backend() {
            super::super::strategy::BackendTag::Simple => {
                self.simple_mut().max_window_size = final_max_window_size;
            }
            super::super::strategy::BackendTag::Dfast => {
                self.dfast_matcher_mut().max_window_size = final_max_window_size;
            }
            super::super::strategy::BackendTag::Row => {
                self.row_matcher_mut().max_window_size = final_max_window_size;
            }
            super::super::strategy::BackendTag::HashChain => {
                self.hc_matcher_mut().table.max_window_size = final_max_window_size;
            }
        }
        if granted_retained_budget > 0 {
            self.dictionary_retained_budget = self
                .dictionary_retained_budget
                .saturating_add(granted_retained_budget);
        }
        if self.active_backend() == super::super::strategy::BackendTag::HashChain {
            // Recompute the attach decision made per-chunk in
            // `skip_matching_for_dictionary_priming` (stable across the prime —
            // `reset_size_log` does not change here).
            //
            // The HC attach/copy mode is deliberately NOT folded into `PrimedKey`
            // (unlike Fast `fast_attach`). Fast attach builds a separate dict
            // table whose dimensions differ from the copy-mode live table, so a
            // cross-mode restore would install mismatched table geometry and the
            // encoder could search past the frame window (undecodable). The two
            // modes share identical window geometry: `max_window_size` and the
            // dictionary limit are both set ABOVE this branch (the same value in
            // either mode), and the live tree dimensions come from the
            // resolved `params` the key already pins. The modes differ only in
            // WHERE the committed dict lives — a separate `dms` tree (attach) vs
            // merged into the live tree (copy) — both producing valid matches at
            // in-window offsets. Upstream zstd makes the same observation: attach
            // (`ZSTD_resetCCtx_byAttachingCDict`) and copy
            // (`ZSTD_resetCCtx_byCopyingCDict`) both keep the caller's
            // `windowLog`; the choice is a memory/speed trade-off, not a wire
            // contract. So restoring an attach snapshot where this frame would
            // have copied (or vice versa) yields a decodable frame that may only
            // differ in which matches are found (ratio) — algorithmic freedom, not
            // a defect. Keying on the mode would instead force a re-prime across
            // the cutoff, re-adding the per-frame cost this snapshot path removes.
            //
            // In practice the public reuse path (`compress_independent_frame`)
            // only ever captures AND restores the COPY-mode snapshot — capture is
            // gated on the above-cutoff source size, so a restored frame always
            // matches the captured mode. `hc_dict_snapshot_reuse_roundtrips` pins
            // that same-mode reuse decodes; the driver-level cross-mode restore is
            // accepted (not refused) per
            // `primed_snapshot_fast_attach_does_not_over_key_non_simple_backends`.
            let attach = self.hc_dict_attach_mode();
            let table = &mut self.hc_matcher_mut().table;
            table.set_dictionary_limit_from_primed_bytes(committed_dict_budget);
            // Build the dictMatchState tree over the committed dict (front of
            // history) so the search dual-probes it with its own compare budget —
            // but ONLY in ATTACH mode. COPY mode (large known source) already
            // merged the dict into the live tree in
            // `skip_matching_for_dictionary_priming`, so it carries no separate
            // dms — drop any stale one.
            if attach {
                table.prime_dms_bt(committed_dict_budget);
            } else {
                table.dms.invalidate();
            }
        }
        // CDict-equivalent: now that every dict chunk is indexed, mark the
        // Fast-backend dict table primed so the next frame's re-prime reuses
        // it (skips the re-hash) while still re-committing the dict bytes to
        // history. No-op when the attach path built no table (copy mode or a
        // sub-8-byte dict) — `mark_dict_primed` self-guards on table presence.
        match self.active_backend() {
            super::super::strategy::BackendTag::Simple => self.simple_mut().mark_dict_primed(),
            super::super::strategy::BackendTag::Dfast => {
                self.dfast_matcher_mut().mark_dict_primed()
            }
            super::super::strategy::BackendTag::Row => self.row_matcher_mut().mark_dict_primed(),
            _ => {}
        }
        // The dictionary is indexed; later inserts are the input's own.
        self.storage.set_dictionary_index_floor(0);
    }

    pub(super) fn restore_primed_dictionary_impl(
        &mut self,
        level: super::super::CompressionLevel,
    ) -> bool {
        // Only the (storage, dictionary_retained_budget) pair is what
        // `prime_with_dictionary` writes; restoring them reproduces the
        // post-prime state exactly. Gated on the FULL resolved key (level + the
        // resolved `LevelParams` + the active backend's table width), not just
        // the level: `reset` resolves the hint into a window/table geometry, so a
        // same-level snapshot taken at a hint that resolved to a different shape
        // carries a `storage.max_window_size` / table dimensions that no longer
        // match this reset. Restoring it would let the encoder search past the
        // frame header's window (an undecodable match), so on a key mismatch we
        // refuse and the caller re-primes.
        let Some((params, table_bits, fast_attach, fast_tagged)) = self.reset_shape else {
            return false;
        };
        let key = PrimedKey {
            level,
            params,
            table_bits,
            fast_attach,
            fast_tagged,
        };
        let Some((snapshot, budget, captured_key)) = &mut self.primed else {
            return false;
        };
        if *captured_key != key {
            return false;
        }
        let budget = *budget;
        match (&mut self.storage, snapshot) {
            // The same for Dfast and Row: the tables and history land in the
            // rooms the reset laid out in the workspace, so a restore is the
            // copy upstream's CDict reuse pays and allocates nothing.
            (MatcherStorage::Dfast(live), MatcherStorage::Dfast(snap)) => {
                live.restore_snapshot(snap);
            }
            (MatcherStorage::Row(live), MatcherStorage::Row(snap)) => {
                live.restore_snapshot(snap);
            }
            // Same-variant Fast restore: copy the snapshot into the retained
            // live storage. `clone_from` reuses the history / hash-table /
            // dict-table buffers, so this is the upstream zstd CDict table-copy
            // regime's cost (pure copies) instead of a full per-frame
            // allocation + copy + drop cycle.
            (MatcherStorage::Simple(live), MatcherStorage::Simple(snap)) => {
                live.clone_from(snap);
            }
            // Binary tree: the history and, in COPY mode, the tree land in the
            // rooms the reset laid out in the workspace. An ATTACH-mode
            // snapshot holds no tree (see `capture_primed_dictionary`), so the
            // laid-out tables are emptied where they are, reproducing the
            // post-prime state (all `HC_EMPTY`) without an allocation.
            (MatcherStorage::HashChain(live), MatcherStorage::HashChain(snap)) => {
                let snapshot_tables = core::mem::take(&mut snap.table.tables);
                let snapshot_history = core::mem::take(&mut snap.table.history);
                let mut tables = core::mem::take(&mut live.table.tables);
                let mut history = core::mem::take(&mut live.table.history);
                history.clone_from(&snapshot_history);
                let tree_in_snapshot = !snapshot_tables.is_empty();
                if tree_in_snapshot {
                    tables.clone_from(&snapshot_tables);
                }
                *live = snap.clone();
                live.table.tables = tables;
                live.table.history = history;
                if !tree_in_snapshot {
                    live.table.ensure_tables();
                }
                snap.table.tables = snapshot_tables;
                snap.table.history = snapshot_history;
            }
            // The key pins the backend, so a snapshot of another variant is
            // never restored; should one be, it replaces the storage whole.
            (live, snapshot_storage) => {
                *live = snapshot_storage.clone();
            }
        }
        self.dictionary_retained_budget = budget;
        #[cfg(feature = "ldm")]
        self.index_dictionary_in_place();
        true
    }

    /// Index the dictionary the reset left in the window into the
    /// long-distance matcher's cleared table, as priming it would. Upstream
    /// re-indexes per frame as well: a CDict carries no long-distance table,
    /// and a frame that loads the dictionary fills one
    /// (`ZSTD_ldm_fillHashTable` from `ZSTD_loadDictionaryContent`). Keeping a
    /// copy of the filled table instead would hold a second table, the
    /// largest the frame has, and copying it costs more than the fill.
    #[cfg(feature = "ldm")]
    fn index_dictionary_in_place(&mut self) {
        self.ldm_committed_end = self.storage.committed_window().len();
        self.ldm_dict_end = self.ldm_committed_end;
        if let Some(producer) = self.ldm.as_mut() {
            let window = self.storage.committed_window();
            producer.fill_table(window, 0, 0, window.len());
        }
    }

    pub(super) fn capture_primed_dictionary_impl(&mut self, level: super::super::CompressionLevel) {
        // No resolved shape means `reset` has not run for this frame — nothing
        // valid to key a snapshot on, so skip the capture.
        let Some((params, table_bits, fast_attach, fast_tagged)) = self.reset_shape else {
            return;
        };
        let key = PrimedKey {
            level,
            params,
            table_bits,
            fast_attach,
            fast_tagged,
        };
        // CDict-equivalent retained state. A binary-tree level in ATTACH mode
        // decouples the dictionary into `dms` (the upstream zstd `dictMatchState`); its
        // live hash / chain / hash3 tables carry NO dict entries
        // (`skip_matching_dict_bt` keeps the dict out of the live tree), so they
        // are pure zeros. Storing them in the snapshot wastes the full table
        // footprint (a second window-tier table set resident for the whole
        // compress). Instead, move the live tables OUT of the working storage,
        // clone only the dict-state (history + `dms` + window/offset/dict-limit),
        // then move the live tables back — the snapshot keeps just what upstream zstd's
        // CDict keeps, and `restore_primed_dictionary` re-allocates the zeroed
        // live tables. COPY mode keeps the dict reachable through the live tree
        // (`dms` invalidated, dict merged in), so its snapshot must retain the
        // full tables (full clone). `dms.is_primed()` is therefore the exact
        // "decoupled" signal.
        let bt_decoupled = matches!(
            &self.storage,
            MatcherStorage::HashChain(hc) if hc.table.dms.is_primed()
        );
        if bt_decoupled {
            let MatcherStorage::HashChain(hc) = &mut self.storage else {
                unreachable!("bt_decoupled implies HashChain storage");
            };
            // One buffer holds all three regions now; its seams describe it,
            // so they are taken with it — a snapshot left holding non-zero
            // seams over an empty buffer would index out of bounds.
            let tables = core::mem::take(&mut hc.table.tables);
            let chain_off = core::mem::take(&mut hc.table.chain_off);
            let hash3_off = core::mem::take(&mut hc.table.hash3_off);
            // Clone the dict-state-only storage (live tables now empty Vecs).
            let snapshot = self.storage.clone();
            // Move the live tables back into the working storage.
            let MatcherStorage::HashChain(hc) = &mut self.storage else {
                unreachable!("storage variant is stable across the take/put");
            };
            hc.table.tables = tables;
            hc.table.chain_off = chain_off;
            hc.table.hash3_off = hash3_off;
            self.primed = Some((snapshot, self.dictionary_retained_budget, key));
        } else {
            // Into the buffers of the snapshot it replaces, or of the one a
            // dictionary change set aside, where the backend copies in place;
            // a fresh clone otherwise. The fast backend's `clone_from` costs
            // more instructions than its clone (a level-1 dictionary-swap loop
            // retired 1.2% more), so it clones.
            let replaced = self
                .primed
                .take()
                .map(|(storage, ..)| storage)
                .or_else(|| self.primed_spare.take());
            let snapshot = match (replaced, &mut self.storage) {
                (Some(MatcherStorage::Dfast(mut snapshot)), MatcherStorage::Dfast(live)) => {
                    snapshot.restore_snapshot(live);
                    MatcherStorage::Dfast(snapshot)
                }
                (Some(MatcherStorage::Row(mut snapshot)), MatcherStorage::Row(live)) => {
                    snapshot.restore_snapshot(live);
                    MatcherStorage::Row(snapshot)
                }
                (_, live) => live.clone(),
            };
            self.primed = Some((snapshot, self.dictionary_retained_budget, key));
        }
    }

    pub(super) fn invalidate_primed_dictionary_impl(&mut self) {
        // The snapshot belongs to the dictionary being replaced; its buffers
        // stay for the next one's where the capture copies into them.
        if let Some((storage, ..)) = self.primed.take() {
            self.primed_spare =
                matches!(storage, MatcherStorage::Dfast(_) | MatcherStorage::Row(_))
                    .then_some(storage);
        }
        // Drop the Fast-backend CDict-equivalent table cache too: it is keyed
        // to the dictionary being removed / replaced. Left in place, the next
        // same-params `reset` would retain it and the kernel would probe a
        // dict region whose bytes are no longer re-committed to history.
        match self.active_backend() {
            super::super::strategy::BackendTag::Simple => self.simple_mut().invalidate_dict_cache(),
            super::super::strategy::BackendTag::Dfast => {
                self.dfast_matcher_mut().invalidate_dict_cache()
            }
            // Row keeps its attach index across frames (like Simple/Dfast),
            // so a dictionary swap must drop its cached dict rows too;
            // otherwise the next small/unknown-size frame reuses stale
            // attach state through `prime_dict_attach_current_block`.
            super::super::strategy::BackendTag::Row => {
                self.row_matcher_mut().invalidate_dict_cache()
            }
            // The BT dms tree is keyed to the dict bytes; `prime_dms_bt`
            // skips the rebuild while its shape matches, so a swapped
            // dictionary of the same length would otherwise keep serving the
            // OLD dictionary's tree.
            super::super::strategy::BackendTag::HashChain => {
                let table = &mut self.hc_matcher_mut().table;
                table.dms.invalidate();
                // Deactivate the dictionary state so the next `reset` does not
                // take the dict-active defer-the-table-clear branch. That branch
                // rewinds the tables to the origin and hands the clear off to a
                // following `prime_with_dictionary` / `restore_primed_dictionary`.
                // After a dictionary is removed (or replaced), the very next
                // frame may carry no dictionary, in which case neither hand-off
                // runs and the deferred clear would never execute — leaving stale
                // dict-region entries at the rewound base. Clearing the flag
                // routes that reset down the no-dictionary path instead; a
                // replacement dictionary re-arms the flag when it re-primes.
                table.dictionary_active = false;
            }
        }
    }

    pub(super) fn seed_dictionary_entropy_impl(
        &mut self,
        huff: Option<&crate::huff0::huff0_encoder::HuffmanTable>,
        ll: Option<&crate::fse::fse_encoder::FSETable>,
        ml: Option<&crate::fse::fse_encoder::FSETable>,
        of: Option<&crate::fse::fse_encoder::FSETable>,
    ) {
        if self.active_backend() == super::super::strategy::BackendTag::HashChain {
            self.hc_matcher_mut()
                .seed_dictionary_entropy(huff, ll, ml, of);
        }
    }
}