structured-zstd 0.0.54

Pure-Rust Zstandard (zstd) compression and decompression: all levels, streaming, dictionaries, no_std and WebAssembly ready — no FFI, no cmake
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
use crate::{
    blocks::block::BlockType,
    common::MAX_BLOCK_SIZE,
    encoding::{
        CompressionLevel, Matcher,
        block_header::BlockHeader,
        blocks::{compress_block, compress_block_with_post_split},
        frame_compressor::CompressState,
        incompressible::{
            block_looks_incompressible, block_looks_incompressible_strict,
            compression_level_allows_raw_fast_path,
        },
        match_generator::MatchGeneratorDriver,
    },
};
use alloc::vec::Vec;

/// Where this block's bytes live before the matcher takes ownership of them.
///
/// The owned block loop used to always stage a block in a scratch `Vec` that
/// the matcher then copied into its history. Backends implementing
/// [`Matcher::fill_in_place`](crate::encoding::Matcher::fill_in_place) instead
/// read straight into that history, so the bytes are already in place and only
/// need claiming — [`BlockInput::InPlace`] carries just the length.
pub(crate) enum BlockInput {
    /// Bytes staged in a caller-owned buffer, handed to the matcher on commit.
    Staged(Vec<u8>),
    /// Bytes already at the head of the matcher's uncommitted region; the
    /// payload is the block length.
    InPlace(usize),
}

impl BlockInput {
    fn len(&self) -> usize {
        match self {
            BlockInput::Staged(v) => v.len(),
            BlockInput::InPlace(n) => *n,
        }
    }

    /// Borrow the block's bytes before they are committed. Shared borrow of
    /// `matcher` for the in-place case, so this composes with the other
    /// read-only matcher queries the classification below performs.
    fn bytes<'a, M: Matcher>(&'a self, matcher: &'a M) -> &'a [u8] {
        match self {
            BlockInput::Staged(v) => v,
            BlockInput::InPlace(n) => &matcher.uncommitted_input()[..*n],
        }
    }

    /// Hand the block to the matcher: move the staged buffer in, or claim the
    /// already-resident bytes.
    fn commit<M: Matcher>(self, matcher: &mut M) {
        match self {
            BlockInput::Staged(v) => matcher.commit_space(v),
            BlockInput::InPlace(n) => matcher.commit_filled(n),
        }
    }
}

/// Compresses a single block using the shared compressed-block pipeline.
///
/// Used by all compressed levels (Fastest, Default, Better, Best, and numeric levels). The actual
/// compression quality is determined by the matcher backend in `state`,
/// not by this function.
///
/// # Parameters
/// - `state`: [`CompressState`] so the compressor can refer to data before
///   the start of this block
/// - `last_block`: Whether or not this block is going to be the last block in the frame
///   (needed because this info is written into the block header)
/// - `uncompressed_data`: A block's worth of uncompressed data, either staged
///   in a caller-owned buffer or already sitting in the matcher's history (see
///   [`BlockInput`])
/// - `output`: As `uncompressed_data` is compressed, it's appended to `output`.
// Mirrors the per-block sidecar plumbing of its borrowed sibling
// (`compress_block_encoded_borrowed`): the lsm decompressed-size and
// optional XXH64 checksum out-params push the arg count past the lint's
// threshold. Bundling them into a struct would diverge from the established
// emit-fn shape for no readability gain.
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn compress_block_encoded<M: Matcher>(
    state: &mut CompressState<M>,
    compression_level: CompressionLevel,
    last_block: bool,
    uncompressed_data: BlockInput,
    output: &mut Vec<u8>,
    // Whether a dictionary is primed for this frame. A high-entropy block is
    // not necessarily incompressible when one is, so the raw-skip below raises
    // its bar rather than trusting the block's own bytes.
    dict_active: bool,
    // Per-physical-block decompressed (regenerated) size sidecar, in
    // block-emit order — 1:1 with `FrameEmitInfo.blocks`, same cardinality
    // discipline as the XXH64 checksum sidecar. Captured under `lsm` alone
    // (not gated on `hash`, not opt-in) because `FrameEmitInfo` is always
    // built under `lsm` and every block needs its `decompressed_size`.
    #[cfg(feature = "lsm")] block_decompressed_sizes: Option<&mut Vec<u32>>,
    #[cfg(all(feature = "lsm", feature = "hash"))] block_checksums: Option<&mut Vec<u32>>,
) -> BlockType {
    let block_size = uncompressed_data.len() as u32;
    // Classify the block while the bytes are still uncommitted. Every query
    // here is read-only, so the `InPlace` borrow of the matcher's history
    // coexists with the `window_size()` / `block_samples_match_dict()` probes.
    let bytes = uncompressed_data.bytes(&state.matcher);
    let rle_byte_opt = bytes
        .first()
        .copied()
        .filter(|f| bytes.iter().all(|x| x == f));
    // Order is by cost, cheapest question first, because every one of these
    // runs on every block: an RLE block and a window that forbids the skip are
    // constants; the dict probe is a constant `true` for the conservative
    // backends and a small table probe for Fast; the classifier reads a sample;
    // the repeat grid reads the WHOLE block.
    //
    // The grid is therefore asked last, and only about blocks the classifier
    // already called incompressible — the only blocks a repeat could save from
    // going out raw. That is also why recording there is enough: a block
    // duplicating a COMPRESSIBLE one is compressible itself, so it is never at
    // risk of being skipped and never needs to find its original here.
    let window_size = state.matcher.window_size();
    // An attached dictionary keeps the block on the search, full stop. Asking a
    // sample of thirty-odd positions whether the dictionary is relevant is the
    // wrong shape for a gate that DISCARDS the search: dictionary matches are
    // external to the block, so nothing else in this chain can see them, and a
    // block whose dictionary runs happen to fall between the sampled offsets
    // goes out raw although the search would have coded nearly all of it from
    // the dictionary. The classifier's own sample is safe in a way this one is
    // not — it measures the block against itself, and a repeat is what the grid
    // then catches.
    let dict_rejects_raw = dict_active;
    // The level and the window decide this for the whole frame, so it also says
    // whether the grid is worth keeping at all: nothing it records can be acted
    // on where no block may go out raw, and recording there would take its
    // tables and hash a run of every block for an answer no one asks for.
    let raw_skip_reachable = compression_level_allows_raw_fast_path(compression_level, window_size);
    let looks_incompressible = rle_byte_opt.is_none()
        && !dict_rejects_raw
        && raw_skip_reachable
        && should_emit_raw_fast_path(compression_level, bytes);
    let repeats_earlier_content = if looks_incompressible {
        state
            .seen_content
            .record_and_report_repeat(bytes, window_size as usize)
    } else {
        // Searched, so the matcher will hold it for as long as the window does.
        // It still has to be RECORDED, or a later block made mostly of this one
        // finds nothing on the grid and goes out raw with the match sitting in
        // history. Recording without probing: the probe is what costs.
        //
        // A block of one repeated byte is the exception: anything that
        // duplicates it is itself one repeated byte, and such a block is
        // answered as RLE above without ever asking the grid. Recording it is a
        // key every `RECORD_STEP` bytes for a question nobody puts.
        if raw_skip_reachable && rle_byte_opt.is_none() {
            state
                .seen_content
                .record_searched(bytes, window_size as usize);
        } else if raw_skip_reachable {
            state.seen_content.skip_recording(bytes.len());
        }
        false
    };
    let raw_fast_path = looks_incompressible && !repeats_earlier_content;
    // Hashed once, from the pre-commit view, and reused by whichever branch
    // wins — the compressed branch covers the same bytes as the RLE and raw
    // ones. This is a whole-block pass, so it is skipped whenever nothing will
    // consume it: when no sink collects checksums (the common case), and when
    // the block is headed for the post-split helper, which emits several
    // physical blocks and records a checksum per partition of its own.
    #[cfg(all(feature = "lsm", feature = "hash"))]
    let post_split_path = rle_byte_opt.is_none()
        && !raw_fast_path
        && matches!(compression_level, CompressionLevel::Level(16..=22))
        && state.matcher.window_size() >= (1 << 17);
    #[cfg(all(feature = "lsm", feature = "hash"))]
    let precomputed_checksum = block_checksums
        .as_ref()
        .filter(|_| !post_split_path)
        .map(|_| crate::encoding::frame_compressor::xxh64_block_low32(bytes));

    // First check to see if run length encoding can be used for the entire block
    if let Some(rle_byte) = rle_byte_opt {
        #[cfg(feature = "lsm")]
        if let Some(sink) = block_decompressed_sizes {
            sink.push(block_size);
        }
        #[cfg(all(feature = "lsm", feature = "hash"))]
        if let Some(sink) = block_checksums {
            sink.push(precomputed_checksum.expect("checksum is hashed whenever a sink exists"));
        }
        uncompressed_data.commit(&mut state.matcher);
        state.matcher.skip_matching_with_hint(Some(false));
        let header = BlockHeader {
            last_block,
            block_type: BlockType::RLE,
            block_size,
        };
        // Write the header, then the block
        header.serialize(output);
        output.push(rle_byte);
        BlockType::RLE
    } else if raw_fast_path {
        #[cfg(feature = "lsm")]
        if let Some(sink) = block_decompressed_sizes {
            sink.push(block_size);
        }
        #[cfg(all(feature = "lsm", feature = "hash"))]
        if let Some(sink) = block_checksums {
            sink.push(precomputed_checksum.expect("checksum is hashed whenever a sink exists"));
        }
        uncompressed_data.commit(&mut state.matcher);
        state.matcher.skip_matching_with_hint(Some(true));
        let header = BlockHeader {
            last_block,
            block_type: BlockType::Raw,
            block_size,
        };
        header.serialize(output);
        output.extend_from_slice(state.matcher.get_last_space());
        BlockType::Raw
    } else {
        // Compress as a standard compressed block
        uncompressed_data.commit(&mut state.matcher);
        if matches!(compression_level, CompressionLevel::Level(16..=22))
            && state.matcher.window_size() >= (1 << 17)
        {
            // This helper may emit multiple physical blocks (compressed or raw)
            // into `output`; the decompressed-size and (if requested) checksum
            // sidecars are pushed per physical block from inside the partition
            // loop so the cardinality matches the decoder's per-block count
            // exactly.
            #[cfg(all(feature = "lsm", feature = "hash"))]
            compress_block_with_post_split(
                state,
                last_block,
                output,
                block_decompressed_sizes,
                block_checksums,
            );
            #[cfg(all(feature = "lsm", not(feature = "hash")))]
            compress_block_with_post_split(state, last_block, output, block_decompressed_sizes);
            #[cfg(not(feature = "lsm"))]
            compress_block_with_post_split(state, last_block, output);
            return BlockType::Compressed;
        }
        #[cfg(feature = "lsm")]
        if let Some(sink) = block_decompressed_sizes {
            sink.push(block_size);
        }
        #[cfg(all(feature = "lsm", feature = "hash"))]
        if let Some(sink) = block_checksums {
            // The pre-commit view covers exactly the bytes the decoder will see
            // for this block, so the hash taken above stands — no second pass
            // over the committed copy.
            sink.push(precomputed_checksum.expect("checksum is hashed whenever a sink exists"));
        }

        // Keep rollback snapshots for the oversize fallback path below:
        // `compress_block` can mutate entropy/history state before we know
        // whether the compressed payload fits `MAX_BLOCK_SIZE`.
        let saved_offset_hist = state.offset_hist;
        // Snapshot the Huffman table into the scratch's persistent rollback
        // slot: `clone_from` reuses the slot's buffers across blocks (a
        // fresh `.clone()` paid a malloc + free pair on both code containers
        // every block). FSE previous tables are `SharedFseTable` handles —
        // their clone is a refcount bump, no slot needed.
        let mut saved_huff_table = core::mem::take(&mut state.block_scratch.huff_rollback);
        // Only when there IS a table to snapshot. `Option::clone_from` from a
        // `None` source drops what the slot holds, which is every buffer this
        // persistent slot exists to keep: a run of blocks that build a table and
        // then lose the size test would free and rebuild it every time.
        let had_prior_huff_table = state.last_huff_table.is_some();
        if let Some(prior) = state.last_huff_table.as_ref() {
            match saved_huff_table.as_mut() {
                Some(slot) => slot.clone_from(prior),
                None => saved_huff_table = Some(prior.clone()),
            }
        }
        let saved_ll_previous = state.fse_tables.ll_previous.clone();
        let saved_ml_previous = state.fse_tables.ml_previous.clone();
        let saved_of_previous = state.fse_tables.of_previous.clone();
        // Compress directly into `output`: reserve the fixed 3-byte block
        // header, append the payload after it, then backfill the header in
        // place once its length is known — no temp `Vec`, no extend-copy.
        let hdr_off = output.len();
        output.extend_from_slice(&[0u8; 3]);
        let payload_off = output.len();
        compress_block(state, output);
        let payload_len = output.len() - payload_off;
        // Fall back to a raw block when the compressed payload is not
        // smaller than the source (`payload >= block_size`) or exceeds the
        // maximum block size. A compressed block that did not shrink is never
        // the right choice: it wastes bytes and, in a single-segment frame
        // (window == content size), can reference past the declared window
        // and fail to decode in a strict decoder. This mirrors the upstream
        // post-hoc raw fallback and applies to every block, dictionary-primed
        // or not — the pre-compression raw-fast-path only catches blocks that
        // already look incompressible, so small inputs that slip past it but
        // fail to shrink still need this post-hoc store-raw.
        if payload_len >= MAX_BLOCK_SIZE as usize || payload_len >= block_size as usize {
            // Roll back the payload + reserved header and the entropy state.
            output.truncate(hdr_off);
            state.offset_hist = saved_offset_hist;
            // Swap (not move) so the slot keeps owning a reusable table
            // allocation for the next block's snapshot. With no prior table
            // there is nothing to restore, and the table this block built goes
            // into the slot rather than being dropped.
            if had_prior_huff_table {
                core::mem::swap(&mut state.last_huff_table, &mut saved_huff_table);
            } else if let Some(built) = state.last_huff_table.take() {
                saved_huff_table = Some(built);
            }
            // A block that built a table and then chose raw or RLE literals
            // parked it on its way there, so the swap above had nothing to give
            // back and the slot would go into the next block empty — which is
            // the per-block allocation of both code buffers this slot exists to
            // avoid, on exactly the run of blocks that keeps failing the size
            // test. Take the parked table instead: the slot is asked for one
            // per block, the spare's other reader once per frame.
            if saved_huff_table.is_none() {
                saved_huff_table = state.huff_table_spare.take();
            }
            state.fse_tables.roll_back_confirmation([
                saved_ll_previous,
                saved_ml_previous,
                saved_of_previous,
            ]);
            state.block_scratch.huff_rollback = saved_huff_table;
            let header = BlockHeader {
                last_block,
                block_type: BlockType::Raw,
                block_size,
            };
            // Write the header, then the block
            header.serialize(output);
            output.extend_from_slice(state.matcher.get_last_space());
            BlockType::Raw
        } else {
            // Return the snapshot to its slot so the next block's
            // `clone_from` reuses the allocation.
            state.block_scratch.huff_rollback = saved_huff_table;
            let header = BlockHeader {
                last_block,
                block_type: BlockType::Compressed,
                block_size: payload_len as u32,
            };
            // Backfill the reserved 3-byte header in place.
            output[hdr_off..hdr_off + 3].copy_from_slice(&header.to_le_bytes());
            BlockType::Compressed
        }
    }
}

/// Borrowed one-shot variant of [`compress_block_encoded`] for the Fast
/// (Simple) backend: the block bytes live at `[block_start, block_end)`
/// of the matcher's registered borrowed window (`set_borrowed_window`),
/// so there is no owned block `Vec` to `commit_space`. Instead the range
/// is staged via `set_borrowed_block`, which routes the subsequent
/// `start_matching` / `skip_matching_with_hint` to the borrowed scan.
///
/// Mirrors `compress_block_encoded`'s RLE / raw-fast-path / compressed
/// branch selection and shares the heavy `compress_block` machinery; the
/// only differences are how the block is acquired (borrowed slice, no
/// copy) and that raw/RLE bodies are emitted straight from `block`. The
/// `Level(16..=22)` post-split branch is unreachable here (the borrowed
/// path is gated to Fast levels), so it is omitted.
#[allow(clippy::too_many_arguments)]
pub(crate) fn compress_block_encoded_borrowed(
    state: &mut CompressState<MatchGeneratorDriver>,
    compression_level: CompressionLevel,
    last_block: bool,
    block: &[u8],
    block_start: usize,
    block_end: usize,
    output: &mut Vec<u8>,
    dict_active: bool,
    #[cfg(feature = "lsm")] block_decompressed_sizes: Option<&mut Vec<u32>>,
    #[cfg(all(feature = "lsm", feature = "hash"))] block_checksums: Option<&mut Vec<u32>>,
) -> BlockType {
    // The borrowed one-shot path emits ONE block per staged range (no
    // pre-split partition loop). `borrowed_supported()` is the single source
    // of truth for which backend + search configs have a borrowed scan
    // (Simple / Dfast / Row, and HashChain's lazy CHAIN parser + btlazy2); the
    // optimal BT search stays on the owned path. `borrowed_eligible` gates on
    // the same predicate, so this only ever fires on a wiring bug. Checked at
    // entry (not per-branch) so RLE / raw-fast / compressed paths all stage
    // their borrowed range under the same invariant.
    debug_assert!(
        state.matcher.borrowed_supported(),
        "borrowed one-shot path reached for an unsupported backend/search config",
    );
    let block_size = block.len() as u32;
    // Same order as the owned path, cheapest question first: the whole-block
    // grid is asked last and only about blocks the classifier already called
    // incompressible.
    let is_rle = !block.is_empty() && block.iter().all(|x| block[0].eq(x));
    let window_size = state.matcher.window_size();
    // As on the owned path: an attached dictionary keeps the block on the
    // search rather than trusting a sample to say the dictionary is irrelevant.
    let dict_rejects_raw = dict_active;
    // As on the owned path: where no block may go out raw, the grid has nothing
    // to answer and is left alone.
    let raw_skip_reachable = compression_level_allows_raw_fast_path(compression_level, window_size);
    let looks_incompressible = !is_rle
        && !dict_rejects_raw
        && raw_skip_reachable
        && should_emit_raw_fast_path(compression_level, block);
    let repeats_earlier_content = if looks_incompressible {
        state
            .seen_content
            .record_and_report_repeat(block, window_size as usize)
    } else {
        // As on the owned path: a searched block is recorded, not merely
        // stepped over — except a block of one repeated byte, which nothing
        // will ever ask the grid about.
        if raw_skip_reachable && !is_rle {
            state
                .seen_content
                .record_searched(block, window_size as usize);
        } else if raw_skip_reachable {
            state.seen_content.skip_recording(block.len());
        }
        false
    };
    if is_rle {
        let rle_byte = block[0];
        #[cfg(feature = "lsm")]
        if let Some(sink) = block_decompressed_sizes {
            sink.push(block_size);
        }
        #[cfg(all(feature = "lsm", feature = "hash"))]
        if let Some(sink) = block_checksums {
            sink.push(crate::encoding::frame_compressor::xxh64_block_low32(block));
        }
        state.matcher.set_borrowed_block(block_start, block_end);
        state.matcher.skip_matching_with_hint(Some(false));
        let header = BlockHeader {
            last_block,
            block_type: BlockType::RLE,
            block_size,
        };
        header.serialize(output);
        output.push(rle_byte);
        BlockType::RLE
    } else if looks_incompressible && !repeats_earlier_content {
        #[cfg(feature = "lsm")]
        if let Some(sink) = block_decompressed_sizes {
            sink.push(block_size);
        }
        #[cfg(all(feature = "lsm", feature = "hash"))]
        if let Some(sink) = block_checksums {
            sink.push(crate::encoding::frame_compressor::xxh64_block_low32(block));
        }
        state.matcher.set_borrowed_block(block_start, block_end);
        state.matcher.skip_matching_with_hint(Some(true));
        let header = BlockHeader {
            last_block,
            block_type: BlockType::Raw,
            block_size,
        };
        header.serialize(output);
        output.extend_from_slice(block);
        BlockType::Raw
    } else {
        // Stage the borrowed range so `compress_block`'s internal
        // `start_matching` scans it in place (no `commit_space` copy).
        state.matcher.set_borrowed_block(block_start, block_end);
        // No post-split branch here: the optimal levels (16-22), the only
        // strategies that post-split, are NOT borrowed-eligible
        // (`borrowed_supported` keeps them owned because the borrowed
        // continuous-index scan yields ratio-worse candidates for their
        // cost-based DP). btlazy2 (L13-15) and every other borrowed backend
        // emit a single block per staged range, handled by the path below.
        #[cfg(feature = "lsm")]
        if let Some(sink) = block_decompressed_sizes {
            sink.push(block_size);
        }
        #[cfg(all(feature = "lsm", feature = "hash"))]
        if let Some(sink) = block_checksums {
            // Hash the block bytes directly: the staged borrowed range is
            // consumed by the `start_matching` inside `compress_block`
            // below, so hashing `block` is both correct and order-safe.
            sink.push(crate::encoding::frame_compressor::xxh64_block_low32(block));
        }
        let saved_offset_hist = state.offset_hist;
        // Persistent rollback slot — same allocation-reuse rationale as the
        // owned `compress_block_encoded` snapshot above.
        let mut saved_huff_table = core::mem::take(&mut state.block_scratch.huff_rollback);
        // Only when there IS a table to snapshot. `Option::clone_from` from a
        // `None` source drops what the slot holds, which is every buffer this
        // persistent slot exists to keep: a run of blocks that build a table and
        // then lose the size test would free and rebuild it every time.
        let had_prior_huff_table = state.last_huff_table.is_some();
        if let Some(prior) = state.last_huff_table.as_ref() {
            match saved_huff_table.as_mut() {
                Some(slot) => slot.clone_from(prior),
                None => saved_huff_table = Some(prior.clone()),
            }
        }
        let saved_ll_previous = state.fse_tables.ll_previous.clone();
        let saved_ml_previous = state.fse_tables.ml_previous.clone();
        let saved_of_previous = state.fse_tables.of_previous.clone();
        // Compress directly into `output`: reserve the fixed 3-byte block
        // header, append the payload after it, then backfill the header in
        // place once its length is known. Avoids the per-block temp `Vec`
        // plus the `output.extend(compressed)` copy (the dominant per-frame
        // memmove on this hot path).
        let hdr_off = output.len();
        output.extend_from_slice(&[0u8; 3]);
        let payload_off = output.len();
        compress_block(state, output);
        let payload_len = output.len() - payload_off;
        if payload_len >= MAX_BLOCK_SIZE as usize || payload_len >= block_size as usize {
            // Incompressible (compressed payload not smaller than the source,
            // or over the max block size): roll back the payload + reserved
            // header and the entropy state, then emit a stored Raw block. A
            // non-shrinking compressed block wastes bytes and can reference
            // past a single-segment frame's window (== content size); storing
            // raw matches the upstream post-hoc fallback.
            output.truncate(hdr_off);
            state.offset_hist = saved_offset_hist;
            // Swap (not move) so the slot keeps owning a reusable table
            // allocation for the next block's snapshot. With no prior table
            // there is nothing to restore, and the table this block built goes
            // into the slot rather than being dropped.
            if had_prior_huff_table {
                core::mem::swap(&mut state.last_huff_table, &mut saved_huff_table);
            } else if let Some(built) = state.last_huff_table.take() {
                saved_huff_table = Some(built);
            }
            // A block that built a table and then chose raw or RLE literals
            // parked it on its way there, so the swap above had nothing to give
            // back and the slot would go into the next block empty — which is
            // the per-block allocation of both code buffers this slot exists to
            // avoid, on exactly the run of blocks that keeps failing the size
            // test. Take the parked table instead: the slot is asked for one
            // per block, the spare's other reader once per frame.
            if saved_huff_table.is_none() {
                saved_huff_table = state.huff_table_spare.take();
            }
            state.fse_tables.roll_back_confirmation([
                saved_ll_previous,
                saved_ml_previous,
                saved_of_previous,
            ]);
            state.block_scratch.huff_rollback = saved_huff_table;
            let header = BlockHeader {
                last_block,
                block_type: BlockType::Raw,
                block_size,
            };
            header.serialize(output);
            output.extend_from_slice(block);
            BlockType::Raw
        } else {
            // Return the snapshot to its slot so the next block's
            // `clone_from` reuses the allocation.
            state.block_scratch.huff_rollback = saved_huff_table;
            let header = BlockHeader {
                last_block,
                block_type: BlockType::Compressed,
                block_size: payload_len as u32,
            };
            output[hdr_off..hdr_off + 3].copy_from_slice(&header.to_le_bytes());
            BlockType::Compressed
        }
    }
}

/// Whether this block may go out raw without being searched.
///
/// The classifier answers from the block's own bytes, which cannot see a repeat
/// that lives in the history, so the caller pairs it with a probe of the match
/// table before acting on it. Callers ask it only after the level and window
/// admit a raw skip at all, and only with no dictionary attached — one keeps the
/// block on the search whatever its own bytes look like.
#[inline]
fn should_emit_raw_fast_path(level: CompressionLevel, block: &[u8]) -> bool {
    if matches!(level, CompressionLevel::Best) {
        return block_looks_incompressible_strict(block);
    }
    block_looks_incompressible(block)
}

#[cfg(test)]
mod tests;