1use crate::SOURCE_FORMAT_OPAQUE;
8use crate::adapter::opaque;
9use crate::container::{Descriptor, UNIVERSE};
10use crate::dra::{Op, Program};
11#[cfg(feature = "rans")]
12use crate::entropy::{
13 CODER_ORDER0_BYTE_RANS, CODER_VERSION_1, EntropyChannelDescriptor, EntropyModel, encode_channel,
14};
15use crate::error::Result;
16use crate::integrity::sha256;
17use crate::limits::Limits;
18
19#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
22#[repr(u8)]
23pub enum CandidateKind {
24 Raw = 0,
26 Rle = 1,
28 ByteRans = 2,
30 PdfPhysical = 3,
32 PdfChannels = 4,
34 PdfLayout = 5,
37 PdfLayoutRans = 6,
40 PdfDeflateReplay = 7,
43 PdfDeflateReplayRans = 8,
46 PdfDeflateReplayRansIndexed = 9,
50 PdfLengthRevision = 10,
53 PdfCosTemplate = 11,
57}
58
59impl CandidateKind {
60 pub const fn name(self) -> &'static str {
62 match self {
63 CandidateKind::Raw => "RAW",
64 CandidateKind::Rle => "RLE",
65 CandidateKind::ByteRans => "BYTE_RANS",
66 CandidateKind::PdfPhysical => "PDF_PHYSICAL",
67 CandidateKind::PdfChannels => "PDF_CHANNELS",
68 CandidateKind::PdfLayout => "PDF_LAYOUT",
69 CandidateKind::PdfLayoutRans => "PDF_LAYOUT_RANS",
70 CandidateKind::PdfDeflateReplay => "PDF_DEFLATE_REPLAY",
71 CandidateKind::PdfDeflateReplayRans => "PDF_DEFLATE_REPLAY_RANS",
72 CandidateKind::PdfDeflateReplayRansIndexed => "PDF_DEFLATE_REPLAY_RANS_INDEXED",
73 CandidateKind::PdfLengthRevision => "PDF_LENGTH_REVISION",
74 CandidateKind::PdfCosTemplate => "PDF_COS_TEMPLATE",
75 }
76 }
77}
78
79#[derive(Debug, Clone)]
81pub struct Candidate {
82 pub kind: CandidateKind,
84 pub descriptor: Descriptor,
86}
87
88pub fn propose_all(input: &[u8], limits: Limits) -> Result<Vec<Candidate>> {
99 let mut out = Vec::new();
100 propose_each(input, limits, |c| {
101 out.push(c);
102 Ok(())
103 })?;
104 Ok(out)
105}
106
107#[allow(unreachable_patterns)]
115pub fn propose_forced(
116 input: &[u8],
117 limits: Limits,
118 kind: CandidateKind,
119) -> Result<Option<Candidate>> {
120 match kind {
122 CandidateKind::PdfPhysical
123 | CandidateKind::PdfChannels
124 | CandidateKind::PdfLayout
125 | CandidateKind::PdfLayoutRans
126 | CandidateKind::PdfLengthRevision => {
127 let physical = match crate::adapter::pdf::physical::scan(input, limits) {
128 Ok(p) => p,
129 Err(_) => return Ok(None),
130 };
131 match kind {
132 CandidateKind::PdfPhysical => {
133 crate::adapter::pdf::adapter::propose_pdf_with(input, limits, &physical)
134 }
135 #[cfg(feature = "rans")]
136 CandidateKind::PdfChannels => {
137 crate::adapter::pdf::adapter::propose_pdf_channels_with(
138 input, limits, &physical,
139 )
140 }
141 CandidateKind::PdfLayout => {
142 crate::adapter::pdf::layout::propose_pdf_layout_with(input, limits, &physical)
143 }
144 #[cfg(feature = "rans")]
145 CandidateKind::PdfLayoutRans => {
146 crate::adapter::pdf::layout::propose_pdf_layout_rans_with(
147 input, limits, &physical,
148 )
149 }
150 CandidateKind::PdfLengthRevision => {
151 crate::adapter::pdf::length_revision::propose_pdf_length_revision_with(
152 input, limits, &physical,
153 )
154 }
155 _ => Ok(None),
156 }
157 }
158 CandidateKind::Raw => Ok(Some(Candidate {
159 kind,
160 descriptor: opaque::propose(input, limits)?,
161 })),
162 CandidateKind::Rle => propose_rle(input, limits),
163 #[cfg(feature = "rans")]
164 CandidateKind::ByteRans => propose_byte_rans(input, limits),
165 #[cfg(feature = "deflate-replay")]
166 CandidateKind::PdfDeflateReplay => {
167 crate::adapter::pdf::propose_pdf_deflate_replay(input, limits)
168 }
169 #[cfg(all(feature = "deflate-replay", feature = "rans"))]
170 CandidateKind::PdfDeflateReplayRans => {
171 crate::adapter::pdf::propose_pdf_deflate_replay_rans(input, limits)
172 }
173 #[cfg(all(feature = "deflate-replay", feature = "rans"))]
174 CandidateKind::PdfDeflateReplayRansIndexed => {
175 crate::adapter::pdf::propose_pdf_deflate_replay_rans_indexed(input, limits)
176 }
177 CandidateKind::PdfCosTemplate => {
178 crate::adapter::pdf::propose_pdf_cos_template(input, limits)
179 }
180 _ => Ok(None),
181 }
182}
183
184pub fn propose_each<F>(input: &[u8], limits: Limits, mut emit: F) -> Result<()>
190where
191 F: FnMut(Candidate) -> Result<()>,
192{
193 emit(Candidate {
194 kind: CandidateKind::Raw,
195 descriptor: opaque::propose(input, limits)?,
196 })?;
197 if let Some(rle) = propose_rle(input, limits)? {
198 emit(rle)?;
199 }
200 #[cfg(feature = "rans")]
201 if let Some(byte_rans) = propose_byte_rans(input, limits)? {
202 emit(byte_rans)?;
203 }
204 let physical = crate::adapter::pdf::physical::scan(input, limits).ok();
208 if let Some(p) = physical.as_ref() {
209 if let Some(pdf) = crate::adapter::pdf::adapter::propose_pdf_with(input, limits, p)? {
210 emit(pdf)?;
211 }
212 #[cfg(feature = "rans")]
213 if let Some(pdf_channels) =
214 crate::adapter::pdf::adapter::propose_pdf_channels_with(input, limits, p)?
215 {
216 emit(pdf_channels)?;
217 }
218 if let Some(pdf_layout) =
219 crate::adapter::pdf::layout::propose_pdf_layout_with(input, limits, p)?
220 {
221 emit(pdf_layout)?;
222 }
223 #[cfg(feature = "rans")]
224 if let Some(pdf_layout_rans) =
225 crate::adapter::pdf::layout::propose_pdf_layout_rans_with(input, limits, p)?
226 {
227 emit(pdf_layout_rans)?;
228 }
229 }
230 #[cfg(feature = "deflate-replay")]
231 if let Some(pdf_deflate) = crate::adapter::pdf::propose_pdf_deflate_replay(input, limits)? {
232 emit(pdf_deflate)?;
233 }
234 #[cfg(all(feature = "deflate-replay", feature = "rans"))]
235 if let Some(pdf_deflate_rans) =
236 crate::adapter::pdf::propose_pdf_deflate_replay_rans(input, limits)?
237 {
238 emit(pdf_deflate_rans)?;
239 }
240 #[cfg(all(feature = "deflate-replay", feature = "rans"))]
241 if let Some(pdf_deflate_rans_indexed) =
242 crate::adapter::pdf::propose_pdf_deflate_replay_rans_indexed(input, limits)?
243 {
244 emit(pdf_deflate_rans_indexed)?;
245 }
246 if let Some(p) = physical.as_ref()
247 && let Some(pdf_length_revision) =
248 crate::adapter::pdf::length_revision::propose_pdf_length_revision_with(
249 input, limits, p,
250 )?
251 {
252 emit(pdf_length_revision)?;
253 }
254 if let Some(pdf_cos_template) = crate::adapter::pdf::propose_pdf_cos_template(input, limits)? {
255 emit(pdf_cos_template)?;
256 }
257 Ok(())
258}
259
260pub fn propose(input: &[u8], limits: Limits) -> Result<Vec<Candidate>> {
265 propose_all(input, limits)
266}
267
268pub fn propose_rle(input: &[u8], limits: Limits) -> Result<Option<Candidate>> {
290 let mut run_count: u64 = 0;
293 let mut max_run_len: u64 = 0;
294 let mut prev: Option<u8> = None;
295 let mut len: u64 = 0;
296 for &b in input {
297 if prev == Some(b) {
298 len += 1;
299 } else {
300 max_run_len = max_run_len.max(len);
301 run_count += 1;
302 prev = Some(b);
303 len = 1;
304 }
305 }
306 max_run_len = max_run_len.max(len);
307
308 if run_count.saturating_mul(2) > limits.max_graph_ops as u64 {
310 return Ok(None);
311 }
312
313 let max_extra = max_run_len.saturating_sub(1);
317 if max_extra > limits.max_repeat_count || max_extra > u32::MAX as u64 {
318 return Ok(None);
319 }
320
321 let mut ops: Vec<Op> = Vec::with_capacity(run_count as usize * 2);
323 let mut prev: Option<u8> = None;
324 let mut len: u64 = 0;
325 for &b in input {
326 if prev == Some(b) {
327 len += 1;
328 } else {
329 if let Some(byte) = prev {
330 ops.push(Op::Inline { bytes: vec![byte] });
331 if len > 1 {
332 ops.push(Op::RepeatLast {
333 count: (len - 1) as u32,
334 });
335 }
336 }
337 prev = Some(b);
338 len = 1;
339 }
340 }
341 if let Some(byte) = prev {
342 ops.push(Op::Inline { bytes: vec![byte] });
343 if len > 1 {
344 ops.push(Op::RepeatLast {
345 count: (len - 1) as u32,
346 });
347 }
348 }
349
350 let descriptor = Descriptor {
351 universe: UNIVERSE.to_string(),
352 source_format: SOURCE_FORMAT_OPAQUE,
353 format_basis: "opaque;rle-runs".to_string(),
354 models: vec![],
355 channels: vec![],
356 objects: vec![],
357 program: Program::new(ops),
358 observation_index: None,
359 seek_directory: false,
360 checkpoints: None,
361 source_sha256: sha256(input),
362 source_len: input.len() as u64,
363 };
364 Ok(Some(Candidate {
365 kind: CandidateKind::Rle,
366 descriptor,
367 }))
368}
369
370#[cfg(feature = "rans")]
384pub fn propose_byte_rans(input: &[u8], limits: Limits) -> Result<Option<Candidate>> {
385 if input.is_empty() || input.len() as u64 > limits.max_channel_symbols {
386 return Ok(None);
387 }
388
389 let mut counts = [0u64; 256];
390 for &b in input {
391 counts[b as usize] += 1;
392 }
393 let model = EntropyModel::from_counts(&counts, 12)?;
394 let capsule = encode_channel(&model, input)?;
395
396 let channel = EntropyChannelDescriptor {
397 coder: CODER_ORDER0_BYTE_RANS,
398 coder_version: CODER_VERSION_1,
399 scale_bits: model.scale_bits,
400 lane_count: 1,
401 model_id: 0,
402 symbol_count: capsule.symbol_count,
403 decoded_length: capsule.decoded_length,
404 initial_state: capsule.initial_state,
405 payload: capsule.payload,
406 };
407
408 let descriptor = Descriptor {
409 universe: UNIVERSE.to_string(),
410 source_format: SOURCE_FORMAT_OPAQUE,
411 format_basis: "opaque;byte-rans".to_string(),
412 models: vec![model],
413 channels: vec![channel],
414 objects: vec![],
415 program: Program::new(vec![Op::DecodeChannel { channel_id: 0 }]),
416 observation_index: None,
417 seek_directory: false,
418 checkpoints: None,
419 source_sha256: sha256(input),
420 source_len: input.len() as u64,
421 };
422
423 Ok(Some(Candidate {
424 kind: CandidateKind::ByteRans,
425 descriptor,
426 }))
427}
428
429#[cfg(test)]
430mod tests {
431 use super::*;
432
433 fn xorshift64(state: &mut u64) -> u64 {
435 let mut x = *state;
436 x ^= x << 13;
437 x ^= x >> 7;
438 x ^= x << 17;
439 *state = x;
440 x
441 }
442
443 fn xorshift_bytes(n: usize, seed: u64) -> Vec<u8> {
444 let mut state = seed | 1; let mut out = Vec::with_capacity(n + 8);
446 while out.len() < n {
447 out.extend_from_slice(&xorshift64(&mut state).to_le_bytes());
448 }
449 out.truncate(n);
450 out
451 }
452
453 fn assert_exact(bytes: &[u8], input: &[u8], limits: Limits) {
454 let (out, parsed) = crate::materialize::decode_to_bytes(bytes, limits).unwrap();
455 assert_eq!(out, input, "materialized bytes must equal the source");
456 assert_eq!(out.len() as u64, parsed.descriptor.source_len);
457 assert_eq!(
458 crate::integrity::sha256(&out),
459 crate::integrity::sha256(input)
460 );
461 }
462
463 #[test]
464 fn rle_wins_on_zeros() {
465 let input = vec![0u8; 65536];
466 let (bytes, report) = crate::encode::encode(&input, Limits::DEFAULT).unwrap();
467 assert_eq!(report.kind, CandidateKind::Rle);
468 assert!(
469 report.encoded_len < 512,
470 "RLE encoding of zeros was {} bytes",
471 report.encoded_len
472 );
473 assert_exact(&bytes, &input, Limits::DEFAULT);
474 }
475
476 #[test]
477 fn rle_wins_on_long_runs() {
478 let mut input = Vec::new();
479 input.extend_from_slice(&[0xAAu8; 1000]);
480 input.extend_from_slice(&[0x00, 0x01]);
481 input.extend_from_slice(&[0x55u8; 5000]);
482 input.extend_from_slice(b"tail");
483
484 let (bytes, report) = crate::encode::encode(&input, Limits::DEFAULT).unwrap();
485 assert_eq!(report.kind, CandidateKind::Rle);
486 assert!(report.encoded_len < 512);
487 assert_exact(&bytes, &input, Limits::DEFAULT);
488 }
489
490 #[test]
491 fn raw_wins_on_incompressible() {
492 let input = xorshift_bytes(64 * 1024, 0x9E37_79B9_7F4A_7C15);
493 let (bytes, report) = crate::encode::encode(&input, Limits::DEFAULT).unwrap();
494 assert_eq!(
495 report.kind,
496 CandidateKind::Raw,
497 "incompressible data must be stored RAW"
498 );
499 assert_exact(&bytes, &input, Limits::DEFAULT);
500 }
501
502 #[test]
503 fn single_byte_is_rle() {
504 let input = [7u8];
505 let (bytes, report) = crate::encode::encode(&input, Limits::DEFAULT).unwrap();
506 assert_eq!(report.kind, CandidateKind::Rle);
507 assert!(report.encoded_len < 512);
508 assert_exact(&bytes, &input, Limits::DEFAULT);
509 }
510
511 #[test]
512 fn rle_declines_when_graph_too_big() {
513 let input = vec![0u8; 100];
514 let limits = Limits {
515 max_graph_ops: 1,
516 ..Limits::DEFAULT
517 };
518 assert!(
519 propose_rle(&input, limits).unwrap().is_none(),
520 "100 single-byte runs cannot fit in a one-op graph"
521 );
522 let (bytes, report) = crate::encode::encode(&input, limits).unwrap();
526 assert_ne!(report.kind, CandidateKind::Rle);
527 assert_exact(&bytes, &input, limits);
528 }
529
530 #[test]
531 fn rle_declines_on_large_incompressible() {
532 let input = xorshift_bytes(4 * 1024 * 1024, 0xD1B5_4A32_D192_ED03);
539 assert!(
540 propose_rle(&input, Limits::DEFAULT).unwrap().is_none(),
541 "an incompressible buffer cannot be expressed as an RLE graph"
542 );
543 }
544
545 #[test]
546 fn rle_ops_match_run_list_reference() {
547 let mut input = Vec::new();
551 for &(byte, length) in &[(b'a', 6u64), (b'b', 1), (b'c', 300), (b'a', 2), (b'z', 1)] {
552 input.extend(std::iter::repeat_n(byte, length as usize));
553 }
554 let mut expected: Vec<Op> = Vec::new();
555 for &(byte, length) in &[(b'a', 6u64), (b'b', 1), (b'c', 300), (b'a', 2), (b'z', 1)] {
556 expected.push(Op::Inline { bytes: vec![byte] });
557 if length > 1 {
558 expected.push(Op::RepeatLast {
559 count: (length - 1) as u32,
560 });
561 }
562 }
563 let cand = propose_rle(&input, Limits::DEFAULT)
564 .unwrap()
565 .expect("a 5-run input is expressible");
566 assert_eq!(cand.descriptor.program.ops, expected);
567 }
568
569 #[cfg(feature = "rans")]
570 #[test]
571 fn byte_rans_wins_on_text() {
572 let input = b"The quick brown fox jumps over the lazy dog. ".repeat(1500);
573 let (bytes, report) =
578 crate::encode::encode_with(&input, Limits::DEFAULT, Some(CandidateKind::ByteRans))
579 .unwrap();
580 assert_eq!(report.kind, CandidateKind::ByteRans);
581 assert!(
582 report.encoded_len < report.source_len,
583 "order-0 rANS must beat RAW on low-entropy text: {} vs {}",
584 report.encoded_len,
585 report.source_len
586 );
587 assert_exact(&bytes, &input, Limits::DEFAULT);
588 }
589
590 #[cfg(feature = "rans")]
591 #[test]
592 fn byte_rans_exact_on_all_byte_values() {
593 let mut input: Vec<u8> = (0..=255u8).collect();
598 let mut state = 0x2545_F491_4F6C_DD1D;
599 while input.len() < 64 * 1024 {
600 let r = xorshift64(&mut state);
601 if !r.is_multiple_of(8) {
602 input.push(0x00);
603 } else {
604 input.push((r >> 32) as u8);
605 }
606 }
607 input.truncate(64 * 1024);
608
609 let (bytes, report) = crate::encode::encode(&input, Limits::DEFAULT).unwrap();
610 assert_eq!(report.kind, CandidateKind::ByteRans);
611 assert_exact(&bytes, &input, Limits::DEFAULT);
612 }
613
614 #[test]
615 fn byte_rans_is_deterministic() {
616 let input = b"deterministic byte rANS stream ".repeat(600);
617 let (a, _) = crate::encode::encode(&input, Limits::DEFAULT).unwrap();
618 let (b, _) = crate::encode::encode(&input, Limits::DEFAULT).unwrap();
619 assert_eq!(a, b, ".voldoc bytes must be identical across encodes");
620 }
621
622 #[cfg(feature = "rans")]
623 #[test]
624 fn byte_rans_declines_empty() {
625 assert!(
626 propose_byte_rans(&[], Limits::DEFAULT).unwrap().is_none(),
627 "empty input must decline: RAW is trivially smaller"
628 );
629 }
630
631 #[cfg(feature = "rans")]
632 #[test]
633 fn model_cost_is_charged() {
634 let input = b"ab";
638 let (bytes, report) = crate::encode::encode(input, Limits::DEFAULT).unwrap();
639 assert_ne!(report.kind, CandidateKind::ByteRans);
640 assert_exact(&bytes, input, Limits::DEFAULT);
641 }
642}