use super::*;
use crate::{
DecoderDictionary, decode_all_with_prepared_dict, encode_all, encode_all_with_prepared_dict,
};
fn sample_records(count: usize) -> Vec<Vec<u8>> {
let hosts = ["alpha", "beta", "gamma-node", "delta", "epsilon-7"];
let levels = ["INFO", "WARN", "ERROR", "DEBUG"];
(0..count)
.map(|index| {
let mut record = Vec::new();
for line in 0..12 {
let seed = index * 31 + line * 7;
record.extend_from_slice(
&format!(
"{{\"timestamp\":\"2026-08-{:02}T{:02}:{:02}:{:02}Z\",\
\"host\":\"{}\",\"level\":\"{}\",\"request_id\":{},\
\"duration_ms\":{},\"message\":\"request completed for tenant {}\"}}\n",
(seed % 28) + 1,
seed % 24,
(seed * 13) % 60,
(seed * 17) % 60,
hosts[seed % hosts.len()],
levels[(seed / 3) % levels.len()],
1_000_000 + seed * 991,
(seed * 7919) % 4096,
seed % 97,
)
.into_bytes(),
);
}
record
})
.collect()
}
fn as_slices(samples: &[Vec<u8>]) -> Vec<&[u8]> {
samples.iter().map(Vec::as_slice).collect()
}
#[test]
fn trains_a_dictionary_that_round_trips() {
let samples = sample_records(40);
let dictionary = train_dictionary(&as_slices(&samples), 4096).expect("training");
let prepared = EncoderDictionary::new(&dictionary).expect("parse trained dictionary");
let prepared_decoding =
DecoderDictionary::new(&dictionary).expect("parse trained dictionary for decoding");
assert!(
!prepared.is_raw_content(),
"a trained dictionary carries entropy tables and must parse as formatted"
);
for sample in &samples {
let compressed = encode_all_with_prepared_dict(sample, &prepared).expect("encode");
let decoded =
decode_all_with_prepared_dict(&compressed, &prepared_decoding).expect("decode");
assert_eq!(&decoded, sample);
}
}
#[test]
fn trained_dictionary_beats_no_dictionary_on_short_records() {
let samples = sample_records(40);
let dictionary = train_dictionary(&as_slices(&samples), 4096).expect("training");
let prepared = EncoderDictionary::new(&dictionary).expect("parse");
let held_out = sample_records(60);
let held_out = &held_out[40..];
let mut with_dictionary = 0usize;
let mut without = 0usize;
for sample in held_out {
let record = sample.split(|&b| b == b'\n').next().expect("a record");
with_dictionary += encode_all_with_prepared_dict(record, &prepared)
.expect("encode")
.len();
without += encode_all(record).expect("encode").len();
}
assert!(
with_dictionary * 2 < without,
"a trained dictionary should less than halve short records: \
{with_dictionary} with, {without} without"
);
}
#[test]
fn the_dictionary_id_is_derived_from_the_content() {
let samples = sample_records(40);
let dictionary = train_dictionary(&as_slices(&samples), 4096).expect("training");
let id = EncoderDictionary::new(&dictionary).expect("parse").id();
assert!(
(32768..(1u32 << 31)).contains(&id),
"derived id {id} is outside the non-registered range"
);
let again = train_dictionary(&as_slices(&samples), 4096).expect("training");
assert_eq!(dictionary, again, "training is not deterministic");
}
#[test]
fn an_explicit_dictionary_id_is_recorded_verbatim() {
let samples = sample_records(40);
let trained = train_dictionary_with_parameters(
&as_slices(&samples),
4096,
DictionaryTrainingParameters {
d: 8,
steps: 4,
dictionary_id: 12345,
..DictionaryTrainingParameters::default()
},
)
.expect("training");
assert_eq!(
EncoderDictionary::new(trained.as_bytes())
.expect("parse")
.id(),
12345
);
}
#[test]
fn the_dictionary_never_exceeds_the_requested_capacity() {
let samples = sample_records(40);
for capacity in [DICTIONARY_SIZE_MIN, 512, 1024, 4096, 16384] {
let dictionary = train_dictionary(&as_slices(&samples), capacity)
.unwrap_or_else(|error| panic!("training at capacity {capacity}: {error:?}"));
assert!(
dictionary.len() <= capacity,
"capacity {capacity} produced {} bytes",
dictionary.len()
);
EncoderDictionary::new(&dictionary)
.unwrap_or_else(|error| panic!("capacity {capacity} produced junk: {error:?}"));
}
}
#[test]
fn a_capacity_below_the_minimum_is_rejected() {
let samples = sample_records(40);
assert!(matches!(
train_dictionary(&as_slices(&samples), DICTIONARY_SIZE_MIN - 1),
Err(Error::DstSizeTooSmall)
));
}
#[test]
fn too_few_samples_are_rejected_rather_than_producing_a_weak_dictionary() {
let samples = sample_records(5);
assert!(matches!(
train_dictionary(&as_slices(&samples), 4096),
Err(Error::InvalidParameter(_))
));
assert!(matches!(
train_dictionary(&[], 4096),
Err(Error::InvalidParameter(_))
));
}
#[test]
fn out_of_range_parameters_are_rejected() {
let samples = sample_records(40);
let samples = as_slices(&samples);
let cases = [
DictionaryTrainingParameters {
d: 7,
..Default::default()
},
DictionaryTrainingParameters {
f: 32,
..Default::default()
},
DictionaryTrainingParameters {
accel: 11,
..Default::default()
},
DictionaryTrainingParameters {
split_point: 1.5,
..Default::default()
},
];
for parameters in cases {
assert!(
matches!(
train_dictionary_with_parameters(&samples, 4096, parameters),
Err(Error::InvalidParameter(_))
),
"expected rejection for {parameters:?}"
);
}
}
#[test]
fn a_capacity_below_the_default_search_range_still_trains() {
let samples = sample_records(40);
let dictionary = train_dictionary(&as_slices(&samples), DICTIONARY_SIZE_MIN).expect("training");
assert!(dictionary.len() <= DICTIONARY_SIZE_MIN);
EncoderDictionary::new(&dictionary).expect("parse");
}
#[test]
fn both_substring_lengths_are_searched_when_d_is_unset() {
let samples = sample_records(40);
let trained = train_dictionary_with_parameters(
&as_slices(&samples),
4096,
DictionaryTrainingParameters {
steps: 2,
..Default::default()
},
)
.expect("training");
assert!(
trained.d() == 6 || trained.d() == 8,
"search settled on d={}",
trained.d()
);
assert!(trained.total_compressed_size() > 0);
}
#[test]
fn incompressible_samples_still_produce_a_usable_dictionary() {
let mut state = 0x243f_6a88_85a3_08d3u64;
let samples: Vec<Vec<u8>> = (0..16)
.map(|_| {
(0..2048)
.map(|_| {
state = state
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1_442_695_040_888_963_407);
(state >> 33) as u8
})
.collect()
})
.collect();
let dictionary = train_dictionary(&as_slices(&samples), 1024).expect("training");
let prepared = EncoderDictionary::new(&dictionary).expect("parse");
let prepared_decoding = DecoderDictionary::new(&dictionary).expect("parse for decoding");
let compressed = encode_all_with_prepared_dict(&samples[0], &prepared).expect("encode");
let decoded = decode_all_with_prepared_dict(&compressed, &prepared_decoding).expect("decode");
assert_eq!(decoded, samples[0]);
}
#[test]
fn content_selection_is_independent_of_the_entropy_half() {
let samples = sample_records(40);
let corpus = Corpus::new(&as_slices(&samples));
let context = FrequencyContext::build(&corpus, 8, 20, 0.75, 0).expect("frequencies");
let select = |content: &mut Vec<u8>| {
let mut freqs = context.freqs.clone();
let mut segment_freqs = vec![0u32; 1 << 20];
context.build_content(&mut freqs, &mut segment_freqs, content, 200)
};
let mut content = vec![0u8; 1024];
let tail = select(&mut content);
assert!(tail < content.len(), "no content was selected");
let probe = &content[content.len() - context.d as usize..];
assert!(
corpus
.bytes
.windows(probe.len())
.any(|window| window == probe),
"selected content does not appear in the corpus"
);
let mut again = vec![0u8; 1024];
assert_eq!(select(&mut again), tail);
assert_eq!(content, again, "content selection is not reproducible");
}
#[test]
fn degenerate_sample_shapes_are_rejected_rather_than_read_past() {
let mut samples: Vec<Vec<u8>> = vec![Vec::new(); 6];
samples.push(vec![b'a'; 100_000]);
samples.push(vec![b'b'; 100_000]);
assert!(matches!(
train_dictionary(&as_slices(&samples), 4096),
Err(Error::InvalidParameter(_))
));
let tiny: Vec<Vec<u8>> = (0..12).map(|i| vec![b'x'; i % 7]).collect();
match train_dictionary(&as_slices(&tiny), 4096) {
Ok(dictionary) => {
EncoderDictionary::new(&dictionary).expect("parse");
}
Err(Error::InvalidParameter(_) | Error::SrcSizeWrong | Error::DstSizeTooSmall) => {}
Err(other) => panic!("unexpected error for tiny samples: {other:?}"),
}
let mut mixed = Vec::new();
for (index, record) in sample_records(40).into_iter().enumerate() {
if index % 4 == 0 {
mixed.push(Vec::new());
}
mixed.push(record);
}
let dictionary = train_dictionary(&as_slices(&mixed), 4096).expect("training");
EncoderDictionary::new(&dictionary).expect("parse");
}
#[test]
fn a_segment_size_beyond_a_sixteen_bit_counter_does_not_overflow() {
let samples: Vec<Vec<u8>> = (0..8).map(|_| vec![b'a'; 40_000]).collect();
let trained = train_dictionary_with_parameters(
&as_slices(&samples),
200_000,
DictionaryTrainingParameters {
k: 100_000,
d: 8,
steps: 1,
..Default::default()
},
)
.expect("training");
EncoderDictionary::new(trained.as_bytes()).expect("parse");
}
#[test]
fn an_extreme_segment_size_terminates_instead_of_wrapping() {
let samples = sample_records(40);
assert!(matches!(
train_dictionary_with_parameters(
&as_slices(&samples),
4096,
DictionaryTrainingParameters {
k: u32::MAX,
d: 8,
steps: 1,
..Default::default()
},
),
Err(Error::InvalidParameter(_))
));
assert!(
train_dictionary_with_parameters(
&as_slices(&samples),
4096,
DictionaryTrainingParameters {
k: 500_000_000,
d: 8,
steps: 1,
..Default::default()
},
)
.is_err()
);
}