use std::borrow::Cow;
use thiserror::Error;
use crate::compressor::core::rfc9841::serialized::{
self, Combination, ListRef, MAX_LISTS, NUM_CONTEXTS, SerializedDictionaryData, SerializedError,
SerializedLimits,
};
use crate::compressor::core::rfc9841::transform::{
MAX_STRINGLET_BYTES, MAX_WORD_LENGTH, TransformList as CoreTransformList,
TransformListError as CoreTransformListError, TransformScratch,
};
use crate::compressor::core::rfc9841::words::{
MAX_SIZE_BITS, MIN_WORD_LENGTH, NUM_ENCODED_LENGTHS, WordList as CoreWordList,
WordListError as CoreWordListError,
};
use super::DictionaryLimits;
impl From<DictionaryLimits> for SerializedLimits {
fn from(value: DictionaryLimits) -> Self {
Self {
max_total_bytes: value.max_serialized_bytes(),
max_prefix_bytes: value.max_prefix_bytes(),
max_word_lists: value.max_word_lists(),
max_word_bytes: value.max_word_bytes(),
max_transform_lists: value.max_transform_lists(),
max_transform_bytes: value.max_transform_bytes(),
max_combinations: value.max_combinations(),
}
}
}
pub const CONTEXTS: usize = NUM_CONTEXTS;
pub const MAX_LIST_COUNT: usize = MAX_LISTS;
pub const MAX_TRANSFORMS: usize = 255;
pub const MAX_STRINGLETS: usize = 256;
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct OmitLength(u8);
impl OmitLength {
pub const MIN: Self = Self(1);
pub const MAX: Self = Self(9);
}
impl TryFrom<u8> for OmitLength {
type Error = OmitLengthOutOfRange;
fn try_from(value: u8) -> Result<Self, Self::Error> {
if (Self::MIN.0..=Self::MAX.0).contains(&value) {
Ok(Self(value))
} else {
Err(OmitLengthOutOfRange { value })
}
}
}
impl From<OmitLength> for u8 {
fn from(value: OmitLength) -> Self {
value.0
}
}
impl std::fmt::Display for OmitLength {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(formatter, "{}", self.0)
}
}
#[derive(Error, Debug, Copy, Clone, Eq, PartialEq)]
#[error("an omit transform drops between 1 and 9 bytes, not {value}")]
pub struct OmitLengthOutOfRange {
pub value: u8,
}
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum TransformOperation {
Identity,
OmitLast(OmitLength),
FermentFirst,
FermentAll,
OmitFirst(OmitLength),
ShiftFirst(u16),
ShiftAll(u16),
}
impl TransformOperation {
const fn parameter(self) -> u16 {
match self {
Self::ShiftFirst(parameter) | Self::ShiftAll(parameter) => parameter,
_ => 0,
}
}
const fn shifts(self) -> bool {
matches!(self, Self::ShiftFirst(_) | Self::ShiftAll(_))
}
}
impl From<TransformOperation> for u8 {
fn from(value: TransformOperation) -> Self {
match value {
TransformOperation::Identity => 0,
TransformOperation::OmitLast(length) => length.0,
TransformOperation::FermentFirst => 10,
TransformOperation::FermentAll => 11,
TransformOperation::OmitFirst(length) => 11 + length.0,
TransformOperation::ShiftFirst(_) => 21,
TransformOperation::ShiftAll(_) => 22,
}
}
}
impl TryFrom<u8> for TransformOperation {
type Error = UndefinedTransformOperation;
fn try_from(value: u8) -> Result<Self, Self::Error> {
match value {
0 => Ok(Self::Identity),
1..=9 => Ok(Self::OmitLast(OmitLength(value))),
10 => Ok(Self::FermentFirst),
11 => Ok(Self::FermentAll),
12..=20 => Ok(Self::OmitFirst(OmitLength(value - 11))),
21 => Ok(Self::ShiftFirst(0)),
22 => Ok(Self::ShiftAll(0)),
_ => Err(UndefinedTransformOperation { value }),
}
}
}
#[derive(Error, Debug, Copy, Clone, Eq, PartialEq)]
#[error("RFC 9841 defines transform operations 0 to 22, not {value}")]
pub struct UndefinedTransformOperation {
pub value: u8,
}
#[derive(Copy, Clone, Debug, Default, Eq, PartialEq, Hash)]
pub enum ListSelector {
#[default]
Builtin,
Custom(u8),
}
impl From<u8> for ListSelector {
fn from(index: u8) -> Self {
Self::Custom(index)
}
}
impl From<ListSelector> for ListRef {
fn from(value: ListSelector) -> Self {
match value {
ListSelector::Builtin => Self::Builtin,
ListSelector::Custom(index) => Self::Custom(index),
}
}
}
impl From<ListRef> for ListSelector {
fn from(value: ListRef) -> Self {
match value {
ListRef::Builtin => Self::Builtin,
ListRef::Custom(index) => Self::Custom(index),
}
}
}
#[derive(Copy, Clone, Debug, Default, Eq, PartialEq, Hash)]
pub struct DictionaryCombination {
words: ListSelector,
transforms: ListSelector,
}
impl DictionaryCombination {
#[must_use]
pub const fn new(words: ListSelector, transforms: ListSelector) -> Self {
Self { words, transforms }
}
#[must_use]
pub const fn words(self) -> ListSelector {
self.words
}
#[must_use]
pub const fn transforms(self) -> ListSelector {
self.transforms
}
}
impl From<DictionaryCombination> for Combination {
fn from(value: DictionaryCombination) -> Self {
Self {
words: value.words.into(),
transforms: value.transforms.into(),
}
}
}
impl From<Combination> for DictionaryCombination {
fn from(value: Combination) -> Self {
Self {
words: value.words.into(),
transforms: value.transforms.into(),
}
}
}
#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
pub struct ContextMap([u8; CONTEXTS]);
impl ContextMap {
#[must_use]
pub const fn uniform(combination: u8) -> Self {
Self([combination; CONTEXTS])
}
pub const fn set(&mut self, context: usize, combination: u8) {
if context < CONTEXTS {
self.0[context] = combination;
}
}
}
impl Default for ContextMap {
fn default() -> Self {
Self::uniform(0)
}
}
impl From<[u8; CONTEXTS]> for ContextMap {
fn from(value: [u8; CONTEXTS]) -> Self {
Self(value)
}
}
impl From<ContextMap> for [u8; CONTEXTS] {
fn from(value: ContextMap) -> Self {
value.0
}
}
impl AsRef<[u8]> for ContextMap {
fn as_ref(&self) -> &[u8] {
&self.0
}
}
impl std::ops::Index<usize> for ContextMap {
type Output = u8;
fn index(&self, context: usize) -> &Self::Output {
&self.0[context]
}
}
#[derive(Debug, Clone)]
pub struct WordList {
inner: CoreWordList,
}
impl WordList {
#[must_use]
pub fn builtin() -> Self {
Self {
inner: CoreWordList::builtin(),
}
}
#[must_use]
pub fn builder() -> WordListBuilder {
WordListBuilder::default()
}
#[must_use]
pub const fn as_view(&self) -> WordListView<'_> {
WordListView::new(&self.inner)
}
#[must_use]
pub fn word_count(&self, length: usize) -> usize {
self.as_view().word_count(length)
}
#[must_use]
pub fn word(&self, length: usize, index: usize) -> &[u8] {
self.as_view().word(length, index)
}
#[must_use]
pub fn as_bytes(&self) -> &[u8] {
self.as_view().as_bytes()
}
pub(crate) fn into_inner(self) -> CoreWordList {
self.inner
}
pub(crate) const fn from_inner(inner: CoreWordList) -> Self {
Self { inner }
}
}
impl Default for WordList {
fn default() -> Self {
Self::builtin()
}
}
#[derive(Copy, Clone, Debug)]
pub struct WordListView<'a> {
inner: &'a CoreWordList,
}
impl<'a> WordListView<'a> {
pub(crate) const fn new(inner: &'a CoreWordList) -> Self {
Self { inner }
}
#[must_use]
pub fn word_count(&self, length: usize) -> usize {
self.inner.word_count(length)
}
#[must_use]
pub fn word(&self, length: usize, index: usize) -> &'a [u8] {
self.inner.word(length, index)
}
#[must_use]
pub fn as_bytes(&self) -> &'a [u8] {
self.inner.data()
}
#[must_use]
pub fn to_owned_list(&self) -> WordList {
WordList::from_inner(self.inner.clone())
}
}
#[derive(Debug, Default, Clone)]
pub struct WordListBuilder {
words: Vec<Box<[u8]>>,
}
impl WordListBuilder {
#[must_use]
pub fn add_word<B>(mut self, word: B) -> Self
where
B: AsRef<[u8]>,
{
self.words.push(Box::from(word.as_ref()));
self
}
pub fn build(self) -> Result<WordList, WordListError> {
if self.words.is_empty() {
return Err(WordListError::Empty);
}
let mut groups: Vec<Vec<Box<[u8]>>> = vec![Vec::new(); MAX_WORD_LENGTH + 1];
for word in self.words {
let length = word.len();
let Some(group) = groups.get_mut(length) else {
return Err(WordListError::WordLength {
length,
word: truncate_for_report(&word),
});
};
if length < MIN_WORD_LENGTH {
return Err(WordListError::WordLength {
length,
word: truncate_for_report(&word),
});
}
group.push(word);
}
let mut size_bits = [0u8; NUM_ENCODED_LENGTHS];
let mut data = Vec::new();
for length in MIN_WORD_LENGTH..=MAX_WORD_LENGTH {
let group = &groups[length];
if group.is_empty() {
continue;
}
let bits = ceil_log2(group.len());
if bits > MAX_SIZE_BITS {
return Err(WordListError::TooManyWords {
length,
count: group.len(),
limit: 1usize << MAX_SIZE_BITS,
});
}
size_bits[length - MIN_WORD_LENGTH] = bits;
let target = 1usize << bits;
for index in 0..target {
let word = match group.get(index).or_else(|| group.last()) {
Some(word) => word.as_ref(),
None => &[],
};
data.extend_from_slice(word);
}
}
CoreWordList::from_parts(&size_bits, Cow::Owned(data))
.map(WordList::from_inner)
.map_err(WordListError::from)
}
}
fn ceil_log2(count: usize) -> u8 {
if count <= 2 {
return 1;
}
let bits = usize::BITS - (count - 1).leading_zeros();
u8::try_from(bits).unwrap_or(u8::MAX)
}
fn truncate_for_report(word: &[u8]) -> Box<[u8]> {
Box::from(word.get(..word.len().min(16)).unwrap_or_default())
}
#[derive(Error, Debug, Clone, Eq, PartialEq)]
#[non_exhaustive]
pub enum WordListError {
#[error("a word list must hold at least one word")]
Empty,
#[error("a static dictionary word is 4 to 31 bytes, not {length}")]
WordLength {
length: usize,
word: Box<[u8]>,
},
#[error("length {length} holds {count} words, past the limit of {limit}")]
TooManyWords {
length: usize,
count: usize,
limit: usize,
},
#[error("the size bits describe {expected} bytes of words, but {found} were given")]
DataLength {
expected: usize,
found: usize,
},
#[error("length {length} claims 2^{bits} words, past the limit of 2^15")]
TooManySizeBits {
length: usize,
bits: u8,
},
}
impl From<CoreWordListError> for WordListError {
fn from(value: CoreWordListError) -> Self {
match value {
CoreWordListError::TooManySizeBits { length, bits } => {
Self::TooManySizeBits { length, bits }
}
CoreWordListError::DataLength { expected, found } => {
Self::DataLength { expected, found }
}
}
}
}
#[derive(Debug, Clone)]
pub struct TransformList {
inner: CoreTransformList,
}
impl TransformList {
#[must_use]
pub fn builtin() -> Self {
Self {
inner: CoreTransformList::builtin(),
}
}
#[must_use]
pub fn builder() -> TransformListBuilder {
TransformListBuilder::default()
}
#[must_use]
pub const fn as_view(&self) -> TransformListView<'_> {
TransformListView::new(&self.inner)
}
#[must_use]
pub fn len(&self) -> usize {
self.as_view().len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.as_view().is_empty()
}
#[must_use]
pub fn prefix(&self, index: usize) -> &[u8] {
self.as_view().prefix(index)
}
#[must_use]
pub fn suffix(&self, index: usize) -> &[u8] {
self.as_view().suffix(index)
}
#[must_use]
pub fn operation(&self, index: usize) -> Option<TransformOperation> {
self.as_view().operation(index)
}
#[must_use]
pub fn apply(&self, index: usize, word: &[u8]) -> Vec<u8> {
self.as_view().apply(index, word)
}
pub(crate) fn into_inner(self) -> CoreTransformList {
self.inner
}
pub(crate) const fn from_inner(inner: CoreTransformList) -> Self {
Self { inner }
}
}
impl Default for TransformList {
fn default() -> Self {
Self::builtin()
}
}
#[derive(Copy, Clone, Debug)]
pub struct TransformListView<'a> {
inner: &'a CoreTransformList,
}
impl<'a> TransformListView<'a> {
pub(crate) const fn new(inner: &'a CoreTransformList) -> Self {
Self { inner }
}
#[must_use]
pub fn len(&self) -> usize {
self.inner.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.inner.len() == 0
}
#[must_use]
pub fn prefix(&self, index: usize) -> &'a [u8] {
match self.inner.transform(index) {
Some((prefix, _, _)) => self.inner.stringlet(usize::from(prefix)),
None => &[],
}
}
#[must_use]
pub fn suffix(&self, index: usize) -> &'a [u8] {
match self.inner.transform(index) {
Some((_, _, suffix)) => self.inner.stringlet(usize::from(suffix)),
None => &[],
}
}
#[must_use]
pub fn operation(&self, index: usize) -> Option<TransformOperation> {
let (_, operation, _) = self.inner.transform(index)?;
match TransformOperation::try_from(operation).ok()? {
TransformOperation::ShiftFirst(_) => {
Some(TransformOperation::ShiftFirst(self.inner.parameter(index)))
}
TransformOperation::ShiftAll(_) => {
Some(TransformOperation::ShiftAll(self.inner.parameter(index)))
}
other => Some(other),
}
}
#[must_use]
pub fn apply(&self, index: usize, word: &[u8]) -> Vec<u8> {
let mut scratch = TransformScratch::default();
self.inner.apply(index, word, &mut scratch).to_vec()
}
#[must_use]
pub fn to_owned_list(&self) -> TransformList {
TransformList::from_inner(self.inner.clone())
}
}
#[derive(Debug, Default, Clone)]
pub struct TransformListBuilder {
transforms: Vec<PendingTransform>,
}
#[derive(Debug, Clone)]
struct PendingTransform {
prefix: Box<[u8]>,
operation: TransformOperation,
suffix: Box<[u8]>,
}
impl TransformListBuilder {
#[must_use]
pub fn add_transform<P, S>(
mut self,
prefix: P,
operation: TransformOperation,
suffix: S,
) -> Self
where
P: AsRef<[u8]>,
S: AsRef<[u8]>,
{
self.transforms.push(PendingTransform {
prefix: Box::from(prefix.as_ref()),
operation,
suffix: Box::from(suffix.as_ref()),
});
self
}
pub fn build(self) -> Result<TransformList, TransformListError> {
if self.transforms.len() > MAX_TRANSFORMS {
return Err(TransformListError::TooManyTransforms {
count: self.transforms.len(),
limit: MAX_TRANSFORMS,
});
}
let mut empty_slots: Vec<usize> = Vec::new();
let mut table: Vec<Box<[u8]>> = Vec::new();
let mut triples: Vec<u8> = Vec::with_capacity(self.transforms.len() * 3);
let mut params = Vec::with_capacity(self.transforms.len() * 2);
let mut shifts = false;
for transform in &self.transforms {
let operation = &transform.operation;
let prefix_id = intern(&mut table, &transform.prefix)?;
let suffix_id = intern(&mut table, &transform.suffix)?;
triples.push(prefix_id.unwrap_or_default());
triples.push(u8::from(*operation));
triples.push(suffix_id.unwrap_or_default());
if prefix_id.is_none() {
empty_slots.push(triples.len() - 3);
}
if suffix_id.is_none() {
empty_slots.push(triples.len() - 1);
}
params.extend_from_slice(&operation.parameter().to_le_bytes());
shifts |= operation.shifts();
}
let Ok(terminator) = u8::try_from(table.len()) else {
return Err(TransformListError::TooManyStringlets {
count: table.len() + 1,
limit: MAX_STRINGLETS,
});
};
for slot in empty_slots {
if let Some(id) = triples.get_mut(slot) {
*id = terminator;
}
}
let mut block = Vec::new();
for string in &table {
block.push(u8::try_from(string.len()).unwrap_or(u8::MAX));
block.extend_from_slice(string);
}
block.push(0);
CoreTransformList::from_parts(
Cow::Owned(block),
Cow::Owned(triples),
Cow::Owned(if shifts { params } else { Vec::new() }),
)
.map(TransformList::from_inner)
.map_err(TransformListError::from)
}
}
fn intern(table: &mut Vec<Box<[u8]>>, string: &[u8]) -> Result<Option<u8>, TransformListError> {
if string.len() > MAX_STRINGLET_BYTES {
return Err(TransformListError::StringletTooLong {
length: string.len(),
limit: MAX_STRINGLET_BYTES,
});
}
if string.is_empty() {
return Ok(None);
}
if let Some(index) = table.iter().position(|held| held.as_ref() == string) {
return Ok(u8::try_from(index).ok());
}
if table.len() >= MAX_STRINGLETS - 1 {
return Err(TransformListError::TooManyStringlets {
count: table.len() + 2,
limit: MAX_STRINGLETS,
});
}
table.push(Box::from(string));
Ok(u8::try_from(table.len() - 1).ok())
}
#[derive(Error, Debug, Clone, Eq, PartialEq)]
#[non_exhaustive]
pub enum TransformListError {
#[error("a prefix or suffix is at most {limit} bytes, not {length}")]
StringletTooLong {
length: usize,
limit: usize,
},
#[error("a transform list holds at most {limit} distinct strings, not {count}")]
TooManyStringlets {
count: usize,
limit: usize,
},
#[error("a transform list holds at most {limit} transforms, not {count}")]
TooManyTransforms {
count: usize,
limit: usize,
},
#[error("the prefix and suffix data is malformed: {detail}")]
MalformedStringlets {
detail: String,
},
#[error("a transform refers to something undefined: {detail}")]
UndefinedReference {
detail: String,
},
}
impl From<CoreTransformListError> for TransformListError {
fn from(value: CoreTransformListError) -> Self {
match value {
CoreTransformListError::TooManyTransforms { count } => Self::TooManyTransforms {
count,
limit: MAX_TRANSFORMS,
},
CoreTransformListError::TooManyStringlets => Self::TooManyStringlets {
count: MAX_STRINGLETS + 1,
limit: MAX_STRINGLETS,
},
CoreTransformListError::EmptyStringlets
| CoreTransformListError::StringletOverrun { .. }
| CoreTransformListError::MisplacedTerminator => Self::MalformedStringlets {
detail: value.to_string(),
},
CoreTransformListError::UndefinedStringlet { .. }
| CoreTransformListError::UndefinedOperation { .. }
| CoreTransformListError::ParameterLength { .. }
| CoreTransformListError::UnusedParameter { .. } => Self::UndefinedReference {
detail: value.to_string(),
},
}
}
}
#[derive(Debug, Clone, Default)]
pub struct SerializedDictionary {
inner: SerializedDictionaryData,
}
impl SerializedDictionary {
#[must_use]
pub fn builder() -> SerializedDictionaryBuilder {
SerializedDictionaryBuilder::default()
}
pub fn parse(
bytes: &[u8],
limits: DictionaryLimits,
) -> Result<Self, SerializedDictionaryError> {
let inner = serialized::parse_exact(bytes, &SerializedLimits::from(limits))?;
Ok(Self { inner })
}
#[must_use]
pub fn prefix(&self) -> &[u8] {
self.inner.prefix()
}
#[must_use]
pub fn word_list_count(&self) -> usize {
self.inner.word_lists().len()
}
#[must_use]
pub fn word_list(&self, index: usize) -> Option<WordListView<'_>> {
self.inner.word_lists().get(index).map(WordListView::new)
}
#[must_use]
pub fn transform_list_count(&self) -> usize {
self.inner.transform_lists().len()
}
#[must_use]
pub fn transform_list(&self, index: usize) -> Option<TransformListView<'_>> {
self.inner
.transform_lists()
.get(index)
.map(TransformListView::new)
}
#[must_use]
pub fn combination_count(&self) -> usize {
self.inner.combinations().len()
}
pub fn combinations(&self) -> impl ExactSizeIterator<Item = DictionaryCombination> + '_ {
self.inner
.combinations()
.iter()
.copied()
.map(DictionaryCombination::from)
}
#[must_use]
pub fn context_map(&self) -> Option<ContextMap> {
self.inner.context_map().copied().map(ContextMap::from)
}
#[must_use]
pub fn is_custom_static(&self) -> bool {
self.inner.is_custom_static()
}
#[must_use]
pub fn serialized_len(&self) -> usize {
self.inner.wire_len()
}
#[must_use]
pub fn to_bytes(&self) -> Vec<u8> {
let mut out = Vec::with_capacity(self.serialized_len());
self.write_to(&mut out);
out
}
pub fn write_to(&self, out: &mut Vec<u8>) {
let _ = self.inner.serialize(out);
}
pub(crate) const fn data(&self) -> &SerializedDictionaryData {
&self.inner
}
}
impl TryFrom<&[u8]> for SerializedDictionary {
type Error = SerializedDictionaryError;
fn try_from(bytes: &[u8]) -> Result<Self, Self::Error> {
Self::parse(bytes, DictionaryLimits::default())
}
}
#[derive(Debug, Default, Clone)]
pub struct SerializedDictionaryBuilder {
prefix: Option<Box<[u8]>>,
word_lists: Vec<WordList>,
transform_lists: Vec<TransformList>,
combinations: Vec<DictionaryCombination>,
context_map: Option<ContextMap>,
limits: DictionaryLimits,
}
impl SerializedDictionaryBuilder {
#[must_use]
pub fn with_prefix<B>(mut self, bytes: B) -> Self
where
B: Into<Box<[u8]>>,
{
self.prefix = Some(bytes.into());
self
}
#[must_use]
pub fn add_word_list(mut self, list: WordList) -> Self {
self.word_lists.push(list);
self
}
#[must_use]
pub fn add_transform_list(mut self, list: TransformList) -> Self {
self.transform_lists.push(list);
self
}
#[must_use]
pub fn add_combination(mut self, combination: DictionaryCombination) -> Self {
self.combinations.push(combination);
self
}
#[must_use]
pub const fn with_context_map(mut self, map: ContextMap) -> Self {
self.context_map = Some(map);
self
}
#[must_use]
pub const fn with_limits(mut self, limits: DictionaryLimits) -> Self {
self.limits = limits;
self
}
pub fn build(self) -> Result<SerializedDictionary, SerializedDictionaryError> {
let custom = !self.word_lists.is_empty() || !self.transform_lists.is_empty();
let mut combinations = self.combinations;
if custom && combinations.is_empty() {
combinations.push(DictionaryCombination::new(
if self.word_lists.is_empty() {
ListSelector::Builtin
} else {
ListSelector::Custom(0)
},
if self.transform_lists.is_empty() {
ListSelector::Builtin
} else {
ListSelector::Custom(0)
},
));
}
let inner = SerializedDictionaryData::assemble(
self.prefix,
self.word_lists
.into_iter()
.map(WordList::into_inner)
.collect(),
self.transform_lists
.into_iter()
.map(TransformList::into_inner)
.collect(),
combinations.into_iter().map(Combination::from).collect(),
self.context_map.map(<[u8; CONTEXTS]>::from),
&SerializedLimits::from(self.limits),
)?;
Ok(SerializedDictionary { inner })
}
}
#[derive(Error, Debug, Clone, Eq, PartialEq)]
#[non_exhaustive]
pub enum SerializedDictionaryError {
#[error("a serialized dictionary starts with 0x91 0x00, not {found:02X?}")]
BadMagic {
found: Box<[u8]>,
},
#[error("the dictionary ends after {length} bytes, mid-{field}")]
Truncated {
field: &'static str,
length: usize,
},
#[error("{detail}")]
Malformed {
detail: String,
},
#[error("{detail}")]
UndefinedReference {
detail: String,
},
#[error("the dictionary's {what} of {found} exceeds the limit of {limit}")]
LimitExceeded {
what: &'static str,
found: u64,
limit: u64,
},
#[error("{extra} byte(s) follow the end of the dictionary")]
TrailingBytes {
extra: usize,
},
}
impl From<SerializedError> for SerializedDictionaryError {
fn from(value: SerializedError) -> Self {
match value {
SerializedError::BadMagic { found } => Self::BadMagic {
found: found.into_boxed_slice(),
},
SerializedError::Truncated { field, position } => Self::Truncated {
field,
length: position,
},
SerializedError::LimitExceeded { what, found, limit } => {
Self::LimitExceeded { what, found, limit }
}
SerializedError::TrailingBytes { extra } => Self::TrailingBytes { extra },
SerializedError::UndefinedList { .. }
| SerializedError::UndefinedCombination { .. }
| SerializedError::NoCombinations => Self::UndefinedReference {
detail: value.to_string(),
},
other => Self::Malformed {
detail: other.to_string(),
},
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn rich() -> SerializedDictionary {
let mut map = ContextMap::uniform(0);
map.set(5, 1);
map.set(CONTEXTS, 1);
SerializedDictionary::builder()
.with_prefix(&b"prefix bytes"[..])
.add_word_list(
WordList::builder()
.add_word(b"alpha")
.add_word(b"bravo")
.build()
.expect("well formed"),
)
.add_transform_list(
TransformList::builder()
.add_transform(b"", TransformOperation::Identity, b"")
.add_transform(b"<", TransformOperation::FermentAll, b">")
.build()
.expect("well formed"),
)
.add_combination(DictionaryCombination::new(
ListSelector::Custom(0),
ListSelector::Custom(0),
))
.add_combination(DictionaryCombination::default())
.with_context_map(map)
.with_limits(DictionaryLimits::default())
.build()
.expect("the parts are consistent")
}
#[test]
fn an_omit_length_accepts_one_to_nine() {
assert_eq!(u8::from(OmitLength::try_from(1).expect("in range")), 1);
assert_eq!(u8::from(OmitLength::MIN), 1);
assert_eq!(u8::from(OmitLength::MAX), 9);
assert_eq!(
OmitLength::try_from(0),
Err(OmitLengthOutOfRange { value: 0 })
);
assert_eq!(
OmitLength::try_from(10),
Err(OmitLengthOutOfRange { value: 10 })
);
assert_eq!(OmitLength::MAX.to_string(), "9");
assert_eq!(
OmitLengthOutOfRange { value: 0 }.to_string(),
"an omit transform drops between 1 and 9 bytes, not 0"
);
}
#[test]
fn every_operation_id_round_trips() {
for id in 0..=22u8 {
let operation = TransformOperation::try_from(id).expect("defined");
assert_eq!(u8::from(operation), id, "id {id}");
}
assert_eq!(
TransformOperation::try_from(23),
Err(UndefinedTransformOperation { value: 23 })
);
assert_eq!(
UndefinedTransformOperation { value: 23 }.to_string(),
"RFC 9841 defines transform operations 0 to 22, not 23"
);
}
#[test]
fn a_shift_carries_its_parameter_and_the_rest_carry_none() {
assert_eq!(TransformOperation::ShiftFirst(7).parameter(), 7);
assert_eq!(TransformOperation::ShiftAll(7).parameter(), 7);
assert_eq!(TransformOperation::Identity.parameter(), 0);
assert!(TransformOperation::ShiftAll(0).shifts());
assert!(!TransformOperation::FermentAll.shifts());
}
#[test]
fn a_list_selector_converts_both_ways() {
assert_eq!(ListSelector::from(2), ListSelector::Custom(2));
assert_eq!(ListSelector::default(), ListSelector::Builtin);
assert_eq!(ListRef::from(ListSelector::Builtin), ListRef::Builtin);
assert_eq!(ListRef::from(ListSelector::Custom(1)), ListRef::Custom(1));
assert_eq!(ListSelector::from(ListRef::Builtin), ListSelector::Builtin);
assert_eq!(
ListSelector::from(ListRef::Custom(1)),
ListSelector::Custom(1)
);
}
#[test]
fn a_combination_converts_both_ways() {
let public = DictionaryCombination::new(ListSelector::Custom(1), ListSelector::Builtin);
assert_eq!(public.words(), ListSelector::Custom(1));
assert_eq!(public.transforms(), ListSelector::Builtin);
assert_eq!(
DictionaryCombination::from(Combination::from(public)),
public
);
assert_eq!(
DictionaryCombination::default().words(),
ListSelector::Builtin
);
}
#[test]
fn a_context_map_reads_back_what_was_set() {
let mut map = ContextMap::uniform(2);
map.set(0, 1);
map.set(CONTEXTS, 9);
assert_eq!(map[0], 1);
assert_eq!(map[1], 2);
assert_eq!(map.as_ref().len(), CONTEXTS);
assert_eq!(ContextMap::default()[0], 0);
assert_eq!(<[u8; CONTEXTS]>::from(map)[0], 1);
assert_eq!(ContextMap::from([3u8; CONTEXTS])[63], 3);
}
#[test]
#[should_panic(expected = "the len is 64")]
fn indexing_a_context_map_past_its_end_panics() {
let _ = ContextMap::default()[CONTEXTS];
}
#[test]
fn the_builtin_lists_are_the_defaults() {
assert_eq!(
WordList::default().word(4, 0),
WordList::builtin().word(4, 0)
);
assert_eq!(
TransformList::default().len(),
TransformList::builtin().len()
);
assert_eq!(WordList::builtin().as_bytes().len(), 122_784);
assert!(!TransformList::builtin().is_empty());
}
#[test]
fn a_word_list_is_read_the_same_owned_and_borrowed() {
let list = WordList::builder()
.add_word(b"alpha")
.add_word(b"bravo")
.build()
.expect("well formed");
let view = list.as_view();
assert_eq!(view.word_count(5), list.word_count(5));
assert_eq!(view.word(5, 0), list.word(5, 0));
assert_eq!(view.as_bytes(), list.as_bytes());
assert_eq!(view.to_owned_list().as_bytes(), list.as_bytes());
}
#[test]
fn a_word_list_builder_refuses_what_the_format_cannot_hold() {
assert_eq!(
WordList::builder().build().err(),
Some(WordListError::Empty)
);
assert!(matches!(
WordList::builder().add_word(b"ab").build(),
Err(WordListError::WordLength { length: 2, .. })
));
let long = [b'z'; 32];
assert!(matches!(
WordList::builder().add_word(long).build(),
Err(WordListError::WordLength { length: 32, .. })
));
}
#[test]
fn a_word_list_group_past_the_format_ceiling_is_refused() {
let mut builder = WordList::builder();
for index in 0..=(1usize << MAX_SIZE_BITS) {
builder = builder.add_word(format!("{index:08}"));
}
assert!(matches!(
builder.build(),
Err(WordListError::TooManyWords { length: 8, .. })
));
}
#[test]
fn a_word_reported_in_an_error_is_truncated() {
let word = [b'q'; 40];
assert_eq!(truncate_for_report(&word).len(), 16);
assert_eq!(truncate_for_report(b"short").len(), 5);
}
#[test]
fn a_group_of_one_word_is_stored_as_two() {
assert_eq!(ceil_log2(1), 1);
assert_eq!(ceil_log2(2), 1);
assert_eq!(ceil_log2(3), 2);
assert_eq!(ceil_log2(4), 2);
assert_eq!(ceil_log2(5), 3);
assert_eq!(ceil_log2(1 << 15), 15);
}
#[test]
fn a_transform_list_is_read_the_same_owned_and_borrowed() {
let list = TransformList::builder()
.add_transform(b"<", TransformOperation::FermentAll, b">")
.build()
.expect("well formed");
let view = list.as_view();
assert_eq!(view.len(), list.len());
assert_eq!(view.is_empty(), list.is_empty());
assert_eq!(view.prefix(0), list.prefix(0));
assert_eq!(view.suffix(0), list.suffix(0));
assert_eq!(view.operation(0), list.operation(0));
assert_eq!(view.apply(0, b"loud"), list.apply(0, b"loud"));
assert_eq!(view.to_owned_list().len(), list.len());
assert_eq!(list.apply(0, b"loud"), b"<LOUD>".to_vec());
}
#[test]
fn an_index_past_a_transform_list_reads_as_nothing() {
let list = TransformList::builder()
.add_transform(b"<", TransformOperation::Identity, b">")
.build()
.expect("well formed");
assert_eq!(list.prefix(9), b"");
assert_eq!(list.suffix(9), b"");
assert_eq!(list.operation(9), None);
assert_eq!(list.apply(9, b"word"), b"word".to_vec());
}
#[test]
fn an_empty_transform_list_is_empty() {
let list = TransformList::builder().build().expect("well formed");
assert!(list.is_empty());
assert_eq!(list.len(), 0);
}
#[test]
fn a_shift_transform_keeps_its_parameter_through_a_round_trip() {
let list = TransformList::builder()
.add_transform(b"", TransformOperation::ShiftFirst(0x1234), b"")
.add_transform(b"", TransformOperation::ShiftAll(0x8000), b"")
.add_transform(b"", TransformOperation::Identity, b"")
.build()
.expect("well formed");
assert_eq!(
list.operation(0),
Some(TransformOperation::ShiftFirst(0x1234))
);
assert_eq!(
list.operation(1),
Some(TransformOperation::ShiftAll(0x8000))
);
assert_eq!(list.operation(2), Some(TransformOperation::Identity));
}
#[test]
fn a_transform_list_builder_refuses_what_the_format_cannot_hold() {
let long = vec![b'p'; MAX_STRINGLET_BYTES + 1];
assert!(matches!(
TransformList::builder()
.add_transform(&long[..], TransformOperation::Identity, b"")
.build(),
Err(TransformListError::StringletTooLong { .. })
));
let mut builder = TransformList::builder();
for index in 0..=MAX_TRANSFORMS {
builder = builder.add_transform(b"", TransformOperation::Identity, b"");
let _ = index;
}
assert!(matches!(
builder.build(),
Err(TransformListError::TooManyTransforms { .. })
));
}
#[test]
fn too_many_distinct_strings_are_refused() {
let mut builder = TransformList::builder();
for index in 0..MAX_STRINGLETS / 2 {
builder = builder.add_transform(
format!("p{index:04}"),
TransformOperation::Identity,
format!("s{index:04}"),
);
}
assert!(matches!(
builder.build(),
Err(TransformListError::TooManyStringlets { .. })
));
}
#[test]
fn identical_strings_are_stored_once() {
let list = TransformList::builder()
.add_transform(b"same", TransformOperation::Identity, b"same")
.add_transform(b"same", TransformOperation::FermentAll, b"same")
.build()
.expect("well formed");
assert_eq!(list.prefix(0), b"same");
assert_eq!(list.suffix(1), b"same");
assert_eq!(list.apply(1, b"x"), b"sameXsame".to_vec());
}
#[test]
fn a_dictionary_reports_everything_it_was_built_from() {
let dictionary = rich();
assert_eq!(dictionary.prefix(), b"prefix bytes");
assert_eq!(dictionary.word_list_count(), 1);
assert_eq!(dictionary.transform_list_count(), 1);
assert_eq!(dictionary.combination_count(), 2);
assert!(dictionary.is_custom_static());
assert_eq!(dictionary.context_map().map(|map| map[5]), Some(1));
assert_eq!(
dictionary.word_list(0).map(|list| list.word_count(5)),
Some(2)
);
assert_eq!(dictionary.transform_list(0).map(|list| list.len()), Some(2));
assert!(dictionary.word_list(1).is_none());
assert!(dictionary.transform_list(1).is_none());
assert_eq!(dictionary.combinations().len(), 2);
assert_eq!(
dictionary.combinations().next().map(|c| c.words()),
Some(ListSelector::Custom(0))
);
}
#[test]
fn writing_to_a_buffer_matches_the_owned_bytes() {
let dictionary = rich();
let mut out = vec![0xAA];
dictionary.write_to(&mut out);
assert_eq!(out[0], 0xAA);
assert_eq!(&out[1..], dictionary.to_bytes().as_slice());
assert_eq!(out.len() - 1, dictionary.serialized_len());
assert!(!dictionary.data().prefix().is_empty());
}
#[test]
fn parsing_under_explicit_limits_agrees_with_the_default_ones() {
let bytes = rich().to_bytes();
let parsed = SerializedDictionary::parse(&bytes, DictionaryLimits::default())
.expect("within the defaults");
assert_eq!(
parsed.to_bytes(),
SerializedDictionary::try_from(&bytes[..])
.expect("the same bytes")
.to_bytes()
);
}
#[test]
fn a_builder_with_one_custom_list_implies_its_combination() {
let with_words = SerializedDictionary::builder()
.add_word_list(
WordList::builder()
.add_word(b"word")
.build()
.expect("valid"),
)
.build()
.expect("valid");
let with_transforms = SerializedDictionary::builder()
.add_transform_list(
TransformList::builder()
.add_transform(b"", TransformOperation::Identity, b"")
.build()
.expect("valid"),
)
.build()
.expect("valid");
assert_eq!(
with_words.combinations().next().map(|c| c.words()),
Some(ListSelector::Custom(0))
);
assert_eq!(
with_words.combinations().next().map(|c| c.transforms()),
Some(ListSelector::Builtin)
);
assert_eq!(
with_transforms.combinations().next().map(|c| c.words()),
Some(ListSelector::Builtin)
);
assert_eq!(
with_transforms
.combinations()
.next()
.map(|c| c.transforms()),
Some(ListSelector::Custom(0))
);
}
#[test]
fn an_empty_dictionary_carries_nothing() {
let dictionary = SerializedDictionary::default();
assert!(dictionary.prefix().is_empty());
assert_eq!(dictionary.word_list_count(), 0);
assert_eq!(dictionary.transform_list_count(), 0);
assert_eq!(dictionary.combination_count(), 0);
assert!(dictionary.context_map().is_none());
assert!(!dictionary.is_custom_static());
assert_eq!(dictionary.to_bytes(), vec![0x91, 0x00, 0, 0, 0]);
}
#[test]
fn each_codec_error_lifts_into_its_public_shape() {
let cases = [
(
SerializedError::BadMagic { found: vec![1, 2] },
"a serialized dictionary starts with 0x91 0x00",
),
(
SerializedError::Truncated {
field: "magic",
position: 1,
},
"mid-magic",
),
(
SerializedError::TrailingBytes { extra: 3 },
"follow the end of the dictionary",
),
(SerializedError::NoCombinations, "at least one combination"),
(
SerializedError::LimitExceeded {
what: "word data",
found: 9,
limit: 4,
},
"exceeds the limit of 4",
),
(
SerializedError::NotABoolean {
field: "CONTEXT_ENABLED",
value: 2,
},
"must be 0 or 1",
),
];
for (error, fragment) in cases {
let lifted = SerializedDictionaryError::from(error);
assert!(
lifted.to_string().contains(fragment),
"{lifted} does not mention {fragment}"
);
}
}
#[test]
fn each_word_list_error_lifts_into_its_public_shape() {
assert_eq!(
WordListError::from(CoreWordListError::TooManySizeBits {
length: 4,
bits: 16,
}),
WordListError::TooManySizeBits {
length: 4,
bits: 16
}
);
assert_eq!(
WordListError::from(CoreWordListError::DataLength {
expected: 8,
found: 7,
}),
WordListError::DataLength {
expected: 8,
found: 7,
}
);
}
#[test]
fn each_transform_list_error_lifts_into_its_public_shape() {
assert!(matches!(
TransformListError::from(CoreTransformListError::TooManyTransforms { count: 300 }),
TransformListError::TooManyTransforms { count: 300, .. }
));
assert!(matches!(
TransformListError::from(CoreTransformListError::TooManyStringlets),
TransformListError::TooManyStringlets { .. }
));
assert!(matches!(
TransformListError::from(CoreTransformListError::EmptyStringlets),
TransformListError::MalformedStringlets { .. }
));
assert!(matches!(
TransformListError::from(CoreTransformListError::MisplacedTerminator),
TransformListError::MalformedStringlets { .. }
));
assert!(matches!(
TransformListError::from(CoreTransformListError::StringletOverrun { length: 4 }),
TransformListError::MalformedStringlets { .. }
));
assert!(matches!(
TransformListError::from(CoreTransformListError::UndefinedOperation {
index: 0,
operation: 30,
}),
TransformListError::UndefinedReference { .. }
));
assert!(matches!(
TransformListError::from(CoreTransformListError::UndefinedStringlet {
index: 0,
stringlet: 3,
count: 1,
}),
TransformListError::UndefinedReference { .. }
));
assert!(matches!(
TransformListError::from(CoreTransformListError::ParameterLength {
expected: 2,
found: 0,
}),
TransformListError::UndefinedReference { .. }
));
assert!(matches!(
TransformListError::from(CoreTransformListError::UnusedParameter {
index: 0,
parameter: 7,
}),
TransformListError::UndefinedReference { .. }
));
}
#[test]
fn the_public_limits_lower_into_the_codec_ones() {
let limits = DictionaryLimits::default()
.with_max_serialized_bytes(11)
.with_max_prefix_bytes(12)
.with_max_word_lists(3)
.with_max_word_bytes(13)
.with_max_transform_lists(4)
.with_max_transform_bytes(14)
.with_max_combinations(5);
let lowered = SerializedLimits::from(limits);
assert_eq!(lowered.max_total_bytes, 11);
assert_eq!(lowered.max_prefix_bytes, 12);
assert_eq!(lowered.max_word_lists, 3);
assert_eq!(lowered.max_word_bytes, 13);
assert_eq!(lowered.max_transform_lists, 4);
assert_eq!(lowered.max_transform_bytes, 14);
assert_eq!(lowered.max_combinations, 5);
}
#[test]
fn interning_reports_the_first_use_of_each_string() {
let mut table = Vec::new();
assert_eq!(intern(&mut table, b""), Ok(None));
assert_eq!(intern(&mut table, b"a"), Ok(Some(0)));
assert_eq!(intern(&mut table, b"b"), Ok(Some(1)));
assert_eq!(intern(&mut table, b"a"), Ok(Some(0)));
assert_eq!(table.len(), 2);
}
}