use super::core::rfc9841::context::{Budget, SharedContextInner};
use super::shared::SharedBrotliError;
use thiserror::Error;
#[cfg(feature = "experimental")]
mod serialized;
#[cfg(feature = "experimental")]
pub use serialized::{
CONTEXTS, ContextMap, DictionaryCombination, ListSelector, MAX_LIST_COUNT, MAX_STRINGLETS,
MAX_TRANSFORMS, OmitLength, OmitLengthOutOfRange, SerializedDictionary,
SerializedDictionaryBuilder, SerializedDictionaryError, TransformList, TransformListBuilder,
TransformListError, TransformListView, TransformOperation, UndefinedTransformOperation,
WordList, WordListBuilder, WordListError, WordListView,
};
const MAX_ATTACHMENTS: usize = 15;
#[derive(Debug)]
pub struct PreparedDictionary {
inner: SharedContextInner,
}
impl PreparedDictionary {
#[must_use]
pub fn attachment_count(&self) -> usize {
self.inner.dictionaries().prefix().segment_count()
}
#[must_use]
pub fn source_bytes(&self) -> usize {
let bytes = self.inner.dictionaries().source_size();
#[cfg(feature = "experimental")]
let bytes = bytes
+ self
.inner
.static_index
.as_ref()
.map_or(0, |index| index.source_bytes);
bytes
}
#[must_use]
pub fn retained_bytes(&self) -> usize {
self.inner.allocated_size()
}
#[must_use]
pub fn backward_distance(&self, offset: u64, max_backward: u64) -> Option<u64> {
self.inner
.dictionaries()
.prefix()
.distance_of(offset, max_backward)
}
#[must_use]
pub fn prefix_offset(&self, distance: u64, max_backward: u64) -> Option<u64> {
self.inner
.dictionaries()
.prefix()
.address_of(distance, max_backward)
}
#[cfg(feature = "diagnostics")]
#[must_use]
pub fn longest_match(&self, input: &[u8]) -> Option<PrefixMatch> {
self.inner
.longest_prefix_match(input)
.map(PrefixMatch::from)
}
pub(crate) const fn inner(&self) -> &SharedContextInner {
&self.inner
}
}
#[cfg(feature = "diagnostics")]
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub struct PrefixMatch {
offset: u64,
length: usize,
}
#[cfg(feature = "diagnostics")]
impl PrefixMatch {
#[must_use]
pub const fn prefix_offset(self) -> u64 {
self.offset
}
#[must_use]
pub const fn length(self) -> usize {
self.length
}
}
#[cfg(feature = "diagnostics")]
impl From<super::core::rfc9841::context::PrefixMatch> for PrefixMatch {
fn from(value: super::core::rfc9841::context::PrefixMatch) -> Self {
Self {
offset: value.offset,
length: value.length,
}
}
}
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub struct DictionaryLimits {
max_source_bytes: u64,
max_prefix_bytes: u64,
max_retained_bytes: u64,
max_attachments: usize,
#[cfg(feature = "experimental")]
max_serialized_bytes: u64,
#[cfg(feature = "experimental")]
max_word_bytes: u64,
#[cfg(feature = "experimental")]
max_word_lists: usize,
#[cfg(feature = "experimental")]
max_transform_bytes: u64,
#[cfg(feature = "experimental")]
max_transform_lists: usize,
#[cfg(feature = "experimental")]
max_combinations: usize,
#[cfg(feature = "experimental")]
max_transformed_word_bytes: u64,
#[cfg(feature = "experimental")]
max_static_entries: u64,
}
impl DictionaryLimits {
#[cfg(feature = "experimental")]
#[must_use]
pub const fn with_max_transformed_word_bytes(mut self, bytes: u64) -> Self {
self.max_transformed_word_bytes = bytes;
self
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn max_transformed_word_bytes(self) -> u64 {
self.max_transformed_word_bytes
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn with_max_static_entries(mut self, count: u64) -> Self {
self.max_static_entries = count;
self
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn max_static_entries(self) -> u64 {
self.max_static_entries
}
pub const DEFAULT_MAX_SOURCE_BYTES: u64 = 64 << 20;
pub const DEFAULT_MAX_PREFIX_BYTES: u64 = 64 << 20;
pub const DEFAULT_MAX_RETAINED_BYTES: u64 = 1 << 30;
pub const DEFAULT_MAX_ATTACHMENTS: usize = MAX_ATTACHMENTS;
#[cfg(feature = "experimental")]
pub const DEFAULT_MAX_SERIALIZED_BYTES: u64 = 128 << 20;
#[cfg(feature = "experimental")]
pub const DEFAULT_MAX_WORD_BYTES: u64 = 16 << 20;
#[cfg(feature = "experimental")]
pub const DEFAULT_MAX_WORD_LISTS: usize = MAX_LIST_COUNT;
#[cfg(feature = "experimental")]
pub const DEFAULT_MAX_TRANSFORM_BYTES: u64 = 8 << 20;
#[cfg(feature = "experimental")]
pub const DEFAULT_MAX_TRANSFORM_LISTS: usize = MAX_LIST_COUNT;
#[cfg(feature = "experimental")]
pub const DEFAULT_MAX_COMBINATIONS: usize = MAX_LIST_COUNT;
#[must_use]
pub const fn with_max_source_bytes(mut self, bytes: u64) -> Self {
self.max_source_bytes = bytes;
self
}
#[must_use]
pub const fn max_source_bytes(self) -> u64 {
self.max_source_bytes
}
#[must_use]
pub const fn with_max_prefix_bytes(mut self, bytes: u64) -> Self {
self.max_prefix_bytes = bytes;
self
}
#[must_use]
pub const fn max_prefix_bytes(self) -> u64 {
self.max_prefix_bytes
}
#[must_use]
pub const fn with_max_retained_bytes(mut self, bytes: u64) -> Self {
self.max_retained_bytes = bytes;
self
}
#[must_use]
pub const fn max_retained_bytes(self) -> u64 {
self.max_retained_bytes
}
#[must_use]
pub const fn with_max_attachments(mut self, attachments: usize) -> Self {
self.max_attachments = attachments;
self
}
#[must_use]
pub const fn max_attachments(self) -> usize {
self.max_attachments
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn with_max_serialized_bytes(mut self, bytes: u64) -> Self {
self.max_serialized_bytes = bytes;
self
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn max_serialized_bytes(self) -> u64 {
self.max_serialized_bytes
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn with_max_word_bytes(mut self, bytes: u64) -> Self {
self.max_word_bytes = bytes;
self
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn max_word_bytes(self) -> u64 {
self.max_word_bytes
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn with_max_word_lists(mut self, lists: usize) -> Self {
self.max_word_lists = lists;
self
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn max_word_lists(self) -> usize {
self.max_word_lists
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn with_max_transform_bytes(mut self, bytes: u64) -> Self {
self.max_transform_bytes = bytes;
self
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn max_transform_bytes(self) -> u64 {
self.max_transform_bytes
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn with_max_transform_lists(mut self, lists: usize) -> Self {
self.max_transform_lists = lists;
self
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn max_transform_lists(self) -> usize {
self.max_transform_lists
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn with_max_combinations(mut self, combinations: usize) -> Self {
self.max_combinations = combinations;
self
}
#[cfg(feature = "experimental")]
#[must_use]
pub const fn max_combinations(self) -> usize {
self.max_combinations
}
}
impl Default for DictionaryLimits {
fn default() -> Self {
Self {
max_source_bytes: Self::DEFAULT_MAX_SOURCE_BYTES,
max_prefix_bytes: Self::DEFAULT_MAX_PREFIX_BYTES,
max_retained_bytes: Self::DEFAULT_MAX_RETAINED_BYTES,
max_attachments: Self::DEFAULT_MAX_ATTACHMENTS,
#[cfg(feature = "experimental")]
max_serialized_bytes: Self::DEFAULT_MAX_SERIALIZED_BYTES,
#[cfg(feature = "experimental")]
max_word_bytes: Self::DEFAULT_MAX_WORD_BYTES,
#[cfg(feature = "experimental")]
max_word_lists: Self::DEFAULT_MAX_WORD_LISTS,
#[cfg(feature = "experimental")]
max_transform_bytes: Self::DEFAULT_MAX_TRANSFORM_BYTES,
#[cfg(feature = "experimental")]
max_transform_lists: Self::DEFAULT_MAX_TRANSFORM_LISTS,
#[cfg(feature = "experimental")]
max_combinations: Self::DEFAULT_MAX_COMBINATIONS,
#[cfg(feature = "experimental")]
max_transformed_word_bytes: 128 << 20,
#[cfg(feature = "experimental")]
max_static_entries: 8_000_000,
}
}
}
impl From<DictionaryLimits> for Budget {
fn from(value: DictionaryLimits) -> Self {
Self {
max_total_source_bytes: value.max_source_bytes,
max_prefix_bytes: value.max_prefix_bytes,
max_allocated_bytes: value.max_retained_bytes,
max_attachments: value.max_attachments,
}
}
}
#[derive(Debug, Default)]
pub struct DictionaryBuilder {
limits: DictionaryLimits,
attachments: Vec<Box<[u8]>>,
#[cfg(feature = "experimental")]
custom_static: Option<crate::compressor::core::rfc9841::serialized::SerializedDictionaryData>,
}
impl DictionaryBuilder {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn add_prefix<B>(mut self, bytes: B) -> Self
where
B: Into<Box<[u8]>>,
{
self.attachments.push(bytes.into());
self
}
#[cfg(feature = "experimental")]
#[must_use]
pub fn add_serialized(mut self, dictionary: &SerializedDictionary) -> Self {
if dictionary.data().has_prefix() {
self.attachments.push(Box::from(dictionary.prefix()));
}
if dictionary.is_custom_static() {
self.custom_static = Some(dictionary.data().clone());
}
self
}
#[must_use]
pub const fn with_limits(mut self, limits: DictionaryLimits) -> Self {
self.limits = limits;
self
}
pub fn build(self) -> Result<PreparedDictionary, DictionaryError> {
#[cfg(feature = "experimental")]
let has_static = self.custom_static.is_some();
#[cfg(not(feature = "experimental"))]
let has_static = false;
if !has_static && self.attachments.iter().all(|bytes| bytes.is_empty()) {
return Err(DictionaryError::Empty);
}
#[cfg(feature = "experimental")]
if let Some(data) = &self.custom_static {
let word_bytes = data
.word_lists()
.iter()
.map(|w| w.data().len() as u64)
.sum::<u64>();
let transform_bytes = data
.transform_lists()
.iter()
.map(|t| t.wire_len() as u64)
.sum::<u64>();
for (what, found, limit) in [
(
"serialized bytes",
data.wire_len() as u64,
self.limits.max_serialized_bytes,
),
(
"word lists",
data.word_lists().len() as u64,
self.limits.max_word_lists as u64,
),
("word bytes", word_bytes, self.limits.max_word_bytes),
(
"transform lists",
data.transform_lists().len() as u64,
self.limits.max_transform_lists as u64,
),
(
"transform bytes",
transform_bytes,
self.limits.max_transform_bytes,
),
(
"combinations",
data.combinations().len() as u64,
self.limits.max_combinations as u64,
),
] {
if found > limit {
return Err(DictionaryError::LimitExceeded { what, found, limit });
}
}
let source = self.attachments.iter().map(|p| p.len() as u64).sum::<u64>()
+ word_bytes
+ transform_bytes;
if source > self.limits.max_source_bytes {
return Err(DictionaryError::TooLarge {
bytes: source,
limit: self.limits.max_source_bytes,
});
}
}
let budget = Budget::from(self.limits);
#[cfg(feature = "experimental")]
let description_bytes = self
.custom_static
.as_ref()
.map_or(0, |d| d.allocation_bound()) as u64;
#[cfg(feature = "experimental")]
let construction_bytes = description_bytes
.saturating_add((self.attachments.capacity() * size_of::<Box<[u8]>>()) as u64);
#[cfg(feature = "experimental")]
let budget = Budget {
max_allocated_bytes: budget
.max_allocated_bytes
.saturating_sub(construction_bytes),
..budget
};
let inner = SharedContextInner::new(self.attachments, &budget).map_err(|error| {
#[cfg(feature = "experimental")]
let error = match error {
SharedBrotliError::SharedContextTooLarge { bytes, .. } => {
SharedBrotliError::SharedContextTooLarge {
bytes: bytes.saturating_add(construction_bytes),
limit: self.limits.max_retained_bytes,
}
}
other => other,
};
DictionaryError::from_core(error)
})?;
#[cfg(feature = "experimental")]
let inner = {
let mut inner = inner;
if let Some(data) = self.custom_static {
inner.static_index = Some(
super::core::rfc9841::static_index::StaticIndex::prepare(
&data,
self.limits
.max_retained_bytes
.saturating_sub(inner.allocated_size() as u64)
.saturating_sub(description_bytes),
self.limits.max_transformed_word_bytes,
self.limits.max_static_entries,
)
.map_err(DictionaryError::from_core)?,
);
}
inner
};
Ok(PreparedDictionary { inner })
}
}
#[derive(Error, Debug, Copy, Clone, Eq, PartialEq)]
#[non_exhaustive]
pub enum DictionaryError {
#[cfg(feature = "experimental")]
#[error("dictionary {what} of {found} exceeds the limit of {limit}")]
LimitExceeded {
what: &'static str,
found: u64,
limit: u64,
},
#[error("a dictionary with no bytes in it cannot shorten a stream")]
Empty,
#[error("a dictionary holds at most {limit} attachments, not {attached}")]
TooManyAttachments {
attached: usize,
limit: usize,
},
#[error("{bytes} dictionary bytes exceed the limit of {limit}")]
TooLarge {
bytes: u64,
limit: u64,
},
#[error("preparing a dictionary would allocate {bytes} bytes, past the limit of {limit}")]
PreparationTooLarge {
bytes: u64,
limit: u64,
},
}
impl DictionaryError {
const fn from_core(error: SharedBrotliError) -> Self {
match error {
SharedBrotliError::TooManyPrefixDictionaries { attached, limit } => {
Self::TooManyAttachments { attached, limit }
}
SharedBrotliError::DictionaryTooLarge { bytes, limit } => {
Self::TooLarge { bytes, limit }
}
SharedBrotliError::SharedContextTooLarge { bytes, limit } => {
Self::PreparationTooLarge { bytes, limit }
}
SharedBrotliError::UnsupportedLargeWindow { .. } => Self::Empty,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_attachment_limit_is_configurable_and_never_above_the_format_one() {
let limits = DictionaryLimits::default();
assert_eq!(limits.max_attachments(), MAX_ATTACHMENTS);
assert_eq!(limits.with_max_attachments(2).max_attachments(), 2);
assert_eq!(
limits.max_source_bytes(),
DictionaryLimits::DEFAULT_MAX_SOURCE_BYTES
);
let outcome = DictionaryBuilder::new()
.add_prefix(&b"one"[..])
.add_prefix(&b"two"[..])
.with_limits(limits.with_max_attachments(1))
.build();
assert!(matches!(
outcome,
Err(DictionaryError::TooManyAttachments {
attached: 2,
limit: 1
})
));
let mut builder = DictionaryBuilder::new().with_limits(limits.with_max_attachments(999));
for _ in 0..=MAX_ATTACHMENTS {
builder = builder.add_prefix(&b"payload"[..]);
}
assert!(matches!(
builder.build(),
Err(DictionaryError::TooManyAttachments {
limit: MAX_ATTACHMENTS,
..
})
));
}
#[test]
fn a_prepared_dictionary_is_send_and_sync() {
const fn assert_send<T: Send>() {}
const fn assert_sync<T: Sync>() {}
assert_send::<PreparedDictionary>();
assert_sync::<PreparedDictionary>();
assert_send::<DictionaryBuilder>();
}
#[test]
fn an_empty_builder_is_refused_rather_than_built() {
assert_eq!(
DictionaryBuilder::new().build().unwrap_err(),
DictionaryError::Empty
);
assert_eq!(
DictionaryBuilder::new()
.add_prefix(&b""[..])
.add_prefix(&b""[..])
.build()
.unwrap_err(),
DictionaryError::Empty
);
}
#[test]
fn attachment_order_is_call_order() {
let dictionary = DictionaryBuilder::new()
.add_prefix(&b"oldest"[..])
.add_prefix(&b"middle"[..])
.add_prefix(&b"newest"[..])
.build()
.expect("prepared");
let prefix = dictionary.inner().dictionaries().prefix();
assert_eq!(prefix.segment(0), b"oldest");
assert_eq!(prefix.segment(1), b"middle");
assert_eq!(prefix.segment(2), b"newest");
assert_eq!(dictionary.attachment_count(), 3);
assert_eq!(dictionary.source_bytes(), 18);
}
#[test]
fn every_owned_byte_form_is_accepted() {
let boxed: Box<[u8]> = b"boxed".to_vec().into_boxed_slice();
let dictionary = DictionaryBuilder::new()
.add_prefix(b"vector".to_vec())
.add_prefix(boxed)
.add_prefix(&b"borrowed"[..])
.build()
.expect("prepared");
assert_eq!(dictionary.attachment_count(), 3);
assert_eq!(dictionary.source_bytes(), 6 + 5 + 8);
}
#[test]
fn the_format_limit_on_attachments_is_enforced() {
let mut builder = DictionaryBuilder::new();
for _ in 0..MAX_ATTACHMENTS {
builder = builder.add_prefix(&b"payload"[..]);
}
assert_eq!(
builder
.build()
.expect("fifteen is legal")
.attachment_count(),
MAX_ATTACHMENTS
);
let mut builder = DictionaryBuilder::new();
for _ in 0..=MAX_ATTACHMENTS {
builder = builder.add_prefix(&b"payload"[..]);
}
assert_eq!(
builder.build().unwrap_err(),
DictionaryError::TooManyAttachments {
attached: 16,
limit: 15
}
);
}
#[test]
fn every_limit_is_checked_before_anything_is_built() {
let too_long = DictionaryBuilder::new()
.add_prefix(&b"nine byte"[..])
.with_limits(DictionaryLimits::default().with_max_prefix_bytes(8))
.build();
assert_eq!(
too_long.unwrap_err(),
DictionaryError::TooLarge { bytes: 9, limit: 8 }
);
let too_much_source = DictionaryBuilder::new()
.add_prefix(&b"four"[..])
.add_prefix(&b"five!"[..])
.with_limits(DictionaryLimits::default().with_max_source_bytes(8))
.build();
assert_eq!(
too_much_source.unwrap_err(),
DictionaryError::TooLarge { bytes: 9, limit: 8 }
);
let too_much_index = DictionaryBuilder::new()
.add_prefix(&b"eight!!!"[..])
.with_limits(DictionaryLimits::default().with_max_retained_bytes(1024))
.build();
assert!(matches!(
too_much_index.unwrap_err(),
DictionaryError::PreparationTooLarge { limit: 1024, .. }
));
}
#[test]
fn a_refusal_leaves_nothing_behind() {
assert!(
DictionaryBuilder::new()
.add_prefix(&b"nine byte"[..])
.with_limits(DictionaryLimits::default().with_max_prefix_bytes(8))
.build()
.is_err()
);
assert!(
DictionaryBuilder::new()
.add_prefix(&b"nine byte"[..])
.build()
.is_ok()
);
}
#[test]
fn the_limits_expose_their_documented_defaults() {
let limits = DictionaryLimits::default();
assert_eq!(
limits.max_source_bytes(),
DictionaryLimits::DEFAULT_MAX_SOURCE_BYTES
);
assert_eq!(
limits.max_prefix_bytes(),
DictionaryLimits::DEFAULT_MAX_PREFIX_BYTES
);
assert_eq!(
limits.max_retained_bytes(),
DictionaryLimits::DEFAULT_MAX_RETAINED_BYTES
);
let tightened = limits
.with_max_source_bytes(1)
.with_max_prefix_bytes(2)
.with_max_retained_bytes(3);
assert_eq!(tightened.max_source_bytes(), 1);
assert_eq!(tightened.max_prefix_bytes(), 2);
assert_eq!(tightened.max_retained_bytes(), 3);
assert_ne!(tightened, limits);
let budget = Budget::from(tightened);
assert_eq!(budget.max_total_source_bytes, 1);
assert_eq!(budget.max_prefix_bytes, 2);
assert_eq!(budget.max_allocated_bytes, 3);
}
#[test]
fn addressing_and_its_inverse_agree() {
let dictionary = DictionaryBuilder::new()
.add_prefix(&b"oldest"[..])
.add_prefix(&b"newest"[..])
.build()
.expect("prepared");
let max_backward = 1u64 << 20;
for offset in 0..12u64 {
let distance = dictionary
.backward_distance(offset, max_backward)
.expect("inside the prefix");
assert!(distance > max_backward);
assert_eq!(
dictionary.prefix_offset(distance, max_backward),
Some(offset)
);
}
assert_eq!(
dictionary.backward_distance(11, max_backward),
Some(max_backward + 1)
);
assert_eq!(
dictionary.backward_distance(0, max_backward),
Some(max_backward + 12)
);
assert_eq!(dictionary.backward_distance(12, max_backward), None);
assert_eq!(dictionary.prefix_offset(max_backward, max_backward), None);
assert_eq!(
dictionary.prefix_offset(max_backward + 13, max_backward),
None
);
assert_eq!(dictionary.prefix_offset(u64::MAX, u64::MAX), None);
}
#[test]
fn retained_bytes_cover_the_sources_and_the_indexes() {
let short = DictionaryBuilder::new()
.add_prefix(&b"tiny"[..])
.build()
.expect("prepared");
let indexed = DictionaryBuilder::new()
.add_prefix(&b"long enough to be indexed"[..])
.build()
.expect("prepared");
assert!(short.retained_bytes() > short.source_bytes());
assert!(indexed.retained_bytes() > short.retained_bytes());
}
#[test]
fn a_preparation_error_says_what_was_refused() {
assert!(DictionaryError::Empty.to_string().contains("no bytes"));
assert!(
DictionaryError::TooManyAttachments {
attached: 16,
limit: 15
}
.to_string()
.contains("15")
);
assert!(
DictionaryError::TooLarge {
bytes: 99,
limit: 8
}
.to_string()
.contains("99")
);
assert!(
DictionaryError::PreparationTooLarge {
bytes: 99,
limit: 8
}
.to_string()
.contains("99")
);
}
#[cfg(feature = "diagnostics")]
#[test]
fn the_longest_match_diagnostic_reports_what_it_found() {
let dictionary = DictionaryBuilder::new()
.add_prefix(&b"the quick brown fox"[..])
.build()
.expect("prepared");
let found = dictionary
.longest_match(b"quick brown foxes")
.expect("a match");
assert_eq!(found.prefix_offset(), 4);
assert_eq!(found.length(), 15);
assert!(dictionary.longest_match(b"nothing here at all").is_none());
assert!(dictionary.longest_match(b"the").is_none());
}
}