pub mod defaults;
pub mod error;
pub mod profile;
pub mod toml;
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::config::error::ConfigError;
pub const DEFAULT_TEXT_CODEC: &str = "brotli";
pub const DEFAULT_BINARY_CODEC: &str = "lz4";
pub const DEFAULT_METADATA_CODEC: &str = "brotli";
pub const DEFAULT_METADATA_QUALITY: u8 = 5;
pub const DEFAULT_INLINE_THRESHOLD: u16 = 4096;
pub const DEFAULT_AVG_CHUNK_SIZE: u32 = 8192;
pub const DEFAULT_MIN_CHUNK_SIZE: u32 = 1024;
pub const DEFAULT_MAX_CHUNK_SIZE: u32 = 65_536;
pub const DEFAULT_TOURNAMENT_MIN_SIZE: u32 = 256;
pub const DEFAULT_TOURNAMENT_SKIP_BINARY: bool = true;
pub const DEFAULT_AEAD: &str = "chacha20-poly1305";
pub const DEFAULT_KEY_WRAP: &str = "x25519-hkdf";
pub const DEFAULT_DICT_ENABLED: bool = true;
pub const DEFAULT_DICT_MIN_CLASS_SIZE: u32 = 100;
pub const DEFAULT_DICT_MAX_SIZE: u32 = 65_536;
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct WriteConfig {
pub defaults: Defaults,
#[serde(default, rename = "categorizer")]
pub categorizers: Vec<CategorizerConfig>,
pub chunking: ChunkingConfig,
pub tournament: TournamentConfig,
#[serde(default)]
pub codec_tunables: CodecTunables,
#[serde(default)]
pub mode: ImageMode,
#[serde(default = "default_write_codec")]
pub write_codec: String,
#[serde(default)]
pub turnover_threshold: u32,
#[serde(default)]
pub skip_chunking: bool,
pub encryption: EncryptionConfig,
pub dictionaries: DictionaryConfig,
}
fn default_write_codec() -> String {
"lz4".into()
}
fn default_metadata_externalize_threshold() -> usize {
crate::METADATA_EXTERNALIZE_THRESHOLD
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct Defaults {
pub text_codec: String,
pub binary_codec: String,
pub metadata_codec: String,
pub metadata_quality: u8,
#[serde(default = "default_metadata_externalize_threshold")]
pub metadata_externalize_threshold: usize,
#[serde(default = "default_true")]
pub shared_inline: bool,
pub inline_threshold: u16,
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct CategorizerConfig {
pub name: String,
#[serde(default)]
pub extensions: Vec<String>,
#[serde(default)]
pub magic_bytes: Vec<u8>,
pub codec: String,
#[serde(default)]
pub max_size: Option<u32>,
#[serde(default = "default_true")]
pub enabled: bool,
}
fn default_true() -> bool {
true
}
#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
pub struct ChunkingConfig {
#[serde(default = "default_chunker_name")]
pub name: String,
#[serde(default)]
pub avg_chunk_size: u32,
#[serde(default)]
pub min_chunk_size: u32,
#[serde(default)]
pub max_chunk_size: u32,
}
fn default_chunker_name() -> String {
"fastcdc".into()
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct TournamentConfig {
pub codecs: Vec<String>,
pub min_size_threshold: u32,
pub skip_for_binary: bool,
#[serde(default = "default_short_circuit_threshold")]
pub short_circuit_threshold: u32,
}
fn default_short_circuit_threshold() -> u32 {
250
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct EncryptionConfig {
pub aead: String,
pub key_wrap: String,
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct DictionaryConfig {
pub enabled: bool,
pub min_class_size: u32,
pub max_dict_size: u32,
#[serde(default = "default_trainer")]
pub trainer: String,
}
fn default_trainer() -> String {
"frequency".into()
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
pub enum ImageMode {
#[default]
ReadOnly,
ReadWrite(RWMode),
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
pub enum RWMode {
AppendOnly,
#[default]
UpdateInPlace,
CopyOnWrite,
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
pub struct CodecTunables {
#[serde(default)]
pub ppmd7: Ppmd7Tunables,
#[serde(default)]
pub ppmd8: Ppmd8Tunables,
#[serde(default)]
pub brotli: BrotliTunables,
#[serde(default)]
pub lzma: LzmaTunables,
#[serde(default)]
pub bzip2: Bzip2Tunables,
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct Ppmd7Tunables {
pub order: u8,
pub memory_budget_mb: u32,
}
impl Default for Ppmd7Tunables {
fn default() -> Self {
Self {
order: 4,
memory_budget_mb: 80,
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct Ppmd8Tunables {
pub order: u8,
pub memory_budget_mb: u32,
}
impl Default for Ppmd8Tunables {
fn default() -> Self {
Self {
order: 6,
memory_budget_mb: 64,
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct BrotliTunables {
pub quality: u8,
pub window: u8,
}
impl Default for BrotliTunables {
fn default() -> Self {
Self {
quality: 11,
window: 22,
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct LzmaTunables {
pub lc: u8,
pub lp: u8,
pub pb: u8,
pub dict_size_mb: u32,
pub use_optimal_parser: bool,
}
impl Default for LzmaTunables {
fn default() -> Self {
Self {
lc: 3,
lp: 0,
pb: 2,
dict_size_mb: 16,
use_optimal_parser: false,
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct Bzip2Tunables {
pub block_size_kb: u32,
}
impl Default for Bzip2Tunables {
fn default() -> Self {
Self { block_size_kb: 900 }
}
}
impl WriteConfig {
#[must_use]
pub fn default_v0_1() -> Self {
Self {
defaults: Defaults {
text_codec: DEFAULT_TEXT_CODEC.to_string(),
binary_codec: DEFAULT_BINARY_CODEC.to_string(),
metadata_codec: DEFAULT_METADATA_CODEC.to_string(),
metadata_quality: DEFAULT_METADATA_QUALITY,
metadata_externalize_threshold: crate::METADATA_EXTERNALIZE_THRESHOLD,
shared_inline: true,
inline_threshold: DEFAULT_INLINE_THRESHOLD,
},
categorizers: Vec::new(),
chunking: ChunkingConfig {
name: "fastcdc".into(),
avg_chunk_size: DEFAULT_AVG_CHUNK_SIZE,
min_chunk_size: DEFAULT_MIN_CHUNK_SIZE,
max_chunk_size: DEFAULT_MAX_CHUNK_SIZE,
},
tournament: TournamentConfig {
codecs: vec![
"store".to_string(),
"lz4".to_string(),
"zstd".to_string(),
"brotli".to_string(),
],
min_size_threshold: DEFAULT_TOURNAMENT_MIN_SIZE,
skip_for_binary: DEFAULT_TOURNAMENT_SKIP_BINARY,
short_circuit_threshold: default_short_circuit_threshold(),
},
encryption: EncryptionConfig {
aead: DEFAULT_AEAD.to_string(),
key_wrap: DEFAULT_KEY_WRAP.to_string(),
},
dictionaries: DictionaryConfig {
enabled: DEFAULT_DICT_ENABLED,
min_class_size: DEFAULT_DICT_MIN_CLASS_SIZE,
max_dict_size: DEFAULT_DICT_MAX_SIZE,
trainer: "frequency".into(),
},
codec_tunables: CodecTunables::default(),
mode: ImageMode::ReadOnly,
write_codec: default_write_codec(),
turnover_threshold: 0,
skip_chunking: false,
}
}
#[must_use]
pub fn from_profile(name: &str) -> Option<Self> {
profile::select(name)
}
#[must_use]
pub fn with_text_codec(mut self, codec: &str) -> Self {
self.defaults.text_codec = codec.into();
self
}
#[must_use]
pub fn with_binary_codec(mut self, codec: &str) -> Self {
self.defaults.binary_codec = codec.into();
self
}
#[must_use]
pub fn with_chunk_size(mut self, size: u32) -> Self {
self.chunking.avg_chunk_size = size;
self
}
#[must_use]
pub fn with_mode(mut self, mode: ImageMode) -> Self {
self.mode = mode;
self
}
#[must_use]
pub fn with_brotli_quality(mut self, quality: u8) -> Self {
self.codec_tunables.brotli.quality = quality;
self
}
pub fn build(self) -> Result<Self, ConfigError> {
self.validate()?;
Ok(self)
}
pub fn validate(&self) -> Result<(), ConfigError> {
if self.chunking.min_chunk_size > self.chunking.avg_chunk_size {
return Err(ConfigError::InvalidValue {
field: "chunking.min_chunk_size".into(),
reason: format!(
"min_chunk_size ({}) > avg_chunk_size ({})",
self.chunking.min_chunk_size, self.chunking.avg_chunk_size
),
});
}
if self.chunking.avg_chunk_size > self.chunking.max_chunk_size {
return Err(ConfigError::InvalidValue {
field: "chunking.avg_chunk_size".into(),
reason: format!(
"avg_chunk_size ({}) > max_chunk_size ({})",
self.chunking.avg_chunk_size, self.chunking.max_chunk_size
),
});
}
if self.defaults.metadata_quality < 1 || self.defaults.metadata_quality > 11 {
return Err(ConfigError::InvalidValue {
field: "defaults.metadata_quality".into(),
reason: format!(
"metadata_quality ({}) out of range 1..=11",
self.defaults.metadata_quality
),
});
}
let ceiling = limnifs_core::metadata_reference::DEFAULT_INLINE_METADATA_MAX_BYTES as usize;
if self.defaults.metadata_externalize_threshold == 0
|| self.defaults.metadata_externalize_threshold > ceiling
{
return Err(ConfigError::InvalidValue {
field: "defaults.metadata_externalize_threshold".into(),
reason: format!(
"metadata_externalize_threshold ({}) must be within 1..={ceiling} (the reader inline ceiling; larger inline metadata is unreadable)",
self.defaults.metadata_externalize_threshold
),
});
}
if self.tournament.short_circuit_threshold > 1000 {
return Err(ConfigError::InvalidValue {
field: "tournament.short_circuit_threshold".into(),
reason: format!(
"short_circuit_threshold ({}) out of range 0..=1000 (per-mille)",
self.tournament.short_circuit_threshold
),
});
}
let mut names_seen: BTreeMap<&str, ()> = BTreeMap::new();
for rule in &self.categorizers {
if !names_seen
.insert(rule.name.as_str(), ())
.map_or(true, |()| false)
{
return Err(ConfigError::DuplicateCategorizer(rule.name.clone()));
}
}
Ok(())
}
pub fn codec_registry(&self) -> Result<CodecRegistry, ConfigError> {
let mut registry = CodecRegistry::default();
registry.insert("store", 0x00);
registry.insert("lz4", 0x01);
registry.insert("lz4-hc", 0x13);
registry.insert("zstd", 0x02);
registry.insert("xz", 0x03);
registry.insert("brotli", 0x04);
registry.insert("deflate", 0x05);
registry.insert("snappy", 0x06);
registry.insert("flac", 0x07);
registry.insert("ricepp", 0x08);
registry.insert("fsst+brotli", 0x09);
registry.insert("shuffle+lz4", 0x0A);
registry.insert("zpaq", 0x0B);
registry.insert("ppmd", 0x0C);
registry.insert("glza", 0x0D);
registry.insert("shuffle+zstd", 0x0E);
registry.insert("bitshuffle+lz4", 0x0F);
registry.insert("bzip2", 0x10);
registry.insert("deflate64", 0x11);
registry.insert("libdeflate", 0x14);
if !registry.contains_name(&self.defaults.text_codec) {
return Err(ConfigError::UnknownCodec(self.defaults.text_codec.clone()));
}
if !registry.contains_name(&self.defaults.binary_codec) {
return Err(ConfigError::UnknownCodec(
self.defaults.binary_codec.clone(),
));
}
if !registry.contains_name(&self.defaults.metadata_codec) {
return Err(ConfigError::UnknownCodec(
self.defaults.metadata_codec.clone(),
));
}
for rule in &self.categorizers {
if !registry.contains_name(&rule.codec) {
return Err(ConfigError::UnknownCodec(rule.codec.clone()));
}
}
for codec in &self.tournament.codecs {
if !registry.contains_name(codec) {
return Err(ConfigError::UnknownCodec(codec.clone()));
}
}
Ok(registry)
}
pub fn text_codec_id(&self) -> Result<u8, ConfigError> {
let registry = self.codec_registry()?;
Ok(registry
.lookup_by_name(&self.defaults.text_codec)
.unwrap_or(0x04))
}
pub fn binary_codec_id(&self) -> Result<u8, ConfigError> {
let registry = self.codec_registry()?;
Ok(registry
.lookup_by_name(&self.defaults.binary_codec)
.unwrap_or(0x01))
}
#[must_use]
pub fn to_core_tunables(&self) -> limnifs_core::codec::CodecTunables {
limnifs_core::codec::CodecTunables {
quality: self.codec_tunables.brotli.quality,
ppmd_order: self
.codec_tunables
.ppmd7
.order
.max(self.codec_tunables.ppmd8.order),
ppmd7_budget: (self.codec_tunables.ppmd7.memory_budget_mb as usize)
.saturating_mul(1024 * 1024),
ppmd8_budget: (self.codec_tunables.ppmd8.memory_budget_mb as usize)
.saturating_mul(1024 * 1024),
bzip2_block_kb: self.codec_tunables.bzip2.block_size_kb,
lzma_dict_mb: self.codec_tunables.lzma.dict_size_mb,
}
}
pub fn metadata_codec_id(&self) -> Result<u8, ConfigError> {
let registry = self.codec_registry()?;
Ok(registry
.lookup_by_name(&self.defaults.metadata_codec)
.unwrap_or(0x04))
}
}
#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
pub struct CodecRegistry {
by_name: BTreeMap<String, u8>,
by_id: BTreeMap<u8, String>,
}
impl CodecRegistry {
pub fn insert(&mut self, name: &str, id: u8) {
self.by_name.insert(name.to_string(), id);
self.by_id.insert(id, name.to_string());
}
#[must_use]
pub fn lookup_by_name(&self, name: &str) -> Option<u8> {
self.by_name.get(name).copied()
}
#[must_use]
pub fn lookup_by_id(&self, id: u8) -> Option<&str> {
self.by_id.get(&id).map(String::as_str)
}
#[must_use]
pub fn contains_name(&self, name: &str) -> bool {
self.by_name.contains_key(name)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn default_v0_1_validates() {
let config = WriteConfig::default_v0_1();
config.validate().expect("v0.1 default should validate");
}
#[test]
fn default_v0_1_codec_ids() {
let config = WriteConfig::default_v0_1();
assert_eq!(config.text_codec_id().unwrap(), 0x04); assert_eq!(config.binary_codec_id().unwrap(), 0x01); assert_eq!(config.metadata_codec_id().unwrap(), 0x04); }
#[test]
fn rejects_invalid_chunking() {
let mut config = WriteConfig::default_v0_1();
config.chunking.min_chunk_size = 16_000;
config.chunking.avg_chunk_size = 8_000;
assert!(config.validate().is_err());
}
#[test]
fn rejects_invalid_quality() {
let mut config = WriteConfig::default_v0_1();
config.defaults.metadata_quality = 12;
assert!(config.validate().is_err());
}
#[test]
fn rejects_duplicate_categorizer_names() {
let mut config = WriteConfig::default_v0_1();
config.categorizers.push(CategorizerConfig {
name: "dna".into(),
extensions: vec!["fasta".into()],
magic_bytes: vec![],
codec: "glza".into(),
max_size: None,
enabled: true,
});
config.categorizers.push(CategorizerConfig {
name: "dna".into(),
extensions: vec!["fa".into()],
magic_bytes: vec![],
codec: "glza".into(),
max_size: None,
enabled: true,
});
assert!(config.validate().is_err());
}
#[test]
fn rejects_unknown_codec() {
let mut config = WriteConfig::default_v0_1();
config.defaults.text_codec = "does-not-exist".into();
assert!(config.codec_registry().is_err());
}
}