1pub mod defaults;
28pub mod error;
29pub mod profile;
30pub mod toml;
31
32use std::collections::BTreeMap;
33
34use serde::{Deserialize, Serialize};
35
36use crate::config::error::ConfigError;
37
38pub const DEFAULT_TEXT_CODEC: &str = "brotli";
41pub const DEFAULT_BINARY_CODEC: &str = "lz4";
44pub const DEFAULT_METADATA_CODEC: &str = "brotli";
47pub const DEFAULT_METADATA_QUALITY: u8 = 5;
49pub const DEFAULT_INLINE_THRESHOLD: u16 = 4096;
51pub const DEFAULT_AVG_CHUNK_SIZE: u32 = 8192;
53pub const DEFAULT_MIN_CHUNK_SIZE: u32 = 1024;
55pub const DEFAULT_MAX_CHUNK_SIZE: u32 = 65_536;
57pub const DEFAULT_TOURNAMENT_MIN_SIZE: u32 = 256;
59pub const DEFAULT_TOURNAMENT_SKIP_BINARY: bool = true;
61pub const DEFAULT_AEAD: &str = "chacha20-poly1305";
63pub const DEFAULT_KEY_WRAP: &str = "x25519-hkdf";
65pub const DEFAULT_DICT_ENABLED: bool = true;
67pub const DEFAULT_DICT_MIN_CLASS_SIZE: u32 = 100;
69pub const DEFAULT_DICT_MAX_SIZE: u32 = 65_536;
71
72#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
76pub struct WriteConfig {
77 pub defaults: Defaults,
79 #[serde(default, rename = "categorizer")]
81 pub categorizers: Vec<CategorizerConfig>,
82 pub chunking: ChunkingConfig,
84 pub tournament: TournamentConfig,
86 #[serde(default)]
88 pub codec_tunables: CodecTunables,
89 #[serde(default)]
91 pub mode: ImageMode,
92 #[serde(default = "default_write_codec")]
95 pub write_codec: String,
96 #[serde(default)]
99 pub turnover_threshold: u32,
100 #[serde(default)]
104 pub skip_chunking: bool,
105 pub encryption: EncryptionConfig,
107 pub dictionaries: DictionaryConfig,
109}
110
111fn default_write_codec() -> String {
112 "lz4".into()
113}
114
115#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
117pub struct Defaults {
118 pub text_codec: String,
119 pub binary_codec: String,
120 pub metadata_codec: String,
121 pub metadata_quality: u8,
122 pub inline_threshold: u16,
124}
125
126#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
128pub struct CategorizerConfig {
129 pub name: String,
131 #[serde(default)]
133 pub extensions: Vec<String>,
134 #[serde(default)]
136 pub magic_bytes: Vec<u8>,
137 pub codec: String,
139 #[serde(default)]
141 pub max_size: Option<u32>,
142 #[serde(default = "default_true")]
144 pub enabled: bool,
145}
146
147fn default_true() -> bool {
148 true
149}
150
151#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
153pub struct ChunkingConfig {
154 #[serde(default = "default_chunker_name")]
159 pub name: String,
160 #[serde(default)]
161 pub avg_chunk_size: u32,
162 #[serde(default)]
163 pub min_chunk_size: u32,
164 #[serde(default)]
165 pub max_chunk_size: u32,
166}
167
168fn default_chunker_name() -> String {
169 "fastcdc".into()
170}
171
172#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
174pub struct TournamentConfig {
175 pub codecs: Vec<String>,
179 pub min_size_threshold: u32,
181 pub skip_for_binary: bool,
183 #[serde(default = "default_short_circuit_threshold")]
195 pub short_circuit_threshold: u32,
196}
197
198fn default_short_circuit_threshold() -> u32 {
199 250
200}
201
202#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
204pub struct EncryptionConfig {
205 pub aead: String,
207 pub key_wrap: String,
209}
210
211#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
213pub struct DictionaryConfig {
214 pub enabled: bool,
215 pub min_class_size: u32,
216 pub max_dict_size: u32,
217 #[serde(default = "default_trainer")]
224 pub trainer: String,
225}
226
227fn default_trainer() -> String {
228 "frequency".into()
229}
230
231#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
236pub enum ImageMode {
237 #[default]
241 ReadOnly,
242 ReadWrite(RWMode),
244}
245
246#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
248pub enum RWMode {
249 AppendOnly,
252 #[default]
255 UpdateInPlace,
256 CopyOnWrite,
260}
261
262#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
275pub struct CodecTunables {
276 #[serde(default)]
277 pub ppmd7: Ppmd7Tunables,
278 #[serde(default)]
279 pub ppmd8: Ppmd8Tunables,
280 #[serde(default)]
281 pub brotli: BrotliTunables,
282 #[serde(default)]
283 pub lzma: LzmaTunables,
284 #[serde(default)]
285 pub bzip2: Bzip2Tunables,
286}
287
288#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
290pub struct Ppmd7Tunables {
291 pub order: u8,
292 pub memory_budget_mb: u32,
293}
294
295impl Default for Ppmd7Tunables {
296 fn default() -> Self {
297 Self {
298 order: 4,
299 memory_budget_mb: 80,
300 }
301 }
302}
303
304#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
306pub struct Ppmd8Tunables {
307 pub order: u8,
308 pub memory_budget_mb: u32,
309}
310
311impl Default for Ppmd8Tunables {
312 fn default() -> Self {
313 Self {
314 order: 6,
315 memory_budget_mb: 64,
316 }
317 }
318}
319
320#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
322pub struct BrotliTunables {
323 pub quality: u8,
324 pub window: u8,
325}
326
327impl Default for BrotliTunables {
328 fn default() -> Self {
329 Self {
330 quality: 11,
331 window: 22,
332 }
333 }
334}
335
336#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
338pub struct LzmaTunables {
339 pub lc: u8,
340 pub lp: u8,
341 pub pb: u8,
342 pub dict_size_mb: u32,
343 pub use_optimal_parser: bool,
344}
345
346impl Default for LzmaTunables {
347 fn default() -> Self {
348 Self {
349 lc: 3,
350 lp: 0,
351 pb: 2,
352 dict_size_mb: 16,
353 use_optimal_parser: false,
354 }
355 }
356}
357
358#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
360pub struct Bzip2Tunables {
361 pub block_size_kb: u32,
362}
363
364impl Default for Bzip2Tunables {
365 fn default() -> Self {
366 Self { block_size_kb: 900 }
367 }
368}
369
370impl WriteConfig {
371 #[must_use]
375 pub fn default_v0_1() -> Self {
376 Self {
377 defaults: Defaults {
378 text_codec: DEFAULT_TEXT_CODEC.to_string(),
379 binary_codec: DEFAULT_BINARY_CODEC.to_string(),
380 metadata_codec: DEFAULT_METADATA_CODEC.to_string(),
381 metadata_quality: DEFAULT_METADATA_QUALITY,
382 inline_threshold: DEFAULT_INLINE_THRESHOLD,
383 },
384 categorizers: Vec::new(),
385 chunking: ChunkingConfig {
386 name: "fastcdc".into(),
387 avg_chunk_size: DEFAULT_AVG_CHUNK_SIZE,
388 min_chunk_size: DEFAULT_MIN_CHUNK_SIZE,
389 max_chunk_size: DEFAULT_MAX_CHUNK_SIZE,
390 },
391 tournament: TournamentConfig {
392 codecs: vec![
393 "store".to_string(),
394 "lz4".to_string(),
395 "zstd".to_string(),
396 "brotli".to_string(),
397 ],
398 min_size_threshold: DEFAULT_TOURNAMENT_MIN_SIZE,
399 skip_for_binary: DEFAULT_TOURNAMENT_SKIP_BINARY,
400 short_circuit_threshold: default_short_circuit_threshold(),
401 },
402 encryption: EncryptionConfig {
403 aead: DEFAULT_AEAD.to_string(),
404 key_wrap: DEFAULT_KEY_WRAP.to_string(),
405 },
406 dictionaries: DictionaryConfig {
407 enabled: DEFAULT_DICT_ENABLED,
408 min_class_size: DEFAULT_DICT_MIN_CLASS_SIZE,
409 max_dict_size: DEFAULT_DICT_MAX_SIZE,
410 trainer: "frequency".into(),
411 },
412 codec_tunables: CodecTunables::default(),
413 mode: ImageMode::ReadOnly,
414 write_codec: default_write_codec(),
415 turnover_threshold: 0,
416 skip_chunking: false,
417 }
418 }
419
420 #[must_use]
423 pub fn from_profile(name: &str) -> Option<Self> {
424 profile::select(name)
425 }
426
427 #[must_use]
429 pub fn with_text_codec(mut self, codec: &str) -> Self {
430 self.defaults.text_codec = codec.into();
431 self
432 }
433
434 #[must_use]
436 pub fn with_binary_codec(mut self, codec: &str) -> Self {
437 self.defaults.binary_codec = codec.into();
438 self
439 }
440
441 #[must_use]
443 pub fn with_chunk_size(mut self, size: u32) -> Self {
444 self.chunking.avg_chunk_size = size;
445 self
446 }
447
448 #[must_use]
450 pub fn with_mode(mut self, mode: ImageMode) -> Self {
451 self.mode = mode;
452 self
453 }
454
455 #[must_use]
457 pub fn with_brotli_quality(mut self, quality: u8) -> Self {
458 self.codec_tunables.brotli.quality = quality;
459 self
460 }
461
462 pub fn build(self) -> Result<Self, ConfigError> {
466 self.validate()?;
467 Ok(self)
468 }
469
470 pub fn validate(&self) -> Result<(), ConfigError> {
474 if self.chunking.min_chunk_size > self.chunking.avg_chunk_size {
475 return Err(ConfigError::InvalidValue {
476 field: "chunking.min_chunk_size".into(),
477 reason: format!(
478 "min_chunk_size ({}) > avg_chunk_size ({})",
479 self.chunking.min_chunk_size, self.chunking.avg_chunk_size
480 ),
481 });
482 }
483 if self.chunking.avg_chunk_size > self.chunking.max_chunk_size {
484 return Err(ConfigError::InvalidValue {
485 field: "chunking.avg_chunk_size".into(),
486 reason: format!(
487 "avg_chunk_size ({}) > max_chunk_size ({})",
488 self.chunking.avg_chunk_size, self.chunking.max_chunk_size
489 ),
490 });
491 }
492 if self.defaults.metadata_quality < 1 || self.defaults.metadata_quality > 11 {
493 return Err(ConfigError::InvalidValue {
494 field: "defaults.metadata_quality".into(),
495 reason: format!(
496 "metadata_quality ({}) out of range 1..=11",
497 self.defaults.metadata_quality
498 ),
499 });
500 }
501 if self.tournament.short_circuit_threshold > 1000 {
502 return Err(ConfigError::InvalidValue {
503 field: "tournament.short_circuit_threshold".into(),
504 reason: format!(
505 "short_circuit_threshold ({}) out of range 0..=1000 (per-mille)",
506 self.tournament.short_circuit_threshold
507 ),
508 });
509 }
510 let mut names_seen: BTreeMap<&str, ()> = BTreeMap::new();
512 for rule in &self.categorizers {
513 if !names_seen
514 .insert(rule.name.as_str(), ())
515 .map_or(true, |()| false)
516 {
517 return Err(ConfigError::DuplicateCategorizer(rule.name.clone()));
518 }
519 }
520 Ok(())
521 }
522
523 pub fn codec_registry(&self) -> Result<CodecRegistry, ConfigError> {
526 let mut registry = CodecRegistry::default();
527 registry.insert("store", 0x00);
528 registry.insert("lz4", 0x01);
529 registry.insert("lz4-hc", 0x13);
530 registry.insert("zstd", 0x02);
531 registry.insert("xz", 0x03);
532 registry.insert("brotli", 0x04);
533 registry.insert("deflate", 0x05);
534 registry.insert("snappy", 0x06);
535 registry.insert("flac", 0x07);
536 registry.insert("ricepp", 0x08);
537 registry.insert("fsst+brotli", 0x09);
538 registry.insert("shuffle+lz4", 0x0A);
539 registry.insert("zpaq", 0x0B);
540 registry.insert("ppmd", 0x0C);
541 registry.insert("glza", 0x0D);
542 registry.insert("shuffle+zstd", 0x0E);
543 registry.insert("bitshuffle+lz4", 0x0F);
544 registry.insert("bzip2", 0x10);
545 registry.insert("deflate64", 0x11);
546 registry.insert("libdeflate", 0x14);
547
548 if !registry.contains_name(&self.defaults.text_codec) {
549 return Err(ConfigError::UnknownCodec(self.defaults.text_codec.clone()));
550 }
551 if !registry.contains_name(&self.defaults.binary_codec) {
552 return Err(ConfigError::UnknownCodec(
553 self.defaults.binary_codec.clone(),
554 ));
555 }
556 if !registry.contains_name(&self.defaults.metadata_codec) {
557 return Err(ConfigError::UnknownCodec(
558 self.defaults.metadata_codec.clone(),
559 ));
560 }
561 for rule in &self.categorizers {
562 if !registry.contains_name(&rule.codec) {
563 return Err(ConfigError::UnknownCodec(rule.codec.clone()));
564 }
565 }
566 for codec in &self.tournament.codecs {
567 if !registry.contains_name(codec) {
568 return Err(ConfigError::UnknownCodec(codec.clone()));
569 }
570 }
571 Ok(registry)
572 }
573
574 pub fn text_codec_id(&self) -> Result<u8, ConfigError> {
578 let registry = self.codec_registry()?;
579 Ok(registry
580 .lookup_by_name(&self.defaults.text_codec)
581 .unwrap_or(0x04))
582 }
583
584 pub fn binary_codec_id(&self) -> Result<u8, ConfigError> {
588 let registry = self.codec_registry()?;
589 Ok(registry
590 .lookup_by_name(&self.defaults.binary_codec)
591 .unwrap_or(0x01))
592 }
593
594 #[must_use]
600 pub fn to_core_tunables(&self) -> limnifs_core::codec::CodecTunables {
601 limnifs_core::codec::CodecTunables {
602 quality: self.codec_tunables.brotli.quality,
603 ppmd_order: self
604 .codec_tunables
605 .ppmd7
606 .order
607 .max(self.codec_tunables.ppmd8.order),
608 ppmd7_budget: (self.codec_tunables.ppmd7.memory_budget_mb as usize)
609 .saturating_mul(1024 * 1024),
610 ppmd8_budget: (self.codec_tunables.ppmd8.memory_budget_mb as usize)
611 .saturating_mul(1024 * 1024),
612 bzip2_block_kb: self.codec_tunables.bzip2.block_size_kb,
613 lzma_dict_mb: self.codec_tunables.lzma.dict_size_mb,
614 }
615 }
616
617 pub fn metadata_codec_id(&self) -> Result<u8, ConfigError> {
621 let registry = self.codec_registry()?;
622 Ok(registry
623 .lookup_by_name(&self.defaults.metadata_codec)
624 .unwrap_or(0x04))
625 }
626}
627
628#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
630pub struct CodecRegistry {
631 by_name: BTreeMap<String, u8>,
632 by_id: BTreeMap<u8, String>,
633}
634
635impl CodecRegistry {
636 pub fn insert(&mut self, name: &str, id: u8) {
638 self.by_name.insert(name.to_string(), id);
639 self.by_id.insert(id, name.to_string());
640 }
641
642 #[must_use]
644 pub fn lookup_by_name(&self, name: &str) -> Option<u8> {
645 self.by_name.get(name).copied()
646 }
647
648 #[must_use]
650 pub fn lookup_by_id(&self, id: u8) -> Option<&str> {
651 self.by_id.get(&id).map(String::as_str)
652 }
653
654 #[must_use]
656 pub fn contains_name(&self, name: &str) -> bool {
657 self.by_name.contains_key(name)
658 }
659}
660
661#[cfg(test)]
662mod tests {
663 use super::*;
664
665 #[test]
666 fn default_v0_1_validates() {
667 let config = WriteConfig::default_v0_1();
668 config.validate().expect("v0.1 default should validate");
669 }
670
671 #[test]
672 fn default_v0_1_codec_ids() {
673 let config = WriteConfig::default_v0_1();
674 assert_eq!(config.text_codec_id().unwrap(), 0x04); assert_eq!(config.binary_codec_id().unwrap(), 0x01); assert_eq!(config.metadata_codec_id().unwrap(), 0x04); }
678
679 #[test]
680 fn rejects_invalid_chunking() {
681 let mut config = WriteConfig::default_v0_1();
682 config.chunking.min_chunk_size = 16_000;
683 config.chunking.avg_chunk_size = 8_000;
684 assert!(config.validate().is_err());
685 }
686
687 #[test]
688 fn rejects_invalid_quality() {
689 let mut config = WriteConfig::default_v0_1();
690 config.defaults.metadata_quality = 12;
691 assert!(config.validate().is_err());
692 }
693
694 #[test]
695 fn rejects_duplicate_categorizer_names() {
696 let mut config = WriteConfig::default_v0_1();
697 config.categorizers.push(CategorizerConfig {
698 name: "dna".into(),
699 extensions: vec!["fasta".into()],
700 magic_bytes: vec![],
701 codec: "glza".into(),
702 max_size: None,
703 enabled: true,
704 });
705 config.categorizers.push(CategorizerConfig {
706 name: "dna".into(),
707 extensions: vec!["fa".into()],
708 magic_bytes: vec![],
709 codec: "glza".into(),
710 max_size: None,
711 enabled: true,
712 });
713 assert!(config.validate().is_err());
714 }
715
716 #[test]
717 fn rejects_unknown_codec() {
718 let mut config = WriteConfig::default_v0_1();
719 config.defaults.text_codec = "does-not-exist".into();
720 assert!(config.codec_registry().is_err());
721 }
722}