1pub mod defaults;
28pub mod error;
29pub mod profile;
30pub mod toml;
31
32use std::collections::BTreeMap;
33
34use serde::{Deserialize, Serialize};
35
36use crate::config::error::ConfigError;
37
38pub const DEFAULT_TEXT_CODEC: &str = "brotli";
41pub const DEFAULT_BINARY_CODEC: &str = "lz4";
44pub const DEFAULT_METADATA_CODEC: &str = "brotli";
47pub const DEFAULT_METADATA_QUALITY: u8 = 5;
49pub const DEFAULT_INLINE_THRESHOLD: u16 = 4096;
51
52pub const DEFAULT_MAX_DROP_SIZE: u32 = 4 * 1024 * 1024;
57pub const DEFAULT_AVG_CHUNK_SIZE: u32 = 262_144;
59pub const DEFAULT_MIN_CHUNK_SIZE: u32 = 65_536;
61pub const DEFAULT_MAX_CHUNK_SIZE: u32 = 1_048_576;
63pub const DEFAULT_TOURNAMENT_MIN_SIZE: u32 = 256;
65pub const DEFAULT_TOURNAMENT_SKIP_BINARY: bool = true;
67pub const DEFAULT_AEAD: &str = "chacha20-poly1305";
69pub const DEFAULT_KEY_WRAP: &str = "x25519-hkdf";
71pub const DEFAULT_DICT_ENABLED: bool = true;
73pub const DEFAULT_DICT_MIN_CLASS_SIZE: u32 = 100;
75pub const DEFAULT_DICT_MAX_SIZE: u32 = 65_536;
77
78#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
82pub struct WriteConfig {
83 pub defaults: Defaults,
85 #[serde(default, rename = "categorizer")]
87 pub categorizers: Vec<CategorizerConfig>,
88 pub chunking: ChunkingConfig,
90 pub tournament: TournamentConfig,
92 #[serde(default)]
94 pub codec_tunables: CodecTunables,
95 #[serde(default)]
97 pub mode: ImageMode,
98 #[serde(default = "default_write_codec")]
101 pub write_codec: String,
102 #[serde(default)]
105 pub turnover_threshold: u32,
106 #[serde(default)]
110 pub skip_chunking: bool,
111 pub encryption: EncryptionConfig,
113 pub dictionaries: DictionaryConfig,
115}
116
117fn default_write_codec() -> String {
118 "lz4".into()
119}
120
121fn default_metadata_externalize_threshold() -> usize {
122 crate::METADATA_EXTERNALIZE_THRESHOLD
123}
124
125#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
127pub struct Defaults {
128 pub text_codec: String,
129 pub binary_codec: String,
130 pub metadata_codec: String,
131 pub metadata_quality: u8,
132 #[serde(default = "default_metadata_externalize_threshold")]
139 pub metadata_externalize_threshold: usize,
140 #[serde(default = "default_true")]
145 pub shared_inline: bool,
146 #[serde(default = "default_true")]
152 pub seekable_drops: bool,
153 #[serde(default = "default_max_drop_size")]
162 pub max_drop_size: u32,
163 pub inline_threshold: u16,
165}
166
167#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
169pub struct CategorizerConfig {
170 pub name: String,
172 #[serde(default)]
174 pub extensions: Vec<String>,
175 #[serde(default)]
177 pub magic_bytes: Vec<u8>,
178 pub codec: String,
180 #[serde(default)]
182 pub max_size: Option<u32>,
183 #[serde(default = "default_true")]
185 pub enabled: bool,
186}
187
188fn default_true() -> bool {
189 true
190}
191
192fn default_max_drop_size() -> u32 {
193 DEFAULT_MAX_DROP_SIZE
194}
195
196#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
198pub struct ChunkingConfig {
199 #[serde(default = "default_chunker_name")]
204 pub name: String,
205 #[serde(default)]
206 pub avg_chunk_size: u32,
207 #[serde(default)]
208 pub min_chunk_size: u32,
209 #[serde(default)]
210 pub max_chunk_size: u32,
211}
212
213fn default_chunker_name() -> String {
214 "fastcdc".into()
215}
216
217#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
219pub struct TournamentConfig {
220 pub codecs: Vec<String>,
224 pub min_size_threshold: u32,
226 pub skip_for_binary: bool,
228 #[serde(default = "default_short_circuit_threshold")]
240 pub short_circuit_threshold: u32,
241}
242
243fn default_short_circuit_threshold() -> u32 {
244 250
245}
246
247#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
249pub struct EncryptionConfig {
250 pub aead: String,
252 pub key_wrap: String,
254}
255
256#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
258pub struct DictionaryConfig {
259 pub enabled: bool,
260 pub min_class_size: u32,
261 pub max_dict_size: u32,
262 #[serde(default = "default_trainer")]
269 pub trainer: String,
270}
271
272fn default_trainer() -> String {
273 "frequency".into()
274}
275
276#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
281pub enum ImageMode {
282 #[default]
286 ReadOnly,
287 ReadWrite(RWMode),
289}
290
291#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
293pub enum RWMode {
294 AppendOnly,
297 #[default]
300 UpdateInPlace,
301 CopyOnWrite,
305}
306
307#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
320pub struct CodecTunables {
321 #[serde(default)]
322 pub ppmd7: Ppmd7Tunables,
323 #[serde(default)]
324 pub ppmd8: Ppmd8Tunables,
325 #[serde(default)]
326 pub brotli: BrotliTunables,
327 #[serde(default)]
328 pub lzma: LzmaTunables,
329 #[serde(default)]
330 pub bzip2: Bzip2Tunables,
331}
332
333#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
335pub struct Ppmd7Tunables {
336 pub order: u8,
337 pub memory_budget_mb: u32,
338}
339
340impl Default for Ppmd7Tunables {
341 fn default() -> Self {
342 Self {
343 order: 4,
344 memory_budget_mb: 80,
345 }
346 }
347}
348
349#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
351pub struct Ppmd8Tunables {
352 pub order: u8,
353 pub memory_budget_mb: u32,
354}
355
356impl Default for Ppmd8Tunables {
357 fn default() -> Self {
358 Self {
359 order: 6,
360 memory_budget_mb: 64,
361 }
362 }
363}
364
365#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
367pub struct BrotliTunables {
368 pub quality: u8,
369 pub window: u8,
370}
371
372impl Default for BrotliTunables {
373 fn default() -> Self {
374 Self {
375 quality: 11,
376 window: 22,
377 }
378 }
379}
380
381#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
383pub struct LzmaTunables {
384 pub lc: u8,
385 pub lp: u8,
386 pub pb: u8,
387 pub dict_size_mb: u32,
388 pub use_optimal_parser: bool,
389}
390
391impl Default for LzmaTunables {
392 fn default() -> Self {
393 Self {
394 lc: 3,
395 lp: 0,
396 pb: 2,
397 dict_size_mb: 16,
398 use_optimal_parser: false,
399 }
400 }
401}
402
403#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
405pub struct Bzip2Tunables {
406 pub block_size_kb: u32,
407}
408
409impl Default for Bzip2Tunables {
410 fn default() -> Self {
411 Self { block_size_kb: 900 }
412 }
413}
414
415impl WriteConfig {
416 #[must_use]
420 pub fn default_v0_1() -> Self {
421 Self {
422 defaults: Defaults {
423 text_codec: DEFAULT_TEXT_CODEC.to_string(),
424 binary_codec: DEFAULT_BINARY_CODEC.to_string(),
425 metadata_codec: DEFAULT_METADATA_CODEC.to_string(),
426 metadata_quality: DEFAULT_METADATA_QUALITY,
427 metadata_externalize_threshold: crate::METADATA_EXTERNALIZE_THRESHOLD,
428 shared_inline: true,
429 seekable_drops: true,
430 max_drop_size: DEFAULT_MAX_DROP_SIZE,
431 inline_threshold: DEFAULT_INLINE_THRESHOLD,
432 },
433 categorizers: Vec::new(),
434 chunking: ChunkingConfig {
435 name: "fastcdc".into(),
436 avg_chunk_size: 262_144,
437 min_chunk_size: 65_536,
438 max_chunk_size: 1_048_576,
439 },
440 tournament: TournamentConfig {
441 codecs: vec![
442 "store".to_string(),
443 "lz4".to_string(),
444 "zstd".to_string(),
445 "brotli".to_string(),
446 ],
447 min_size_threshold: DEFAULT_TOURNAMENT_MIN_SIZE,
448 skip_for_binary: DEFAULT_TOURNAMENT_SKIP_BINARY,
449 short_circuit_threshold: default_short_circuit_threshold(),
450 },
451 encryption: EncryptionConfig {
452 aead: DEFAULT_AEAD.to_string(),
453 key_wrap: DEFAULT_KEY_WRAP.to_string(),
454 },
455 dictionaries: DictionaryConfig {
456 enabled: DEFAULT_DICT_ENABLED,
457 min_class_size: DEFAULT_DICT_MIN_CLASS_SIZE,
458 max_dict_size: DEFAULT_DICT_MAX_SIZE,
459 trainer: "frequency".into(),
460 },
461 codec_tunables: CodecTunables::default(),
462 mode: ImageMode::ReadOnly,
463 write_codec: default_write_codec(),
464 turnover_threshold: 0,
465 skip_chunking: false,
466 }
467 }
468
469 #[must_use]
472 pub fn from_profile(name: &str) -> Option<Self> {
473 profile::select(name)
474 }
475
476 #[must_use]
478 pub fn with_text_codec(mut self, codec: &str) -> Self {
479 self.defaults.text_codec = codec.into();
480 self
481 }
482
483 #[must_use]
485 pub fn with_binary_codec(mut self, codec: &str) -> Self {
486 self.defaults.binary_codec = codec.into();
487 self
488 }
489
490 #[must_use]
492 pub fn with_chunk_size(mut self, size: u32) -> Self {
493 self.chunking.avg_chunk_size = size;
494 self
495 }
496
497 #[must_use]
499 pub fn with_mode(mut self, mode: ImageMode) -> Self {
500 self.mode = mode;
501 self
502 }
503
504 #[must_use]
506 pub fn with_brotli_quality(mut self, quality: u8) -> Self {
507 self.codec_tunables.brotli.quality = quality;
508 self
509 }
510
511 pub fn build(self) -> Result<Self, ConfigError> {
515 self.validate()?;
516 Ok(self)
517 }
518
519 pub fn validate(&self) -> Result<(), ConfigError> {
523 if self.chunking.min_chunk_size > self.chunking.avg_chunk_size {
524 return Err(ConfigError::InvalidValue {
525 field: "chunking.min_chunk_size".into(),
526 reason: format!(
527 "min_chunk_size ({}) > avg_chunk_size ({})",
528 self.chunking.min_chunk_size, self.chunking.avg_chunk_size
529 ),
530 });
531 }
532 if self.chunking.avg_chunk_size > self.chunking.max_chunk_size {
533 return Err(ConfigError::InvalidValue {
534 field: "chunking.avg_chunk_size".into(),
535 reason: format!(
536 "avg_chunk_size ({}) > max_chunk_size ({})",
537 self.chunking.avg_chunk_size, self.chunking.max_chunk_size
538 ),
539 });
540 }
541 if self.defaults.metadata_quality < 1 || self.defaults.metadata_quality > 11 {
542 return Err(ConfigError::InvalidValue {
543 field: "defaults.metadata_quality".into(),
544 reason: format!(
545 "metadata_quality ({}) out of range 1..=11",
546 self.defaults.metadata_quality
547 ),
548 });
549 }
550 let ceiling = limnifs_core::metadata_reference::DEFAULT_INLINE_METADATA_MAX_BYTES as usize;
551 if self.defaults.metadata_externalize_threshold == 0
552 || self.defaults.metadata_externalize_threshold > ceiling
553 {
554 return Err(ConfigError::InvalidValue {
555 field: "defaults.metadata_externalize_threshold".into(),
556 reason: format!(
557 "metadata_externalize_threshold ({}) must be within 1..={ceiling} (the reader inline ceiling; larger inline metadata is unreadable)",
558 self.defaults.metadata_externalize_threshold
559 ),
560 });
561 }
562 if self.defaults.max_drop_size != 0 && self.defaults.max_drop_size < 1024 {
563 return Err(ConfigError::InvalidValue {
564 field: "defaults.max_drop_size".into(),
565 reason: format!(
566 "max_drop_size ({}) is nonzero but below 1 KiB — no whole-file \
567 drop can form under the cap; raise it or set 0 (unlimited)",
568 self.defaults.max_drop_size
569 ),
570 });
571 }
572 if self.tournament.short_circuit_threshold > 1000 {
573 return Err(ConfigError::InvalidValue {
574 field: "tournament.short_circuit_threshold".into(),
575 reason: format!(
576 "short_circuit_threshold ({}) out of range 0..=1000 (per-mille)",
577 self.tournament.short_circuit_threshold
578 ),
579 });
580 }
581 let mut names_seen: BTreeMap<&str, ()> = BTreeMap::new();
583 for rule in &self.categorizers {
584 if !names_seen
585 .insert(rule.name.as_str(), ())
586 .map_or(true, |()| false)
587 {
588 return Err(ConfigError::DuplicateCategorizer(rule.name.clone()));
589 }
590 }
591 Ok(())
592 }
593
594 pub fn codec_registry(&self) -> Result<CodecRegistry, ConfigError> {
597 let mut registry = CodecRegistry::default();
598 registry.insert("store", 0x00);
599 registry.insert("lz4", 0x01);
600 registry.insert("lz4-hc", 0x13);
601 registry.insert("zstd", 0x02);
602 registry.insert("xz", 0x03);
603 registry.insert("brotli", 0x04);
604 registry.insert("deflate", 0x05);
605 registry.insert("snappy", 0x06);
606 registry.insert("flac", 0x07);
607 registry.insert("ricepp", 0x08);
608 registry.insert("fsst+brotli", 0x09);
609 registry.insert("shuffle+lz4", 0x0A);
610 registry.insert("zpaq", 0x0B);
611 registry.insert("ppmd", 0x0C);
612 registry.insert("glza", 0x0D);
613 registry.insert("shuffle+zstd", 0x0E);
614 registry.insert("bitshuffle+lz4", 0x0F);
615 registry.insert("bzip2", 0x10);
616 registry.insert("deflate64", 0x11);
617 registry.insert("libdeflate", 0x14);
618
619 if !registry.contains_name(&self.defaults.text_codec) {
620 return Err(ConfigError::UnknownCodec(self.defaults.text_codec.clone()));
621 }
622 if !registry.contains_name(&self.defaults.binary_codec) {
623 return Err(ConfigError::UnknownCodec(
624 self.defaults.binary_codec.clone(),
625 ));
626 }
627 if !registry.contains_name(&self.defaults.metadata_codec) {
628 return Err(ConfigError::UnknownCodec(
629 self.defaults.metadata_codec.clone(),
630 ));
631 }
632 for rule in &self.categorizers {
633 if !registry.contains_name(&rule.codec) {
634 return Err(ConfigError::UnknownCodec(rule.codec.clone()));
635 }
636 }
637 for codec in &self.tournament.codecs {
638 if !registry.contains_name(codec) {
639 return Err(ConfigError::UnknownCodec(codec.clone()));
640 }
641 }
642 Ok(registry)
643 }
644
645 pub fn text_codec_id(&self) -> Result<u8, ConfigError> {
649 let registry = self.codec_registry()?;
650 Ok(registry
651 .lookup_by_name(&self.defaults.text_codec)
652 .unwrap_or(0x04))
653 }
654
655 pub fn binary_codec_id(&self) -> Result<u8, ConfigError> {
659 let registry = self.codec_registry()?;
660 Ok(registry
661 .lookup_by_name(&self.defaults.binary_codec)
662 .unwrap_or(0x01))
663 }
664
665 #[must_use]
671 pub fn to_core_tunables(&self) -> limnifs_core::codec::CodecTunables {
672 limnifs_core::codec::CodecTunables {
673 quality: self.codec_tunables.brotli.quality,
674 ppmd_order: self
675 .codec_tunables
676 .ppmd7
677 .order
678 .max(self.codec_tunables.ppmd8.order),
679 ppmd7_budget: (self.codec_tunables.ppmd7.memory_budget_mb as usize)
680 .saturating_mul(1024 * 1024),
681 ppmd8_budget: (self.codec_tunables.ppmd8.memory_budget_mb as usize)
682 .saturating_mul(1024 * 1024),
683 bzip2_block_kb: self.codec_tunables.bzip2.block_size_kb,
684 lzma_dict_mb: self.codec_tunables.lzma.dict_size_mb,
685 }
686 }
687
688 pub fn metadata_codec_id(&self) -> Result<u8, ConfigError> {
692 let registry = self.codec_registry()?;
693 Ok(registry
694 .lookup_by_name(&self.defaults.metadata_codec)
695 .unwrap_or(0x04))
696 }
697}
698
699#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
701pub struct CodecRegistry {
702 by_name: BTreeMap<String, u8>,
703 by_id: BTreeMap<u8, String>,
704}
705
706impl CodecRegistry {
707 pub fn insert(&mut self, name: &str, id: u8) {
709 self.by_name.insert(name.to_string(), id);
710 self.by_id.insert(id, name.to_string());
711 }
712
713 #[must_use]
715 pub fn lookup_by_name(&self, name: &str) -> Option<u8> {
716 self.by_name.get(name).copied()
717 }
718
719 #[must_use]
721 pub fn lookup_by_id(&self, id: u8) -> Option<&str> {
722 self.by_id.get(&id).map(String::as_str)
723 }
724
725 #[must_use]
727 pub fn contains_name(&self, name: &str) -> bool {
728 self.by_name.contains_key(name)
729 }
730}
731
732#[cfg(test)]
733mod tests {
734 use super::*;
735
736 #[test]
737 fn default_v0_1_validates() {
738 let config = WriteConfig::default_v0_1();
739 config.validate().expect("v0.1 default should validate");
740 }
741
742 #[test]
743 fn default_v0_1_codec_ids() {
744 let config = WriteConfig::default_v0_1();
745 assert_eq!(config.text_codec_id().unwrap(), 0x04); assert_eq!(config.binary_codec_id().unwrap(), 0x01); assert_eq!(config.metadata_codec_id().unwrap(), 0x04); }
749
750 #[test]
751 fn rejects_invalid_chunking() {
752 let mut config = WriteConfig::default_v0_1();
753 config.chunking.min_chunk_size = 16_000;
754 config.chunking.avg_chunk_size = 8_000;
755 assert!(config.validate().is_err());
756 }
757
758 #[test]
759 fn rejects_invalid_quality() {
760 let mut config = WriteConfig::default_v0_1();
761 config.defaults.metadata_quality = 12;
762 assert!(config.validate().is_err());
763 }
764
765 #[test]
766 fn rejects_duplicate_categorizer_names() {
767 let mut config = WriteConfig::default_v0_1();
768 config.categorizers.push(CategorizerConfig {
769 name: "dna".into(),
770 extensions: vec!["fasta".into()],
771 magic_bytes: vec![],
772 codec: "glza".into(),
773 max_size: None,
774 enabled: true,
775 });
776 config.categorizers.push(CategorizerConfig {
777 name: "dna".into(),
778 extensions: vec!["fa".into()],
779 magic_bytes: vec![],
780 codec: "glza".into(),
781 max_size: None,
782 enabled: true,
783 });
784 assert!(config.validate().is_err());
785 }
786
787 #[test]
788 fn rejects_unknown_codec() {
789 let mut config = WriteConfig::default_v0_1();
790 config.defaults.text_codec = "does-not-exist".into();
791 assert!(config.codec_registry().is_err());
792 }
793}