1pub mod defaults;
28pub mod error;
29pub mod profile;
30pub mod toml;
31
32use std::collections::BTreeMap;
33
34use serde::{Deserialize, Serialize};
35
36use crate::config::error::ConfigError;
37
38pub const DEFAULT_TEXT_CODEC: &str = "brotli";
41pub const DEFAULT_BINARY_CODEC: &str = "lz4";
44pub const DEFAULT_METADATA_CODEC: &str = "brotli";
47pub const DEFAULT_METADATA_QUALITY: u8 = 5;
49pub const DEFAULT_INLINE_THRESHOLD: u16 = 4096;
51
52pub const DEFAULT_MAX_DROP_SIZE: u32 = 4 * 1024 * 1024;
57pub const DEFAULT_AVG_CHUNK_SIZE: u32 = 262_144;
59pub const DEFAULT_MIN_CHUNK_SIZE: u32 = 65_536;
61pub const DEFAULT_MAX_CHUNK_SIZE: u32 = 1_048_576;
63pub const DEFAULT_TOURNAMENT_MIN_SIZE: u32 = 256;
65pub const DEFAULT_TOURNAMENT_SKIP_BINARY: bool = true;
67pub const DEFAULT_AEAD: &str = "chacha20-poly1305";
69pub const DEFAULT_KEY_WRAP: &str = "x25519-hkdf";
71pub const DEFAULT_DICT_ENABLED: bool = true;
73pub const DEFAULT_DICT_MIN_CLASS_SIZE: u32 = 100;
75pub const DEFAULT_DICT_MAX_SIZE: u32 = 65_536;
77
78#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
82pub struct WriteConfig {
83 pub defaults: Defaults,
85 #[serde(default, rename = "categorizer")]
87 pub categorizers: Vec<CategorizerConfig>,
88 pub chunking: ChunkingConfig,
90 pub tournament: TournamentConfig,
92 #[serde(default)]
94 pub codec_tunables: CodecTunables,
95 #[serde(default)]
97 pub mode: ImageMode,
98 #[serde(default = "default_write_codec")]
101 pub write_codec: String,
102 #[serde(default)]
105 pub turnover_threshold: u32,
106 #[serde(default)]
110 pub skip_chunking: bool,
111 pub encryption: EncryptionConfig,
113 pub dictionaries: DictionaryConfig,
115}
116
117fn default_write_codec() -> String {
118 "lz4".into()
119}
120
121fn default_metadata_externalize_threshold() -> usize {
122 crate::METADATA_EXTERNALIZE_THRESHOLD
123}
124
125#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
127pub struct Defaults {
128 pub text_codec: String,
129 pub binary_codec: String,
130 pub metadata_codec: String,
131 pub metadata_quality: u8,
132 #[serde(default = "default_metadata_externalize_threshold")]
139 pub metadata_externalize_threshold: usize,
140 #[serde(default = "default_true")]
145 pub shared_inline: bool,
146 #[serde(default = "default_true")]
152 pub seekable_drops: bool,
153 #[serde(default = "default_max_drop_size")]
162 pub max_drop_size: u32,
163 pub inline_threshold: u16,
165}
166
167#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
169pub struct CategorizerConfig {
170 pub name: String,
172 #[serde(default)]
174 pub extensions: Vec<String>,
175 #[serde(default)]
177 pub magic_bytes: Vec<u8>,
178 pub codec: String,
180 #[serde(default)]
182 pub max_size: Option<u32>,
183 #[serde(default = "default_true")]
185 pub enabled: bool,
186}
187
188fn default_true() -> bool {
189 true
190}
191
192fn default_max_drop_size() -> u32 {
193 DEFAULT_MAX_DROP_SIZE
194}
195
196#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
198pub struct ChunkingConfig {
199 #[serde(default = "default_chunker_name")]
204 pub name: String,
205 #[serde(default)]
206 pub avg_chunk_size: u32,
207 #[serde(default)]
208 pub min_chunk_size: u32,
209 #[serde(default)]
210 pub max_chunk_size: u32,
211}
212
213fn default_chunker_name() -> String {
214 "fastcdc".into()
215}
216
217#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
219pub struct TournamentConfig {
220 pub codecs: Vec<String>,
224 pub min_size_threshold: u32,
226 pub skip_for_binary: bool,
228 #[serde(default = "default_short_circuit_threshold")]
240 pub short_circuit_threshold: u32,
241}
242
243fn default_short_circuit_threshold() -> u32 {
244 250
245}
246
247#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
249pub struct EncryptionConfig {
250 pub aead: String,
252 pub key_wrap: String,
254}
255
256#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
258pub struct DictionaryConfig {
259 pub enabled: bool,
260 pub min_class_size: u32,
261 pub max_dict_size: u32,
262 #[serde(default = "default_trainer")]
269 pub trainer: String,
270}
271
272fn default_trainer() -> String {
273 "frequency".into()
274}
275
276#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
281pub enum ImageMode {
282 #[default]
286 ReadOnly,
287 ReadWrite(RWMode),
289}
290
291#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
293pub enum RWMode {
294 AppendOnly,
297 #[default]
300 UpdateInPlace,
301 CopyOnWrite,
305}
306
307#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
320pub struct CodecTunables {
321 #[serde(default)]
322 pub ppmd7: Ppmd7Tunables,
323 #[serde(default)]
324 pub ppmd8: Ppmd8Tunables,
325 #[serde(default)]
326 pub brotli: BrotliTunables,
327 #[serde(default)]
328 pub zstd: ZstdTunables,
329 #[serde(default)]
330 pub lzma: LzmaTunables,
331 #[serde(default)]
332 pub bzip2: Bzip2Tunables,
333}
334
335#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
337pub struct Ppmd7Tunables {
338 pub order: u8,
339 pub memory_budget_mb: u32,
340}
341
342impl Default for Ppmd7Tunables {
343 fn default() -> Self {
344 Self {
345 order: 4,
346 memory_budget_mb: 80,
347 }
348 }
349}
350
351#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
353pub struct Ppmd8Tunables {
354 pub order: u8,
355 pub memory_budget_mb: u32,
356}
357
358impl Default for Ppmd8Tunables {
359 fn default() -> Self {
360 Self {
361 order: 6,
362 memory_budget_mb: 64,
363 }
364 }
365}
366
367#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
369pub struct BrotliTunables {
370 pub quality: u8,
371 pub window: u8,
372}
373
374impl Default for BrotliTunables {
375 fn default() -> Self {
376 Self {
377 quality: 11,
378 window: 22,
379 }
380 }
381}
382
383#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
389pub struct ZstdTunables {
390 pub quality: u8,
391}
392
393impl Default for ZstdTunables {
394 fn default() -> Self {
395 Self { quality: 2 }
396 }
397}
398
399#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
401pub struct LzmaTunables {
402 pub lc: u8,
403 pub lp: u8,
404 pub pb: u8,
405 pub dict_size_mb: u32,
406 pub use_optimal_parser: bool,
407}
408
409impl Default for LzmaTunables {
410 fn default() -> Self {
411 Self {
412 lc: 3,
413 lp: 0,
414 pb: 2,
415 dict_size_mb: 16,
416 use_optimal_parser: false,
417 }
418 }
419}
420
421#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
423pub struct Bzip2Tunables {
424 pub block_size_kb: u32,
425}
426
427impl Default for Bzip2Tunables {
428 fn default() -> Self {
429 Self { block_size_kb: 900 }
430 }
431}
432
433impl WriteConfig {
434 #[must_use]
438 pub fn default_v0_1() -> Self {
439 Self {
440 defaults: Defaults {
441 text_codec: DEFAULT_TEXT_CODEC.to_string(),
442 binary_codec: DEFAULT_BINARY_CODEC.to_string(),
443 metadata_codec: DEFAULT_METADATA_CODEC.to_string(),
444 metadata_quality: DEFAULT_METADATA_QUALITY,
445 metadata_externalize_threshold: crate::METADATA_EXTERNALIZE_THRESHOLD,
446 shared_inline: true,
447 seekable_drops: true,
448 max_drop_size: DEFAULT_MAX_DROP_SIZE,
449 inline_threshold: DEFAULT_INLINE_THRESHOLD,
450 },
451 categorizers: Vec::new(),
452 chunking: ChunkingConfig {
453 name: "fastcdc".into(),
454 avg_chunk_size: 262_144,
455 min_chunk_size: 65_536,
456 max_chunk_size: 1_048_576,
457 },
458 tournament: TournamentConfig {
459 codecs: vec![
460 "store".to_string(),
461 "lz4".to_string(),
462 "zstd".to_string(),
463 "brotli".to_string(),
464 ],
465 min_size_threshold: DEFAULT_TOURNAMENT_MIN_SIZE,
466 skip_for_binary: DEFAULT_TOURNAMENT_SKIP_BINARY,
467 short_circuit_threshold: default_short_circuit_threshold(),
468 },
469 encryption: EncryptionConfig {
470 aead: DEFAULT_AEAD.to_string(),
471 key_wrap: DEFAULT_KEY_WRAP.to_string(),
472 },
473 dictionaries: DictionaryConfig {
474 enabled: DEFAULT_DICT_ENABLED,
475 min_class_size: DEFAULT_DICT_MIN_CLASS_SIZE,
476 max_dict_size: DEFAULT_DICT_MAX_SIZE,
477 trainer: "frequency".into(),
478 },
479 codec_tunables: CodecTunables::default(),
480 mode: ImageMode::ReadOnly,
481 write_codec: default_write_codec(),
482 turnover_threshold: 0,
483 skip_chunking: false,
484 }
485 }
486
487 #[must_use]
490 pub fn from_profile(name: &str) -> Option<Self> {
491 profile::select(name)
492 }
493
494 #[must_use]
496 pub fn with_text_codec(mut self, codec: &str) -> Self {
497 self.defaults.text_codec = codec.into();
498 self
499 }
500
501 #[must_use]
503 pub fn with_binary_codec(mut self, codec: &str) -> Self {
504 self.defaults.binary_codec = codec.into();
505 self
506 }
507
508 #[must_use]
510 pub fn with_chunk_size(mut self, size: u32) -> Self {
511 self.chunking.avg_chunk_size = size;
512 self
513 }
514
515 #[must_use]
517 pub fn with_mode(mut self, mode: ImageMode) -> Self {
518 self.mode = mode;
519 self
520 }
521
522 #[must_use]
524 pub fn with_brotli_quality(mut self, quality: u8) -> Self {
525 self.codec_tunables.brotli.quality = quality;
526 self
527 }
528
529 pub fn build(self) -> Result<Self, ConfigError> {
533 self.validate()?;
534 Ok(self)
535 }
536
537 pub fn validate(&self) -> Result<(), ConfigError> {
541 if self.chunking.min_chunk_size > self.chunking.avg_chunk_size {
542 return Err(ConfigError::InvalidValue {
543 field: "chunking.min_chunk_size".into(),
544 reason: format!(
545 "min_chunk_size ({}) > avg_chunk_size ({})",
546 self.chunking.min_chunk_size, self.chunking.avg_chunk_size
547 ),
548 });
549 }
550 if self.chunking.avg_chunk_size > self.chunking.max_chunk_size {
551 return Err(ConfigError::InvalidValue {
552 field: "chunking.avg_chunk_size".into(),
553 reason: format!(
554 "avg_chunk_size ({}) > max_chunk_size ({})",
555 self.chunking.avg_chunk_size, self.chunking.max_chunk_size
556 ),
557 });
558 }
559 if self.defaults.metadata_quality < 1 || self.defaults.metadata_quality > 11 {
560 return Err(ConfigError::InvalidValue {
561 field: "defaults.metadata_quality".into(),
562 reason: format!(
563 "metadata_quality ({}) out of range 1..=11",
564 self.defaults.metadata_quality
565 ),
566 });
567 }
568 let ceiling = limnifs_core::metadata_reference::DEFAULT_INLINE_METADATA_MAX_BYTES as usize;
569 if self.defaults.metadata_externalize_threshold == 0
570 || self.defaults.metadata_externalize_threshold > ceiling
571 {
572 return Err(ConfigError::InvalidValue {
573 field: "defaults.metadata_externalize_threshold".into(),
574 reason: format!(
575 "metadata_externalize_threshold ({}) must be within 1..={ceiling} (the reader inline ceiling; larger inline metadata is unreadable)",
576 self.defaults.metadata_externalize_threshold
577 ),
578 });
579 }
580 if self.defaults.max_drop_size != 0 && self.defaults.max_drop_size < 1024 {
581 return Err(ConfigError::InvalidValue {
582 field: "defaults.max_drop_size".into(),
583 reason: format!(
584 "max_drop_size ({}) is nonzero but below 1 KiB — no whole-file \
585 drop can form under the cap; raise it or set 0 (unlimited)",
586 self.defaults.max_drop_size
587 ),
588 });
589 }
590 if self.tournament.short_circuit_threshold > 1000 {
591 return Err(ConfigError::InvalidValue {
592 field: "tournament.short_circuit_threshold".into(),
593 reason: format!(
594 "short_circuit_threshold ({}) out of range 0..=1000 (per-mille)",
595 self.tournament.short_circuit_threshold
596 ),
597 });
598 }
599 let mut names_seen: BTreeMap<&str, ()> = BTreeMap::new();
601 for rule in &self.categorizers {
602 if !names_seen
603 .insert(rule.name.as_str(), ())
604 .map_or(true, |()| false)
605 {
606 return Err(ConfigError::DuplicateCategorizer(rule.name.clone()));
607 }
608 }
609 Ok(())
610 }
611
612 pub fn codec_registry(&self) -> Result<CodecRegistry, ConfigError> {
615 let mut registry = CodecRegistry::default();
616 registry.insert("store", 0x00);
617 registry.insert("lz4", 0x01);
618 registry.insert("lz4-hc", 0x13);
619 registry.insert("zstd", 0x02);
620 registry.insert("xz", 0x03);
621 registry.insert("brotli", 0x04);
622 registry.insert("deflate", 0x05);
623 registry.insert("snappy", 0x06);
624 registry.insert("flac", 0x07);
625 registry.insert("ricepp", 0x08);
626 registry.insert("fsst+brotli", 0x09);
627 registry.insert("shuffle+lz4", 0x0A);
628 registry.insert("zpaq", 0x0B);
629 registry.insert("ppmd", 0x0C);
630 registry.insert("glza", 0x0D);
631 registry.insert("shuffle+zstd", 0x0E);
632 registry.insert("bitshuffle+lz4", 0x0F);
633 registry.insert("bzip2", 0x10);
634 registry.insert("deflate64", 0x11);
635 registry.insert("libdeflate", 0x14);
636 registry.insert("bcj-x86-lz4", 0x20);
637 registry.insert("bcj-x86-zstd", 0x21);
638 registry.insert("bcj-arm64-lz4", 0x23);
639 registry.insert("bcj-arm64-zstd", 0x24);
640
641 if !registry.contains_name(&self.defaults.text_codec) {
642 return Err(ConfigError::UnknownCodec(self.defaults.text_codec.clone()));
643 }
644 if !registry.contains_name(&self.defaults.binary_codec) {
645 return Err(ConfigError::UnknownCodec(
646 self.defaults.binary_codec.clone(),
647 ));
648 }
649 if !registry.contains_name(&self.defaults.metadata_codec) {
650 return Err(ConfigError::UnknownCodec(
651 self.defaults.metadata_codec.clone(),
652 ));
653 }
654 for rule in &self.categorizers {
655 if !registry.contains_name(&rule.codec) {
656 return Err(ConfigError::UnknownCodec(rule.codec.clone()));
657 }
658 }
659 for codec in &self.tournament.codecs {
660 if !registry.contains_name(codec) {
661 return Err(ConfigError::UnknownCodec(codec.clone()));
662 }
663 }
664 Ok(registry)
665 }
666
667 pub fn text_codec_id(&self) -> Result<u8, ConfigError> {
671 let registry = self.codec_registry()?;
672 Ok(registry
673 .lookup_by_name(&self.defaults.text_codec)
674 .unwrap_or(0x04))
675 }
676
677 pub fn binary_codec_id(&self) -> Result<u8, ConfigError> {
681 let registry = self.codec_registry()?;
682 Ok(registry
683 .lookup_by_name(&self.defaults.binary_codec)
684 .unwrap_or(0x01))
685 }
686
687 #[must_use]
693 pub fn to_core_tunables(&self) -> limnifs_core::codec::CodecTunables {
694 limnifs_core::codec::CodecTunables {
695 quality: self.codec_tunables.brotli.quality,
696 zstd_quality: self.codec_tunables.zstd.quality,
697 xz_level: 0,
698 ppmd_order: self
699 .codec_tunables
700 .ppmd7
701 .order
702 .max(self.codec_tunables.ppmd8.order),
703 ppmd7_budget: (self.codec_tunables.ppmd7.memory_budget_mb as usize)
704 .saturating_mul(1024 * 1024),
705 ppmd8_budget: (self.codec_tunables.ppmd8.memory_budget_mb as usize)
706 .saturating_mul(1024 * 1024),
707 bzip2_block_kb: self.codec_tunables.bzip2.block_size_kb,
708 lzma_dict_mb: self.codec_tunables.lzma.dict_size_mb,
709 }
710 }
711
712 pub fn metadata_codec_id(&self) -> Result<u8, ConfigError> {
716 let registry = self.codec_registry()?;
717 Ok(registry
718 .lookup_by_name(&self.defaults.metadata_codec)
719 .unwrap_or(0x04))
720 }
721}
722
723#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
725pub struct CodecRegistry {
726 by_name: BTreeMap<String, u8>,
727 by_id: BTreeMap<u8, String>,
728}
729
730impl CodecRegistry {
731 pub fn insert(&mut self, name: &str, id: u8) {
733 self.by_name.insert(name.to_string(), id);
734 self.by_id.insert(id, name.to_string());
735 }
736
737 #[must_use]
739 pub fn lookup_by_name(&self, name: &str) -> Option<u8> {
740 self.by_name.get(name).copied()
741 }
742
743 #[must_use]
745 pub fn lookup_by_id(&self, id: u8) -> Option<&str> {
746 self.by_id.get(&id).map(String::as_str)
747 }
748
749 #[must_use]
751 pub fn contains_name(&self, name: &str) -> bool {
752 self.by_name.contains_key(name)
753 }
754}
755
756#[cfg(test)]
757mod tests {
758 use super::*;
759
760 #[test]
761 fn default_v0_1_validates() {
762 let config = WriteConfig::default_v0_1();
763 config.validate().expect("v0.1 default should validate");
764 }
765
766 #[test]
767 fn default_v0_1_codec_ids() {
768 let config = WriteConfig::default_v0_1();
769 assert_eq!(config.text_codec_id().unwrap(), 0x04); assert_eq!(config.binary_codec_id().unwrap(), 0x01); assert_eq!(config.metadata_codec_id().unwrap(), 0x04); }
773
774 #[test]
775 fn rejects_invalid_chunking() {
776 let mut config = WriteConfig::default_v0_1();
777 config.chunking.min_chunk_size = 16_000;
778 config.chunking.avg_chunk_size = 8_000;
779 assert!(config.validate().is_err());
780 }
781
782 #[test]
783 fn rejects_invalid_quality() {
784 let mut config = WriteConfig::default_v0_1();
785 config.defaults.metadata_quality = 12;
786 assert!(config.validate().is_err());
787 }
788
789 #[test]
790 fn rejects_duplicate_categorizer_names() {
791 let mut config = WriteConfig::default_v0_1();
792 config.categorizers.push(CategorizerConfig {
793 name: "dna".into(),
794 extensions: vec!["fasta".into()],
795 magic_bytes: vec![],
796 codec: "glza".into(),
797 max_size: None,
798 enabled: true,
799 });
800 config.categorizers.push(CategorizerConfig {
801 name: "dna".into(),
802 extensions: vec!["fa".into()],
803 magic_bytes: vec![],
804 codec: "glza".into(),
805 max_size: None,
806 enabled: true,
807 });
808 assert!(config.validate().is_err());
809 }
810
811 #[test]
812 fn rejects_unknown_codec() {
813 let mut config = WriteConfig::default_v0_1();
814 config.defaults.text_codec = "does-not-exist".into();
815 assert!(config.codec_registry().is_err());
816 }
817}