1pub mod defaults;
28pub mod error;
29pub mod profile;
30pub mod toml;
31
32use std::collections::BTreeMap;
33
34use serde::{Deserialize, Serialize};
35
36use crate::config::error::ConfigError;
37
38pub const DEFAULT_TEXT_CODEC: &str = "brotli";
41pub const DEFAULT_BINARY_CODEC: &str = "lz4";
44pub const DEFAULT_METADATA_CODEC: &str = "brotli";
47pub const DEFAULT_METADATA_QUALITY: u8 = 5;
49pub const DEFAULT_INLINE_THRESHOLD: u16 = 4096;
51
52pub const DEFAULT_MAX_DROP_SIZE: u32 = 4 * 1024 * 1024;
57pub const DEFAULT_AVG_CHUNK_SIZE: u32 = 262_144;
59pub const DEFAULT_MIN_CHUNK_SIZE: u32 = 65_536;
61pub const DEFAULT_MAX_CHUNK_SIZE: u32 = 1_048_576;
63pub const DEFAULT_TOURNAMENT_MIN_SIZE: u32 = 256;
65pub const DEFAULT_TOURNAMENT_SKIP_BINARY: bool = true;
67pub const DEFAULT_AEAD: &str = "chacha20-poly1305";
69pub const DEFAULT_KEY_WRAP: &str = "x25519-hkdf";
71pub const DEFAULT_DICT_ENABLED: bool = true;
73pub const DEFAULT_DICT_MIN_CLASS_SIZE: u32 = 100;
75pub const DEFAULT_DICT_MAX_SIZE: u32 = 65_536;
77
78#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
82pub struct WriteConfig {
83 pub defaults: Defaults,
85 #[serde(default, rename = "categorizer")]
87 pub categorizers: Vec<CategorizerConfig>,
88 pub chunking: ChunkingConfig,
90 pub tournament: TournamentConfig,
92 #[serde(default)]
94 pub codec_tunables: CodecTunables,
95 #[serde(default)]
97 pub mode: ImageMode,
98 #[serde(default = "default_write_codec")]
101 pub write_codec: String,
102 #[serde(default)]
105 pub turnover_threshold: u32,
106 #[serde(default)]
110 pub skip_chunking: bool,
111 pub encryption: EncryptionConfig,
113 pub dictionaries: DictionaryConfig,
115}
116
117fn default_write_codec() -> String {
118 "lz4".into()
119}
120
121fn default_metadata_externalize_threshold() -> usize {
122 crate::METADATA_EXTERNALIZE_THRESHOLD
123}
124
125#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
127pub struct Defaults {
128 pub text_codec: String,
129 pub binary_codec: String,
130 pub metadata_codec: String,
131 pub metadata_quality: u8,
132 #[serde(default = "default_metadata_externalize_threshold")]
139 pub metadata_externalize_threshold: usize,
140 #[serde(default = "default_true")]
145 pub shared_inline: bool,
146 #[serde(default = "default_true")]
152 pub seekable_drops: bool,
153 #[serde(default = "default_max_drop_size")]
162 pub max_drop_size: u32,
163 pub inline_threshold: u16,
165}
166
167#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
169pub struct CategorizerConfig {
170 pub name: String,
172 #[serde(default)]
174 pub extensions: Vec<String>,
175 #[serde(default)]
177 pub magic_bytes: Vec<u8>,
178 pub codec: String,
180 #[serde(default)]
182 pub max_size: Option<u32>,
183 #[serde(default = "default_true")]
185 pub enabled: bool,
186}
187
188fn default_true() -> bool {
189 true
190}
191
192fn default_max_drop_size() -> u32 {
193 DEFAULT_MAX_DROP_SIZE
194}
195
196#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
198pub struct ChunkingConfig {
199 #[serde(default = "default_chunker_name")]
204 pub name: String,
205 #[serde(default)]
206 pub avg_chunk_size: u32,
207 #[serde(default)]
208 pub min_chunk_size: u32,
209 #[serde(default)]
210 pub max_chunk_size: u32,
211}
212
213fn default_chunker_name() -> String {
214 "fastcdc".into()
215}
216
217#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
219pub struct TournamentConfig {
220 pub codecs: Vec<String>,
224 pub min_size_threshold: u32,
226 pub skip_for_binary: bool,
228 #[serde(default = "default_short_circuit_threshold")]
240 pub short_circuit_threshold: u32,
241}
242
243fn default_short_circuit_threshold() -> u32 {
244 250
245}
246
247#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
249pub struct EncryptionConfig {
250 pub aead: String,
252 pub key_wrap: String,
254}
255
256#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
258pub struct DictionaryConfig {
259 pub enabled: bool,
260 pub min_class_size: u32,
261 pub max_dict_size: u32,
262 #[serde(default = "default_trainer")]
269 pub trainer: String,
270}
271
272fn default_trainer() -> String {
273 "frequency".into()
274}
275
276#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
281pub enum ImageMode {
282 #[default]
286 ReadOnly,
287 ReadWrite(RWMode),
289}
290
291#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
293pub enum RWMode {
294 AppendOnly,
297 #[default]
300 UpdateInPlace,
301 CopyOnWrite,
305}
306
307#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
320pub struct CodecTunables {
321 #[serde(default)]
322 pub ppmd7: Ppmd7Tunables,
323 #[serde(default)]
324 pub ppmd8: Ppmd8Tunables,
325 #[serde(default)]
326 pub brotli: BrotliTunables,
327 #[serde(default)]
328 pub lzma: LzmaTunables,
329 #[serde(default)]
330 pub bzip2: Bzip2Tunables,
331}
332
333#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
335pub struct Ppmd7Tunables {
336 pub order: u8,
337 pub memory_budget_mb: u32,
338}
339
340impl Default for Ppmd7Tunables {
341 fn default() -> Self {
342 Self {
343 order: 4,
344 memory_budget_mb: 80,
345 }
346 }
347}
348
349#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
351pub struct Ppmd8Tunables {
352 pub order: u8,
353 pub memory_budget_mb: u32,
354}
355
356impl Default for Ppmd8Tunables {
357 fn default() -> Self {
358 Self {
359 order: 6,
360 memory_budget_mb: 64,
361 }
362 }
363}
364
365#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
367pub struct BrotliTunables {
368 pub quality: u8,
369 pub window: u8,
370}
371
372impl Default for BrotliTunables {
373 fn default() -> Self {
374 Self {
375 quality: 11,
376 window: 22,
377 }
378 }
379}
380
381#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
383pub struct LzmaTunables {
384 pub lc: u8,
385 pub lp: u8,
386 pub pb: u8,
387 pub dict_size_mb: u32,
388 pub use_optimal_parser: bool,
389}
390
391impl Default for LzmaTunables {
392 fn default() -> Self {
393 Self {
394 lc: 3,
395 lp: 0,
396 pb: 2,
397 dict_size_mb: 16,
398 use_optimal_parser: false,
399 }
400 }
401}
402
403#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
405pub struct Bzip2Tunables {
406 pub block_size_kb: u32,
407}
408
409impl Default for Bzip2Tunables {
410 fn default() -> Self {
411 Self { block_size_kb: 900 }
412 }
413}
414
415impl WriteConfig {
416 #[must_use]
420 pub fn default_v0_1() -> Self {
421 Self {
422 defaults: Defaults {
423 text_codec: DEFAULT_TEXT_CODEC.to_string(),
424 binary_codec: DEFAULT_BINARY_CODEC.to_string(),
425 metadata_codec: DEFAULT_METADATA_CODEC.to_string(),
426 metadata_quality: DEFAULT_METADATA_QUALITY,
427 metadata_externalize_threshold: crate::METADATA_EXTERNALIZE_THRESHOLD,
428 shared_inline: true,
429 seekable_drops: true,
430 max_drop_size: DEFAULT_MAX_DROP_SIZE,
431 inline_threshold: DEFAULT_INLINE_THRESHOLD,
432 },
433 categorizers: Vec::new(),
434 chunking: ChunkingConfig {
435 name: "fastcdc".into(),
436 avg_chunk_size: 262_144,
437 min_chunk_size: 65_536,
438 max_chunk_size: 1_048_576,
439 },
440 tournament: TournamentConfig {
441 codecs: vec![
442 "store".to_string(),
443 "lz4".to_string(),
444 "zstd".to_string(),
445 "brotli".to_string(),
446 ],
447 min_size_threshold: DEFAULT_TOURNAMENT_MIN_SIZE,
448 skip_for_binary: DEFAULT_TOURNAMENT_SKIP_BINARY,
449 short_circuit_threshold: default_short_circuit_threshold(),
450 },
451 encryption: EncryptionConfig {
452 aead: DEFAULT_AEAD.to_string(),
453 key_wrap: DEFAULT_KEY_WRAP.to_string(),
454 },
455 dictionaries: DictionaryConfig {
456 enabled: DEFAULT_DICT_ENABLED,
457 min_class_size: DEFAULT_DICT_MIN_CLASS_SIZE,
458 max_dict_size: DEFAULT_DICT_MAX_SIZE,
459 trainer: "frequency".into(),
460 },
461 codec_tunables: CodecTunables::default(),
462 mode: ImageMode::ReadOnly,
463 write_codec: default_write_codec(),
464 turnover_threshold: 0,
465 skip_chunking: false,
466 }
467 }
468
469 #[must_use]
472 pub fn from_profile(name: &str) -> Option<Self> {
473 profile::select(name)
474 }
475
476 #[must_use]
478 pub fn with_text_codec(mut self, codec: &str) -> Self {
479 self.defaults.text_codec = codec.into();
480 self
481 }
482
483 #[must_use]
485 pub fn with_binary_codec(mut self, codec: &str) -> Self {
486 self.defaults.binary_codec = codec.into();
487 self
488 }
489
490 #[must_use]
492 pub fn with_chunk_size(mut self, size: u32) -> Self {
493 self.chunking.avg_chunk_size = size;
494 self
495 }
496
497 #[must_use]
499 pub fn with_mode(mut self, mode: ImageMode) -> Self {
500 self.mode = mode;
501 self
502 }
503
504 #[must_use]
506 pub fn with_brotli_quality(mut self, quality: u8) -> Self {
507 self.codec_tunables.brotli.quality = quality;
508 self
509 }
510
511 pub fn build(self) -> Result<Self, ConfigError> {
515 self.validate()?;
516 Ok(self)
517 }
518
519 pub fn validate(&self) -> Result<(), ConfigError> {
523 if self.chunking.min_chunk_size > self.chunking.avg_chunk_size {
524 return Err(ConfigError::InvalidValue {
525 field: "chunking.min_chunk_size".into(),
526 reason: format!(
527 "min_chunk_size ({}) > avg_chunk_size ({})",
528 self.chunking.min_chunk_size, self.chunking.avg_chunk_size
529 ),
530 });
531 }
532 if self.chunking.avg_chunk_size > self.chunking.max_chunk_size {
533 return Err(ConfigError::InvalidValue {
534 field: "chunking.avg_chunk_size".into(),
535 reason: format!(
536 "avg_chunk_size ({}) > max_chunk_size ({})",
537 self.chunking.avg_chunk_size, self.chunking.max_chunk_size
538 ),
539 });
540 }
541 if self.defaults.metadata_quality < 1 || self.defaults.metadata_quality > 11 {
542 return Err(ConfigError::InvalidValue {
543 field: "defaults.metadata_quality".into(),
544 reason: format!(
545 "metadata_quality ({}) out of range 1..=11",
546 self.defaults.metadata_quality
547 ),
548 });
549 }
550 let ceiling = limnifs_core::metadata_reference::DEFAULT_INLINE_METADATA_MAX_BYTES as usize;
551 if self.defaults.metadata_externalize_threshold == 0
552 || self.defaults.metadata_externalize_threshold > ceiling
553 {
554 return Err(ConfigError::InvalidValue {
555 field: "defaults.metadata_externalize_threshold".into(),
556 reason: format!(
557 "metadata_externalize_threshold ({}) must be within 1..={ceiling} (the reader inline ceiling; larger inline metadata is unreadable)",
558 self.defaults.metadata_externalize_threshold
559 ),
560 });
561 }
562 if self.defaults.max_drop_size != 0 && self.defaults.max_drop_size < 1024 {
563 return Err(ConfigError::InvalidValue {
564 field: "defaults.max_drop_size".into(),
565 reason: format!(
566 "max_drop_size ({}) is nonzero but below 1 KiB — no whole-file \
567 drop can form under the cap; raise it or set 0 (unlimited)",
568 self.defaults.max_drop_size
569 ),
570 });
571 }
572 if self.tournament.short_circuit_threshold > 1000 {
573 return Err(ConfigError::InvalidValue {
574 field: "tournament.short_circuit_threshold".into(),
575 reason: format!(
576 "short_circuit_threshold ({}) out of range 0..=1000 (per-mille)",
577 self.tournament.short_circuit_threshold
578 ),
579 });
580 }
581 let mut names_seen: BTreeMap<&str, ()> = BTreeMap::new();
583 for rule in &self.categorizers {
584 if !names_seen
585 .insert(rule.name.as_str(), ())
586 .map_or(true, |()| false)
587 {
588 return Err(ConfigError::DuplicateCategorizer(rule.name.clone()));
589 }
590 }
591 Ok(())
592 }
593
594 pub fn codec_registry(&self) -> Result<CodecRegistry, ConfigError> {
597 let mut registry = CodecRegistry::default();
598 registry.insert("store", 0x00);
599 registry.insert("lz4", 0x01);
600 registry.insert("lz4-hc", 0x13);
601 registry.insert("zstd", 0x02);
602 registry.insert("xz", 0x03);
603 registry.insert("brotli", 0x04);
604 registry.insert("deflate", 0x05);
605 registry.insert("snappy", 0x06);
606 registry.insert("flac", 0x07);
607 registry.insert("ricepp", 0x08);
608 registry.insert("fsst+brotli", 0x09);
609 registry.insert("shuffle+lz4", 0x0A);
610 registry.insert("zpaq", 0x0B);
611 registry.insert("ppmd", 0x0C);
612 registry.insert("glza", 0x0D);
613 registry.insert("shuffle+zstd", 0x0E);
614 registry.insert("bitshuffle+lz4", 0x0F);
615 registry.insert("bzip2", 0x10);
616 registry.insert("deflate64", 0x11);
617 registry.insert("libdeflate", 0x14);
618 registry.insert("bcj-x86-lz4", 0x20);
619 registry.insert("bcj-x86-zstd", 0x21);
620 registry.insert("bcj-arm64-lz4", 0x23);
621 registry.insert("bcj-arm64-zstd", 0x24);
622
623 if !registry.contains_name(&self.defaults.text_codec) {
624 return Err(ConfigError::UnknownCodec(self.defaults.text_codec.clone()));
625 }
626 if !registry.contains_name(&self.defaults.binary_codec) {
627 return Err(ConfigError::UnknownCodec(
628 self.defaults.binary_codec.clone(),
629 ));
630 }
631 if !registry.contains_name(&self.defaults.metadata_codec) {
632 return Err(ConfigError::UnknownCodec(
633 self.defaults.metadata_codec.clone(),
634 ));
635 }
636 for rule in &self.categorizers {
637 if !registry.contains_name(&rule.codec) {
638 return Err(ConfigError::UnknownCodec(rule.codec.clone()));
639 }
640 }
641 for codec in &self.tournament.codecs {
642 if !registry.contains_name(codec) {
643 return Err(ConfigError::UnknownCodec(codec.clone()));
644 }
645 }
646 Ok(registry)
647 }
648
649 pub fn text_codec_id(&self) -> Result<u8, ConfigError> {
653 let registry = self.codec_registry()?;
654 Ok(registry
655 .lookup_by_name(&self.defaults.text_codec)
656 .unwrap_or(0x04))
657 }
658
659 pub fn binary_codec_id(&self) -> Result<u8, ConfigError> {
663 let registry = self.codec_registry()?;
664 Ok(registry
665 .lookup_by_name(&self.defaults.binary_codec)
666 .unwrap_or(0x01))
667 }
668
669 #[must_use]
675 pub fn to_core_tunables(&self) -> limnifs_core::codec::CodecTunables {
676 limnifs_core::codec::CodecTunables {
677 quality: self.codec_tunables.brotli.quality,
678 ppmd_order: self
679 .codec_tunables
680 .ppmd7
681 .order
682 .max(self.codec_tunables.ppmd8.order),
683 ppmd7_budget: (self.codec_tunables.ppmd7.memory_budget_mb as usize)
684 .saturating_mul(1024 * 1024),
685 ppmd8_budget: (self.codec_tunables.ppmd8.memory_budget_mb as usize)
686 .saturating_mul(1024 * 1024),
687 bzip2_block_kb: self.codec_tunables.bzip2.block_size_kb,
688 lzma_dict_mb: self.codec_tunables.lzma.dict_size_mb,
689 }
690 }
691
692 pub fn metadata_codec_id(&self) -> Result<u8, ConfigError> {
696 let registry = self.codec_registry()?;
697 Ok(registry
698 .lookup_by_name(&self.defaults.metadata_codec)
699 .unwrap_or(0x04))
700 }
701}
702
703#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
705pub struct CodecRegistry {
706 by_name: BTreeMap<String, u8>,
707 by_id: BTreeMap<u8, String>,
708}
709
710impl CodecRegistry {
711 pub fn insert(&mut self, name: &str, id: u8) {
713 self.by_name.insert(name.to_string(), id);
714 self.by_id.insert(id, name.to_string());
715 }
716
717 #[must_use]
719 pub fn lookup_by_name(&self, name: &str) -> Option<u8> {
720 self.by_name.get(name).copied()
721 }
722
723 #[must_use]
725 pub fn lookup_by_id(&self, id: u8) -> Option<&str> {
726 self.by_id.get(&id).map(String::as_str)
727 }
728
729 #[must_use]
731 pub fn contains_name(&self, name: &str) -> bool {
732 self.by_name.contains_key(name)
733 }
734}
735
736#[cfg(test)]
737mod tests {
738 use super::*;
739
740 #[test]
741 fn default_v0_1_validates() {
742 let config = WriteConfig::default_v0_1();
743 config.validate().expect("v0.1 default should validate");
744 }
745
746 #[test]
747 fn default_v0_1_codec_ids() {
748 let config = WriteConfig::default_v0_1();
749 assert_eq!(config.text_codec_id().unwrap(), 0x04); assert_eq!(config.binary_codec_id().unwrap(), 0x01); assert_eq!(config.metadata_codec_id().unwrap(), 0x04); }
753
754 #[test]
755 fn rejects_invalid_chunking() {
756 let mut config = WriteConfig::default_v0_1();
757 config.chunking.min_chunk_size = 16_000;
758 config.chunking.avg_chunk_size = 8_000;
759 assert!(config.validate().is_err());
760 }
761
762 #[test]
763 fn rejects_invalid_quality() {
764 let mut config = WriteConfig::default_v0_1();
765 config.defaults.metadata_quality = 12;
766 assert!(config.validate().is_err());
767 }
768
769 #[test]
770 fn rejects_duplicate_categorizer_names() {
771 let mut config = WriteConfig::default_v0_1();
772 config.categorizers.push(CategorizerConfig {
773 name: "dna".into(),
774 extensions: vec!["fasta".into()],
775 magic_bytes: vec![],
776 codec: "glza".into(),
777 max_size: None,
778 enabled: true,
779 });
780 config.categorizers.push(CategorizerConfig {
781 name: "dna".into(),
782 extensions: vec!["fa".into()],
783 magic_bytes: vec![],
784 codec: "glza".into(),
785 max_size: None,
786 enabled: true,
787 });
788 assert!(config.validate().is_err());
789 }
790
791 #[test]
792 fn rejects_unknown_codec() {
793 let mut config = WriteConfig::default_v0_1();
794 config.defaults.text_codec = "does-not-exist".into();
795 assert!(config.codec_registry().is_err());
796 }
797}