1pub mod defaults;
28pub mod error;
29pub mod profile;
30pub mod toml;
31
32use std::collections::BTreeMap;
33
34use serde::{Deserialize, Serialize};
35
36use crate::config::error::ConfigError;
37
38pub const DEFAULT_TEXT_CODEC: &str = "brotli";
41pub const DEFAULT_BINARY_CODEC: &str = "lz4";
44pub const DEFAULT_METADATA_CODEC: &str = "brotli";
47pub const DEFAULT_METADATA_QUALITY: u8 = 5;
49pub const DEFAULT_INLINE_THRESHOLD: u16 = 4096;
51pub const DEFAULT_AVG_CHUNK_SIZE: u32 = 8192;
53pub const DEFAULT_MIN_CHUNK_SIZE: u32 = 1024;
55pub const DEFAULT_MAX_CHUNK_SIZE: u32 = 65_536;
57pub const DEFAULT_TOURNAMENT_MIN_SIZE: u32 = 256;
59pub const DEFAULT_TOURNAMENT_SKIP_BINARY: bool = true;
61pub const DEFAULT_AEAD: &str = "chacha20-poly1305";
63pub const DEFAULT_KEY_WRAP: &str = "x25519-hkdf";
65pub const DEFAULT_DICT_ENABLED: bool = true;
67pub const DEFAULT_DICT_MIN_CLASS_SIZE: u32 = 100;
69pub const DEFAULT_DICT_MAX_SIZE: u32 = 65_536;
71
72#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
76pub struct WriteConfig {
77 pub defaults: Defaults,
79 #[serde(default, rename = "categorizer")]
81 pub categorizers: Vec<CategorizerConfig>,
82 pub chunking: ChunkingConfig,
84 pub tournament: TournamentConfig,
86 #[serde(default)]
88 pub codec_tunables: CodecTunables,
89 #[serde(default)]
91 pub mode: ImageMode,
92 #[serde(default = "default_write_codec")]
95 pub write_codec: String,
96 #[serde(default)]
99 pub turnover_threshold: u32,
100 #[serde(default)]
104 pub skip_chunking: bool,
105 pub encryption: EncryptionConfig,
107 pub dictionaries: DictionaryConfig,
109}
110
111fn default_write_codec() -> String {
112 "lz4".into()
113}
114
115fn default_metadata_externalize_threshold() -> usize {
116 crate::METADATA_EXTERNALIZE_THRESHOLD
117}
118
119#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
121pub struct Defaults {
122 pub text_codec: String,
123 pub binary_codec: String,
124 pub metadata_codec: String,
125 pub metadata_quality: u8,
126 #[serde(default = "default_metadata_externalize_threshold")]
133 pub metadata_externalize_threshold: usize,
134 pub inline_threshold: u16,
136}
137
138#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
140pub struct CategorizerConfig {
141 pub name: String,
143 #[serde(default)]
145 pub extensions: Vec<String>,
146 #[serde(default)]
148 pub magic_bytes: Vec<u8>,
149 pub codec: String,
151 #[serde(default)]
153 pub max_size: Option<u32>,
154 #[serde(default = "default_true")]
156 pub enabled: bool,
157}
158
159fn default_true() -> bool {
160 true
161}
162
163#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
165pub struct ChunkingConfig {
166 #[serde(default = "default_chunker_name")]
171 pub name: String,
172 #[serde(default)]
173 pub avg_chunk_size: u32,
174 #[serde(default)]
175 pub min_chunk_size: u32,
176 #[serde(default)]
177 pub max_chunk_size: u32,
178}
179
180fn default_chunker_name() -> String {
181 "fastcdc".into()
182}
183
184#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
186pub struct TournamentConfig {
187 pub codecs: Vec<String>,
191 pub min_size_threshold: u32,
193 pub skip_for_binary: bool,
195 #[serde(default = "default_short_circuit_threshold")]
207 pub short_circuit_threshold: u32,
208}
209
210fn default_short_circuit_threshold() -> u32 {
211 250
212}
213
214#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
216pub struct EncryptionConfig {
217 pub aead: String,
219 pub key_wrap: String,
221}
222
223#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
225pub struct DictionaryConfig {
226 pub enabled: bool,
227 pub min_class_size: u32,
228 pub max_dict_size: u32,
229 #[serde(default = "default_trainer")]
236 pub trainer: String,
237}
238
239fn default_trainer() -> String {
240 "frequency".into()
241}
242
243#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
248pub enum ImageMode {
249 #[default]
253 ReadOnly,
254 ReadWrite(RWMode),
256}
257
258#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
260pub enum RWMode {
261 AppendOnly,
264 #[default]
267 UpdateInPlace,
268 CopyOnWrite,
272}
273
274#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
287pub struct CodecTunables {
288 #[serde(default)]
289 pub ppmd7: Ppmd7Tunables,
290 #[serde(default)]
291 pub ppmd8: Ppmd8Tunables,
292 #[serde(default)]
293 pub brotli: BrotliTunables,
294 #[serde(default)]
295 pub lzma: LzmaTunables,
296 #[serde(default)]
297 pub bzip2: Bzip2Tunables,
298}
299
300#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
302pub struct Ppmd7Tunables {
303 pub order: u8,
304 pub memory_budget_mb: u32,
305}
306
307impl Default for Ppmd7Tunables {
308 fn default() -> Self {
309 Self {
310 order: 4,
311 memory_budget_mb: 80,
312 }
313 }
314}
315
316#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
318pub struct Ppmd8Tunables {
319 pub order: u8,
320 pub memory_budget_mb: u32,
321}
322
323impl Default for Ppmd8Tunables {
324 fn default() -> Self {
325 Self {
326 order: 6,
327 memory_budget_mb: 64,
328 }
329 }
330}
331
332#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
334pub struct BrotliTunables {
335 pub quality: u8,
336 pub window: u8,
337}
338
339impl Default for BrotliTunables {
340 fn default() -> Self {
341 Self {
342 quality: 11,
343 window: 22,
344 }
345 }
346}
347
348#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
350pub struct LzmaTunables {
351 pub lc: u8,
352 pub lp: u8,
353 pub pb: u8,
354 pub dict_size_mb: u32,
355 pub use_optimal_parser: bool,
356}
357
358impl Default for LzmaTunables {
359 fn default() -> Self {
360 Self {
361 lc: 3,
362 lp: 0,
363 pb: 2,
364 dict_size_mb: 16,
365 use_optimal_parser: false,
366 }
367 }
368}
369
370#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
372pub struct Bzip2Tunables {
373 pub block_size_kb: u32,
374}
375
376impl Default for Bzip2Tunables {
377 fn default() -> Self {
378 Self { block_size_kb: 900 }
379 }
380}
381
382impl WriteConfig {
383 #[must_use]
387 pub fn default_v0_1() -> Self {
388 Self {
389 defaults: Defaults {
390 text_codec: DEFAULT_TEXT_CODEC.to_string(),
391 binary_codec: DEFAULT_BINARY_CODEC.to_string(),
392 metadata_codec: DEFAULT_METADATA_CODEC.to_string(),
393 metadata_quality: DEFAULT_METADATA_QUALITY,
394 metadata_externalize_threshold: crate::METADATA_EXTERNALIZE_THRESHOLD,
395 inline_threshold: DEFAULT_INLINE_THRESHOLD,
396 },
397 categorizers: Vec::new(),
398 chunking: ChunkingConfig {
399 name: "fastcdc".into(),
400 avg_chunk_size: DEFAULT_AVG_CHUNK_SIZE,
401 min_chunk_size: DEFAULT_MIN_CHUNK_SIZE,
402 max_chunk_size: DEFAULT_MAX_CHUNK_SIZE,
403 },
404 tournament: TournamentConfig {
405 codecs: vec![
406 "store".to_string(),
407 "lz4".to_string(),
408 "zstd".to_string(),
409 "brotli".to_string(),
410 ],
411 min_size_threshold: DEFAULT_TOURNAMENT_MIN_SIZE,
412 skip_for_binary: DEFAULT_TOURNAMENT_SKIP_BINARY,
413 short_circuit_threshold: default_short_circuit_threshold(),
414 },
415 encryption: EncryptionConfig {
416 aead: DEFAULT_AEAD.to_string(),
417 key_wrap: DEFAULT_KEY_WRAP.to_string(),
418 },
419 dictionaries: DictionaryConfig {
420 enabled: DEFAULT_DICT_ENABLED,
421 min_class_size: DEFAULT_DICT_MIN_CLASS_SIZE,
422 max_dict_size: DEFAULT_DICT_MAX_SIZE,
423 trainer: "frequency".into(),
424 },
425 codec_tunables: CodecTunables::default(),
426 mode: ImageMode::ReadOnly,
427 write_codec: default_write_codec(),
428 turnover_threshold: 0,
429 skip_chunking: false,
430 }
431 }
432
433 #[must_use]
436 pub fn from_profile(name: &str) -> Option<Self> {
437 profile::select(name)
438 }
439
440 #[must_use]
442 pub fn with_text_codec(mut self, codec: &str) -> Self {
443 self.defaults.text_codec = codec.into();
444 self
445 }
446
447 #[must_use]
449 pub fn with_binary_codec(mut self, codec: &str) -> Self {
450 self.defaults.binary_codec = codec.into();
451 self
452 }
453
454 #[must_use]
456 pub fn with_chunk_size(mut self, size: u32) -> Self {
457 self.chunking.avg_chunk_size = size;
458 self
459 }
460
461 #[must_use]
463 pub fn with_mode(mut self, mode: ImageMode) -> Self {
464 self.mode = mode;
465 self
466 }
467
468 #[must_use]
470 pub fn with_brotli_quality(mut self, quality: u8) -> Self {
471 self.codec_tunables.brotli.quality = quality;
472 self
473 }
474
475 pub fn build(self) -> Result<Self, ConfigError> {
479 self.validate()?;
480 Ok(self)
481 }
482
483 pub fn validate(&self) -> Result<(), ConfigError> {
487 if self.chunking.min_chunk_size > self.chunking.avg_chunk_size {
488 return Err(ConfigError::InvalidValue {
489 field: "chunking.min_chunk_size".into(),
490 reason: format!(
491 "min_chunk_size ({}) > avg_chunk_size ({})",
492 self.chunking.min_chunk_size, self.chunking.avg_chunk_size
493 ),
494 });
495 }
496 if self.chunking.avg_chunk_size > self.chunking.max_chunk_size {
497 return Err(ConfigError::InvalidValue {
498 field: "chunking.avg_chunk_size".into(),
499 reason: format!(
500 "avg_chunk_size ({}) > max_chunk_size ({})",
501 self.chunking.avg_chunk_size, self.chunking.max_chunk_size
502 ),
503 });
504 }
505 if self.defaults.metadata_quality < 1 || self.defaults.metadata_quality > 11 {
506 return Err(ConfigError::InvalidValue {
507 field: "defaults.metadata_quality".into(),
508 reason: format!(
509 "metadata_quality ({}) out of range 1..=11",
510 self.defaults.metadata_quality
511 ),
512 });
513 }
514 let ceiling = limnifs_core::metadata_reference::DEFAULT_INLINE_METADATA_MAX_BYTES as usize;
515 if self.defaults.metadata_externalize_threshold == 0
516 || self.defaults.metadata_externalize_threshold > ceiling
517 {
518 return Err(ConfigError::InvalidValue {
519 field: "defaults.metadata_externalize_threshold".into(),
520 reason: format!(
521 "metadata_externalize_threshold ({}) must be within 1..={ceiling} (the reader inline ceiling; larger inline metadata is unreadable)",
522 self.defaults.metadata_externalize_threshold
523 ),
524 });
525 }
526 if self.tournament.short_circuit_threshold > 1000 {
527 return Err(ConfigError::InvalidValue {
528 field: "tournament.short_circuit_threshold".into(),
529 reason: format!(
530 "short_circuit_threshold ({}) out of range 0..=1000 (per-mille)",
531 self.tournament.short_circuit_threshold
532 ),
533 });
534 }
535 let mut names_seen: BTreeMap<&str, ()> = BTreeMap::new();
537 for rule in &self.categorizers {
538 if !names_seen
539 .insert(rule.name.as_str(), ())
540 .map_or(true, |()| false)
541 {
542 return Err(ConfigError::DuplicateCategorizer(rule.name.clone()));
543 }
544 }
545 Ok(())
546 }
547
548 pub fn codec_registry(&self) -> Result<CodecRegistry, ConfigError> {
551 let mut registry = CodecRegistry::default();
552 registry.insert("store", 0x00);
553 registry.insert("lz4", 0x01);
554 registry.insert("lz4-hc", 0x13);
555 registry.insert("zstd", 0x02);
556 registry.insert("xz", 0x03);
557 registry.insert("brotli", 0x04);
558 registry.insert("deflate", 0x05);
559 registry.insert("snappy", 0x06);
560 registry.insert("flac", 0x07);
561 registry.insert("ricepp", 0x08);
562 registry.insert("fsst+brotli", 0x09);
563 registry.insert("shuffle+lz4", 0x0A);
564 registry.insert("zpaq", 0x0B);
565 registry.insert("ppmd", 0x0C);
566 registry.insert("glza", 0x0D);
567 registry.insert("shuffle+zstd", 0x0E);
568 registry.insert("bitshuffle+lz4", 0x0F);
569 registry.insert("bzip2", 0x10);
570 registry.insert("deflate64", 0x11);
571 registry.insert("libdeflate", 0x14);
572
573 if !registry.contains_name(&self.defaults.text_codec) {
574 return Err(ConfigError::UnknownCodec(self.defaults.text_codec.clone()));
575 }
576 if !registry.contains_name(&self.defaults.binary_codec) {
577 return Err(ConfigError::UnknownCodec(
578 self.defaults.binary_codec.clone(),
579 ));
580 }
581 if !registry.contains_name(&self.defaults.metadata_codec) {
582 return Err(ConfigError::UnknownCodec(
583 self.defaults.metadata_codec.clone(),
584 ));
585 }
586 for rule in &self.categorizers {
587 if !registry.contains_name(&rule.codec) {
588 return Err(ConfigError::UnknownCodec(rule.codec.clone()));
589 }
590 }
591 for codec in &self.tournament.codecs {
592 if !registry.contains_name(codec) {
593 return Err(ConfigError::UnknownCodec(codec.clone()));
594 }
595 }
596 Ok(registry)
597 }
598
599 pub fn text_codec_id(&self) -> Result<u8, ConfigError> {
603 let registry = self.codec_registry()?;
604 Ok(registry
605 .lookup_by_name(&self.defaults.text_codec)
606 .unwrap_or(0x04))
607 }
608
609 pub fn binary_codec_id(&self) -> Result<u8, ConfigError> {
613 let registry = self.codec_registry()?;
614 Ok(registry
615 .lookup_by_name(&self.defaults.binary_codec)
616 .unwrap_or(0x01))
617 }
618
619 #[must_use]
625 pub fn to_core_tunables(&self) -> limnifs_core::codec::CodecTunables {
626 limnifs_core::codec::CodecTunables {
627 quality: self.codec_tunables.brotli.quality,
628 ppmd_order: self
629 .codec_tunables
630 .ppmd7
631 .order
632 .max(self.codec_tunables.ppmd8.order),
633 ppmd7_budget: (self.codec_tunables.ppmd7.memory_budget_mb as usize)
634 .saturating_mul(1024 * 1024),
635 ppmd8_budget: (self.codec_tunables.ppmd8.memory_budget_mb as usize)
636 .saturating_mul(1024 * 1024),
637 bzip2_block_kb: self.codec_tunables.bzip2.block_size_kb,
638 lzma_dict_mb: self.codec_tunables.lzma.dict_size_mb,
639 }
640 }
641
642 pub fn metadata_codec_id(&self) -> Result<u8, ConfigError> {
646 let registry = self.codec_registry()?;
647 Ok(registry
648 .lookup_by_name(&self.defaults.metadata_codec)
649 .unwrap_or(0x04))
650 }
651}
652
653#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
655pub struct CodecRegistry {
656 by_name: BTreeMap<String, u8>,
657 by_id: BTreeMap<u8, String>,
658}
659
660impl CodecRegistry {
661 pub fn insert(&mut self, name: &str, id: u8) {
663 self.by_name.insert(name.to_string(), id);
664 self.by_id.insert(id, name.to_string());
665 }
666
667 #[must_use]
669 pub fn lookup_by_name(&self, name: &str) -> Option<u8> {
670 self.by_name.get(name).copied()
671 }
672
673 #[must_use]
675 pub fn lookup_by_id(&self, id: u8) -> Option<&str> {
676 self.by_id.get(&id).map(String::as_str)
677 }
678
679 #[must_use]
681 pub fn contains_name(&self, name: &str) -> bool {
682 self.by_name.contains_key(name)
683 }
684}
685
686#[cfg(test)]
687mod tests {
688 use super::*;
689
690 #[test]
691 fn default_v0_1_validates() {
692 let config = WriteConfig::default_v0_1();
693 config.validate().expect("v0.1 default should validate");
694 }
695
696 #[test]
697 fn default_v0_1_codec_ids() {
698 let config = WriteConfig::default_v0_1();
699 assert_eq!(config.text_codec_id().unwrap(), 0x04); assert_eq!(config.binary_codec_id().unwrap(), 0x01); assert_eq!(config.metadata_codec_id().unwrap(), 0x04); }
703
704 #[test]
705 fn rejects_invalid_chunking() {
706 let mut config = WriteConfig::default_v0_1();
707 config.chunking.min_chunk_size = 16_000;
708 config.chunking.avg_chunk_size = 8_000;
709 assert!(config.validate().is_err());
710 }
711
712 #[test]
713 fn rejects_invalid_quality() {
714 let mut config = WriteConfig::default_v0_1();
715 config.defaults.metadata_quality = 12;
716 assert!(config.validate().is_err());
717 }
718
719 #[test]
720 fn rejects_duplicate_categorizer_names() {
721 let mut config = WriteConfig::default_v0_1();
722 config.categorizers.push(CategorizerConfig {
723 name: "dna".into(),
724 extensions: vec!["fasta".into()],
725 magic_bytes: vec![],
726 codec: "glza".into(),
727 max_size: None,
728 enabled: true,
729 });
730 config.categorizers.push(CategorizerConfig {
731 name: "dna".into(),
732 extensions: vec!["fa".into()],
733 magic_bytes: vec![],
734 codec: "glza".into(),
735 max_size: None,
736 enabled: true,
737 });
738 assert!(config.validate().is_err());
739 }
740
741 #[test]
742 fn rejects_unknown_codec() {
743 let mut config = WriteConfig::default_v0_1();
744 config.defaults.text_codec = "does-not-exist".into();
745 assert!(config.codec_registry().is_err());
746 }
747}