1pub mod defaults;
28pub mod error;
29pub mod profile;
30pub mod toml;
31
32use std::collections::BTreeMap;
33
34use serde::{Deserialize, Serialize};
35
36use crate::config::error::ConfigError;
37
38pub const DEFAULT_TEXT_CODEC: &str = "brotli";
41pub const DEFAULT_BINARY_CODEC: &str = "lz4";
44pub const DEFAULT_METADATA_CODEC: &str = "brotli";
47pub const DEFAULT_METADATA_QUALITY: u8 = 5;
49pub const DEFAULT_INLINE_THRESHOLD: u16 = 4096;
51pub const DEFAULT_AVG_CHUNK_SIZE: u32 = 8192;
53pub const DEFAULT_MIN_CHUNK_SIZE: u32 = 1024;
55pub const DEFAULT_MAX_CHUNK_SIZE: u32 = 65_536;
57pub const DEFAULT_TOURNAMENT_MIN_SIZE: u32 = 256;
59pub const DEFAULT_TOURNAMENT_SKIP_BINARY: bool = true;
61pub const DEFAULT_AEAD: &str = "chacha20-poly1305";
63pub const DEFAULT_KEY_WRAP: &str = "x25519-hkdf";
65pub const DEFAULT_DICT_ENABLED: bool = true;
67pub const DEFAULT_DICT_MIN_CLASS_SIZE: u32 = 100;
69pub const DEFAULT_DICT_MAX_SIZE: u32 = 65_536;
71
72#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
76pub struct WriteConfig {
77 pub defaults: Defaults,
79 #[serde(default, rename = "categorizer")]
81 pub categorizers: Vec<CategorizerConfig>,
82 pub chunking: ChunkingConfig,
84 pub tournament: TournamentConfig,
86 #[serde(default)]
88 pub codec_tunables: CodecTunables,
89 #[serde(default)]
91 pub mode: ImageMode,
92 #[serde(default = "default_write_codec")]
95 pub write_codec: String,
96 #[serde(default)]
99 pub turnover_threshold: u32,
100 #[serde(default)]
104 pub skip_chunking: bool,
105 pub encryption: EncryptionConfig,
107 pub dictionaries: DictionaryConfig,
109}
110
111fn default_write_codec() -> String {
112 "lz4".into()
113}
114
115fn default_metadata_externalize_threshold() -> usize {
116 crate::METADATA_EXTERNALIZE_THRESHOLD
117}
118
119#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
121pub struct Defaults {
122 pub text_codec: String,
123 pub binary_codec: String,
124 pub metadata_codec: String,
125 pub metadata_quality: u8,
126 #[serde(default = "default_metadata_externalize_threshold")]
133 pub metadata_externalize_threshold: usize,
134 #[serde(default = "default_true")]
139 pub shared_inline: bool,
140 pub inline_threshold: u16,
142}
143
144#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
146pub struct CategorizerConfig {
147 pub name: String,
149 #[serde(default)]
151 pub extensions: Vec<String>,
152 #[serde(default)]
154 pub magic_bytes: Vec<u8>,
155 pub codec: String,
157 #[serde(default)]
159 pub max_size: Option<u32>,
160 #[serde(default = "default_true")]
162 pub enabled: bool,
163}
164
165fn default_true() -> bool {
166 true
167}
168
169#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
171pub struct ChunkingConfig {
172 #[serde(default = "default_chunker_name")]
177 pub name: String,
178 #[serde(default)]
179 pub avg_chunk_size: u32,
180 #[serde(default)]
181 pub min_chunk_size: u32,
182 #[serde(default)]
183 pub max_chunk_size: u32,
184}
185
186fn default_chunker_name() -> String {
187 "fastcdc".into()
188}
189
190#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
192pub struct TournamentConfig {
193 pub codecs: Vec<String>,
197 pub min_size_threshold: u32,
199 pub skip_for_binary: bool,
201 #[serde(default = "default_short_circuit_threshold")]
213 pub short_circuit_threshold: u32,
214}
215
216fn default_short_circuit_threshold() -> u32 {
217 250
218}
219
220#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
222pub struct EncryptionConfig {
223 pub aead: String,
225 pub key_wrap: String,
227}
228
229#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
231pub struct DictionaryConfig {
232 pub enabled: bool,
233 pub min_class_size: u32,
234 pub max_dict_size: u32,
235 #[serde(default = "default_trainer")]
242 pub trainer: String,
243}
244
245fn default_trainer() -> String {
246 "frequency".into()
247}
248
249#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
254pub enum ImageMode {
255 #[default]
259 ReadOnly,
260 ReadWrite(RWMode),
262}
263
264#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
266pub enum RWMode {
267 AppendOnly,
270 #[default]
273 UpdateInPlace,
274 CopyOnWrite,
278}
279
280#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Default)]
293pub struct CodecTunables {
294 #[serde(default)]
295 pub ppmd7: Ppmd7Tunables,
296 #[serde(default)]
297 pub ppmd8: Ppmd8Tunables,
298 #[serde(default)]
299 pub brotli: BrotliTunables,
300 #[serde(default)]
301 pub lzma: LzmaTunables,
302 #[serde(default)]
303 pub bzip2: Bzip2Tunables,
304}
305
306#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
308pub struct Ppmd7Tunables {
309 pub order: u8,
310 pub memory_budget_mb: u32,
311}
312
313impl Default for Ppmd7Tunables {
314 fn default() -> Self {
315 Self {
316 order: 4,
317 memory_budget_mb: 80,
318 }
319 }
320}
321
322#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
324pub struct Ppmd8Tunables {
325 pub order: u8,
326 pub memory_budget_mb: u32,
327}
328
329impl Default for Ppmd8Tunables {
330 fn default() -> Self {
331 Self {
332 order: 6,
333 memory_budget_mb: 64,
334 }
335 }
336}
337
338#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
340pub struct BrotliTunables {
341 pub quality: u8,
342 pub window: u8,
343}
344
345impl Default for BrotliTunables {
346 fn default() -> Self {
347 Self {
348 quality: 11,
349 window: 22,
350 }
351 }
352}
353
354#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
356pub struct LzmaTunables {
357 pub lc: u8,
358 pub lp: u8,
359 pub pb: u8,
360 pub dict_size_mb: u32,
361 pub use_optimal_parser: bool,
362}
363
364impl Default for LzmaTunables {
365 fn default() -> Self {
366 Self {
367 lc: 3,
368 lp: 0,
369 pb: 2,
370 dict_size_mb: 16,
371 use_optimal_parser: false,
372 }
373 }
374}
375
376#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
378pub struct Bzip2Tunables {
379 pub block_size_kb: u32,
380}
381
382impl Default for Bzip2Tunables {
383 fn default() -> Self {
384 Self { block_size_kb: 900 }
385 }
386}
387
388impl WriteConfig {
389 #[must_use]
393 pub fn default_v0_1() -> Self {
394 Self {
395 defaults: Defaults {
396 text_codec: DEFAULT_TEXT_CODEC.to_string(),
397 binary_codec: DEFAULT_BINARY_CODEC.to_string(),
398 metadata_codec: DEFAULT_METADATA_CODEC.to_string(),
399 metadata_quality: DEFAULT_METADATA_QUALITY,
400 metadata_externalize_threshold: crate::METADATA_EXTERNALIZE_THRESHOLD,
401 shared_inline: true,
402 inline_threshold: DEFAULT_INLINE_THRESHOLD,
403 },
404 categorizers: Vec::new(),
405 chunking: ChunkingConfig {
406 name: "fastcdc".into(),
407 avg_chunk_size: DEFAULT_AVG_CHUNK_SIZE,
408 min_chunk_size: DEFAULT_MIN_CHUNK_SIZE,
409 max_chunk_size: DEFAULT_MAX_CHUNK_SIZE,
410 },
411 tournament: TournamentConfig {
412 codecs: vec![
413 "store".to_string(),
414 "lz4".to_string(),
415 "zstd".to_string(),
416 "brotli".to_string(),
417 ],
418 min_size_threshold: DEFAULT_TOURNAMENT_MIN_SIZE,
419 skip_for_binary: DEFAULT_TOURNAMENT_SKIP_BINARY,
420 short_circuit_threshold: default_short_circuit_threshold(),
421 },
422 encryption: EncryptionConfig {
423 aead: DEFAULT_AEAD.to_string(),
424 key_wrap: DEFAULT_KEY_WRAP.to_string(),
425 },
426 dictionaries: DictionaryConfig {
427 enabled: DEFAULT_DICT_ENABLED,
428 min_class_size: DEFAULT_DICT_MIN_CLASS_SIZE,
429 max_dict_size: DEFAULT_DICT_MAX_SIZE,
430 trainer: "frequency".into(),
431 },
432 codec_tunables: CodecTunables::default(),
433 mode: ImageMode::ReadOnly,
434 write_codec: default_write_codec(),
435 turnover_threshold: 0,
436 skip_chunking: false,
437 }
438 }
439
440 #[must_use]
443 pub fn from_profile(name: &str) -> Option<Self> {
444 profile::select(name)
445 }
446
447 #[must_use]
449 pub fn with_text_codec(mut self, codec: &str) -> Self {
450 self.defaults.text_codec = codec.into();
451 self
452 }
453
454 #[must_use]
456 pub fn with_binary_codec(mut self, codec: &str) -> Self {
457 self.defaults.binary_codec = codec.into();
458 self
459 }
460
461 #[must_use]
463 pub fn with_chunk_size(mut self, size: u32) -> Self {
464 self.chunking.avg_chunk_size = size;
465 self
466 }
467
468 #[must_use]
470 pub fn with_mode(mut self, mode: ImageMode) -> Self {
471 self.mode = mode;
472 self
473 }
474
475 #[must_use]
477 pub fn with_brotli_quality(mut self, quality: u8) -> Self {
478 self.codec_tunables.brotli.quality = quality;
479 self
480 }
481
482 pub fn build(self) -> Result<Self, ConfigError> {
486 self.validate()?;
487 Ok(self)
488 }
489
490 pub fn validate(&self) -> Result<(), ConfigError> {
494 if self.chunking.min_chunk_size > self.chunking.avg_chunk_size {
495 return Err(ConfigError::InvalidValue {
496 field: "chunking.min_chunk_size".into(),
497 reason: format!(
498 "min_chunk_size ({}) > avg_chunk_size ({})",
499 self.chunking.min_chunk_size, self.chunking.avg_chunk_size
500 ),
501 });
502 }
503 if self.chunking.avg_chunk_size > self.chunking.max_chunk_size {
504 return Err(ConfigError::InvalidValue {
505 field: "chunking.avg_chunk_size".into(),
506 reason: format!(
507 "avg_chunk_size ({}) > max_chunk_size ({})",
508 self.chunking.avg_chunk_size, self.chunking.max_chunk_size
509 ),
510 });
511 }
512 if self.defaults.metadata_quality < 1 || self.defaults.metadata_quality > 11 {
513 return Err(ConfigError::InvalidValue {
514 field: "defaults.metadata_quality".into(),
515 reason: format!(
516 "metadata_quality ({}) out of range 1..=11",
517 self.defaults.metadata_quality
518 ),
519 });
520 }
521 let ceiling = limnifs_core::metadata_reference::DEFAULT_INLINE_METADATA_MAX_BYTES as usize;
522 if self.defaults.metadata_externalize_threshold == 0
523 || self.defaults.metadata_externalize_threshold > ceiling
524 {
525 return Err(ConfigError::InvalidValue {
526 field: "defaults.metadata_externalize_threshold".into(),
527 reason: format!(
528 "metadata_externalize_threshold ({}) must be within 1..={ceiling} (the reader inline ceiling; larger inline metadata is unreadable)",
529 self.defaults.metadata_externalize_threshold
530 ),
531 });
532 }
533 if self.tournament.short_circuit_threshold > 1000 {
534 return Err(ConfigError::InvalidValue {
535 field: "tournament.short_circuit_threshold".into(),
536 reason: format!(
537 "short_circuit_threshold ({}) out of range 0..=1000 (per-mille)",
538 self.tournament.short_circuit_threshold
539 ),
540 });
541 }
542 let mut names_seen: BTreeMap<&str, ()> = BTreeMap::new();
544 for rule in &self.categorizers {
545 if !names_seen
546 .insert(rule.name.as_str(), ())
547 .map_or(true, |()| false)
548 {
549 return Err(ConfigError::DuplicateCategorizer(rule.name.clone()));
550 }
551 }
552 Ok(())
553 }
554
555 pub fn codec_registry(&self) -> Result<CodecRegistry, ConfigError> {
558 let mut registry = CodecRegistry::default();
559 registry.insert("store", 0x00);
560 registry.insert("lz4", 0x01);
561 registry.insert("lz4-hc", 0x13);
562 registry.insert("zstd", 0x02);
563 registry.insert("xz", 0x03);
564 registry.insert("brotli", 0x04);
565 registry.insert("deflate", 0x05);
566 registry.insert("snappy", 0x06);
567 registry.insert("flac", 0x07);
568 registry.insert("ricepp", 0x08);
569 registry.insert("fsst+brotli", 0x09);
570 registry.insert("shuffle+lz4", 0x0A);
571 registry.insert("zpaq", 0x0B);
572 registry.insert("ppmd", 0x0C);
573 registry.insert("glza", 0x0D);
574 registry.insert("shuffle+zstd", 0x0E);
575 registry.insert("bitshuffle+lz4", 0x0F);
576 registry.insert("bzip2", 0x10);
577 registry.insert("deflate64", 0x11);
578 registry.insert("libdeflate", 0x14);
579
580 if !registry.contains_name(&self.defaults.text_codec) {
581 return Err(ConfigError::UnknownCodec(self.defaults.text_codec.clone()));
582 }
583 if !registry.contains_name(&self.defaults.binary_codec) {
584 return Err(ConfigError::UnknownCodec(
585 self.defaults.binary_codec.clone(),
586 ));
587 }
588 if !registry.contains_name(&self.defaults.metadata_codec) {
589 return Err(ConfigError::UnknownCodec(
590 self.defaults.metadata_codec.clone(),
591 ));
592 }
593 for rule in &self.categorizers {
594 if !registry.contains_name(&rule.codec) {
595 return Err(ConfigError::UnknownCodec(rule.codec.clone()));
596 }
597 }
598 for codec in &self.tournament.codecs {
599 if !registry.contains_name(codec) {
600 return Err(ConfigError::UnknownCodec(codec.clone()));
601 }
602 }
603 Ok(registry)
604 }
605
606 pub fn text_codec_id(&self) -> Result<u8, ConfigError> {
610 let registry = self.codec_registry()?;
611 Ok(registry
612 .lookup_by_name(&self.defaults.text_codec)
613 .unwrap_or(0x04))
614 }
615
616 pub fn binary_codec_id(&self) -> Result<u8, ConfigError> {
620 let registry = self.codec_registry()?;
621 Ok(registry
622 .lookup_by_name(&self.defaults.binary_codec)
623 .unwrap_or(0x01))
624 }
625
626 #[must_use]
632 pub fn to_core_tunables(&self) -> limnifs_core::codec::CodecTunables {
633 limnifs_core::codec::CodecTunables {
634 quality: self.codec_tunables.brotli.quality,
635 ppmd_order: self
636 .codec_tunables
637 .ppmd7
638 .order
639 .max(self.codec_tunables.ppmd8.order),
640 ppmd7_budget: (self.codec_tunables.ppmd7.memory_budget_mb as usize)
641 .saturating_mul(1024 * 1024),
642 ppmd8_budget: (self.codec_tunables.ppmd8.memory_budget_mb as usize)
643 .saturating_mul(1024 * 1024),
644 bzip2_block_kb: self.codec_tunables.bzip2.block_size_kb,
645 lzma_dict_mb: self.codec_tunables.lzma.dict_size_mb,
646 }
647 }
648
649 pub fn metadata_codec_id(&self) -> Result<u8, ConfigError> {
653 let registry = self.codec_registry()?;
654 Ok(registry
655 .lookup_by_name(&self.defaults.metadata_codec)
656 .unwrap_or(0x04))
657 }
658}
659
660#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
662pub struct CodecRegistry {
663 by_name: BTreeMap<String, u8>,
664 by_id: BTreeMap<u8, String>,
665}
666
667impl CodecRegistry {
668 pub fn insert(&mut self, name: &str, id: u8) {
670 self.by_name.insert(name.to_string(), id);
671 self.by_id.insert(id, name.to_string());
672 }
673
674 #[must_use]
676 pub fn lookup_by_name(&self, name: &str) -> Option<u8> {
677 self.by_name.get(name).copied()
678 }
679
680 #[must_use]
682 pub fn lookup_by_id(&self, id: u8) -> Option<&str> {
683 self.by_id.get(&id).map(String::as_str)
684 }
685
686 #[must_use]
688 pub fn contains_name(&self, name: &str) -> bool {
689 self.by_name.contains_key(name)
690 }
691}
692
693#[cfg(test)]
694mod tests {
695 use super::*;
696
697 #[test]
698 fn default_v0_1_validates() {
699 let config = WriteConfig::default_v0_1();
700 config.validate().expect("v0.1 default should validate");
701 }
702
703 #[test]
704 fn default_v0_1_codec_ids() {
705 let config = WriteConfig::default_v0_1();
706 assert_eq!(config.text_codec_id().unwrap(), 0x04); assert_eq!(config.binary_codec_id().unwrap(), 0x01); assert_eq!(config.metadata_codec_id().unwrap(), 0x04); }
710
711 #[test]
712 fn rejects_invalid_chunking() {
713 let mut config = WriteConfig::default_v0_1();
714 config.chunking.min_chunk_size = 16_000;
715 config.chunking.avg_chunk_size = 8_000;
716 assert!(config.validate().is_err());
717 }
718
719 #[test]
720 fn rejects_invalid_quality() {
721 let mut config = WriteConfig::default_v0_1();
722 config.defaults.metadata_quality = 12;
723 assert!(config.validate().is_err());
724 }
725
726 #[test]
727 fn rejects_duplicate_categorizer_names() {
728 let mut config = WriteConfig::default_v0_1();
729 config.categorizers.push(CategorizerConfig {
730 name: "dna".into(),
731 extensions: vec!["fasta".into()],
732 magic_bytes: vec![],
733 codec: "glza".into(),
734 max_size: None,
735 enabled: true,
736 });
737 config.categorizers.push(CategorizerConfig {
738 name: "dna".into(),
739 extensions: vec!["fa".into()],
740 magic_bytes: vec![],
741 codec: "glza".into(),
742 max_size: None,
743 enabled: true,
744 });
745 assert!(config.validate().is_err());
746 }
747
748 #[test]
749 fn rejects_unknown_codec() {
750 let mut config = WriteConfig::default_v0_1();
751 config.defaults.text_codec = "does-not-exist".into();
752 assert!(config.codec_registry().is_err());
753 }
754}