limnifs_write/file_categorizer/
mod.rs

1//! File-level categorizer framework.
2//!
3//! The seine chunk classifier (`crate::classifier`) operates on
4//! chunks AFTER `FastCDC` has split a file. By that point file-level
5//! signal is gone — a FITS header lives in chunk 0; chunk 50 looks
6//! like generic binary. Specialized codecs (FLAC for PCM audio,
7//! ricepp for FITS) need that file-level signal to route correctly.
8//!
9//! This module runs file-level categorizers BEFORE `FastCDC`. If a
10//! categorizer claims the file, the whole file becomes one drop
11//! compressed with the categorizer's chosen codec. Otherwise the
12//! file falls through to the existing `FastCDC` path unchanged.
13//!
14//! ## Architecture
15//!
16//! - [`FileCategorizer`] trait: synchronous, pure-functional,
17//!   deterministic. Same `(path, data)` → same `Categorization`.
18//! - [`FileCategorizerRegistry`]: OCP. Adding a categorizer is one
19//!   new file + one `register()` call. Dispatch code never changes.
20//! - The registry is consulted by `process_file` before `FastCDC`.
21//!
22//! ## Current state
23//!
24//! The registry ships EMPTY today. Categorizers for FLAC (PCM audio),
25//! ricepp (FITS), and FSST (CSV/JSON) will be added when the
26//! corresponding omnizip codec crates ship. The framework is in
27//! place so the integration is a one-file PR per codec.
28
29use std::path::Path;
30use std::sync::OnceLock;
31
32pub mod csv_text;
33pub mod executable;
34pub mod fits;
35pub mod pcm_audio;
36pub mod registry;
37
38pub use registry::FileCategorizerRegistry;
39
40/// Process-wide default registry. Populated on first access with
41/// every shipped categorizer (`pcm_audio`, fits, `csv_text`). The
42/// writer calls `default_registry().categorize(...)` from
43/// `process_file` before `FastCDC`; categorizers that aren't
44/// enabled (their `*_ENABLED` flag is false) return `None`
45/// internally and fall through.
46///
47/// Adding a categorizer: implement `FileCategorizer`, push an
48/// instance here. Dispatch code never changes.
49#[must_use]
50pub fn default_registry() -> &'static FileCategorizerRegistry {
51    static REGISTRY: OnceLock<FileCategorizerRegistry> = OnceLock::new();
52    REGISTRY.get_or_init(|| {
53        FileCategorizerRegistry::new()
54            .register(Box::new(fits::FitsCategorizer))
55            .register(Box::new(pcm_audio::PcmAudioCategorizer))
56            .register(Box::new(csv_text::CsvTextCategorizer))
57            .register(Box::new(executable::ExecutableCategorizer))
58    })
59}
60
61/// A categorizer's decision for one file.
62///
63/// `codec_id` selects the codec; `codec_params` carries any
64/// codec-specific parameters the categorizer extracted from the
65/// file header (e.g. PCM sample format for FLAC, bitpix for
66/// ricepp). The codec crate owns its parameter format; the
67/// categorizer just hands opaque bytes through.
68#[derive(Clone, Debug)]
69pub struct Categorization {
70    /// Codec id from `limnifs_core::codec` (e.g. `CODEC_FLAC`).
71    pub codec_id: u8,
72    /// Codec-specific parameters extracted from the file header.
73    /// Encoded format is owned by the codec crate; opaque to the
74    /// framework.
75    pub codec_params: Vec<u8>,
76    /// Human-readable category name for diagnostics
77    /// (e.g. `"pcmaudio/waveform"`, `"fits/image"`).
78    pub category: &'static str,
79}
80
81/// One file-level categorizer.
82///
83/// Implementations should be:
84/// - **Pure-functional**: same input → same output, no I/O.
85/// - **Deterministic**: no clocks, no RNG, no system state.
86/// - **Cheap to refuse**: header parsing should bail on the first
87///   mismatched magic byte, not scan the whole file.
88///
89/// Categorizers are tried in registration order. The first one to
90/// return `Some(Categorization)` wins; later categorizers are not
91/// consulted. Order matters: register specific categorizers before
92/// generic ones.
93pub trait FileCategorizer: Sync + Send {
94    /// Unique name for logging/diagnostics.
95    fn name(&self) -> &'static str;
96
97    /// Categories this categorizer can emit. Used for diagnostic
98    /// dumps; does not affect dispatch.
99    fn categories(&self) -> &'static [&'static str];
100
101    /// Categorize a file by its path and full contents.
102    ///
103    /// Returns `Some(Categorization)` if this categorizer claims the
104    /// file, `None` to defer to the next categorizer in the registry
105    /// (or to the `FastCDC` fallback path).
106    ///
107    /// Implementations should not read the file from disk — `data`
108    /// is already in hand. Path is provided for extension-based
109    /// hints when magic-byte detection is ambiguous.
110    fn categorize(&self, path: &Path, data: &[u8]) -> Option<Categorization>;
111
112    /// The set of first bytes that this categorizer can possibly
113    /// match. The registry uses this to skip categorizers without
114    /// a function call when `data[0]` isn't in the set.
115    ///
116    /// Return `None` (default) to opt out of the early-exit
117    /// optimisation — the categorizer is always tried. Return
118    /// `Some(&[bytes])` to enable early-exit: the registry checks
119    /// `data[0]` and skips this categorizer if it's not in the set.
120    ///
121    /// Example: ELF categorizer returns `Some(&[0x7F])` — it can
122    /// only match files whose first byte is 0x7F.
123    fn first_byte_hint(&self) -> Option<&'static [u8]> {
124        None
125    }
126}
limnifs_write/file_categorizer/mod.rs

limnifs_write/file_categorizer/
mod.rs
