limnifs_write/file_categorizer/
csv_text.rs

1//! CSV/text categorizer — routes CSV/JSON/TSV to FSST+Brotli.
2//!
3//! **Status:** DETECTION READY, ROUTING DISABLED.
4//!
5//! FSST (Fast Static Symbol Table) is a preprocessor that finds the
6//! most common substrings in a block and replaces each with a single
7//! byte. Reported 1.2–1.5× ratio improvement on text-heavy workloads
8//! (CSV columns, JSON keys, log files).
9//!
10//! This categorizer detects CSV/JSON/TSV by file extension and
11//! content sniffing. When `omnizip-fsst` ships and the
12//! `limnifs-core::codec::fsst_brotli` composite codec is wired in,
13//! flip `FSST_ENABLED` to `true` and the categorizer will claim
14//! these files.
15
16use std::path::Path;
17
18use super::{Categorization, FileCategorizer};
19use limnifs_core::codec::CODEC_FSST_BROTLI;
20
21/// Flip to `false` if FSST starts hurting a workload. Currently
22/// always on because `omnizip-fsst` 0.4 is shipped and the composite
23/// codec at `limnifs-core::codec::fsst_brotli` falls back to plain
24/// Brotli when FSST doesn't help.
25const FSST_ENABLED: bool = true;
26
27/// Minimum size to bother routing through FSST. Below this, FSST's
28/// dictionary overhead exceeds the gain.
29const MIN_FSST_SIZE: usize = 4 * 1024;
30
31pub struct CsvTextCategorizer;
32
33impl FileCategorizer for CsvTextCategorizer {
34    fn name(&self) -> &'static str {
35        "csv-text"
36    }
37
38    fn categories(&self) -> &'static [&'static str] {
39        &["csv-text/composite"]
40    }
41
42    fn categorize(&self, path: &Path, data: &[u8]) -> Option<Categorization> {
43        if !FSST_ENABLED {
44            return None;
45        }
46        if data.len() < MIN_FSST_SIZE {
47            return None;
48        }
49        if !looks_like_csv_text(path, data) {
50            return None;
51        }
52        Some(Categorization {
53            codec_id: CODEC_FSST_BROTLI,
54            codec_params: Vec::new(),
55            category: "csv-text/composite",
56        })
57    }
58}
59
60/// Heuristic: does this file look like CSV/JSON/TSV?
61///
62/// **Extension-required.** Content sniffing alone is too unreliable —
63/// PHP/Python/JS source has plenty of commas, quotes, and braces,
64/// which trips a pure-content heuristic and routes gigabytes of
65/// source code through FSST (extremely slow, marginal ratio gain).
66/// The categorizer only claims files with a recognised extension
67/// AND a printable majority AND CSV/JSON punctuation present.
68#[must_use]
69fn looks_like_csv_text(path: &Path, data: &[u8]) -> bool {
70    let ext = match path.extension().and_then(|e| e.to_str()) {
71        Some(e) => e.to_ascii_lowercase(),
72        None => return false,
73    };
74    if !matches!(ext.as_str(), "csv" | "tsv" | "json" | "jsonl" | "ndjson") {
75        return false;
76    }
77    let sample = if data.len() > 4096 {
78        &data[..4096]
79    } else {
80        data
81    };
82    let printables = sample
83        .iter()
84        .filter(|&&b| (0x20..=0x7E).contains(&b) || b == b'\n' || b == b'\r' || b == b'\t')
85        .count();
86    if (printables as f32 / sample.len() as f32) < 0.95 {
87        return false;
88    }
89    let punct_count = sample
90        .iter()
91        .filter(|&&b| matches!(b, b',' | b'"' | b'{' | b'}' | b'[' | b']'))
92        .count();
93    (punct_count as f32 / sample.len() as f32) > 0.01
94}
95
96#[cfg(test)]
97mod tests {
98    use super::*;
99    use std::path::PathBuf;
100
101    #[test]
102    fn csv_extension_routes_when_enabled() {
103        let c = CsvTextCategorizer;
104        let csv = "a,b,c\n".repeat(2000);
105        let cat = c
106            .categorize(&PathBuf::from("/x.csv"), csv.as_bytes())
107            .expect("csv extension claims");
108        assert_eq!(cat.codec_id, limnifs_core::codec::CODEC_FSST_BROTLI);
109    }
110
111    #[test]
112    fn csv_extension_detected_when_enabled() {
113        // Toggle the const manually via a re-evaluation would require
114        // a feature flag; for now just verify the heuristic.
115        let csv = "a,b,c\n".repeat(2000);
116        assert!(looks_like_csv_text(
117            &PathBuf::from("/x.csv"),
118            csv.as_bytes()
119        ));
120        assert!(looks_like_csv_text(
121            &PathBuf::from("/x.json"),
122            csv.as_bytes()
123        ));
124    }
125
126    #[test]
127    fn binary_data_not_misdetected() {
128        // Use high-entropy random-ish bytes rather than (0..255).cycle(),
129        // which includes all printable ASCII and trips the heuristic.
130        let mut state: u64 = 0xDEAD_BEEF_CAFE_BABE;
131        let mut binary = Vec::with_capacity(8192);
132        for _ in 0..8192 {
133            state ^= state << 13;
134            state ^= state >> 7;
135            state ^= state << 17;
136            binary.push(u8::try_from(state & 0xFF).unwrap());
137        }
138        assert!(!looks_like_csv_text(&PathBuf::from("/x.csv"), &binary));
139    }
140
141    #[test]
142    fn empty_extension_does_not_trigger_fsst() {
143        // Without a recognised extension, we DON'T claim the file —
144        // content sniffing alone is too unreliable (catches source
145        // code, emails, etc.).
146        let csv = "alpha,beta,gamma\n".repeat(500);
147        assert!(!looks_like_csv_text(
148            &PathBuf::from("/noext"),
149            csv.as_bytes()
150        ));
151    }
152}
limnifs_write/file_categorizer/csv_text.rs

limnifs_write/file_categorizer/
csv_text.rs
