vyre_primitives/text/
encoding_classify.rs

1//! Encoding classifier over a precomputed 256-bin byte histogram.
2
3use std::sync::Arc;
4
5use vyre_foundation::ir::model::expr::{GeneratorRef, Ident};
6use vyre_foundation::ir::{BufferAccess, BufferDecl, DataType, Expr, Node, Program};
7
8use crate::reduce::range_counts::range_counts_u32_child;
9use crate::text::utf8_shape_counts::{utf8_shape_counts_child, utf8_shape_counts_from_histogram};
10
11/// Canonical op id for histogram-based encoding classification.
12pub const ENCODING_CLASSIFY_OP_ID: &str = "vyre-primitives::text::encoding_classify";
13/// Single-result workgroup for standalone histogram classification.
14pub const ENCODING_CLASSIFY_WORKGROUP_SIZE: [u32; 3] = [1, 1, 1];
15
16/// Encoding-id for pure ASCII input.
17pub const ENC_ASCII: u32 = 0;
18/// Encoding-id for UTF-8 input.
19pub const ENC_UTF8: u32 = 1;
20/// Encoding-id for UTF-16 little-endian input.
21pub const ENC_UTF16LE: u32 = 2;
22/// Encoding-id for UTF-16 big-endian input.
23pub const ENC_UTF16BE: u32 = 3;
24/// Encoding-id for ISO-8859-1 / Windows-1252-like high-byte input.
25pub const ENC_ISO8859_1: u32 = 4;
26/// Encoding-id for unknown or binary input.
27pub const ENC_BINARY: u32 = 255;
28
29/// Build the reusable classifier body.
30#[must_use]
31pub fn encoding_classify_body(histogram: &str, output: &str, count: u32) -> Vec<Node> {
32    vec![Node::if_then(
33        Expr::eq(Expr::InvocationId { axis: 0 }, Expr::u32(0)),
34        {
35            let mut body = vec![
36                Node::let_bind("null_count", Expr::load(histogram, Expr::u32(0))),
37                Node::let_bind("ascii_count", Expr::u32(0)),
38                range_counts_u32_child(ENCODING_CLASSIFY_OP_ID, histogram, "ascii_count", 0, 128),
39                Node::let_bind(
40                    "high_count",
41                    Expr::sub(Expr::u32(count), Expr::var("ascii_count")),
42                ),
43                Node::let_bind("enc_id", Expr::u32(ENC_BINARY)),
44                Node::if_then(
45                    Expr::eq(Expr::var("high_count"), Expr::u32(0)),
46                    vec![Node::assign("enc_id", Expr::u32(ENC_ASCII))],
47                ),
48                Node::if_then(
49                    Expr::gt(
50                        Expr::var("null_count"),
51                        Expr::div(Expr::u32(count), Expr::u32(8)),
52                    ),
53                    vec![Node::assign("enc_id", Expr::u32(ENC_UTF16LE))],
54                ),
55                Node::let_bind("continuation", Expr::u32(0)),
56                Node::let_bind("expected_continuation", Expr::u32(0)),
57                utf8_shape_counts_child(
58                    ENCODING_CLASSIFY_OP_ID,
59                    histogram,
60                    "continuation",
61                    "expected_continuation",
62                ),
63            ];
64
65            body.push(Node::if_then(
66                Expr::and(
67                    Expr::gt(Expr::var("high_count"), Expr::u32(0)),
68                    Expr::lt(
69                        Expr::abs_diff(
70                            Expr::var("continuation"),
71                            Expr::var("expected_continuation"),
72                        ),
73                        Expr::div(Expr::u32(count.saturating_add(19)), Expr::u32(20)),
74                    ),
75                ),
76                vec![Node::assign("enc_id", Expr::u32(ENC_UTF8))],
77            ));
78            body.push(Node::if_then(
79                Expr::and(
80                    Expr::gt(Expr::var("high_count"), Expr::u32(0)),
81                    Expr::ne(Expr::var("enc_id"), Expr::u32(ENC_UTF8)),
82                ),
83                vec![Node::if_then(
84                    Expr::ne(Expr::var("enc_id"), Expr::u32(ENC_UTF16LE)),
85                    vec![Node::assign("enc_id", Expr::u32(ENC_ISO8859_1))],
86                )],
87            ));
88            body.push(Node::store(output, Expr::u32(0), Expr::var("enc_id")));
89            body
90        },
91    )]
92}
93
94/// Wrap the classifier body as a child of `parent_op_id`.
95#[must_use]
96pub fn encoding_classify_child(
97    parent_op_id: &str,
98    histogram: &str,
99    output: &str,
100    count: u32,
101) -> Node {
102    Node::Region {
103        generator: Ident::from(ENCODING_CLASSIFY_OP_ID),
104        source_region: Some(GeneratorRef {
105            name: parent_op_id.to_string(),
106        }),
107        body: Arc::new(encoding_classify_body(histogram, output, count)),
108    }
109}
110
111/// Standalone classifier program for primitive-level conformance.
112#[must_use]
113pub fn encoding_classify(histogram: &str, output: &str, count: u32) -> Program {
114    Program::wrapped(
115        vec![
116            BufferDecl::storage(histogram, 0, BufferAccess::ReadOnly, DataType::U32)
117                .with_count(256),
118            BufferDecl::output(output, 1, DataType::U32)
119                .with_count(1)
120                .with_output_byte_range(0..4),
121        ],
122        ENCODING_CLASSIFY_WORKGROUP_SIZE,
123        vec![Node::Region {
124            generator: Ident::from(ENCODING_CLASSIFY_OP_ID),
125            source_region: None,
126            body: Arc::new(encoding_classify_body(histogram, output, count)),
127        }],
128    )
129}
130
131/// Reference oracle for [`encoding_classify`].
132#[must_use]
133pub fn classify_from_histogram(histogram: &[u32; 256], count: u32) -> u32 {
134    if count == 0 {
135        return ENC_ASCII;
136    }
137    let null_count = histogram[0];
138    let ascii_count: u32 = histogram[0..128].iter().sum();
139    let high_count = count - ascii_count;
140
141    if null_count > count / 8 {
142        return ENC_UTF16LE;
143    }
144    if high_count == 0 {
145        return ENC_ASCII;
146    }
147
148    let (continuation, expected_continuation) = utf8_shape_counts_from_histogram(histogram);
149
150    let tolerance = count.saturating_add(19) / 20;
151    if continuation.abs_diff(expected_continuation) < tolerance {
152        return ENC_UTF8;
153    }
154
155    ENC_ISO8859_1
156}
157
158#[cfg(feature = "inventory-registry")]
159inventory::submit! {
160    vyre_foundation::operation::OperationRegistration::primitive(
161        ENCODING_CLASSIFY_OP_ID,
162        || encoding_classify("histogram", "encoding", 5),
163        Some(|| {
164            let mut histogram = vec![0u8; 256 * 4];
165            for (slot, value) in [(b'H' as usize, 1u32), (b'e' as usize, 1), (b'l' as usize, 2), (b'o' as usize, 1)] {
166                histogram[slot * 4..slot * 4 + 4].copy_from_slice(&value.to_le_bytes());
167            }
168            vec![vec![histogram, vec![0; 4]]]
169        }),
170        Some(|| vec![vec![ENC_ASCII.to_le_bytes().to_vec()]]),
171    )
172}
173
174#[cfg(test)]
175mod tests {
176    use super::*;
177
178    #[test]
179    fn classifies_ascii_histogram() {
180        let mut histogram = [0u32; 256];
181        histogram[usize::from(b'H')] = 1;
182        histogram[usize::from(b'e')] = 1;
183        histogram[usize::from(b'l')] = 2;
184        histogram[usize::from(b'o')] = 1;
185        assert_eq!(classify_from_histogram(&histogram, 5), ENC_ASCII);
186    }
187
188    #[test]
189    fn classifies_utf8_shape() {
190        let mut histogram = [0u32; 256];
191        histogram[0xC3] = 2;
192        histogram[0xA9] = 2;
193        assert_eq!(classify_from_histogram(&histogram, 4), ENC_UTF8);
194    }
195
196    #[test]
197    fn program_uses_single_result_workgroup() {
198        let program = encoding_classify("histogram", "encoding", 0);
199        assert_eq!(program.workgroup_size(), ENCODING_CLASSIFY_WORKGROUP_SIZE);
200    }
201}
vyre_primitives/text/encoding_classify.rs

vyre_primitives/text/
encoding_classify.rs
