docvortex_core/
text_classification.rs

1//! 原始字符分类统计，与 canonical 去重后的页面字符严格分离。
2use std::collections::HashSet;
3
4#[derive(Default)]
5pub struct Counts {
6    pub null: usize,
7    pub replacement: usize,
8    pub control: usize,
9    pub private: usize,
10    pub map_error: usize,
11    pub non_generated: usize,
12    pub fonts: Vec<(usize, usize, usize)>,
13    pub font_first: Vec<[usize; 3]>,
14}
15
16/// 保留原判定分支优先级，并按原始字体 ID 累积全部、非生成及非生成 CJK 字符数。
17pub fn count(
18    records: &[(u32, bool, bool, usize)],
19    font_count: usize,
20    cjk: &[(u32, u32)],
21    allowed: &HashSet<u32>,
22    private: (u32, u32),
23) -> Counts {
24    let mut result = Counts {
25        fonts: vec![(0, 0, 0); font_count],
26        font_first: vec![[usize::MAX; 3]; font_count],
27        ..Counts::default()
28    };
29    for (position, &(code, generated, error, font)) in records.iter().enumerate() {
30        if code == 0 {
31            result.null += 1;
32        } else if code == 0xfffd {
33            result.replacement += 1;
34        } else if (code < 32 || (127..=159).contains(&code)) && !allowed.contains(&code) {
35            result.control += 1;
36        } else if private.0 <= code && code <= private.1 {
37            result.private += 1;
38        }
39        result.map_error += usize::from(error);
40        result.fonts[font].0 += 1;
41        result.font_first[font][0] = result.font_first[font][0].min(position);
42        if !generated {
43            result.non_generated += 1;
44            result.fonts[font].1 += 1;
45            result.font_first[font][1] = result.font_first[font][1].min(position);
46            if cjk.iter().any(|&(start, end)| start <= code && code <= end) {
47                result.fonts[font].2 += 1;
48                result.font_first[font][2] = result.font_first[font][2].min(position);
49            }
50        }
51    }
52    result
53}
54
55#[cfg(test)]
56mod tests {
57    use super::*;
58
59    /// 控制字符优先级不受重叠 PUA 范围影响，generated 与错误码统计独立保留。
60    #[test]
61    fn raw_priority_and_generated_counts() {
62        let records = [
63            (0, false, false, 0),
64            (0xfffd, true, true, 0),
65            (2, false, false, 0),
66            (65, false, true, 1),
67            (0xffffffff, false, false, 1),
68        ];
69        let result = count(&records, 2, &[(65, 90)], &HashSet::new(), (0, 100));
70        assert_eq!(
71            (
72                result.null,
73                result.replacement,
74                result.control,
75                result.private,
76                result.map_error,
77                result.non_generated
78            ),
79            (1, 1, 1, 1, 2, 4)
80        );
81        assert_eq!(result.fonts, vec![(3, 2, 0), (2, 2, 1)]);
82    }
83}
docvortex_core/text_classification.rs

docvortex_core/
text_classification.rs
