Skip to main content

docvortex_core/
text_pipeline.rs

1//! 字符到基础文本行的连续原生管线;不携带 Python 对象或 PDFium 指针。
2use std::collections::HashMap;
3
4/// 保存当前 Python 解释器的非 ASCII 字符属性,纯计算阶段不回调 Python。
5#[derive(Default)]
6pub struct UnicodeProperties(pub HashMap<char, u8>);
7impl UnicodeProperties {
8    /// ASCII 使用固定语义,其他字符读取绑定层去重后的 Unicode 属性。
9    fn flags(&self, ch: char) -> u8 {
10        if ch.is_ascii() {
11            (u8::from(ch.is_ascii_alphanumeric()))
12                | (u8::from(ch.is_ascii_digit()) << 1)
13                | (u8::from(ch.is_ascii_digit()) << 5)
14                | (u8::from(matches!(ch, '+' | '<' | '=' | '>' | '|' | '~')) << 2)
15                | (u8::from((' '..='~').contains(&ch)) << 4)
16                | (u8::from(matches!(ch, '\t'..='\r' | '\u{1c}'..='\u{1f}' | ' ')) << 3)
17        } else {
18            self.0.get(&ch).copied().unwrap_or(0)
19        }
20    }
21    /// 保持 Python str.strip 的空白定义。
22    fn is_space(&self, ch: char) -> bool {
23        self.flags(ch) & 8 != 0
24    }
25}
26
27#[derive(Clone, Debug)]
28pub struct TextChar {
29    pub bbox: [f64; 4],
30    pub text: String,
31    pub rotation: f64,
32    pub font_id: usize,
33    pub font_size: Option<f64>,
34    pub signature_id: Option<usize>,
35    pub font_weight: Option<f64>,
36}
37
38#[derive(Debug)]
39pub struct TextSpan {
40    pub bbox: [f64; 4],
41    pub text: String,
42    pub start: usize,
43    pub end: usize,
44    pub font_id: usize,
45    pub rotation: f64,
46    pub superscript: bool,
47    pub subscript: bool,
48}
49
50#[derive(Debug)]
51pub struct TextLine {
52    pub bbox: [f64; 4],
53    pub rotation: f64,
54    pub spans: Vec<TextSpan>,
55}
56
57/// 按 Python 累加矩形的比较顺序合并,保留非有限输入的既有行为。
58fn merge(a: &mut [f64; 4], b: [f64; 4]) {
59    if b[0] < a[0] {
60        a[0] = b[0];
61    }
62    if b[1] < a[1] {
63        a[1] = b[1];
64    }
65    if b[2] > a[2] {
66        a[2] = b[2];
67    }
68    if b[3] > a[3] {
69        a[3] = b[3];
70    }
71}
72
73/// 一次遍历生成字体片段,再连续执行组行与上下标标记。
74pub fn group_text_lines(
75    chars: &[TextChar],
76    height_threshold: f64,
77    distance: f64,
78    unicode: &UnicodeProperties,
79) -> Vec<TextLine> {
80    let mut spans: Vec<TextSpan> = Vec::new();
81    for (index, ch) in chars.iter().enumerate() {
82        let split = match spans.last() {
83            None => true,
84            Some(span) => {
85                let height = span.bbox[3] - span.bbox[1];
86                ch.font_id != span.font_id
87                    || ch.rotation != span.rotation
88                    || matches!(chars[index - 1].text.as_str(), "\x02" | "\n")
89                    || (ch.bbox[1] < span.bbox[1] - height * distance
90                        && ch.bbox[3] < height * height_threshold + span.bbox[1]
91                        && ch.bbox[0] > span.bbox[2])
92            }
93        };
94        if split {
95            spans.push(TextSpan {
96                bbox: ch.bbox,
97                text: ch.text.clone(),
98                start: index,
99                end: index + 1,
100                font_id: ch.font_id,
101                rotation: ch.rotation,
102                superscript: false,
103                subscript: false,
104            });
105        } else {
106            let span = spans.last_mut().unwrap();
107            merge(&mut span.bbox, ch.bbox);
108            span.text.push_str(&ch.text);
109            span.end = index + 1;
110        }
111    }
112    let mut lines: Vec<TextLine> = Vec::new();
113    for span in spans {
114        let split = match lines.last() {
115            None => true,
116            Some(line) => {
117                let text = &line.spans.last().unwrap().text;
118                let diff = (span.rotation - line.rotation).abs() % std::f64::consts::TAU;
119                let diff = diff.min(std::f64::consts::TAU - diff);
120                text.ends_with('\n')
121                    || text.ends_with('\x02')
122                    || (span.rotation != line.rotation
123                        && (std::f64::consts::FRAC_PI_4..=3.0 * std::f64::consts::FRAC_PI_4)
124                            .contains(&diff))
125                    || span.bbox[1] > line.bbox[3]
126            }
127        };
128        if split {
129            lines.push(TextLine {
130                bbox: span.bbox,
131                rotation: span.rotation,
132                spans: vec![span],
133            });
134        } else {
135            let line = lines.last_mut().unwrap();
136            merge(&mut line.bbox, span.bbox);
137            line.spans.push(span);
138        }
139    }
140    for line in &mut lines {
141        assign_scripts(line, height_threshold, distance, unicode);
142    }
143    lines
144}
145
146/// 在线性时间取得两个极值,保持同值时的首索引规则。
147fn extreme_two(values: impl Iterator<Item = f64>, maximum: bool) -> (f64, usize, f64) {
148    let initial = if maximum {
149        f64::NEG_INFINITY
150    } else {
151        f64::INFINITY
152    };
153    let (mut first, mut at, mut second) = (initial, usize::MAX, initial);
154    for (i, v) in values.enumerate() {
155        let better = if maximum { v > first } else { v < first };
156        if better {
157            second = first;
158            first = v;
159            at = i;
160        } else if if maximum { v > second } else { v < second } {
161            second = v;
162        }
163    }
164    (first, at, second)
165}
166
167/// 按基础片段几何标记上下标,Unicode 判定使用与参考后端一致的类别表。
168fn assign_scripts(line: &mut TextLine, threshold: f64, distance: f64, unicode: &UnicodeProperties) {
169    let n = line.spans.len();
170    let line_height = line.bbox[3] - line.bbox[1];
171    if n < 2 || line_height > line.bbox[2] - line.bbox[0] {
172        return;
173    }
174    let heights: Vec<_> = line.spans.iter().map(|s| s.bbox[3] - s.bbox[1]).collect();
175    let above = extreme_two(
176        line.spans
177            .iter()
178            .zip(&heights)
179            .map(|(s, h)| s.bbox[1] - h * distance),
180        true,
181    );
182    let below = extreme_two(
183        line.spans
184            .iter()
185            .zip(&heights)
186            .map(|(s, h)| s.bbox[3] + h * distance),
187        false,
188    );
189    for i in 0..n {
190        let spans = &line.spans;
191        let first = i == 0
192            || spans[i - 1]
193                .text
194                .trim_matches(|ch| unicode.is_space(ch))
195                .is_empty();
196        let last = i == n - 1
197            || spans[i + 1]
198                .text
199                .trim_matches(|ch| unicode.is_space(ch))
200                .is_empty();
201        let span = &spans[i];
202        let height = heights[i];
203        let full = height / line_height.max(1.0) <= threshold;
204        let neighbor_full = (first || height / heights[i - 1].max(1.0) <= threshold)
205            || (last || height / heights[i + 1].max(1.0) <= threshold);
206        let top = span.bbox[1];
207        let bottom = span.bbox[3];
208        let text = span.text.trim_matches(|ch| unicode.is_space(ch));
209        let count = text.chars().count();
210        let okay = (count == 1
211            || (!text.is_empty() && text.chars().all(|ch| unicode.flags(ch) & 2 != 0)))
212            && ((!text.is_empty() && text.chars().all(|ch| unicode.flags(ch) & 1 != 0))
213                || (count == 1 && text.chars().all(|ch| unicode.flags(ch) & 4 != 0)));
214        let super_flag = neighbor_full
215            && full
216            && okay
217            && top < if i == above.1 { above.2 } else { above.0 }
218            && ((first || top < spans[i - 1].bbox[1]) || (last || top < spans[i + 1].bbox[1]));
219        let sub_flag = !super_flag
220            && neighbor_full
221            && full
222            && okay
223            && bottom > if i == below.1 { below.2 } else { below.0 }
224            && ((first || bottom > spans[i - 1].bbox[3])
225                || (last || bottom > spans[i + 1].bbox[3]));
226        line.spans[i].superscript = super_flag;
227        line.spans[i].subscript = sub_flag;
228    }
229}
230
231#[derive(Debug)]
232pub struct VisualTextRun {
233    pub text: String,
234    pub bbox: [f64; 4],
235    pub angle: i32,
236    pub indices: Vec<usize>,
237    pub row_id: usize,
238    pub run_index: usize,
239    pub split: bool,
240    pub formula: bool,
241    pub coarse_fallback: bool,
242    pub paragraph_terminal: bool,
243    pub typography: Option<crate::statistics::Typography>,
244    pub typographic_scale: Option<f64>,
245}
246
247/// 按 Python 规则缓存正数字体尺寸中位数,缺失时留给调用方使用行高回退。
248fn typographic_scale(indices: &[usize], chars: &[TextChar]) -> Option<f64> {
249    let sizes: Vec<_> = indices
250        .iter()
251        .filter_map(|&index| chars[index].font_size)
252        .filter(|size| size.is_finite() && *size > 0.0)
253        .collect();
254    (!sizes.is_empty()).then(|| 0.1_f64.max(crate::median(sizes)))
255}
256
257/// 返回 Python 字符串语义下的可打印、全空白标志。
258fn glyph_flags(text: &str, unicode: &UnicodeProperties) -> u8 {
259    let printable = text.chars().all(|ch| unicode.flags(ch) & 16 != 0);
260    let space = !text.is_empty() && text.chars().all(|ch| unicode.is_space(ch));
261    u8::from(printable && !space) | (u8::from(space) << 1)
262}
263
264/// 将弧度换成非负圆周角,匹配 Python 对非法方向的零度处理。
265fn angle_degrees(value: f64) -> f64 {
266    if value.is_finite() {
267        value.to_degrees().rem_euclid(360.0)
268    } else {
269        0.0
270    }
271}
272
273/// 计算两个角度的最短圆周距离。
274fn angle_distance(a: f64, b: f64) -> f64 {
275    ((a - b + 180.0).rem_euclid(360.0) - 180.0).abs()
276}
277
278/// 用字符中心及中位字高识别仿斜体矩阵,不把真实斜排误判成正文。
279fn horizontal_baseline(
280    spans: &[TextSpan],
281    chars: &[TextChar],
282    unicode: &UnicodeProperties,
283) -> bool {
284    use crate::geometry;
285    let mut boxes: Vec<_> = spans
286        .iter()
287        .flat_map(|s| s.start..s.end)
288        .filter(|&i| glyph_flags(&chars[i].text, unicode) & 1 != 0)
289        .filter_map(|i| geometry::normalize(Some(chars[i].bbox), false))
290        .collect();
291    if boxes.len() < 4 {
292        return false;
293    }
294    let bbox = geometry::union(&boxes).unwrap();
295    let width = bbox[2] - bbox[0];
296    let height = bbox[3] - bbox[1];
297    let median = crate::median(boxes.iter().map(|b| b[3] - b[1]).collect());
298    if height <= 0.0 || median <= 0.0 || width / height < 3.0 {
299        return false;
300    }
301    boxes.sort_by(|a, b| {
302        ((a[0] + a[2]) / 2.0)
303            .total_cmp(&((b[0] + b[2]) / 2.0))
304            .then_with(|| ((a[1] + a[3]) / 2.0).total_cmp(&((b[1] + b[3]) / 2.0)))
305    });
306    let first = boxes.first().unwrap();
307    let last = boxes.last().unwrap();
308    let baseline_width = (last[0] + last[2]) / 2.0 - (first[0] + first[2]) / 2.0;
309    if baseline_width <= 0.0 {
310        return false;
311    }
312    let dy = (last[1] + last[3]) / 2.0 - (first[1] + first[3]) / 2.0;
313    let angle = dy.atan2(baseline_width).to_degrees().abs();
314    let minimum = boxes
315        .iter()
316        .map(|b| (b[1] + b[3]) / 2.0)
317        .fold(f64::INFINITY, f64::min);
318    let maximum = boxes
319        .iter()
320        .map(|b| (b[1] + b[3]) / 2.0)
321        .fold(f64::NEG_INFINITY, f64::max);
322    angle <= 2.0 && maximum - minimum <= 0.75 * median
323}
324
325/// 负字号必须同时得到源字符正向基线证实才纠正半周翻转,单字不作推断。
326fn correct_negative_scale_angle(
327    spans: &[TextSpan],
328    visual: f64,
329    chars: &[TextChar],
330    unicode: &UnicodeProperties,
331) -> f64 {
332    let candidate = (visual + 180.0).rem_euclid(360.0);
333    let cardinal = [0.0, 90.0, 180.0, 270.0]
334        .into_iter()
335        .min_by(|a, b| angle_distance(candidate, *a).total_cmp(&angle_distance(candidate, *b)))
336        .unwrap();
337    if angle_distance(candidate, cardinal) > 0.1 {
338        return visual;
339    }
340    let (axis_x, axis_y) = match cardinal as i32 {
341        0 => (1.0, 0.0),
342        90 => (0.0, 1.0),
343        180 => (-1.0, 0.0),
344        _ => (0.0, -1.0),
345    };
346    let mut positions = Vec::new();
347    let mut heights = Vec::new();
348    // 字符、字体签名和字号共同确定描粗副层的去重身份。
349    type GlyphIdentity<'a> = (&'a str, Option<usize>, u64);
350    let mut seen: HashMap<GlyphIdentity<'_>, Vec<[f64; 4]>> = HashMap::new();
351    for index in spans.iter().flat_map(|span| span.start..span.end) {
352        let ch = &chars[index];
353        if ch.text.is_empty() || !ch.text.chars().all(|c| unicode.flags(c) & 1 != 0) {
354            continue;
355        }
356        let Some(bbox) = crate::geometry::normalize(Some(ch.bbox), false) else {
357            continue;
358        };
359        if bbox[0] >= bbox[2] || bbox[1] >= bbox[3] {
360            continue;
361        }
362        if !ch
363            .font_size
364            .is_some_and(|size| size.is_finite() && size < 0.0)
365        {
366            return visual;
367        }
368        let area = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]);
369        let previous = seen
370            .entry((
371                ch.text.as_str(),
372                ch.signature_id,
373                ch.font_size.unwrap().to_bits(),
374            ))
375            .or_default();
376        if previous.iter().any(|old| {
377            let intersection = (bbox[2].min(old[2]) - bbox[0].max(old[0])).max(0.0)
378                * (bbox[3].min(old[3]) - bbox[1].max(old[1])).max(0.0);
379            let union = area + (old[2] - old[0]) * (old[3] - old[1]) - intersection;
380            intersection >= 0.5 * union
381        }) {
382            continue;
383        }
384        // 描粗副层只从方向证据去重,不删除或重排原始字符成员。
385        previous.push(bbox);
386        let (x, y) = ((bbox[0] + bbox[2]) / 2.0, (bbox[1] + bbox[3]) / 2.0);
387        positions.push((x * axis_x + y * axis_y, -x * axis_y + y * axis_x));
388        heights.push(if axis_x != 0.0 {
389            bbox[3] - bbox[1]
390        } else {
391            bbox[2] - bbox[0]
392        });
393    }
394    if positions.len() < 2 {
395        return visual;
396    }
397    let median_height = crate::median(heights);
398    let first = positions.first().unwrap();
399    let last = positions.last().unwrap();
400    let (forward, sideways) = (last.0 - first.0, last.1 - first.1);
401    let minimum = positions.iter().map(|p| p.1).fold(f64::INFINITY, f64::min);
402    let maximum = positions
403        .iter()
404        .map(|p| p.1)
405        .fold(f64::NEG_INFINITY, f64::max);
406    if forward < 0.5 * median_height
407        || sideways.atan2(forward).to_degrees().abs() > 2.0
408        || maximum - minimum > 0.75 * median_height
409        || positions
410            .windows(2)
411            .any(|pair| pair[1].0 < pair[0].0 - 0.1 * median_height)
412    {
413        return visual;
414    }
415    cardinal
416}
417
418/// 解析常规方向及公式专用小角度候选,在白名单前恢复已证实的负尺度方向。
419fn visual_angle(
420    spans: &[TextSpan],
421    rotation: f64,
422    chars: &[TextChar],
423    unicode: &UnicodeProperties,
424    page_rotation: i32,
425    supported: &[f64],
426) -> Option<(i32, bool)> {
427    let visual = (angle_degrees(rotation) + page_rotation as f64).rem_euclid(360.0);
428    let visual = correct_negative_scale_angle(spans, visual, chars, unicode);
429    for &angle in supported {
430        if angle_distance(visual, angle) <= 0.1 {
431            return Some((angle as i32, false));
432        }
433    }
434    if supported
435        .iter()
436        .any(|&angle| angle_distance(0.0, angle) <= 0.1)
437        && angle_distance(visual, 0.0) <= 30.0
438        && horizontal_baseline(spans, chars, unicode)
439    {
440        return Some((0, false));
441    }
442    let nearest = supported
443        .iter()
444        .copied()
445        .min_by(|a, b| angle_distance(visual, *a).total_cmp(&angle_distance(visual, *b)))?;
446    if angle_distance(visual, nearest) > 30.0 {
447        return None;
448    }
449    let compact: String = spans
450        .iter()
451        .flat_map(|s| s.text.chars())
452        .filter(|&ch| unicode.flags(ch) & 16 != 0 && !unicode.is_space(ch))
453        .collect();
454    if compact.is_empty() {
455        return None;
456    }
457    let script = spans.iter().any(|s| s.superscript || s.subscript);
458    let math = compact.chars().any(|ch| "=∑∫√±×÷".contains(ch));
459    let sizes: Vec<_> = spans
460        .iter()
461        .filter_map(|s| chars[s.start].font_size)
462        .filter(|v| *v > 0.0)
463        .collect();
464    let mixed = !sizes.is_empty()
465        && sizes.iter().copied().fold(f64::NEG_INFINITY, f64::max)
466            >= 1.35 * sizes.iter().copied().fold(f64::INFINITY, f64::min);
467    let identifier =
468        compact.chars().count() <= 3 && compact.chars().all(|ch| unicode.flags(ch) & 1 != 0);
469    if script || math || mixed || identifier {
470        Some((nearest as i32, true))
471    } else {
472        None
473    }
474}
475
476/// 将连续基础组行、旋转拆分、方向筛选及视觉间隙分段融合,避免中间 Python 片段对象。
477pub fn prepare_visual_lines(
478    chars: &[TextChar],
479    size: [f64; 2],
480    page_rotation: i32,
481    supported: &[f64],
482    unicode: &UnicodeProperties,
483    families: &[Option<usize>],
484) -> Vec<VisualTextRun> {
485    use crate::geometry;
486    let lines = group_text_lines(chars, 0.7, 0.1, unicode);
487    let mut output = Vec::new();
488    let mut row_id = 0;
489    for line in lines {
490        let mut start = 0;
491        let mut angle = angle_degrees(line.rotation);
492        let mut chunks = Vec::new();
493        for (i, span) in line.spans.iter().enumerate() {
494            let span_angle = angle_degrees(span.rotation);
495            let visible = span
496                .text
497                .chars()
498                .any(|ch| unicode.flags(ch) & 16 != 0 && !unicode.is_space(ch));
499            if i > start && visible && angle_distance(span_angle, angle) >= 44.9 {
500                chunks.push((start, i, angle));
501                start = i;
502                angle = span_angle;
503            }
504        }
505        chunks.push((start, line.spans.len(), angle));
506        for (start, end, angle) in chunks {
507            let spans = &line.spans[start..end];
508            let Some((visual_angle, formula)) = visual_angle(
509                spans,
510                angle.to_radians(),
511                chars,
512                unicode,
513                page_rotation,
514                supported,
515            ) else {
516                continue;
517            };
518            let this_row = row_id;
519            row_id += 1;
520            let span_boxes: Vec<_> = spans
521                .iter()
522                .filter_map(|s| geometry::normalize(Some(s.bbox), false))
523                .collect();
524            let Some(bbox) = geometry::clip(geometry::union(&span_boxes).or(Some(line.bbox)), size)
525            else {
526                continue;
527            };
528            let member_capacity = spans.iter().map(|s| s.end - s.start).sum();
529            let mut indices = Vec::with_capacity(member_capacity);
530            indices.extend(
531                spans
532                    .iter()
533                    .flat_map(|s| s.start..s.end)
534                    .filter(|&i| !matches!(chars[i].text.as_str(), "\r" | "\n")),
535            );
536            let mut raw = Vec::with_capacity(indices.len());
537            raw.extend(indices.iter().map(|&i| Some(chars[i].bbox)));
538            let mut flags = Vec::with_capacity(indices.len());
539            flags.extend(
540                indices
541                    .iter()
542                    .map(|&i| glyph_flags(&chars[i].text, unicode)),
543            );
544            let ranges =
545                geometry::visual_runs(raw, vec![None; indices.len()], flags, size, visual_angle);
546            if ranges.is_empty() {
547                output.push(VisualTextRun {
548                    text: spans.iter().map(|s| s.text.as_str()).collect(),
549                    bbox,
550                    angle: visual_angle,
551                    indices: spans.iter().flat_map(|s| s.start..s.end).collect(),
552                    row_id: this_row,
553                    run_index: 0,
554                    split: false,
555                    formula,
556                    coarse_fallback: true,
557                    paragraph_terminal: false,
558                    typography: None,
559                    typographic_scale: None,
560                });
561                continue;
562            }
563            let split = ranges.len() > 1;
564            for (run_index, (a, b, run_box)) in ranges.into_iter().enumerate() {
565                let Some(run_box) = run_box else {
566                    continue;
567                };
568                let mut selected = Vec::with_capacity(b - a);
569                selected.extend_from_slice(&indices[a..b]);
570                output.push(VisualTextRun {
571                    text: selected.iter().map(|&i| chars[i].text.as_str()).collect(),
572                    bbox: run_box,
573                    angle: visual_angle,
574                    indices: selected,
575                    row_id: this_row,
576                    run_index,
577                    split,
578                    formula,
579                    coarse_fallback: false,
580                    paragraph_terminal: false,
581                    typography: None,
582                    typographic_scale: None,
583                });
584            }
585        }
586    }
587    for run in &mut output {
588        run.text = normalize_run_text(&run.text, unicode);
589        if run.coarse_fallback || run.text.is_empty() {
590            continue;
591        }
592        run.paragraph_terminal = sentence_terminal(run, chars, unicode);
593        let mut boxes = Vec::with_capacity(run.indices.len());
594        boxes.extend(run.indices.iter().map(|&i| {
595            if glyph_flags(&chars[i].text, unicode) & 1 == 0 {
596                return None;
597            }
598            geometry::clip(geometry::normalize(Some(chars[i].bbox), false), size)
599                .map(|b| geometry::rotate(b, size, run.angle))
600        }));
601        let mut font_ids = Vec::with_capacity(run.indices.len());
602        font_ids.extend(run.indices.iter().map(|&i| chars[i].signature_id));
603        let mut weights = Vec::with_capacity(run.indices.len());
604        weights.extend(run.indices.iter().map(|&i| chars[i].font_weight));
605        let local = geometry::rotate(run.bbox, size, run.angle);
606        run.typography = crate::statistics::typography(
607            boxes,
608            font_ids,
609            weights,
610            families,
611            0.1_f64.max(local[3] - local[1]),
612        );
613        run.typographic_scale = typographic_scale(&run.indices, chars);
614    }
615    output.retain(|run| !run.text.is_empty());
616    output
617}
618
619/// 按 Python 原规则顺序清洗 run 文本,不改变字符成员或视觉分段编号。
620fn normalize_run_text(text: &str, unicode: &UnicodeProperties) -> String {
621    let mut translated = Vec::with_capacity(text.len());
622    let mut source = text.chars().peekable();
623    while let Some(ch) = source.next() {
624        let ch = match ch {
625            '\r' => {
626                if source.peek() == Some(&'\n') {
627                    source.next();
628                }
629                '\n'
630            }
631            '\u{85}' | '\u{2028}' | '\u{2029}' => '\n',
632            '\u{a0}'
633            | '\u{1680}'
634            | '\u{2000}'..='\u{200a}'
635            | '\u{202f}'
636            | '\u{205f}'
637            | '\u{3000}' => ' ',
638            '\u{200b}' | '\u{2060}' | '\u{feff}' => continue,
639            _ => ch,
640        };
641        translated.push(ch);
642    }
643    let mut normalized = String::with_capacity(text.len());
644    let mut previous_space = false;
645    for (index, &original) in translated.iter().enumerate() {
646        let mut ch = original;
647        // 软断词的前后断言必须在删除控制字符、换行和软连字符之前计算。
648        if matches!(ch, '\u{2}' | '\u{ad}')
649            && index > 0
650            && translated[index - 1].is_ascii_alphabetic()
651        {
652            let mut after = index + 1;
653            while after < translated.len() && matches!(translated[after], '\t' | ' ') {
654                after += 1;
655            }
656            if after == translated.len() || translated[after] == '\n' {
657                ch = '-';
658            }
659        }
660        if matches!(ch, '\u{ad}' | '\n' | '\u{0}'..='\u{8}' | '\u{b}' | '\u{c}' | '\u{e}'..='\u{1f}' | '\u{7f}'..='\u{9f}')
661        {
662            continue;
663        }
664        if ch == '\t' {
665            ch = ' ';
666        }
667        if ch == ' ' {
668            if previous_space {
669                continue;
670            }
671            previous_space = true;
672        } else {
673            previous_space = false;
674        }
675        normalized.push(ch);
676    }
677    normalized
678        .trim_matches(|ch| unicode.is_space(ch))
679        .to_owned()
680}
681
682/// 基于规范化文本和原字符记录判断句尾,保留引用长度按记录切片的既有语义。
683fn sentence_terminal(run: &VisualTextRun, chars: &[TextChar], unicode: &UnicodeProperties) -> bool {
684    let text = run.text.trim_end_matches(|ch| unicode.is_space(ch));
685    let mut ending = text.chars().rev();
686    let last_body = ending.find(|&ch| {
687        !matches!(
688            ch,
689            ']' | ')' | '}' | ')' | '】' | '》' | '”' | '’' | '\'' | '"'
690        )
691    });
692    if last_body.is_some_and(|ch| matches!(ch, '.' | '!' | '?' | '。' | '!' | '?')) {
693        return true;
694    }
695    let mut references = 0;
696    let mut first_is_decimal = false;
697    let mut punctuation = false;
698    for ch in text.chars().rev() {
699        let decimal = unicode.flags(ch) & 32 != 0;
700        if decimal || matches!(ch, ',' | '–' | '—' | '-') {
701            references += 1;
702            first_is_decimal = decimal;
703        } else {
704            punctuation = matches!(ch, '.' | '!' | '?' | '。' | '!' | '?');
705            break;
706        }
707    }
708    if references == 0 || !first_is_decimal || !punctuation {
709        return false;
710    }
711    let axis = if matches!(run.angle, 90 | 270) { 0 } else { 1 };
712    let sizes: Vec<_> = run
713        .indices
714        .iter()
715        .filter_map(|&index| {
716            let ch = &chars[index];
717            if ch
718                .text
719                .trim_matches(|value| unicode.is_space(value))
720                .is_empty()
721            {
722                return None;
723            }
724            Some(ch.bbox[axis + 2] - ch.bbox[axis])
725        })
726        .collect();
727    if sizes.len() <= references {
728        return false;
729    }
730    let cut = sizes.len() - references;
731    let body: Vec<_> = sizes[..cut]
732        .iter()
733        .copied()
734        .filter(|size| *size > 0.0)
735        .collect();
736    if body.is_empty() {
737        return false;
738    }
739    let limit = 0.8 * crate::median(body);
740    sizes[cut..]
741        .iter()
742        .all(|size| *size > 0.0 && *size <= limit)
743}