rapiddoc_native/
text_boundary.rs

1//! 移植 foundation/_text.py 的物理行连接规则；仅接受有效 Unicode，保留原始行供上层审计。
2use crate::error;
3use rapiddoc_core::{Error, ErrorCode};
4
5fn flags(c: char) -> u16 {
6    let n = c as u32;
7    let values = super::text_unicode_data::RANGES;
8    let i = values.partition_point(|range| range.0 <= n);
9    if i > 0 && n <= values[i - 1].1 {
10        values[i - 1].2
11    } else {
12        0
13    }
14}
15/// 查询固定 Unicode 16.0.0 的 Python isspace 属性，不依赖宿主 Python。
16pub fn python_space(c: char) -> bool {
17    flags(c) & 1 != 0
18}
19/// 查询固定 Unicode 16.0.0 的 Python isdigit 属性，包含非十进制数字。
20pub fn python_digit(c: char) -> bool {
21    flags(c) & 4 != 0
22}
23pub fn python_alpha(c: char) -> bool {
24    flags(c) & 512 != 0
25}
26pub fn python_alnum(c: char) -> bool {
27    flags(c) & 128 != 0
28}
29/// 查询固定 Unicode 16.0.0 的单字符 Python isupper 属性。
30pub fn python_upper(c: char) -> bool {
31    flags(c) & 256 != 0
32}
33pub fn python_lower(c: char) -> bool {
34    flags(c) & 2 != 0
35}
36/// 逐字母投票移植 uses_unspaced_script，恰好一半 CJK 时也使用无空格阈值。
37pub fn uses_unspaced_script(text: &str) -> bool {
38    let mut letters = 0;
39    let mut cjk = 0;
40    for c in text.chars().filter(|c| flags(*c) & 512 != 0) {
41        letters += 1;
42        cjk += usize::from(unspaced(c));
43    }
44    letters > 0 && cjk * 2 >= letters
45}
46fn unspaced(c: char) -> bool {
47    "，。！？；：、（）【】《》〈〉「」『』〔〕［］｛｝".contains(c)
48        || matches!(c as u32,0x3000..=0x303f|0x3040..=0x30ff|0x31f0..=0x31ff|0x3400..=0x4dbf|
49            0x4e00..=0x9fff|0xf900..=0xfaff|0xff61..=0xff9f|0x1aff0..=0x1afff|0x1b000..=0x1b16f|
50            0x20000..=0x2ee5f|0x2f800..=0x2fa1f|0x30000..=0x323af)
51}
52fn fold(c: char) -> char {
53    match c {
54        'İ' | 'ı' => 'i',
55        'ſ' => 's',
56        'K' => 'k',
57        _ => c.to_ascii_lowercase(),
58    }
59}
60fn visible(text: &str) -> String {
61    let mut output = String::with_capacity(text.len());
62    let final_close = text.rfind('>');
63    let mut i = 0;
64    while i < text.len() {
65        let rest = &text[i..];
66        if let Some(after) = rest.strip_prefix('<') {
67            let after = after.strip_prefix('/').unwrap_or(after);
68            let mut name = String::new();
69            let mut length = 0;
70            for c in after.chars() {
71                let c_folded = fold(c);
72                if !c_folded.is_ascii_alphabetic() {
73                    break;
74                }
75                name.push(c_folded);
76                length += c.len_utf8();
77            }
78            let word_boundary = after[length..]
79                .chars()
80                .next()
81                .is_none_or(|c| flags(c) & 128 == 0 && c != '_');
82            let end = if word_boundary
83                && final_close.is_some_and(|end| i <= end)
84                && [
85                    "sup", "sub", "b", "strong", "i", "em", "u", "s", "del", "strike",
86                ]
87                .contains(&name.as_str())
88            {
89                rest.find('>')
90            } else {
91                None
92            };
93            if let Some(end) = end {
94                i += end + 1;
95                continue;
96            }
97        }
98        let c = rest.chars().next().unwrap();
99        output.push(c);
100        i += c.len_utf8();
101    }
102    output
103}
104fn boundary(text: &str, end: bool) -> Option<char> {
105    let value = visible(text);
106    let mut chars = value
107        .trim_matches(python_space)
108        .chars()
109        .filter(|c| flags(*c) & 16 == 0);
110    if end {
111        chars.next_back()
112    } else {
113        chars.next()
114    }
115}
116fn url_prefix(s: &[u8]) -> Option<usize> {
117    [b"http://".as_slice(), b"https://", b"ftp://", b"www."]
118        .iter()
119        .find(|prefix| s.len() >= prefix.len() && s[..prefix.len()].eq_ignore_ascii_case(prefix))
120        .map(|prefix| prefix.len())
121}
122fn url_start(s: &[u8]) -> bool {
123    url_prefix(s).is_some()
124}
125fn url_crosses(previous: &str, next: &str) -> bool {
126    let candidate = format!("{previous}{next}");
127    let bytes = candidate.as_bytes();
128    let boundary = previous.len();
129    let mut i = 0;
130    while i < bytes.len() {
131        if (i == 0 || !bytes[i - 1].is_ascii_alphanumeric()) && url_start(&bytes[i..]) {
132            let start = i;
133            let suffix = i + url_prefix(&bytes[i..]).unwrap();
134            i = suffix;
135            while i < bytes.len()
136                && (bytes[i].is_ascii_alphanumeric()
137                    || b"._~:/?#[]@!$&'()*+,;=%-".contains(&bytes[i]))
138            {
139                i += 1;
140            }
141            if i > suffix && start < boundary && boundary < i {
142                return true;
143            }
144        } else {
145            i += 1;
146        }
147    }
148    false
149}
150/// 已校验的输出层复用同一旧边界规则；保留断词后的前文和分隔符。
151pub fn resolve(previous: &str, next: &str) -> (String, &'static str) {
152    let previous = previous.trim_end_matches(python_space);
153    let next = next.trim_start_matches(python_space);
154    if previous.is_empty() || next.is_empty() {
155        return (previous.into(), "");
156    }
157    if url_crosses(previous, next) {
158        return (
159            previous.into(),
160            if url_start(next.as_bytes()) { " " } else { "" },
161        );
162    }
163    let last = previous.chars().next_back().unwrap();
164    if "-\u{ad}\u{2010}\u{2011}\u{2043}".contains(last) {
165        let word_end = previous.len() - last.len_utf8();
166        let mut start = word_end;
167        for (index, c) in previous[..word_end].char_indices().rev() {
168            if !c.is_ascii_alphabetic() {
169                break;
170            }
171            start = index;
172        }
173        if start < word_end {
174            let word = &previous[start..word_end];
175            let lower = word.to_ascii_lowercase();
176            let acronym = word.len() > 1 && word.bytes().all(|b| b.is_ascii_uppercase());
177            let compound = last == '-'
178                && ([
179                    "open", "non", "self", "cross", "anti", "pre", "post", "co", "semi", "multi",
180                    "quasi",
181                ]
182                .contains(&lower.as_str())
183                    || (["of", "the", "to", "and", "in", "for", "on", "by", "with"]
184                        .contains(&lower.as_str())
185                        && previous[..start].ends_with('-')));
186            if flags(next.chars().next().unwrap()) & 2 != 0 && !acronym && !compound {
187                return (previous[..word_end].into(), "");
188            }
189            return (previous.into(), "");
190        }
191    }
192    if matches!(last, '-' | '–')
193        && previous
194            .chars()
195            .rev()
196            .nth(1)
197            .is_some_and(|c| flags(c) & 8 != 0)
198        && python_digit(next.chars().next().unwrap())
199    {
200        return (previous.into(), "");
201    }
202    let (Some(a), Some(b)) = (boundary(previous, true), boundary(next, false)) else {
203        return (previous.into(), "");
204    };
205    let separator = if unspaced(a)
206        || unspaced(b)
207        || flags(a) & 32 != 0
208        || flags(b) & 64 != 0
209        || ",.;:!?%".contains(b)
210    {
211        ""
212    } else {
213        " "
214    };
215    (previous.into(), separator)
216}
217/// 使用累计上下文连接，保留横跨三行以上的 URL；先限制最坏扫描工作量。
218pub fn merge(lines: &[String]) -> Result<String, Error> {
219    if lines.len() > rapiddoc_core::resources::limit(1000) {
220        return Err(error(ErrorCode::ResourceLimit, "文本连接行数超过预算"));
221    }
222    let lines: Vec<_> = lines
223        .iter()
224        .map(|s| s.trim_matches(python_space))
225        .filter(|s| !s.is_empty())
226        .collect();
227    let mut bytes = 0usize;
228    let mut work = 0usize;
229    for line in &lines {
230        bytes += line.len();
231        work += bytes;
232        if bytes > rapiddoc_core::resources::limit(1_000_000)
233            || work > rapiddoc_core::resources::limit(32_000_000)
234        {
235            return Err(error(
236                ErrorCode::ResourceLimit,
237                "文本连接字节或累计扫描超过预算",
238            ));
239        }
240    }
241    let mut result = String::new();
242    for line in lines {
243        if result.is_empty() {
244            result = line.into();
245            continue;
246        }
247        let (adjusted, separator) = resolve(&result, line);
248        result = adjusted;
249        result.push_str(separator);
250        result.push_str(line);
251    }
252    Ok(result.trim_matches(python_space).into())
253}
rapiddoc_native/text_boundary.rs

rapiddoc_native/
text_boundary.rs
