novel_segment/
service.rs

1//! Shared CLI / API / MCP segmentation helpers.
2
3use crate::options::DoSegmentOptions;
4use crate::segment::Segment;
5use crate::word::{stringify, Word};
6use serde::{Deserialize, Serialize};
7use serde_json::{json, Value};
8use std::path::Path;
9use std::sync::{Mutex, OnceLock};
10
11#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
12pub enum SegmentMode {
13    /// API / file-process profile (`autoCjk` + `all_mod` + `convertSynonym`).
14    #[default]
15    Novel,
16    /// MCP / test-CLI profile (`nodeNovelMode` synonym files).
17    NodeNovel,
18}
19
20fn novel_segment() -> &'static Mutex<Segment> {
21    static S: OnceLock<Mutex<Segment>> = OnceLock::new();
22    S.get_or_init(|| Mutex::new(Segment::with_novel_default().expect("load novel default")))
23}
24
25fn node_novel_segment() -> &'static Mutex<Segment> {
26    static S: OnceLock<Mutex<Segment>> = OnceLock::new();
27    S.get_or_init(|| {
28        Mutex::new(Segment::with_node_novel_default().expect("load node novel default"))
29    })
30}
31
32pub fn segment_words(text: &str, mode: SegmentMode, opts: DoSegmentOptions) -> Vec<Word> {
33    let lock = match mode {
34        SegmentMode::Novel => novel_segment(),
35        SegmentMode::NodeNovel => node_novel_segment(),
36    };
37    let seg = lock.lock().expect("segment mutex");
38    seg.do_segment(text, opts)
39}
40
41#[derive(Clone, Debug, Default, Serialize, Deserialize)]
42#[serde(rename_all = "camelCase")]
43pub struct TestRequest {
44    pub text: Option<String>,
45    pub file: Option<String>,
46    pub expected_full: Option<String>,
47    pub expected_full_file: Option<String>,
48    pub expected_contains: Option<Vec<ExpectedItem>>,
49    pub expected_contains_not: Option<Vec<ExpectedItem>>,
50    pub expected_index_of: Option<Vec<ExpectedItem>>,
51    pub expected_index_of_not: Option<Vec<ExpectedItem>>,
52    pub dict_entries: Option<Vec<Vec<Value>>>,
53    pub synonym_entries: Option<Vec<Vec<String>>>,
54    pub blacklist_words: Option<Vec<String>>,
55    pub debug_each: Option<bool>,
56    pub output_file: Option<String>,
57    pub output_format: Option<String>,
58}
59
60#[derive(Clone, Debug, Serialize, Deserialize)]
61#[serde(untagged)]
62pub enum ExpectedItem {
63    One(String),
64    Any(Vec<String>),
65}
66
67impl ExpectedItem {
68    pub fn label(&self) -> String {
69        match self {
70            ExpectedItem::One(s) => s.clone(),
71            ExpectedItem::Any(v) => v.join("/"),
72        }
73    }
74}
75
76#[derive(Clone, Debug, Serialize, Deserialize)]
77#[serde(rename_all = "camelCase")]
78pub struct TestResult {
79    pub success: bool,
80    pub changed: bool,
81    pub match_results: MatchResults,
82    pub result: Vec<Value>,
83    pub output_text: String,
84    pub output_words: Vec<String>,
85    pub message: String,
86    #[serde(skip_serializing_if = "Option::is_none")]
87    pub diff: Option<Value>,
88    #[serde(skip_serializing_if = "Option::is_none")]
89    pub match_failures: Option<Value>,
90    #[serde(skip_serializing_if = "Option::is_none")]
91    pub error: Option<String>,
92}
93
94#[derive(Clone, Debug, Default, Serialize, Deserialize)]
95#[serde(rename_all = "camelCase")]
96pub struct MatchResults {
97    pub match_expected_full: Option<bool>,
98    pub match_expected_contains: Option<bool>,
99    pub match_expected_contains_not: Option<bool>,
100    pub match_expected_index_of: Option<bool>,
101    pub match_expected_index_of_not: Option<bool>,
102}
103
104pub fn apply_extras(
105    dict_entries: &[Vec<Value>],
106    synonym_entries: &[Vec<String>],
107    blacklist: &[String],
108) {
109    let mut seg = node_novel_segment().lock().expect("segment mutex");
110    for row in dict_entries {
111        let spec = row
112            .first()
113            .and_then(|v| v.as_str())
114            .unwrap_or("")
115            .to_string();
116        if spec.is_empty() {
117            continue;
118        }
119        let p = row.get(1).and_then(|v| {
120            v.as_u64()
121                .map(|n| n as u32)
122                .or_else(|| v.as_f64().map(|n| n as u32))
123        });
124        let f = row.get(2).and_then(|v| v.as_f64());
125        let _ = seg.add_word(&spec, p, f);
126    }
127    for row in synonym_entries {
128        if row.len() < 2 {
129            continue;
130        }
131        let refs: Vec<&str> = row[1..].iter().map(|s| s.as_str()).collect();
132        seg.add_synonym(&row[0], &refs);
133    }
134    for w in blacklist {
135        seg.add_blacklist(w);
136    }
137}
138
139pub fn run_test(req: TestRequest) -> TestResult {
140    let mut text = req.text.unwrap_or_default();
141    if let Some(path) = req.file.as_deref() {
142        match std::fs::read_to_string(path) {
143            Ok(s) => text = s,
144            Err(e) => {
145                return error_result(format!("Failed to read input file: {e}"), e.to_string())
146            }
147        }
148    }
149    let mut expected_full = req.expected_full.clone();
150    if let Some(path) = req.expected_full_file.as_deref() {
151        match std::fs::read_to_string(path) {
152            Ok(s) => expected_full = Some(s),
153            Err(e) => {
154                return error_result(format!("Failed to read expected file: {e}"), e.to_string())
155            }
156        }
157    }
158    if text.trim().is_empty() {
159        return error_result(
160            "No text provided for segmentation",
161            "No text provided for segmentation",
162        );
163    }
164
165    if let Some(rows) = req.dict_entries.as_deref() {
166        apply_extras(rows, &[], &[]);
167    }
168    if let Some(rows) = req.synonym_entries.as_deref() {
169        apply_extras(&[], rows, &[]);
170    }
171    if let Some(rows) = req.blacklist_words.as_deref() {
172        apply_extras(&[], &[], rows);
173    }
174
175    let words = if req.debug_each.unwrap_or(false) {
176        debug_each_segment(&text)
177    } else {
178        segment_words(&text, SegmentMode::NodeNovel, DoSegmentOptions::default())
179    };
180    let output_words: Vec<String> = words.iter().map(|w| w.w.clone()).collect();
181    let output_text = stringify(&words);
182    let changed = normalize_text(&output_text) != normalize_text(&text);
183
184    let mut match_results = MatchResults::default();
185    let mut failures = json!({});
186    let mut diff = None;
187
188    if let Some(exp) = expected_full.as_deref() {
189        let ne = normalize_text(exp);
190        let na = normalize_text(&output_text);
191        let ok = ne == na;
192        match_results.match_expected_full = Some(ok);
193        if !ok {
194            diff = Some(json!({
195                "expected": ne,
196                "actual": na,
197                "positions": diff_positions(&ne, &na),
198            }));
199        }
200    }
201
202    if let Some(exp) = req.expected_contains.as_deref() {
203        if !exp.is_empty() {
204            let (ok, failed) = ordered_contains(&output_words, exp);
205            match_results.match_expected_contains = Some(ok);
206            if !ok {
207                failures["contains"] = json!(failed);
208            }
209        }
210    }
211    if let Some(exp) = req.expected_contains_not.as_deref() {
212        if !exp.is_empty() {
213            let (hit, failed) = ordered_contains(&output_words, exp);
214            match_results.match_expected_contains_not = Some(!hit);
215            if hit {
216                failures["containsNot"] = json!(failed);
217            }
218        }
219    }
220    if let Some(exp) = req.expected_index_of.as_deref() {
221        if !exp.is_empty() {
222            let (ok, failed) = index_of_all(&output_text, exp);
223            match_results.match_expected_index_of = Some(ok);
224            if !ok {
225                failures["indexOf"] = json!(failed);
226            }
227        }
228    }
229    if let Some(exp) = req.expected_index_of_not.as_deref() {
230        if !exp.is_empty() {
231            let failed = index_of_not_failed(&output_text, exp);
232            let ok = failed.is_empty();
233            match_results.match_expected_index_of_not = Some(ok);
234            if !ok {
235                failures["indexOfNot"] = json!(failed);
236            }
237        }
238    }
239
240    let success = calculate_success(&match_results, changed);
241    let message = build_message(&match_results, changed);
242    let match_failures = if failures.as_object().map(|o| !o.is_empty()).unwrap_or(false) {
243        Some(failures)
244    } else {
245        None
246    };
247
248    let result = TestResult {
249        success,
250        changed,
251        match_results,
252        result: words
253            .iter()
254            .map(|w| json!({"w": w.w, "p": w.p, "f": w.f}))
255            .collect(),
256        output_text,
257        output_words,
258        message,
259        diff,
260        match_failures,
261        error: None,
262    };
263
264    if let Some(path) = req.output_file.as_deref() {
265        let _ = std::fs::write(path, serde_json::to_string_pretty(&result).unwrap_or_default());
266    }
267    result
268}
269
270fn debug_each_segment(text: &str) -> Vec<Word> {
271    let re = regex::Regex::new(r"([\n\p{Punctuation}])").expect("split regex");
272    let mut out = Vec::new();
273    for part in split_keep(&re, text) {
274        out.extend(segment_words(
275            &part,
276            SegmentMode::NodeNovel,
277            DoSegmentOptions::default(),
278        ));
279    }
280    out
281}
282
283fn split_keep(re: &regex::Regex, text: &str) -> Vec<String> {
284    let mut out = Vec::new();
285    let mut last = 0;
286    for m in re.find_iter(text) {
287        if m.start() > last {
288            out.push(text[last..m.start()].to_string());
289        }
290        out.push(m.as_str().to_string());
291        last = m.end();
292    }
293    if last < text.len() {
294        out.push(text[last..].to_string());
295    }
296    out
297}
298
299pub fn convert_joined(text: &str, tw2cn: bool) -> String {
300    #[cfg(feature = "default-dict")]
301    {
302        if tw2cn {
303            return novel_segment_dict::convert_tw2cn(text);
304        }
305        return novel_segment_dict::convert_cn2tw(text);
306    }
307    #[cfg(not(feature = "default-dict"))]
308    {
309        let _ = tw2cn;
310        text.to_string()
311    }
312}
313
314pub fn process_text(text: &str, convert_to_zh_tw: bool, crlf: bool) -> String {
315    let words = segment_words(text, SegmentMode::Novel, DoSegmentOptions::default());
316    let mut out = stringify(&words);
317    if convert_to_zh_tw {
318        #[cfg(feature = "default-dict")]
319        {
320            out = novel_segment_dict::convert_cn2tw(&out);
321        }
322    }
323    if crlf {
324        out = crlf_normalize(&out);
325    }
326    out
327}
328
329pub fn crlf_normalize(text: &str) -> String {
330    text.replace("\r\n", "\n").replace('\r', "\n")
331}
332
333pub fn normalize_text(text: &str) -> String {
334    crlf_normalize(text.trim())
335}
336
337fn ordered_contains(got: &[String], expected: &[ExpectedItem]) -> (bool, Vec<String>) {
338    let mut i = 0;
339    let mut failed = Vec::new();
340    for exp in expected {
341        let mut found = None;
342        for (idx, w) in got.iter().enumerate().skip(i) {
343            if item_hit(w, exp) {
344                found = Some(idx);
345                break;
346            }
347        }
348        if let Some(idx) = found {
349            i = idx + 1;
350        } else {
351            failed.push(exp.label());
352        }
353    }
354    (failed.is_empty(), failed)
355}
356
357fn item_hit(w: &str, exp: &ExpectedItem) -> bool {
358    match exp {
359        ExpectedItem::One(s) => w == s,
360        ExpectedItem::Any(v) => v.iter().any(|s| s == w),
361    }
362}
363
364fn index_of_all(joined: &str, expected: &[ExpectedItem]) -> (bool, Vec<String>) {
365    let mut pos = 0;
366    let mut failed = Vec::new();
367    for exp in expected {
368        match find_item(joined, pos, exp) {
369            Some((at, len)) => pos = at + len,
370            None => failed.push(exp.label()),
371        }
372    }
373    (failed.is_empty(), failed)
374}
375
376fn find_item(joined: &str, from: usize, exp: &ExpectedItem) -> Option<(usize, usize)> {
377    let tail = &joined[from.min(joined.len())..];
378    match exp {
379        ExpectedItem::One(s) => tail.find(s).map(|i| (from + i, s.len())),
380        ExpectedItem::Any(v) => v
381            .iter()
382            .filter_map(|s| tail.find(s).map(|i| (from + i, s.len())))
383            .min_by_key(|(i, _)| *i),
384    }
385}
386
387fn index_of_not_failed(joined: &str, expected: &[ExpectedItem]) -> Vec<String> {
388    let mut failed = Vec::new();
389    for exp in expected {
390        let hit = match exp {
391            ExpectedItem::One(s) => joined.contains(s),
392            ExpectedItem::Any(v) => v.iter().any(|s| joined.contains(s)),
393        };
394        if hit {
395            failed.push(exp.label());
396        }
397    }
398    failed
399}
400
401fn calculate_success(m: &MatchResults, changed: bool) -> bool {
402    if m.match_expected_full == Some(false)
403        || m.match_expected_contains == Some(false)
404        || m.match_expected_contains_not == Some(false)
405        || m.match_expected_index_of == Some(false)
406        || m.match_expected_index_of_not == Some(false)
407    {
408        return false;
409    }
410    if m.match_expected_full.is_none()
411        && m.match_expected_contains.is_none()
412        && m.match_expected_contains_not.is_none()
413        && m.match_expected_index_of.is_none()
414        && m.match_expected_index_of_not.is_none()
415    {
416        return !changed;
417    }
418    true
419}
420
421fn build_message(m: &MatchResults, changed: bool) -> String {
422    let mut messages = Vec::new();
423    push_msg(&mut messages, m.match_expected_full, "Full match");
424    push_msg(&mut messages, m.match_expected_contains, "Contains match");
425    push_msg(
426        &mut messages,
427        m.match_expected_contains_not,
428        "Contains-not match",
429    );
430    push_msg(&mut messages, m.match_expected_index_of, "Index-of match");
431    push_msg(
432        &mut messages,
433        m.match_expected_index_of_not,
434        "Index-of-not match",
435    );
436    if messages.is_empty() {
437        if changed {
438            "Text was changed during segmentation (no validation tests provided)".into()
439        } else {
440            "Text was not changed during segmentation".into()
441        }
442    } else {
443        messages.join("; ")
444    }
445}
446
447fn push_msg(out: &mut Vec<String>, v: Option<bool>, label: &str) {
448    if let Some(ok) = v {
449        out.push(format!(
450            "{label}: {}",
451            if ok { "PASSED" } else { "FAILED" }
452        ));
453    }
454}
455
456fn diff_positions(expected: &str, actual: &str) -> Vec<Value> {
457    let ev: Vec<char> = expected.chars().collect();
458    let av: Vec<char> = actual.chars().collect();
459    let max = ev.len().max(av.len());
460    let mut i = 0;
461    let mut positions = Vec::new();
462    while i < max {
463        let e = ev.get(i).copied();
464        let a = av.get(i).copied();
465        if e != a {
466            let start = i;
467            let mut exp = String::new();
468            let mut act = String::new();
469            while i < max && ev.get(i).copied() != av.get(i).copied() {
470                if let Some(c) = ev.get(i) {
471                    exp.push(*c);
472                }
473                if let Some(c) = av.get(i) {
474                    act.push(*c);
475                }
476                i += 1;
477                if i - start > 32 {
478                    break;
479                }
480            }
481            positions.push(json!({"start": start, "end": i, "expected": exp, "actual": act}));
482        } else {
483            i += 1;
484        }
485    }
486    positions
487}
488
489fn error_result(message: impl Into<String>, error: impl Into<String>) -> TestResult {
490    TestResult {
491        success: false,
492        changed: false,
493        match_results: MatchResults::default(),
494        result: Vec::new(),
495        output_text: String::new(),
496        output_words: Vec::new(),
497        message: message.into(),
498        diff: None,
499        match_failures: None,
500        error: Some(error.into()),
501    }
502}
503
504pub fn load_json_config(path: &Path) -> std::result::Result<TestRequest, String> {
505    let text = std::fs::read_to_string(path).map_err(|e| e.to_string())?;
506    serde_json::from_str(&text).map_err(|e| e.to_string())
507}
508
509#[cfg(test)]
510mod tests {
511    use super::*;
512
513    #[test]
514    fn normalize_crlf() {
515        assert_eq!(normalize_text("  a\r\nb  "), "a\nb");
516    }
517
518    #[test]
519    fn ordered_contains_any() {
520        let got = vec!["兩個".into(), "中國".into()];
521        let exp = vec![ExpectedItem::Any(vec!["兩個".into(), "两个".into()])];
522        assert!(ordered_contains(&got, &exp).0);
523    }
524}
novel_segment/service.rs

novel_segment/
service.rs
