khmer_tokenizer_core/
strategy.rs

1//! Selects which boundary-choosing algorithm [`segment`](crate::KhmerTokenizer::segment) uses.
2
3/// Which segmentation algorithm [`KhmerTokenizer::segment`](crate::KhmerTokenizer::segment) uses.
4///
5/// Set at construction with [`KhmerTokenizer::with_strategy`](crate::KhmerTokenizer::with_strategy).
6#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
7pub enum Strategy {
8    /// Greedy longest-match, left to right. Fast, deterministic — the
9    /// long-standing default.
10    #[default]
11    ForwardMaxMatch,
12    /// Bidirectional maximum matching: runs forward and backward max-match
13    /// and picks between them on disagreement — fewer tokens wins; ties
14    /// broken by fewer single-cluster tokens; a remaining tie favors the
15    /// forward result. The canonical tie-break from Bi & Taing (APSIPA
16    /// 2014); see `docs/RESEARCH-2.md` §3b.
17    BiMaxMatch,
18    /// Unigram max-probability path (jieba-style): builds a DAG of every
19    /// dictionary match over the cluster run, then dynamic-programs
20    /// right-to-left for the path with the highest cumulative
21    /// log-probability, using word frequencies set with
22    /// [`KhmerTokenizer::with_frequencies`](crate::KhmerTokenizer::with_frequencies).
23    /// Falls back to [`Strategy::ForwardMaxMatch`] if no frequencies were
24    /// set — there's nothing to score without them.
25    UnigramDp,
26}
khmer_tokenizer_core/strategy.rs

khmer_tokenizer_core/
strategy.rs
