plugmem_core/tokenizer/
emit.rs

1//! Canonical token emission and byte-budget handling.
2
3use super::{tables::is_word_joiner, unicode::UnicodeBackend};
4
5/// Upper bound on an emitted token, in bytes.
6pub const MAX_TOKEN_BYTES: usize = 64;
7
8/// Sends the assembled token, truncated to [`MAX_TOKEN_BYTES`] at a char
9/// boundary. Unicode marks and word joiners are meaningful only after a
10/// lexical base, so leading marks and joiners are removed without allocating.
11/// Keeping either would make re-tokenization context-sensitive (for example,
12/// `\u{300}word` → `word`, `_word` → `word`, and `word.` → `word`). A mark-only
13/// token is retained when it has Unicode Alphabetic semantics; this keeps
14/// valid standalone script marks such as U+0F71 searchable. Empty and
15/// non-lexical tokens are guarded against rather than asserted.
16pub(super) fn emit_truncated(token: &str, sink: &mut dyn FnMut(&str)) {
17    let token = trim_leading_contextual_chars(token);
18    // Some Unicode marks carry the `Alphabetic` property and are
19    // valid standalone search tokens (for example U+0F71). Others, such as
20    // U+0300, are only an Extend character: ICU will not tokenize them when
21    // presented alone. Never emit a mark-only/non-alphabetic token, because
22    // it cannot satisfy the tokenizer's fixed-point contract.
23    if token.is_empty()
24        || !token
25            .chars()
26            .any(|c| c.is_alphanumeric() || UnicodeBackend::is_alphabetic(c))
27    {
28        return;
29    }
30    let mut end = token.len().min(MAX_TOKEN_BYTES);
31    while !token.is_char_boundary(end) {
32        end -= 1;
33    }
34
35    // A mark adjacent to a trailing joiner is not lexical content of the
36    // token: in isolation UAX #29 breaks it away after the joiner is trimmed.
37    // Remove the whole contextual suffix, regardless of whether the mark or
38    // joiner comes last (`word_◌`, `word◌_`, or `word_◌_`). Marks following a
39    // real base without a trailing joiner are retained.
40    let original_end = end;
41    let mut saw_trailing_joiner = false;
42    while end > 0 {
43        let Some(c) = token[..end].chars().next_back() else {
44            break;
45        };
46        if is_word_joiner(c) {
47            saw_trailing_joiner = true;
48            end -= c.len_utf8();
49        } else if UnicodeBackend::is_mark(c) {
50            end -= c.len_utf8();
51        } else {
52            if !saw_trailing_joiner {
53                end = original_end;
54            }
55            break;
56        }
57    }
58    if end == 0 && !saw_trailing_joiner {
59        end = original_end;
60    }
61    if end == 0 {
62        return;
63    }
64    sink(&token[..end]);
65}
66
67/// Removes Unicode marks that precede a lexical base without allocating.
68///
69/// A leading mark can be attached to a preceding segment by UAX #29, while
70/// retokenizing the emitted string sees it as a standalone prefix. Removing
71/// that prefix makes emission independent of the surrounding input. A token
72/// made entirely of a Unicode-alphabetic mark is kept because it
73/// is a valid standalone token for scripts that use such marks as letters.
74fn trim_leading_marks(token: &str) -> &str {
75    let mut prefix = 0usize;
76    for (offset, c) in token.char_indices() {
77        if UnicodeBackend::is_mark(c) {
78            prefix = offset + c.len_utf8();
79        } else {
80            break;
81        }
82    }
83    // Leave a mark-only token untouched. The lexical-content guard in the
84    // caller decides whether that standalone mark is meaningful.
85    if prefix == token.len() {
86        token
87    } else {
88        &token[prefix..]
89    }
90}
91
92/// Removes any sequence of leading marks and word joiners. The loop matters
93/// for inputs such as `'.' + U+0300 + 'o'`: removing the joiner exposes a
94/// leading mark that must be removed in the next pass.
95fn trim_leading_contextual_chars(mut token: &str) -> &str {
96    loop {
97        let next = trim_leading_marks(token).trim_start_matches(is_word_joiner);
98        if next.len() == token.len() {
99            return token;
100        }
101        token = next;
102    }
103}
plugmem_core/tokenizer/emit.rs

plugmem_core/tokenizer/
emit.rs
