yuru_core/
candidate.rs

1use std::collections::HashSet;
2
3use crate::{KeyKind, LanguageBackend, SearchConfig};
4use rayon::prelude::*;
5
6#[cfg(not(test))]
7const PARALLEL_INDEX_THRESHOLD: usize = 50_000;
8#[cfg(test)]
9const PARALLEL_INDEX_THRESHOLD: usize = 4;
10
11/// Character span in the original candidate text that produced a generated key part.
12#[derive(Clone, Copy, Debug, Eq, PartialEq)]
13pub struct SourceSpan {
14    /// Inclusive character offset where the source span starts.
15    pub start_char: usize,
16    /// Exclusive character offset where the source span ends.
17    pub end_char: usize,
18}
19
20/// Indexed input row with display text and searchable keys.
21#[derive(Clone, Debug, Eq, PartialEq)]
22pub struct Candidate {
23    /// Stable input-order identifier.
24    pub id: usize,
25    /// Text shown to the user and emitted on selection.
26    pub display: String,
27    /// Searchable forms for this candidate.
28    pub keys: Vec<SearchKey>,
29}
30
31/// One searchable form for a candidate.
32#[derive(Clone, Debug, Eq, PartialEq)]
33pub struct SearchKey {
34    /// Text that the matcher scores against query variants.
35    pub text: String,
36    /// Semantic key type used for query compatibility checks.
37    pub kind: KeyKind,
38    /// Score adjustment for this key type.
39    pub weight: i32,
40    /// True when case folding alone reproduces this key from the candidate's display text,
41    /// which makes the key redundant for a case-insensitive search whose matcher folds case
42    /// the same way (see [`crate::MatcherBackend::folds_case`]). A matcher that does not
43    /// claim to fold case is still offered the key.
44    pub case_fold_only: bool,
45    /// Optional map from key character positions back to original source spans.
46    pub source_map: Option<Box<[Option<SourceSpan>]>>,
47}
48
49/// Text plus a per-character source map.
50#[derive(Clone, Debug, Default, Eq, PartialEq)]
51pub struct MappedText {
52    /// Generated text.
53    pub text: String,
54    /// Map from generated text character positions back to source spans.
55    pub source_map: Vec<Option<SourceSpan>>,
56}
57
58/// Helper for constructing mapped generated text.
59#[derive(Clone, Debug, Default)]
60pub struct MappedTextBuilder {
61    mapped: MappedText,
62}
63
64impl MappedTextBuilder {
65    /// Creates an empty mapped text builder.
66    pub fn new() -> Self {
67        Self::default()
68    }
69
70    /// Appends text and maps every appended character to the same source span.
71    pub fn push_str(&mut self, text: &str, source: Option<SourceSpan>) {
72        self.mapped.text.push_str(text);
73        self.mapped.source_map.extend(text.chars().map(|_| source));
74    }
75
76    /// Appends one mapped character.
77    pub fn push_char(&mut self, ch: char, source: Option<SourceSpan>) {
78        self.mapped.text.push(ch);
79        self.mapped.source_map.push(source);
80    }
81
82    /// Appends one unmapped separator character.
83    pub fn push_unmapped_char(&mut self, ch: char) {
84        self.mapped.text.push(ch);
85        self.mapped.source_map.push(None);
86    }
87
88    /// Finishes the builder and returns the mapped text.
89    pub fn finish(self) -> MappedText {
90        self.mapped
91    }
92}
93
94impl SearchKey {
95    /// Creates a search key using the default weight for its kind.
96    pub fn new(kind: KeyKind, text: impl Into<String>) -> Self {
97        Self {
98            text: text.into(),
99            kind,
100            weight: Self::default_weight(kind),
101            case_fold_only: false,
102            source_map: None,
103        }
104    }
105
106    /// Returns the default score adjustment for a key kind.
107    pub fn default_weight(kind: KeyKind) -> i32 {
108        match kind {
109            KeyKind::Original => 3000,
110            KeyKind::Normalized => 2800,
111            KeyKind::KanaReading => 1700,
112            KeyKind::RomajiReading => 1800,
113            KeyKind::PinyinFull => 1750,
114            KeyKind::PinyinJoined => 1800,
115            KeyKind::PinyinInitials => 1850,
116            KeyKind::KoreanRomanized => 1800,
117            KeyKind::KoreanInitials => 1850,
118            KeyKind::KoreanKeyboard => 1750,
119            KeyKind::LearnedAlias => 2500,
120        }
121    }
122
123    /// Marks whether case folding alone reproduces this key from the display text.
124    pub fn with_case_fold_only(mut self, case_fold_only: bool) -> Self {
125        self.case_fold_only = case_fold_only;
126        self
127    }
128
129    /// Attaches a source map to this key.
130    pub fn with_source_map(mut self, source_map: Vec<Option<SourceSpan>>) -> Self {
131        self.source_map = Some(source_map.into_boxed_slice());
132        self
133    }
134
135    /// Creates an original-display search key.
136    pub fn original(text: impl Into<String>) -> Self {
137        Self::new(KeyKind::Original, text)
138    }
139
140    /// Creates a normalized-display search key.
141    pub fn normalized(text: impl Into<String>) -> Self {
142        Self::new(KeyKind::Normalized, text)
143    }
144
145    /// Creates a Japanese kana-reading search key.
146    pub fn kana_reading(text: impl Into<String>) -> Self {
147        Self::new(KeyKind::KanaReading, text)
148    }
149
150    /// Creates a Japanese romaji-reading search key.
151    pub fn romaji_reading(text: impl Into<String>) -> Self {
152        Self::new(KeyKind::RomajiReading, text)
153    }
154
155    /// Creates a Chinese pinyin search key with separated syllables.
156    pub fn pinyin_full(text: impl Into<String>) -> Self {
157        Self::new(KeyKind::PinyinFull, text)
158    }
159
160    /// Creates a Chinese pinyin search key with syllables joined.
161    pub fn pinyin_joined(text: impl Into<String>) -> Self {
162        Self::new(KeyKind::PinyinJoined, text)
163    }
164
165    /// Creates a Chinese pinyin initials search key.
166    pub fn pinyin_initials(text: impl Into<String>) -> Self {
167        Self::new(KeyKind::PinyinInitials, text)
168    }
169
170    /// Creates a Korean romanized Hangul search key.
171    pub fn korean_romanized(text: impl Into<String>) -> Self {
172        Self::new(KeyKind::KoreanRomanized, text)
173    }
174
175    /// Creates a Korean initial-consonant search key.
176    pub fn korean_initials(text: impl Into<String>) -> Self {
177        Self::new(KeyKind::KoreanInitials, text)
178    }
179
180    /// Creates a Korean keyboard-layout search key.
181    pub fn korean_keyboard(text: impl Into<String>) -> Self {
182        Self::new(KeyKind::KoreanKeyboard, text)
183    }
184
185    /// Creates a user-learned alias search key.
186    pub fn learned_alias(text: impl Into<String>) -> Self {
187        Self::new(KeyKind::LearnedAlias, text)
188    }
189}
190
191/// Builds one indexed candidate using base and language-specific keys.
192pub fn build_candidate(
193    id: usize,
194    display: impl Into<String>,
195    backend: &dyn LanguageBackend,
196    config: &SearchConfig,
197) -> Candidate {
198    let display = display.into();
199    let mut keys = vec![SearchKey::original(display.clone())];
200    if config.normalize {
201        keys.push(normalized_base_key(&display, backend));
202    }
203    keys.extend(backend.build_candidate_keys(&display, config.key_budget()));
204    let keys = dedup_and_limit_keys(keys, config);
205
206    Candidate { id, display, keys }
207}
208
209/// Builds the base normalized key for `display`.
210///
211/// The key is flagged as case-fold-only when normalization changed nothing that
212/// [`crate::matcher::fold_case_char`] does not already do, which is the common case for
213/// plain ASCII text. A case-insensitive search whose matcher folds with that same mapping
214/// then scores only the original key instead of scoring both keys with the same result; a
215/// matcher that does not claim that folding keeps being offered both keys.
216fn normalized_base_key(display: &str, backend: &dyn LanguageBackend) -> SearchKey {
217    let normalized = backend.normalize_candidate(display);
218    // The byte comparison already covers unchanged text and ASCII case folding, which is
219    // the common case; the character comparison only runs for text whose non-ASCII
220    // characters changed, such as folded case, width, or kana.
221    let case_fold_only = normalized
222        .as_bytes()
223        .eq_ignore_ascii_case(display.as_bytes())
224        || normalized
225            .chars()
226            .eq(display.chars().map(crate::matcher::fold_case_char));
227
228    SearchKey::normalized(normalized).with_case_fold_only(case_fold_only)
229}
230
231/// Builds an index from input strings, using Rayon for large inputs.
232pub fn build_index<I, S>(
233    items: I,
234    backend: &dyn LanguageBackend,
235    config: &SearchConfig,
236) -> Vec<Candidate>
237where
238    I: IntoIterator<Item = S>,
239    S: Into<String>,
240{
241    let items: Vec<_> = items.into_iter().map(Into::into).collect();
242    if should_build_index_parallel(items.len()) {
243        return items
244            .into_par_iter()
245            .enumerate()
246            .map(|(id, item)| build_candidate(id, item, backend, config))
247            .collect();
248    }
249
250    items
251        .into_iter()
252        .enumerate()
253        .map(|(id, item)| build_candidate(id, item, backend, config))
254        .collect()
255}
256
257fn should_build_index_parallel(len: usize) -> bool {
258    len >= PARALLEL_INDEX_THRESHOLD && rayon::current_num_threads() > 1
259}
260
261/// Removes duplicate keys and caps generated key growth.
262pub fn dedup_and_limit_keys(keys: Vec<SearchKey>, config: &SearchConfig) -> Vec<SearchKey> {
263    let mut seen = HashSet::new();
264    let mut out = Vec::new();
265    let mut total_bytes = 0usize;
266
267    for key in keys {
268        if !seen.insert((key.kind, key.text.clone())) {
269            continue;
270        }
271
272        let required_base_key = matches!(key.kind, KeyKind::Original | KeyKind::Normalized);
273        let would_exceed_count = out.len() >= config.max_search_keys_per_candidate;
274        let would_exceed_bytes =
275            total_bytes + key.text.len() > config.max_total_key_bytes_per_candidate;
276
277        if !required_base_key && (would_exceed_count || would_exceed_bytes) {
278            continue;
279        }
280
281        total_bytes += key.text.len();
282        out.push(key);
283    }
284
285    out
286}
287
288#[cfg(test)]
289mod tests;
yuru_core/candidate.rs

yuru_core/
candidate.rs
