1use std::collections::{HashMap, HashSet};
4
5use serde::{Deserialize, Serialize};
6use uuid::Uuid;
7
8#[derive(Debug, Clone, Serialize, Deserialize)]
12#[serde(tag = "type", rename_all = "snake_case")]
13pub enum AdjustmentCondition {
14 MemoryType { kind: String },
16 AgeRange {
18 #[serde(default)]
19 min_days: Option<f32>,
20 #[serde(default)]
21 max_days: Option<f32>,
22 },
23 SalienceRange {
25 #[serde(default)]
26 min: Option<f32>,
27 #[serde(default)]
28 max: Option<f32>,
29 },
30 EntityMatch,
32 EntityMiss,
34 All {
36 conditions: Vec<AdjustmentCondition>,
37 },
38}
39
40#[derive(Debug, Clone, Serialize, Deserialize)]
42#[serde(tag = "type", rename_all = "snake_case")]
43pub enum AdjustmentOp {
44 Add { value: f32 },
46 Subtract { value: f32 },
48 Multiply { factor: f32 },
50}
51
52#[derive(Debug, Clone, Serialize, Deserialize)]
54pub struct ScoreAdjustment {
55 pub condition: AdjustmentCondition,
56 pub operation: AdjustmentOp,
57}
58
59pub struct CandidateContext<'a> {
61 pub memory_type: &'a str,
62 pub age_days: f32,
63 pub salience: f32,
64 pub content: &'a str,
65 pub entity_names: &'a [String],
66}
67
68fn contains_at_word_boundary(haystack: &str, needle: &str) -> bool {
71 if needle.is_empty() {
72 return false;
73 }
74 if needle.chars().all(is_cjk_char) {
75 return haystack.contains(needle);
76 }
77 let haystack_chars: Vec<char> = haystack.chars().collect();
78 let needle_chars: Vec<char> = needle.chars().collect();
79 let n = needle_chars.len();
80 if n == 0 || haystack_chars.len() < n {
81 return false;
82 }
83 for start in 0..=(haystack_chars.len() - n) {
84 if haystack_chars[start..start + n] != needle_chars[..] {
85 continue;
86 }
87 let before_ok = start == 0 || !haystack_chars[start - 1].is_alphanumeric();
88 let after_idx = start + n;
89 let after_ok =
90 after_idx >= haystack_chars.len() || !haystack_chars[after_idx].is_alphanumeric();
91 if before_ok && after_ok {
92 return true;
93 }
94 }
95 false
96}
97
98impl AdjustmentCondition {
99 pub fn matches(&self, ctx: &CandidateContext<'_>) -> bool {
101 match self {
102 Self::MemoryType { kind } => ctx.memory_type == kind.as_str(),
103 Self::AgeRange { min_days, max_days } => {
104 if let Some(min) = min_days {
105 if ctx.age_days < *min {
106 return false;
107 }
108 }
109 if let Some(max) = max_days {
110 if ctx.age_days > *max {
111 return false;
112 }
113 }
114 true
115 }
116 Self::SalienceRange { min, max } => {
117 if let Some(lo) = min {
118 if ctx.salience < *lo {
119 return false;
120 }
121 }
122 if let Some(hi) = max {
123 if ctx.salience > *hi {
124 return false;
125 }
126 }
127 true
128 }
129 Self::EntityMatch => {
130 if ctx.entity_names.is_empty() {
131 return false;
132 }
133 let lower = ctx.content.to_lowercase();
134 ctx.entity_names
135 .iter()
136 .any(|e| contains_at_word_boundary(&lower, e))
137 }
138 Self::EntityMiss => {
139 if ctx.entity_names.is_empty() {
140 return false;
141 }
142 let lower = ctx.content.to_lowercase();
143 !ctx.entity_names
144 .iter()
145 .any(|e| contains_at_word_boundary(&lower, e))
146 }
147 Self::All { conditions } => conditions.iter().all(|c| c.matches(ctx)),
148 }
149 }
150}
151
152impl AdjustmentOp {
153 pub fn apply(&self, score: f32) -> f32 {
155 match self {
156 Self::Add { value } => score + value,
157 Self::Subtract { value } => score - value,
158 Self::Multiply { factor } => score * factor,
159 }
160 }
161}
162
163impl ScoreAdjustment {
164 pub fn apply(&self, score: f32, ctx: &CandidateContext<'_>) -> f32 {
166 if self.condition.matches(ctx) {
167 self.operation.apply(score)
168 } else {
169 score
170 }
171 }
172}
173
174pub fn default_adjustments() -> Vec<ScoreAdjustment> {
176 vec![
177 ScoreAdjustment {
179 condition: AdjustmentCondition::All {
180 conditions: vec![
181 AdjustmentCondition::MemoryType {
182 kind: "episodic".into(),
183 },
184 AdjustmentCondition::AgeRange {
185 min_days: None,
186 max_days: Some(7.0),
187 },
188 ],
189 },
190 operation: AdjustmentOp::Add { value: 0.05 },
191 },
192 ScoreAdjustment {
195 condition: AdjustmentCondition::All {
196 conditions: vec![
197 AdjustmentCondition::MemoryType {
198 kind: "semantic".into(),
199 },
200 AdjustmentCondition::AgeRange {
201 min_days: Some(30.0),
202 max_days: None,
203 },
204 AdjustmentCondition::SalienceRange {
205 min: Some(0.85),
206 max: None,
207 },
208 ],
209 },
210 operation: AdjustmentOp::Subtract { value: 0.05 },
211 },
212 ScoreAdjustment {
214 condition: AdjustmentCondition::EntityMatch,
215 operation: AdjustmentOp::Multiply { factor: 1.3 },
216 },
217 ]
218}
219
220const ENTITY_STOPWORDS: &[&str] = &[
224 "a", "an", "the", "and", "or", "but", "of", "in", "on", "at", "to", "for", "with", "by",
225 "from", "is", "are", "was", "were", "be", "been", "being", "this", "that", "these", "those",
226 "it", "its", "he", "she", "his", "her", "him", "they", "them", "their", "we", "us", "our",
227 "you", "your", "i", "my", "me", "as", "if", "so", "not", "no", "yes", "do", "does", "did",
228 "has", "have", "had", "will", "would", "can", "could", "should", "may", "might", "about",
229 "into", "than", "then", "there", "here", "what", "which", "who", "whom", "when", "where",
230 "why", "how",
231];
232
233pub const MAX_AUTO_ENTITY_NAMES: usize = 8;
235
236fn strip_token_punctuation(token: &str) -> &str {
240 token.trim_matches(|c: char| !c.is_alphanumeric())
241}
242
243pub fn extract_entity_candidates(query: &str) -> Vec<String> {
249 let mut seen: HashSet<String> = HashSet::new();
250 let mut out: Vec<String> = Vec::new();
251
252 for raw in query.split_whitespace() {
253 let stripped = strip_token_punctuation(raw);
254 if stripped.is_empty() {
255 continue;
256 }
257 if !stripped.chars().next().is_some_and(char::is_uppercase) {
258 continue;
259 }
260 let lower = stripped.to_lowercase();
261 if ENTITY_STOPWORDS.contains(&lower.as_str()) {
262 continue;
263 }
264 if seen.insert(lower.clone()) {
265 out.push(lower);
266 if out.len() >= MAX_AUTO_ENTITY_NAMES {
267 break;
268 }
269 }
270 }
271 out
272}
273
274pub const MAX_ENTITY_LOOKUP_CANDIDATES: usize = 64;
276
277const MAX_BIGRAM_LOOKUP_CANDIDATES: usize = MAX_ENTITY_LOOKUP_CANDIDATES / 4;
278const MIN_CJK_LOOKUP_CHARS: usize = 2;
279const MAX_CJK_LOOKUP_CHARS: usize = 8;
280
281fn entity_lookup_case_variants(candidate: String) -> [Option<String>; 2] {
282 let lower = candidate.to_ascii_lowercase();
283 if candidate.is_ascii() {
284 [None, Some(lower)]
285 } else {
286 [Some(candidate), Some(lower)]
287 }
288}
289
290pub fn entity_lookup_candidates(query: &str) -> Vec<String> {
296 let tokens: Vec<String> = query
297 .split_whitespace()
298 .map(strip_token_punctuation)
299 .filter(|t| !t.is_empty())
300 .map(str::to_owned)
301 .collect();
302
303 let mut seen: HashSet<String> = HashSet::new();
304 let mut out: Vec<String> = Vec::new();
305
306 let chars: Vec<char> = query.chars().collect();
307 let mut cjk_runs = Vec::new();
308 let mut run_start = 0;
309 while run_start < chars.len() {
310 if !is_cjk_char(chars[run_start]) {
311 run_start += 1;
312 continue;
313 }
314 let mut run_end = run_start + 1;
315 while run_end < chars.len() && is_cjk_char(chars[run_end]) {
316 run_end += 1;
317 }
318 cjk_runs.push((run_start, run_end));
319 run_start = run_end;
320 }
321
322 let length_count = MAX_CJK_LOOKUP_CHARS - MIN_CJK_LOOKUP_CHARS + 1;
323 let base_quota = MAX_ENTITY_LOOKUP_CANDIDATES / length_count;
324 let quota_remainder = MAX_ENTITY_LOOKUP_CANDIDATES % length_count;
325
326 let cjk_candidates: Vec<Vec<String>> = (MIN_CJK_LOOKUP_CHARS..=MAX_CJK_LOOKUP_CHARS)
327 .map(|len| {
328 let mut last_start_by_candidate = std::collections::HashMap::new();
329 for &(run_start, run_end) in &cjk_runs {
330 if run_end - run_start < len {
331 continue;
332 }
333 for start in run_start..=run_end - len {
334 let candidate: String = chars[start..start + len].iter().collect();
335 last_start_by_candidate.insert(candidate, start);
336 }
337 }
338
339 let mut positioned: Vec<(usize, String)> = last_start_by_candidate
340 .into_iter()
341 .map(|(candidate, start)| (start, candidate))
342 .collect();
343 positioned.sort_unstable_by_key(|(start, _)| *start);
344 positioned
345 .into_iter()
346 .map(|(_, candidate)| candidate)
347 .collect()
348 })
349 .collect();
350
351 let mut quotas: Vec<usize> = (0..length_count)
352 .map(|index| base_quota + usize::from(index < quota_remainder))
353 .collect();
354 let mut unused = 0;
355 for (quota, candidates) in quotas.iter_mut().zip(&cjk_candidates) {
356 if candidates.len() < *quota {
357 unused += *quota - candidates.len();
358 *quota = candidates.len();
359 }
360 }
361 while unused > 0 {
362 let mut redistributed = false;
363 for (quota, candidates) in quotas.iter_mut().zip(&cjk_candidates) {
364 if *quota < candidates.len() {
365 *quota += 1;
366 unused -= 1;
367 redistributed = true;
368 if unused == 0 {
369 break;
370 }
371 }
372 }
373 if !redistributed {
374 break;
375 }
376 }
377
378 for (candidates, quota) in cjk_candidates.iter().zip(quotas) {
379 if quota == 0 {
380 continue;
381 }
382 let num_positions = candidates.len();
383 let mut sampled_indices = if quota == 1 {
384 vec![0]
385 } else {
386 (0..quota)
387 .map(|index| index * (num_positions - 1) / (quota - 1))
388 .collect::<Vec<_>>()
389 };
390 sampled_indices.dedup();
391 for index in sampled_indices {
392 let candidate = candidates[index].clone();
393 if seen.insert(candidate.clone()) {
394 out.push(candidate);
395 }
396 }
397 }
398 if out.len() >= MAX_ENTITY_LOOKUP_CANDIDATES {
399 return out;
400 }
401
402 let mut bigrams = tokens
403 .windows(2)
404 .map(|pair| format!("{} {}", pair[0], pair[1]));
405 for bigram in bigrams.by_ref().take(MAX_BIGRAM_LOOKUP_CANDIDATES) {
406 for candidate in entity_lookup_case_variants(bigram).into_iter().flatten() {
407 if seen.insert(candidate.clone()) {
408 out.push(candidate);
409 if out.len() >= MAX_ENTITY_LOOKUP_CANDIDATES {
410 return out;
411 }
412 }
413 }
414 }
415
416 for token in tokens.iter().filter(|token| {
417 !ENTITY_STOPWORDS.contains(&token.to_ascii_lowercase().as_str())
418 && !token.chars().all(is_cjk_char)
419 }) {
420 for candidate in entity_lookup_case_variants(token.clone())
421 .into_iter()
422 .flatten()
423 {
424 if seen.insert(candidate.clone()) {
425 out.push(candidate);
426 if out.len() >= MAX_ENTITY_LOOKUP_CANDIDATES {
427 return out;
428 }
429 }
430 }
431 }
432
433 for bigram in bigrams {
435 for candidate in entity_lookup_case_variants(bigram).into_iter().flatten() {
436 if seen.insert(candidate.clone()) {
437 out.push(candidate);
438 if out.len() >= MAX_ENTITY_LOOKUP_CANDIDATES {
439 return out;
440 }
441 }
442 }
443 }
444 out
445}
446
447#[derive(Debug, Clone, Serialize, Deserialize)]
451#[serde(default)]
452pub struct ScoringWeights {
453 pub salience: f32,
455 pub temporal: f32,
457 pub relevance: f32,
459}
460
461impl Default for ScoringWeights {
462 fn default() -> Self {
463 Self {
464 salience: 0.2,
465 temporal: 0.1,
466 relevance: 0.7,
467 }
468 }
469}
470
471#[derive(Debug, Clone, Serialize, Deserialize)]
476#[serde(default)]
477pub struct ScoringConfig {
478 pub weights: ScoringWeights,
480
481 pub min_raw_relevance: f32,
485 pub min_rrf_relevance: f32,
487 pub baseline_relevance: f32,
489
490 pub decay_cap: f32,
494
495 pub max_recall_candidates: usize,
498 pub default_recall_limit: usize,
501 pub default_token_budget: usize,
503 pub chars_per_token: usize,
505
506 pub mmr_penalty: f32,
510 pub mmr_prefix_len: usize,
512
513 pub enable_supersedes_suppression: bool,
517 pub enable_multilingual_routing: bool,
521 pub multilingual_model: Option<String>,
525
526 pub adjustments: Vec<ScoreAdjustment>,
530}
531
532impl Default for ScoringConfig {
533 fn default() -> Self {
534 Self {
535 weights: ScoringWeights::default(),
536
537 min_raw_relevance: 0.10,
538 min_rrf_relevance: 0.0,
539 baseline_relevance: 0.15,
540
541 decay_cap: 0.05,
542
543 max_recall_candidates: 200,
544 default_recall_limit: 10,
545 default_token_budget: 4000,
546 chars_per_token: 4,
547
548 mmr_penalty: 0.1,
549 mmr_prefix_len: 100,
550
551 enable_supersedes_suppression: true,
552 enable_multilingual_routing: true,
553 multilingual_model: None,
554
555 adjustments: default_adjustments(),
556 }
557 }
558}
559
560pub const MAX_RECALL_CANDIDATES: usize = 500;
564pub const MAX_TOKEN_BUDGET: usize = 16_000;
566pub const MAX_RECALL_LIMIT: usize = 200;
568
569impl ScoringConfig {
570 pub fn apply_dos_caps(&mut self) {
572 self.max_recall_candidates = self.max_recall_candidates.min(MAX_RECALL_CANDIDATES);
573 self.default_token_budget = self.default_token_budget.min(MAX_TOKEN_BUDGET);
574 self.default_recall_limit = self.default_recall_limit.min(MAX_RECALL_LIMIT);
575 }
576}
577
578#[inline]
583pub fn is_cjk_char(c: char) -> bool {
584 matches!(c,
585 '\u{4E00}'..='\u{9FFF}' | '\u{3400}'..='\u{4DBF}' | '\u{F900}'..='\u{FAFF}' | '\u{3040}'..='\u{309F}' | '\u{30A0}'..='\u{30FF}' | '\u{20000}'..='\u{2A6DF}' | '\u{AC00}'..='\u{D7AF}' )
593}
594
595pub fn contains_cjk(text: &str) -> bool {
597 let chars: Vec<char> = text.chars().collect();
598 if chars.is_empty() {
599 return false;
600 }
601 let cjk = chars.iter().filter(|&&c| is_cjk_char(c)).count();
602 (cjk as f32) / (chars.len() as f32) > 0.15
603}
604
605pub fn needs_multilingual(text: &str) -> bool {
610 let alpha_chars: Vec<char> = text.chars().filter(|c| c.is_alphabetic()).collect();
611 if alpha_chars.is_empty() {
612 return false;
613 }
614 let non_ascii_alpha = alpha_chars
615 .iter()
616 .filter(|&&c| !c.is_ascii_alphabetic())
617 .count();
618 (non_ascii_alpha as f32) / (alpha_chars.len() as f32) > 0.15
619}
620
621pub fn normalize_min_score(score: f64) -> Result<f32, crate::config::MinScoreError> {
623 if !score.is_finite() {
624 return Err(crate::config::MinScoreError::NotFinite);
625 }
626 if (0.0..=1.0).contains(&score) {
627 return Ok(score as f32);
628 }
629 if (1.0..=100.0).contains(&score) {
630 return Ok((score / 100.0) as f32);
631 }
632 Err(crate::config::MinScoreError::OutOfRange(score))
633}
634
635pub fn is_meaningful_query(query: &str) -> bool {
637 let trimmed = query.trim();
638 if trimmed.is_empty() {
639 return false;
640 }
641
642 let is_alpha_or_cjk = |c: char| c.is_alphabetic() || is_cjk_char(c);
643
644 let meaningful_chars: usize = trimmed.chars().filter(|c| is_alpha_or_cjk(*c)).count();
645 if meaningful_chars == 0 {
646 return false;
647 }
648
649 let cjk_chars: usize = trimmed.chars().filter(|c| is_cjk_char(*c)).count();
650 if meaningful_chars < 2 && cjk_chars == 0 {
651 return false;
652 }
653
654 let words: Vec<&str> = trimmed
656 .split_whitespace()
657 .filter(|w| w.chars().any(is_alpha_or_cjk))
658 .collect();
659 if !words.is_empty() {
660 let all_repeated = words.iter().all(|w| {
661 let chars: Vec<char> = w.chars().filter(|c| is_alpha_or_cjk(*c)).collect();
662 if chars.len() <= 2 {
663 return false;
664 }
665 let unique: std::collections::HashSet<char> = chars
666 .iter()
667 .map(|c| c.to_lowercase().next().unwrap_or(*c))
668 .collect();
669 (unique.len() as f32) / (chars.len() as f32) < 0.4
670 });
671 if all_repeated {
672 return false;
673 }
674 }
675
676 true
677}
678
679const NORMALIZED_RELEVANCE_CEILING: f32 = 0.82;
686
687const RRF_SIGNAL_THRESHOLD: f32 = 0.025;
690
691pub fn normalize_rank_fusion_scores(
693 scores: Vec<(Uuid, f32)>,
694 config: &ScoringConfig,
695) -> HashMap<Uuid, f32> {
696 if scores.is_empty() {
697 return HashMap::new();
698 }
699 let min_rrf = config.min_rrf_relevance;
700 let filtered: Vec<(Uuid, f32)> = scores
701 .into_iter()
702 .filter(|(_, score)| score.is_finite() && *score >= min_rrf)
703 .collect();
704 if filtered.is_empty() {
705 return HashMap::new();
706 }
707 let max_score = filtered
708 .iter()
709 .map(|(_, s)| *s)
710 .fold(f32::NEG_INFINITY, f32::max);
711 if !max_score.is_finite() || max_score <= 0.0 {
712 return HashMap::new();
713 }
714 let min_score_seen = filtered
715 .iter()
716 .map(|(_, s)| *s)
717 .fold(f32::INFINITY, f32::min);
718 let span = max_score - min_score_seen;
719 let floor = config
720 .baseline_relevance
721 .clamp(0.0, NORMALIZED_RELEVANCE_CEILING);
722 let range = NORMALIZED_RELEVANCE_CEILING - floor;
723
724 let signal_strength = (max_score / RRF_SIGNAL_THRESHOLD).min(1.0);
725
726 filtered
727 .into_iter()
728 .map(|(id, score)| {
729 let calibrated = if span <= f32::EPSILON {
730 max_score.clamp(floor, NORMALIZED_RELEVANCE_CEILING)
731 } else {
732 let percentile = ((score - min_score_seen) / span).clamp(0.0, 1.0);
733 floor + percentile * range
734 };
735 (id, calibrated * signal_strength)
736 })
737 .collect()
738}
739
740pub fn normalize_rrf_scores(
742 scores: Vec<(Uuid, f32)>,
743 config: &ScoringConfig,
744) -> HashMap<Uuid, f32> {
745 if scores.is_empty() {
746 return HashMap::new();
747 }
748 let min_rrf = config.min_rrf_relevance;
749 let filtered: Vec<(Uuid, f32)> = scores
750 .into_iter()
751 .filter(|(_, score)| score.is_finite() && *score >= min_rrf)
752 .collect();
753 if filtered.is_empty() {
754 return HashMap::new();
755 }
756 let max_score = filtered
757 .iter()
758 .map(|(_, s)| *s)
759 .fold(f32::NEG_INFINITY, f32::max);
760 if !max_score.is_finite() || max_score <= 0.0 {
761 return HashMap::new();
762 }
763 let min_score_seen = filtered
764 .iter()
765 .map(|(_, s)| *s)
766 .fold(f32::INFINITY, f32::min);
767 let span = max_score - min_score_seen;
768 let floor = config
769 .baseline_relevance
770 .clamp(0.0, NORMALIZED_RELEVANCE_CEILING);
771 let range = NORMALIZED_RELEVANCE_CEILING - floor;
772
773 filtered
774 .into_iter()
775 .map(|(id, score)| {
776 let calibrated = if span <= f32::EPSILON {
777 floor + range
778 } else {
779 let percentile = ((score - min_score_seen) / span).clamp(0.0, 1.0);
780 floor + percentile * range
781 };
782 (id, calibrated)
783 })
784 .collect()
785}
786
787pub struct ScoreInput<'a> {
792 pub salience: f32,
793 pub memory_type_str: &'a str,
794 pub content: &'a str,
795 pub created_at_millis: i64,
796 pub decay_factor: f32,
797 pub now_millis: i64,
798 pub relevance_score: f32,
799 pub entity_names: &'a [String],
800}
801
802pub fn calculate_score(input: &ScoreInput<'_>, config: &ScoringConfig) -> f32 {
810 let w = &config.weights;
811 let semantic_base = w.relevance * input.relevance_score;
812
813 let time_diff_days = ((input.now_millis - input.created_at_millis) as f32
814 / (24.0 * 60.0 * 60.0 * 1000.0))
815 .max(0.0);
816
817 let capped_decay = input.decay_factor.min(config.decay_cap);
818 let temporal_recency = (-capped_decay * time_diff_days).exp();
819
820 let temporal_boost = 1.0 + w.temporal * temporal_recency;
821 let salience_boost = 1.0 + w.salience * input.salience;
822
823 let mut score = semantic_base * temporal_boost * salience_boost;
824
825 let ctx = CandidateContext {
826 memory_type: input.memory_type_str,
827 age_days: time_diff_days,
828 salience: input.salience,
829 content: input.content,
830 entity_names: input.entity_names,
831 };
832
833 for adj in &config.adjustments {
834 score = adj.apply(score, &ctx);
835 }
836
837 score.clamp(0.0, 1.0)
838}
839
840pub const ENTITY_POSTERIOR_WEIGHT: f32 = 0.3;
846
847pub const ENTITY_POSTERIOR_CLAMP_MIN: f32 = 0.85;
850
851pub const ENTITY_POSTERIOR_CLAMP_MAX: f32 = 1.15;
854
855pub fn entity_posterior_term(entity_posterior_mean: Option<f64>, w_ent: f32) -> f32 {
862 match entity_posterior_mean {
863 Some(mean) => (1.0 + w_ent * (mean as f32 - 0.5))
864 .clamp(ENTITY_POSTERIOR_CLAMP_MIN, ENTITY_POSTERIOR_CLAMP_MAX),
865 None => 1.0,
866 }
867}
868
869#[cfg(test)]
872mod tests {
873 use super::*;
874
875 #[test]
876 fn is_meaningful_query_rejects_empty() {
877 assert!(!is_meaningful_query(""));
878 assert!(!is_meaningful_query(" "));
879 }
880
881 #[test]
882 fn is_meaningful_query_rejects_symbols_only() {
883 assert!(!is_meaningful_query("!@#$%"));
884 assert!(!is_meaningful_query("..."));
885 }
886
887 #[test]
888 fn is_meaningful_query_rejects_single_latin_char() {
889 assert!(!is_meaningful_query("a"));
890 assert!(!is_meaningful_query("Z"));
891 }
892
893 #[test]
894 fn is_meaningful_query_rejects_repeated_gibberish() {
895 assert!(!is_meaningful_query("aaaa bbbb cccc"));
896 }
897
898 #[test]
899 fn is_meaningful_query_accepts_normal_queries() {
900 assert!(is_meaningful_query("what is the capital of France"));
901 assert!(is_meaningful_query("rust async runtime"));
902 assert!(is_meaningful_query("hello"));
903 }
904
905 #[test]
906 fn contains_cjk_detects_chinese() {
907 assert!(contains_cjk("你好世界"));
909 assert!(contains_cjk("世界 hi"));
911 assert!(!contains_cjk("hello 世界 world"));
913 }
914
915 #[test]
916 fn contains_cjk_ignores_latin() {
917 assert!(!contains_cjk("hello world"));
918 assert!(!contains_cjk(""));
919 }
920
921 #[test]
922 fn normalize_min_score_fraction_passthrough() {
923 let v = normalize_min_score(0.5).unwrap();
924 assert!((v - 0.5f32).abs() < 1e-6);
925 }
926
927 #[test]
928 fn normalize_min_score_percent_form() {
929 let v = normalize_min_score(50.0).unwrap();
930 assert!((v - 0.5f32).abs() < 1e-6);
931 }
932
933 #[test]
934 fn normalize_min_score_rejects_out_of_range() {
935 assert!(normalize_min_score(200.0).is_err());
936 assert!(normalize_min_score(-1.0).is_err());
937 assert!(normalize_min_score(f64::NAN).is_err());
938 }
939
940 #[test]
941 fn calculate_score_returns_unit_interval() {
942 let config = ScoringConfig::default();
943 let score = calculate_score(
944 &ScoreInput {
945 salience: 0.9,
946 memory_type_str: "episodic",
947 content: "test content",
948 created_at_millis: 0,
949 decay_factor: 0.01,
950 now_millis: 1000,
951 relevance_score: 0.8,
952 entity_names: &[],
953 },
954 &config,
955 );
956 assert!((0.0..=1.0).contains(&score), "score {score} out of [0,1]");
957 }
958
959 #[test]
960 fn calculate_score_high_salience_ranks_higher() {
961 let config = ScoringConfig {
962 adjustments: vec![],
963 ..ScoringConfig::default()
964 };
965 let now_ms = 1_000_000i64;
966 let score_high = calculate_score(
967 &ScoreInput {
968 salience: 0.9,
969 memory_type_str: "episodic",
970 content: "content",
971 created_at_millis: 0,
972 decay_factor: 0.01,
973 now_millis: now_ms,
974 relevance_score: 0.7,
975 entity_names: &[],
976 },
977 &config,
978 );
979 let score_low = calculate_score(
980 &ScoreInput {
981 salience: 0.1,
982 memory_type_str: "episodic",
983 content: "content",
984 created_at_millis: 0,
985 decay_factor: 0.01,
986 now_millis: now_ms,
987 relevance_score: 0.7,
988 entity_names: &[],
989 },
990 &config,
991 );
992 assert!(score_high > score_low, "high salience should rank higher");
993 }
994
995 #[test]
996 fn calculate_score_entity_match_from_auto_extraction_lifts_equal_relevance_candidate() {
997 let config = ScoringConfig::default();
1003 let now_ms = 1_000_000i64;
1004
1005 let query = "sibling transfer to Gamma university";
1010 let entity_names = extract_entity_candidates(query);
1011 assert_eq!(
1012 entity_names,
1013 vec!["gamma"],
1014 "capitalized-signal query must extract exactly the one proper noun: {entity_names:?}"
1015 );
1016
1017 let matching_score = calculate_score(
1018 &ScoreInput {
1019 salience: 0.5,
1020 memory_type_str: "semantic",
1021 content: "sibling transferred to gamma university this fall",
1022 created_at_millis: 0,
1023 decay_factor: 0.005,
1024 now_millis: now_ms,
1025 relevance_score: 0.5,
1026 entity_names: &entity_names,
1027 },
1028 &config,
1029 );
1030 let non_matching_score = calculate_score(
1031 &ScoreInput {
1032 salience: 0.5,
1033 memory_type_str: "semantic",
1034 content: "sibling transferred to a different university this fall",
1035 created_at_millis: 0,
1036 decay_factor: 0.005,
1037 now_millis: now_ms,
1038 relevance_score: 0.5,
1039 entity_names: &entity_names,
1040 },
1041 &config,
1042 );
1043
1044 assert!(
1045 matching_score > non_matching_score,
1046 "memory matching an auto-extracted entity name must outrank an \
1047 equal-relevance memory that doesn't: matching={matching_score} \
1048 non_matching={non_matching_score}"
1049 );
1050 let ratio = matching_score / non_matching_score;
1055 assert!(
1056 (ratio - 1.3).abs() < 0.01,
1057 "expected ~1.3x lift from the EntityMatch adjustment, got ratio {ratio}"
1058 );
1059 }
1060
1061 #[test]
1062 fn dos_caps_enforce_limits() {
1063 let mut config = ScoringConfig {
1064 max_recall_candidates: 9999,
1065 default_token_budget: 99999,
1066 default_recall_limit: 9999,
1067 ..ScoringConfig::default()
1068 };
1069 config.apply_dos_caps();
1070 assert_eq!(config.max_recall_candidates, MAX_RECALL_CANDIDATES);
1071 assert_eq!(config.default_token_budget, MAX_TOKEN_BUDGET);
1072 assert_eq!(config.default_recall_limit, MAX_RECALL_LIMIT);
1073 }
1074
1075 #[test]
1076 fn normalize_rrf_scores_preserves_ordering() {
1077 let config = ScoringConfig::default();
1078 let input = vec![
1079 (Uuid::new_v4(), 0.030f32),
1080 (Uuid::new_v4(), 0.020f32),
1081 (Uuid::new_v4(), 0.015f32),
1082 ];
1083 let ids: Vec<Uuid> = input.iter().map(|(id, _)| *id).collect();
1084 let output = normalize_rrf_scores(input, &config);
1085 let s0 = output[&ids[0]];
1086 let s1 = output[&ids[1]];
1087 let s2 = output[&ids[2]];
1088 assert!(s0 > s1 && s1 > s2, "ordering must be preserved");
1089 assert!(s0 <= 1.0 && s2 >= 0.0, "scores must be in [0,1]");
1090 }
1091
1092 #[test]
1095 fn needs_multilingual_ascii_english_routes_primary() {
1096 assert!(!needs_multilingual("hello world"));
1097 assert!(!needs_multilingual("rust programming language"));
1098 assert!(!needs_multilingual(""));
1099 assert!(!needs_multilingual("42 items in the list"));
1100 }
1101
1102 #[test]
1103 fn needs_multilingual_cjk_routes_multilingual() {
1104 assert!(needs_multilingual("你好世界"));
1106 assert!(needs_multilingual("abc你好"));
1108 }
1109
1110 #[test]
1111 fn needs_multilingual_cyrillic_routes_multilingual() {
1112 assert!(needs_multilingual("Привет мир"));
1114 assert!(needs_multilingual("Привет hello")); }
1118
1119 #[test]
1120 fn needs_multilingual_arabic_routes_multilingual() {
1121 assert!(needs_multilingual("مرحبا بالعالم"));
1123 }
1124
1125 #[test]
1126 fn needs_multilingual_devanagari_routes_multilingual() {
1127 assert!(needs_multilingual("नमस्ते दुनिया"));
1129 }
1130
1131 #[test]
1132 fn needs_multilingual_accented_latin_routes_multilingual() {
1133 assert!(needs_multilingual("café résumé"));
1136 assert!(needs_multilingual("Müller"));
1138 assert!(needs_multilingual("niño"));
1140 }
1141
1142 #[test]
1143 fn needs_multilingual_alphabetic_denominator_is_stable_under_punctuation() {
1144 assert!(needs_multilingual("Müller"));
1147 assert!(needs_multilingual("Müller?"));
1149 assert!(needs_multilingual("Müller!!!"));
1151 assert!(needs_multilingual("42 Ü"));
1153 assert!(!needs_multilingual(""));
1155 assert!(!needs_multilingual("42 100 ???"));
1156 }
1157
1158 #[test]
1159 fn needs_multilingual_pure_ascii_accented_below_threshold_routes_primary() {
1160 assert!(!needs_multilingual("hello naive")); assert!(needs_multilingual("über"));
1166 }
1167
1168 #[test]
1169 fn needs_multilingual_ascii_only_non_english_latin_routes_primary_known_limitation() {
1170 assert!(!needs_multilingual("bonjour le monde"));
1173 assert!(!needs_multilingual("como estas"));
1174 assert!(!needs_multilingual("ich suche einen buchhalter"));
1175 }
1176
1177 #[test]
1180 fn extract_entity_candidates_capitalized_tokens_only() {
1181 let out = extract_entity_candidates("Alex sibling college Acme Beta Gamma");
1185 assert_eq!(out, vec!["alex", "acme", "beta", "gamma"]);
1186 }
1187
1188 #[test]
1189 fn extract_entity_candidates_all_lowercase_query_extracts_nothing() {
1190 let out = extract_entity_candidates("alex sibling college acme beta gamma");
1195 assert!(out.is_empty());
1196 }
1197
1198 #[test]
1199 fn extract_entity_candidates_strips_stopwords() {
1200 let out = extract_entity_candidates("what is the capital of France");
1201 assert_eq!(out, vec!["france"]);
1204 }
1205
1206 #[test]
1207 fn extract_entity_candidates_strips_punctuation() {
1208 let out = extract_entity_candidates("Alex's sibling, Acme!");
1209 assert_eq!(out, vec!["alex's", "acme"]);
1210 }
1211
1212 #[test]
1213 fn extract_entity_candidates_caps_at_max_auto_entity_names() {
1214 let query = "Alpha Bravo Charlie Delta Echo Foxtrot Golf Hotel India Juliet";
1215 let out = extract_entity_candidates(query);
1216 assert_eq!(out.len(), MAX_AUTO_ENTITY_NAMES);
1217 assert_eq!(
1218 out,
1219 vec!["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel"]
1220 );
1221 }
1222
1223 #[test]
1224 fn extract_entity_candidates_dedupes_case_insensitively() {
1225 let out = extract_entity_candidates("Acme ACME college");
1226 assert_eq!(out, vec!["acme"]);
1229 }
1230
1231 #[test]
1232 fn extract_entity_candidates_empty_query_returns_empty() {
1233 assert!(extract_entity_candidates("").is_empty());
1234 assert!(extract_entity_candidates(" ").is_empty());
1235 }
1236
1237 #[test]
1238 fn extract_entity_candidates_all_stopwords_returns_empty() {
1239 assert!(extract_entity_candidates("is the a of").is_empty());
1240 }
1241
1242 #[test]
1243 fn entity_lookup_candidates_is_length_and_position_fair_for_long_cjk_run() {
1244 let run: String = (0..65)
1245 .map(|offset| char::from_u32(0x4e00 + offset).expect("valid CJK character"))
1246 .collect();
1247 let candidates = entity_lookup_candidates(&run);
1248
1249 assert_eq!(candidates.len(), MAX_ENTITY_LOOKUP_CANDIDATES);
1250 for len in MIN_CJK_LOOKUP_CHARS..=MAX_CJK_LOOKUP_CHARS {
1251 let expected_quota = MAX_ENTITY_LOOKUP_CANDIDATES
1252 / (MAX_CJK_LOOKUP_CHARS - MIN_CJK_LOOKUP_CHARS + 1)
1253 + usize::from(len == MIN_CJK_LOOKUP_CHARS);
1254 assert_eq!(
1255 candidates
1256 .iter()
1257 .filter(|candidate| candidate.chars().count() == len)
1258 .count(),
1259 expected_quota,
1260 "candidate set must reserve the exact quota for length {len}"
1261 );
1262
1263 let chars: Vec<char> = run.chars().collect();
1264 let has_late_candidate = (chars.len() - 10..=chars.len() - len).any(|start| {
1265 let expected: String = chars[start..start + len].iter().collect();
1266 candidates.contains(&expected)
1267 });
1268 assert!(
1269 has_late_candidate,
1270 "length {len} must include a candidate starting in the final 10 positions"
1271 );
1272 }
1273 }
1274
1275 #[test]
1276 fn entity_lookup_candidates_retains_final_bigram_independent_of_ascii_case() {
1277 let lowercase = "one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen final entity";
1278 let title_case = "One Two Three Four Five Six Seven Eight Nine Ten Eleven Twelve Thirteen Fourteen Fifteen Sixteen Final Entity";
1279
1280 let lowercase_candidates = entity_lookup_candidates(lowercase);
1281 let title_case_candidates = entity_lookup_candidates(title_case);
1282 let stored_name_ci = "Final Entity".to_ascii_lowercase();
1283
1284 assert_eq!(title_case_candidates, lowercase_candidates);
1285 assert!(lowercase_candidates.contains(&stored_name_ci));
1286 assert!(title_case_candidates.contains(&stored_name_ci));
1287 assert!(!title_case_candidates.contains(&"Final Entity".to_string()));
1288 }
1289
1290 fn entity_match_ctx<'a>(content: &'a str, entity_names: &'a [String]) -> CandidateContext<'a> {
1293 CandidateContext {
1294 memory_type: "episodic",
1295 age_days: 0.0,
1296 salience: 0.5,
1297 content,
1298 entity_names,
1299 }
1300 }
1301
1302 #[test]
1303 fn contains_at_word_boundary_rejects_substring_inside_another_word() {
1304 assert!(!contains_at_word_boundary("alphabet soup", "beta"));
1305 assert!(!contains_at_word_boundary("buy a betamax player", "beta"));
1306 assert!(!contains_at_word_boundary(
1307 "water scarcity is rising",
1308 "car"
1309 ));
1310 }
1311
1312 #[test]
1313 fn contains_at_word_boundary_accepts_the_word_itself() {
1314 assert!(contains_at_word_boundary("beta release notes", "beta"));
1315 assert!(contains_at_word_boundary("drove the car home", "car"));
1316 assert!(contains_at_word_boundary("beta", "beta"));
1318 assert!(contains_at_word_boundary("notes beta", "beta"));
1319 }
1320
1321 #[test]
1322 fn contains_at_word_boundary_accepts_multi_word_phrase() {
1323 assert!(contains_at_word_boundary(
1327 "the knowledge graph shows this",
1328 "knowledge graph"
1329 ));
1330 assert!(!contains_at_word_boundary(
1331 "prior knowledge, graphical view",
1332 "knowledge graph"
1333 ));
1334 }
1335
1336 #[test]
1337 fn entity_match_condition_rejects_substring_false_positives() {
1338 let entity_names = vec!["beta".to_string()];
1339 let ctx = entity_match_ctx("alphabet soup for dinner", &entity_names);
1340 assert!(
1341 !AdjustmentCondition::EntityMatch.matches(&ctx),
1342 "\"beta\" must not match inside \"alphabet\""
1343 );
1344
1345 let entity_names = vec!["car".to_string()];
1346 let ctx = entity_match_ctx("water scarcity is rising", &entity_names);
1347 assert!(
1348 !AdjustmentCondition::EntityMatch.matches(&ctx),
1349 "\"car\" must not match inside \"scarcity\""
1350 );
1351 }
1352
1353 #[test]
1354 fn entity_match_condition_matches_multi_word_explicit_name() {
1355 let entity_names = vec!["knowledge graph".to_string()];
1356 let ctx = entity_match_ctx("notes on the knowledge graph design", &entity_names);
1357 assert!(
1358 AdjustmentCondition::EntityMatch.matches(&ctx),
1359 "explicit multi-word entity name must still match on its own boundaries"
1360 );
1361 }
1362
1363 #[test]
1364 fn entity_match_condition_still_matches_real_word_occurrence() {
1365 let entity_names = vec!["beta".to_string()];
1366 let ctx = entity_match_ctx("the beta release ships tomorrow", &entity_names);
1367 assert!(
1368 AdjustmentCondition::EntityMatch.matches(&ctx),
1369 "boundary anchoring must not break matching the actual word"
1370 );
1371 }
1372
1373 #[test]
1374 fn entity_match_condition_matches_contiguous_cjk_with_cjk_on_both_sides() {
1375 let entity_names = vec!["北京大学".to_string()];
1376 let ctx = entity_match_ctx("我在北京大学学习", &entity_names);
1377 assert!(AdjustmentCondition::EntityMatch.matches(&ctx));
1378 }
1379
1380 #[test]
1381 fn entity_match_condition_keeps_alphabetic_word_boundaries() {
1382 let entity_names = vec!["rust".to_string()];
1383 let ctx = entity_match_ctx("trust requires evidence", &entity_names);
1384 assert!(!AdjustmentCondition::EntityMatch.matches(&ctx));
1385 }
1386
1387 #[test]
1390 fn entity_posterior_term_neutral_when_no_posterior() {
1391 assert_eq!(entity_posterior_term(None, ENTITY_POSTERIOR_WEIGHT), 1.0);
1392 }
1393
1394 #[test]
1395 fn entity_posterior_term_neutral_at_uninformative_prior_mean() {
1396 let term = entity_posterior_term(Some(0.5), ENTITY_POSTERIOR_WEIGHT);
1399 assert!((term - 1.0).abs() < 1e-6, "got {term}");
1400 }
1401
1402 #[test]
1403 fn entity_posterior_term_reaches_low_clamp_bound_at_mean_zero() {
1404 let term = entity_posterior_term(Some(0.0), ENTITY_POSTERIOR_WEIGHT);
1405 assert!(
1406 (term - ENTITY_POSTERIOR_CLAMP_MIN).abs() < 1e-6,
1407 "w_ent=0.3 at mean=0.0 must land exactly on the 0.85 clamp bound, got {term}"
1408 );
1409 }
1410
1411 #[test]
1412 fn entity_posterior_term_reaches_high_clamp_bound_at_mean_one() {
1413 let term = entity_posterior_term(Some(1.0), ENTITY_POSTERIOR_WEIGHT);
1414 assert!(
1415 (term - ENTITY_POSTERIOR_CLAMP_MAX).abs() < 1e-6,
1416 "w_ent=0.3 at mean=1.0 must land exactly on the 1.15 clamp bound, got {term}"
1417 );
1418 }
1419
1420 #[test]
1421 fn entity_posterior_term_never_exceeds_clamp_bounds_for_out_of_range_input() {
1422 let low = entity_posterior_term(Some(-5.0), ENTITY_POSTERIOR_WEIGHT);
1426 let high = entity_posterior_term(Some(5.0), ENTITY_POSTERIOR_WEIGHT);
1427 assert_eq!(low, ENTITY_POSTERIOR_CLAMP_MIN);
1428 assert_eq!(high, ENTITY_POSTERIOR_CLAMP_MAX);
1429 }
1430
1431 #[test]
1432 fn entity_posterior_term_moves_monotonically_with_mean() {
1433 let low = entity_posterior_term(Some(0.2), ENTITY_POSTERIOR_WEIGHT);
1434 let mid = entity_posterior_term(Some(0.5), ENTITY_POSTERIOR_WEIGHT);
1435 let high = entity_posterior_term(Some(0.8), ENTITY_POSTERIOR_WEIGHT);
1436 assert!(low < mid, "low={low} mid={mid}");
1437 assert!(mid < high, "mid={mid} high={high}");
1438 }
1439}