fetchium_core/collab/
merge.rs1use crate::error::FetchiumError;
4use serde::{Deserialize, Serialize};
5
6#[derive(Debug, Clone, Serialize, Deserialize)]
7pub struct Finding {
8 pub id: String,
9 pub session_id: String,
10 pub content: String,
11 pub source_url: Option<String>,
12}
13
14#[derive(Debug)]
15pub struct MergeResult {
16 pub merged: Vec<Finding>,
17 pub added: usize,
18 pub deduplicated: usize,
19}
20
21pub fn merge_sessions(
25 sessions_dir: &std::path::Path,
26 session_a: &str,
27 session_b: &str,
28) -> Result<MergeResult, FetchiumError> {
29 let findings_a = load_findings(sessions_dir, session_a)?;
30 let findings_b = load_findings(sessions_dir, session_b)?;
31
32 let mut merged = findings_a;
33 let mut added = 0;
34 let mut deduplicated = 0;
35
36 for finding in findings_b {
37 let is_duplicate = merged
38 .iter()
39 .any(|existing| bigram_similarity(&existing.content, &finding.content) > 0.85);
40 if is_duplicate {
41 deduplicated += 1;
42 } else {
43 merged.push(finding);
44 added += 1;
45 }
46 }
47
48 Ok(MergeResult {
49 merged,
50 added,
51 deduplicated,
52 })
53}
54
55fn load_findings(
56 sessions_dir: &std::path::Path,
57 session_id: &str,
58) -> Result<Vec<Finding>, FetchiumError> {
59 let path = sessions_dir.join(session_id).join("findings.json");
60 if !path.exists() {
61 return Ok(vec![]);
62 }
63 let content = std::fs::read_to_string(&path)?;
64 Ok(serde_json::from_str(&content).unwrap_or_default())
65}
66
67fn bigram_similarity(a: &str, b: &str) -> f64 {
68 let bigrams = |s: &str| -> std::collections::HashSet<[char; 2]> {
69 let chars: Vec<char> = s.to_lowercase().chars().collect();
70 chars.windows(2).map(|w| [w[0], w[1]]).collect()
71 };
72 let a_set = bigrams(a);
73 let b_set = bigrams(b);
74 if a_set.is_empty() && b_set.is_empty() {
75 return 1.0;
76 }
77 if a_set.is_empty() || b_set.is_empty() {
78 return 0.0;
79 }
80 let intersection = a_set.intersection(&b_set).count();
81 let union = a_set.union(&b_set).count();
82 intersection as f64 / union as f64
83}
84
85#[cfg(test)]
86mod tests {
87 use super::*;
88
89 #[test]
90 fn identical_findings_are_deduplicated() {
91 let content = "Rust is fast and safe".to_string();
92 let merged: Vec<Finding> = vec![Finding {
93 id: "a".into(),
94 session_id: "s1".into(),
95 content: content.clone(),
96 source_url: None,
97 }];
98 let new_f = Finding {
99 id: "b".into(),
100 session_id: "s2".into(),
101 content: content.clone(),
102 source_url: None,
103 };
104 let is_dup = merged
105 .iter()
106 .any(|e| bigram_similarity(&e.content, &new_f.content) > 0.85);
107 assert!(is_dup);
108 assert_eq!(merged.len(), 1);
109 }
110}