Skip to main content

fetchium_core/collab/
merge.rs

1//! Research session merging with deduplication (PRD §37).
2
3use crate::error::FetchiumError;
4use serde::{Deserialize, Serialize};
5
6#[derive(Debug, Clone, Serialize, Deserialize)]
7pub struct Finding {
8    pub id: String,
9    pub session_id: String,
10    pub content: String,
11    pub source_url: Option<String>,
12}
13
14#[derive(Debug)]
15pub struct MergeResult {
16    pub merged: Vec<Finding>,
17    pub added: usize,
18    pub deduplicated: usize,
19}
20
21/// Merge findings from two sessions with bigram-Jaccard deduplication.
22///
23/// Threshold of 0.85 similarity is treated as a duplicate.
24pub fn merge_sessions(
25    sessions_dir: &std::path::Path,
26    session_a: &str,
27    session_b: &str,
28) -> Result<MergeResult, FetchiumError> {
29    let findings_a = load_findings(sessions_dir, session_a)?;
30    let findings_b = load_findings(sessions_dir, session_b)?;
31
32    let mut merged = findings_a;
33    let mut added = 0;
34    let mut deduplicated = 0;
35
36    for finding in findings_b {
37        let is_duplicate = merged
38            .iter()
39            .any(|existing| bigram_similarity(&existing.content, &finding.content) > 0.85);
40        if is_duplicate {
41            deduplicated += 1;
42        } else {
43            merged.push(finding);
44            added += 1;
45        }
46    }
47
48    Ok(MergeResult {
49        merged,
50        added,
51        deduplicated,
52    })
53}
54
55fn load_findings(
56    sessions_dir: &std::path::Path,
57    session_id: &str,
58) -> Result<Vec<Finding>, FetchiumError> {
59    let path = sessions_dir.join(session_id).join("findings.json");
60    if !path.exists() {
61        return Ok(vec![]);
62    }
63    let content = std::fs::read_to_string(&path)?;
64    Ok(serde_json::from_str(&content).unwrap_or_default())
65}
66
67fn bigram_similarity(a: &str, b: &str) -> f64 {
68    let bigrams = |s: &str| -> std::collections::HashSet<[char; 2]> {
69        let chars: Vec<char> = s.to_lowercase().chars().collect();
70        chars.windows(2).map(|w| [w[0], w[1]]).collect()
71    };
72    let a_set = bigrams(a);
73    let b_set = bigrams(b);
74    if a_set.is_empty() && b_set.is_empty() {
75        return 1.0;
76    }
77    if a_set.is_empty() || b_set.is_empty() {
78        return 0.0;
79    }
80    let intersection = a_set.intersection(&b_set).count();
81    let union = a_set.union(&b_set).count();
82    intersection as f64 / union as f64
83}
84
85#[cfg(test)]
86mod tests {
87    use super::*;
88
89    #[test]
90    fn identical_findings_are_deduplicated() {
91        let content = "Rust is fast and safe".to_string();
92        let merged: Vec<Finding> = vec![Finding {
93            id: "a".into(),
94            session_id: "s1".into(),
95            content: content.clone(),
96            source_url: None,
97        }];
98        let new_f = Finding {
99            id: "b".into(),
100            session_id: "s2".into(),
101            content: content.clone(),
102            source_url: None,
103        };
104        let is_dup = merged
105            .iter()
106            .any(|e| bigram_similarity(&e.content, &new_f.content) > 0.85);
107        assert!(is_dup);
108        assert_eq!(merged.len(), 1);
109    }
110}