use std::collections::{BTreeMap, HashSet};
use rusqlite::Connection;
use strsim::levenshtein;
use crate::collect::identity::resolver::email_domain_matches;
use crate::core::errors::Result;
pub const HIGH_CONFIDENCE_CUTOFF: f64 = 0.85;
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct Suggestion {
pub src: String,
pub dst: String,
pub confidence: f64,
pub reason: String,
}
pub fn detect_from_authors(
conn: &Connection,
canonical_domain: Option<&str>,
floor: f64,
) -> Result<Vec<Suggestion>> {
let mut out = Vec::new();
out.extend(detect_same_name_pairs(conn)?);
out.extend(detect_edit_distance_pairs(conn)?);
out.extend(detect_noise_patterns(conn, canonical_domain)?);
Ok(dedupe_and_rank(out, floor))
}
pub fn detect_all(
conn: &Connection,
canonical_domain: Option<&str>,
floor: f64,
) -> Result<Vec<Suggestion>> {
let mut out = Vec::new();
out.extend(detect_same_name_pairs(conn)?);
out.extend(detect_edit_distance_pairs(conn)?);
out.extend(detect_noise_patterns(conn, canonical_domain)?);
out.extend(detect_commit_sha_cooccurrence(conn)?);
Ok(dedupe_and_rank(out, floor))
}
fn normalize_display_name(name: &str) -> String {
let spaced: String = name
.chars()
.map(|c| if c.is_alphanumeric() { c } else { ' ' })
.collect();
spaced
.split_whitespace()
.map(str::to_lowercase)
.collect::<Vec<_>>()
.join(" ")
}
fn detect_same_name_pairs(conn: &Connection) -> Result<Vec<Suggestion>> {
let mut stmt = conn.prepare(
"SELECT canonical_name, canonical_email FROM authors \
ORDER BY canonical_name, canonical_email",
)?;
let rows = stmt.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})?;
let mut groups: BTreeMap<String, Vec<(String, String)>> = BTreeMap::new();
for r in rows {
let (name, email) = r?;
let key = normalize_display_name(&name);
if key.is_empty() {
continue;
}
groups
.entry(key)
.or_default()
.push((email, name.to_lowercase()));
}
let mut out: Vec<Suggestion> = Vec::new();
for (_key, mut members) in groups {
members.sort();
members.dedup_by(|a, b| a.0 == b.0);
if members.len() < 2 {
continue;
}
let (dst, dst_name) = members[0].clone();
for (src, src_name) in members.into_iter().skip(1) {
let exact = src_name == dst_name;
out.push(Suggestion {
src,
dst: dst.clone(),
confidence: if exact { 0.95 } else { 0.90 },
reason: if exact {
"same canonical_name".to_string()
} else {
"same canonical_name after normalisation".to_string()
},
});
}
}
Ok(out)
}
fn detect_edit_distance_pairs(conn: &Connection) -> Result<Vec<Suggestion>> {
let mut stmt = conn.prepare("SELECT canonical_email FROM authors")?;
let rows = stmt.query_map([], |row| row.get::<_, String>(0))?;
let emails: Vec<String> = rows.filter_map(|r| r.ok()).collect();
let mut out: Vec<Suggestion> = Vec::new();
let mut seen: HashSet<(String, String)> = HashSet::new();
for i in 0..emails.len() {
for j in (i + 1)..emails.len() {
let a = &emails[i];
let b = &emails[j];
let Some((la, da)) = split_email(a) else {
continue;
};
let Some((lb, db_)) = split_email(b) else {
continue;
};
if la == lb || la.len() < 3 || lb.len() < 3 {
continue;
}
let dist = levenshtein(&la, &lb);
if dist == 0 || dist > 2 {
continue;
}
let domains_match = da == db_ || da.ends_with(&db_) || db_.ends_with(&da);
if !domains_match {
continue;
}
let confidence = if dist == 1 { 0.85 } else { 0.78 };
let (src, dst) = if a < b {
(b.clone(), a.clone())
} else {
(a.clone(), b.clone())
};
if seen.insert((src.clone(), dst.clone())) {
out.push(Suggestion {
src,
dst,
confidence,
reason: format!("edit-distance {dist} on local-part"),
});
}
}
}
Ok(out)
}
fn detect_noise_patterns(
conn: &Connection,
canonical_domain: Option<&str>,
) -> Result<Vec<Suggestion>> {
let mut stmt = conn.prepare("SELECT canonical_email FROM authors")?;
let rows = stmt.query_map([], |row| row.get::<_, String>(0))?;
let emails: Vec<String> = rows.filter_map(|r| r.ok()).collect();
let mut out: Vec<Suggestion> = Vec::new();
for email in &emails {
let Some((local, domain)) = split_email(email) else {
continue;
};
if domain.ends_with(".local") {
if let Some(canon_dom) = canonical_domain {
let target = format!("{local}@{canon_dom}");
if emails.iter().any(|e| e.eq_ignore_ascii_case(&target)) {
out.push(Suggestion {
src: email.clone(),
dst: target,
confidence: 0.90,
reason: ".local hostname → org email".to_string(),
});
}
}
}
if domain == "users.noreply.github.com" {
let login = local.split_once('+').map_or(local.as_str(), |(_, l)| l);
for other in &emails {
if other == email {
continue;
}
let Some((other_local, _)) = split_email(other) else {
continue;
};
if other_local == login {
out.push(Suggestion {
src: email.clone(),
dst: other.clone(),
confidence: 0.90,
reason: format!("GitHub noreply login '{login}'"),
});
} else if other_local.contains(login) || login.contains(&other_local) {
out.push(Suggestion {
src: email.clone(),
dst: other.clone(),
confidence: 0.78,
reason: format!("GitHub noreply login '{login}' (partial)"),
});
}
}
}
if let Some(canon_dom) = canonical_domain {
if !email_domain_matches(email, canon_dom) {
let dist = levenshtein(&domain, canon_dom);
if dist > 0 && dist <= 2 {
let target = format!("{local}@{canon_dom}");
if emails.iter().any(|e| e.eq_ignore_ascii_case(&target)) {
out.push(Suggestion {
src: email.clone(),
dst: target,
confidence: 0.88,
reason: format!("domain typo '{domain}' (dist {dist})"),
});
}
}
}
}
}
Ok(out)
}
fn detect_commit_sha_cooccurrence(conn: &Connection) -> Result<Vec<Suggestion>> {
let mut stmt =
conn.prepare("SELECT sha, author_email FROM commits ORDER BY sha, author_email")?;
let rows = stmt.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})?;
let mut groups: BTreeMap<String, Vec<String>> = BTreeMap::new();
for r in rows {
let (sha, email) = r?;
groups.entry(sha).or_default().push(email);
}
let mut out: Vec<Suggestion> = Vec::new();
for (sha, mut emails) in groups {
emails.sort();
emails.dedup();
if emails.len() < 2 {
continue;
}
let dst = emails[0].clone();
for src in emails.into_iter().skip(1) {
out.push(Suggestion {
src,
dst: dst.clone(),
confidence: 0.92,
reason: format!("same SHA {short}", short = &sha[..sha.len().min(8)]),
});
}
}
Ok(out)
}
pub fn dedupe_and_rank(input: Vec<Suggestion>, floor: f64) -> Vec<Suggestion> {
let mut by_pair: BTreeMap<(String, String), Suggestion> = BTreeMap::new();
for s in input {
let key = (s.src.clone(), s.dst.clone());
match by_pair.get(&key) {
Some(existing) if existing.confidence >= s.confidence => {}
_ => {
by_pair.insert(key, s);
}
}
}
let mut out: Vec<Suggestion> = by_pair
.into_values()
.filter(|s| s.confidence >= floor)
.collect();
out.sort_by(|a, b| {
b.confidence
.partial_cmp(&a.confidence)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| a.src.cmp(&b.src))
});
out
}
pub fn split_email(email: &str) -> Option<(String, String)> {
let at = email.rfind('@')?;
let local = email[..at].to_lowercase();
let domain = email[at + 1..].to_lowercase();
if local.is_empty() || domain.is_empty() {
return None;
}
Some((local, domain))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::core::db::Database;
use rusqlite::params;
fn insert_author(db: &Database, name: &str, email: &str) -> i64 {
db.connection()
.execute(
"INSERT INTO authors (canonical_name, canonical_email, aliases) \
VALUES (?1, ?2, '[]')",
params![name, email],
)
.expect("insert author");
db.connection().last_insert_rowid()
}
fn insert_commit(db: &Database, sha: &str, author_id: i64) {
db.connection()
.execute(
"INSERT INTO commits (sha, author_id, author_name, author_email, timestamp, \
message, repository) VALUES (?1, ?2, 'n', 'e', '2024-01-01T00:00:00Z', 'm', 'r')",
params![sha, author_id],
)
.expect("insert commit");
}
#[test]
fn same_name_different_email_detected() {
let db = Database::open_in_memory().expect("open");
insert_author(&db, "Bob Matsuoka", "bob@matsuoka.com");
insert_author(&db, "Bob Matsuoka", "robert.matsuoka@duettoresearch.com");
let out = detect_same_name_pairs(db.connection()).expect("detect");
assert_eq!(out.len(), 1, "exactly one pair expected, got {out:?}");
assert!(out[0].confidence >= 0.9);
assert!(out[0].reason.contains("same canonical_name"));
}
#[test]
fn a_dotted_display_name_matches_its_spaced_form() {
let db = Database::open_in_memory().expect("open");
insert_author(&db, "Ada Lovelace", "ada.lovelace@example.com");
insert_author(&db, "ada.lovelace", "ada.lovelace@users.noreply.github.com");
let out = detect_same_name_pairs(db.connection()).expect("detect");
assert_eq!(out.len(), 1, "exactly one pair expected, got {out:?}");
assert_eq!(out[0].src, "ada.lovelace@users.noreply.github.com");
assert_eq!(out[0].dst, "ada.lovelace@example.com");
assert!(
out[0].confidence >= HIGH_CONFIDENCE_CUTOFF,
"a normalised name match must stay HIGH, got {}",
out[0].confidence
);
assert!(out[0].reason.contains("normalisation"), "got {out:?}");
}
#[test]
fn similar_but_distinct_display_names_are_not_paired() {
let db = Database::open_in_memory().expect("open");
insert_author(&db, "Ada Lovelace", "ada.lovelace@example.com");
insert_author(&db, "Ada Lovelaces", "ada.lovelaces@example.com");
let out = detect_same_name_pairs(db.connection()).expect("detect");
assert!(out.is_empty(), "two distinct people must not pair: {out:?}");
}
#[test]
fn a_legacy_github_noreply_address_routes_to_its_login() {
let db = Database::open_in_memory().expect("open");
insert_author(&db, "A", "ada.lovelace@users.noreply.github.com");
insert_author(&db, "B", "ada.lovelace@example.com");
let out = detect_noise_patterns(db.connection(), Some("example.com")).expect("detect");
let hit = out
.iter()
.find(|s| s.src == "ada.lovelace@users.noreply.github.com")
.unwrap_or_else(|| panic!("expected a noreply suggestion, got {out:?}"));
assert_eq!(hit.dst, "ada.lovelace@example.com");
assert!(
hit.confidence >= HIGH_CONFIDENCE_CUTOFF,
"an exact login match must stay HIGH, got {}",
hit.confidence
);
}
#[test]
fn edit_distance_local_part_detected() {
let db = Database::open_in_memory().expect("open");
insert_author(&db, "Alice", "alice@example.com");
insert_author(&db, "Other", "alicea@example.com");
let out = detect_edit_distance_pairs(db.connection()).expect("detect");
assert!(
out.iter().any(|s| s.reason.contains("edit-distance")),
"expected an edit-distance suggestion, got {out:?}"
);
}
#[test]
fn dotlocal_email_routed_to_canonical_domain() {
let db = Database::open_in_memory().expect("open");
insert_author(&db, "Bob", "bob@HOST.local");
insert_author(&db, "Bob", "bob@researchco.example");
let out =
detect_noise_patterns(db.connection(), Some("researchco.example")).expect("detect");
assert!(
out.iter().any(|s| s.reason.contains(".local hostname")),
"expected .local suggestion, got {out:?}"
);
}
#[test]
fn github_noreply_routed_to_login() {
let db = Database::open_in_memory().expect("open");
insert_author(
&db,
"A",
"100000001+pedrosilvaextern@users.noreply.github.com",
);
insert_author(&db, "B", "pedrosilvaextern@researchco.example");
let out =
detect_noise_patterns(db.connection(), Some("researchco.example")).expect("detect");
assert!(
out.iter().any(|s| s.reason.contains("GitHub noreply")),
"expected github noreply suggestion, got {out:?}"
);
}
#[test]
fn domain_typo_detected_against_canonical_domain() {
let db = Database::open_in_memory().expect("open");
insert_author(&db, "Carol", "carol@researhco.example"); insert_author(&db, "Carol", "carol@researchco.example");
let out =
detect_noise_patterns(db.connection(), Some("researchco.example")).expect("detect");
assert!(
out.iter().any(|s| s.reason.contains("domain typo")),
"expected domain-typo suggestion, got {out:?}"
);
}
#[test]
fn same_sha_two_emails_detected() {
let conn = rusqlite::Connection::open_in_memory().expect("open");
conn.execute("CREATE TABLE commits (sha TEXT, author_email TEXT)", [])
.expect("create commits");
conn.execute(
"INSERT INTO commits (sha, author_email) VALUES \
('shared-sha', 'a@example.com'), ('shared-sha', 'b@example.com')",
[],
)
.expect("insert");
let out = detect_commit_sha_cooccurrence(&conn).expect("detect");
assert!(
out.iter().any(|s| s.reason.contains("same SHA")),
"expected commit-SHA co-occurrence, got {out:?}"
);
}
#[test]
fn dedupe_keeps_highest_confidence() {
let input = vec![
Suggestion {
src: "x".into(),
dst: "y".into(),
confidence: 0.7,
reason: "weak".into(),
},
Suggestion {
src: "x".into(),
dst: "y".into(),
confidence: 0.95,
reason: "strong".into(),
},
];
let out = dedupe_and_rank(input, 0.5);
assert_eq!(out.len(), 1);
assert_eq!(out[0].reason, "strong");
assert!((out[0].confidence - 0.95).abs() < 1e-9);
}
#[test]
fn confidence_floor_filters() {
let input = vec![
Suggestion {
src: "a".into(),
dst: "b".into(),
confidence: 0.6,
reason: "weak".into(),
},
Suggestion {
src: "c".into(),
dst: "d".into(),
confidence: 0.95,
reason: "strong".into(),
},
];
let out = dedupe_and_rank(input, 0.85);
assert_eq!(out.len(), 1);
assert_eq!(out[0].src, "c");
}
#[test]
fn the_weakest_noise_signal_stays_below_the_high_cutoff() {
let db = Database::open_in_memory().expect("open");
insert_author(&db, "A", "1+alicedev@users.noreply.github.com");
insert_author(&db, "B", "alicedevlead@corp.com");
let below = detect_from_authors(db.connection(), Some("corp.com"), 0.5).expect("detect");
assert!(
below.iter().any(|s| s.reason.contains("(partial)")),
"the fixture must produce the partial-match signal, got {below:?}"
);
let at_cutoff =
detect_from_authors(db.connection(), Some("corp.com"), HIGH_CONFIDENCE_CUTOFF)
.expect("detect");
assert!(
!at_cutoff.iter().any(|s| s.reason.contains("(partial)")),
"a 0.78 partial match must not reach the HIGH floor, got {at_cutoff:?}"
);
}
#[test]
fn detect_from_authors_finds_same_name_pair() {
let db = Database::open_in_memory().expect("open");
insert_author(&db, "Alice", "alice@corp.com");
insert_author(&db, "Alice", "alice@personal.com");
let out =
detect_from_authors(db.connection(), None, HIGH_CONFIDENCE_CUTOFF).expect("detect");
assert_eq!(out.len(), 1, "one HIGH pair expected, got {out:?}");
assert_eq!(out[0].src, "alice@personal.com");
assert_eq!(out[0].dst, "alice@corp.com");
}
#[test]
fn detect_all_adds_the_sha_signal() {
let db = Database::open_in_memory().expect("open");
let a = insert_author(&db, "A", "aaa@example.com");
let b = insert_author(&db, "B", "bbb@example.com");
insert_commit(&db, "sha-a", a);
insert_commit(&db, "sha-b", b);
let from_authors = detect_from_authors(db.connection(), None, 0.5).expect("authors pass");
assert!(from_authors.is_empty(), "got {from_authors:?}");
let all = detect_all(db.connection(), None, 0.5).expect("all passes");
assert!(all.is_empty(), "got {all:?}");
}
#[test]
fn split_email_basic() {
assert_eq!(
split_email("Bob@Example.COM"),
Some(("bob".to_string(), "example.com".to_string()))
);
assert_eq!(split_email("no-at"), None);
assert_eq!(split_email("@nolocal.com"), None);
assert_eq!(split_email("local@"), None);
}
}