paper-gap 0.3.2

use serde::{Deserialize, Serialize};
use std::collections::HashSet;

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Paper {
    pub id: String,
    pub title: String,
    pub abstract_text: String,
    pub authors: Vec<String>,
    pub published_date: Option<String>,
    pub updated_date: Option<String>,
    pub doi: Option<String>,
    pub arxiv_id: Option<String>,
    pub url: String,
    pub categories: Vec<String>,
    pub extracted_methods: Vec<String>,
    pub source_provider: String,
}

pub fn deduplicate(papers: Vec<Paper>) -> Vec<Paper> {
    let mut seen_doi = HashSet::new();
    let mut seen_arxiv = HashSet::new();
    let mut seen_title = HashSet::new();
    let mut unique = Vec::new();

    for paper in papers {
        let doi = paper.doi.as_deref().map(normalize_identifier);
        let arxiv = paper
            .arxiv_id
            .as_deref()
            .map(trim_arxiv_version)
            .map(normalize_identifier);
        let title = normalize_title(&paper.title);
        let duplicate = doi.as_ref().is_some_and(|value| seen_doi.contains(value))
            || arxiv
                .as_ref()
                .is_some_and(|value| seen_arxiv.contains(value))
            || (!title.is_empty() && seen_title.contains(&title));
        if duplicate {
            continue;
        }
        if let Some(doi) = doi {
            seen_doi.insert(doi);
        }
        if let Some(arxiv) = arxiv {
            seen_arxiv.insert(arxiv);
        }
        if !title.is_empty() {
            seen_title.insert(title);
        }
        unique.push(paper);
    }
    unique
}

fn normalize_identifier(value: &str) -> String {
    value
        .trim()
        .trim_start_matches("https://doi.org/")
        .to_ascii_lowercase()
}

fn trim_arxiv_version(value: &str) -> &str {
    value.rsplit_once('v').map_or(value, |(id, version)| {
        if version.chars().all(|c| c.is_ascii_digit()) {
            id
        } else {
            value
        }
    })
}

fn normalize_title(value: &str) -> String {
    value
        .chars()
        .filter(|c| c.is_alphanumeric())
        .flat_map(char::to_lowercase)
        .collect()
}