paper_gap/
paper.rs

1use serde::{Deserialize, Serialize};
2use std::collections::HashSet;
3
4#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
5pub struct Paper {
6    pub id: String,
7    pub title: String,
8    pub abstract_text: String,
9    pub authors: Vec<String>,
10    pub published_date: Option<String>,
11    pub updated_date: Option<String>,
12    pub doi: Option<String>,
13    pub arxiv_id: Option<String>,
14    pub url: String,
15    pub categories: Vec<String>,
16    pub extracted_methods: Vec<String>,
17    pub source_provider: String,
18}
19
20pub fn deduplicate(papers: Vec<Paper>) -> Vec<Paper> {
21    let mut seen_doi = HashSet::new();
22    let mut seen_arxiv = HashSet::new();
23    let mut seen_title = HashSet::new();
24    let mut unique = Vec::new();
25
26    for paper in papers {
27        let doi = paper.doi.as_deref().map(normalize_identifier);
28        let arxiv = paper
29            .arxiv_id
30            .as_deref()
31            .map(trim_arxiv_version)
32            .map(normalize_identifier);
33        let title = normalize_title(&paper.title);
34        let duplicate = doi.as_ref().is_some_and(|value| seen_doi.contains(value))
35            || arxiv
36                .as_ref()
37                .is_some_and(|value| seen_arxiv.contains(value))
38            || (!title.is_empty() && seen_title.contains(&title));
39        if duplicate {
40            continue;
41        }
42        if let Some(doi) = doi {
43            seen_doi.insert(doi);
44        }
45        if let Some(arxiv) = arxiv {
46            seen_arxiv.insert(arxiv);
47        }
48        if !title.is_empty() {
49            seen_title.insert(title);
50        }
51        unique.push(paper);
52    }
53    unique
54}
55
56fn normalize_identifier(value: &str) -> String {
57    value
58        .trim()
59        .trim_start_matches("https://doi.org/")
60        .to_ascii_lowercase()
61}
62
63fn trim_arxiv_version(value: &str) -> &str {
64    value.rsplit_once('v').map_or(value, |(id, version)| {
65        if version.chars().all(|c| c.is_ascii_digit()) {
66            id
67        } else {
68            value
69        }
70    })
71}
72
73fn normalize_title(value: &str) -> String {
74    value
75        .chars()
76        .filter(|c| c.is_alphanumeric())
77        .flat_map(char::to_lowercase)
78        .collect()
79}
paper_gap/paper.rs

paper_gap/
paper.rs
