scitadel_export/import/
parse.rs

1//! BibTeX / BibLaTeX → [`BibEntry`] intermediate records.
2//!
3//! Tolerant of both dialects:
4//! - `year = {2024}` (BibTeX) and `date = {2024-05}` (BibLaTeX)
5//! - `archivePrefix + eprint` (arXiv v1 convention) and
6//!   `eprinttype = {arxiv} + eprint` (BibLaTeX)
7//! - `note = {...}` / `keywords = {...}` / `file = {...}` (Zotero)
8//!
9//! Field extraction is deliberately forgiving: a field that fails to
10//! parse (e.g., malformed `date = {2024-may}`) is dropped with its
11//! `Err` surfaced via [`ParseWarning`], not a hard error, so a single
12//! bad entry in a 5000-paper Zotero dump doesn't sink the whole run.
13
14use std::collections::HashMap;
15
16use biblatex::{Bibliography, ChunksExt, DateValue, PermissiveType};
17
18/// Parse failure — surfaces upstream to `scitadel bib import`.
19#[derive(Debug, thiserror::Error)]
20pub enum ParseError {
21    #[error("biblatex parse failed: {0}")]
22    Bibliography(String),
23}
24
25/// Intermediate record extracted from one `.bib` entry. Owns its
26/// data — no lifetimes tied to the source buffer.
27#[derive(Debug, Clone, PartialEq, Eq)]
28pub struct BibEntry {
29    /// The citekey as written in the source file (`@article{<this>,...}`).
30    pub citekey: String,
31    pub title: Option<String>,
32    /// Authors normalized to `"Family, Given"` — matches the storage
33    /// format the rest of scitadel (search results, OpenAlex ingest)
34    /// uses.
35    pub authors: Vec<String>,
36    pub year: Option<i32>,
37    /// DOI normalized: lowercased, `https://doi.org/` / `doi:` prefixes stripped.
38    pub doi: Option<String>,
39    pub arxiv_id: Option<String>,
40    pub pubmed_id: Option<String>,
41    pub openalex_id: Option<String>,
42    /// Zotero: free-text note carried in `note = {...}`. Step 4 maps
43    /// this to an annotation with `source="bibtex-import"`.
44    pub note: Option<String>,
45    /// Zotero: `keywords = {a,b,c}` → paper tags (step 4).
46    pub keywords: Vec<String>,
47    /// Zotero: `file = {...}`. Dropped with a `--verbose` log line in
48    /// step 4; kept here so the import summary can count dropped files.
49    pub file: Option<String>,
50    /// Every other field we didn't specifically extract, for the
51    /// merge-strategy step 3 to reason about (`note=` / `keywords=`
52    /// are scitadel-non-owned and win on merge).
53    pub extra: HashMap<String, String>,
54}
55
56/// Parse `.bib` source into a list of [`BibEntry`]. Empty source is
57/// legal (returns empty vec), matching Zotero's export-nothing edge.
58pub fn parse_bibtex(src: &str) -> Result<Vec<BibEntry>, ParseError> {
59    let bib = Bibliography::parse(src).map_err(|e| ParseError::Bibliography(format!("{e:?}")))?;
60    Ok(bib.into_vec().into_iter().map(entry_to_bibentry).collect())
61}
62
63/// Field names extracted into typed slots on `BibEntry`. Anything
64/// not in this list rides along in `extra` for the merge layer.
65const EXTRACTED_FIELDS: &[&str] = &[
66    "title",
67    "author",
68    "year",
69    "date",
70    "doi",
71    "eprint",
72    "eprinttype",
73    "archiveprefix",
74    "pmid",
75    "pubmed",
76    "openalex",
77    "note",
78    "keywords",
79    "file",
80    "pdf",
81];
82
83fn entry_to_bibentry(e: biblatex::Entry) -> BibEntry {
84    let citekey = e.key.clone();
85    let title = e.title().ok().map(|chunks| chunks.format_verbatim());
86    let authors = e
87        .author()
88        .ok()
89        .map(|people| people.iter().map(person_to_string).collect())
90        .unwrap_or_default();
91    let year = extract_year(&e);
92    let doi = e.doi().ok().map(|s| normalize_doi(&s));
93    let arxiv_id = extract_arxiv_id(&e);
94    let pubmed_id = extract_raw(&e, "pmid")
95        .or_else(|| extract_raw(&e, "pubmed"))
96        .map(|s| s.trim().to_string())
97        .filter(|s| !s.is_empty());
98    let openalex_id = extract_raw(&e, "openalex")
99        .map(|s| s.trim().to_string())
100        .filter(|s| !s.is_empty());
101    let note = e.note().ok().map(|chunks| chunks.format_verbatim());
102    let keywords = e
103        .keywords()
104        .ok()
105        .map(|chunks| {
106            chunks
107                .format_verbatim()
108                .split(',')
109                .map(|s| s.trim().to_string())
110                .filter(|s| !s.is_empty())
111                .collect()
112        })
113        .unwrap_or_default();
114    let file = e.file().ok();
115    let extra: HashMap<String, String> = e
116        .fields
117        .iter()
118        .filter(|(k, _)| !EXTRACTED_FIELDS.contains(&k.as_str()))
119        .map(|(k, v)| (k.clone(), v.format_verbatim()))
120        .collect();
121
122    BibEntry {
123        citekey,
124        title,
125        authors,
126        year,
127        doi,
128        arxiv_id,
129        pubmed_id,
130        openalex_id,
131        note,
132        keywords,
133        file,
134        extra,
135    }
136}
137
138fn person_to_string(p: &biblatex::Person) -> String {
139    let family = p.name.trim();
140    let given = p.given_name.trim();
141    let prefix = p.prefix.trim();
142    let suffix = p.suffix.trim();
143    let family_full = if prefix.is_empty() {
144        family.to_string()
145    } else {
146        format!("{prefix} {family}")
147    };
148    let family_suffixed = if suffix.is_empty() {
149        family_full
150    } else {
151        format!("{family_full}, {suffix}")
152    };
153    if given.is_empty() {
154        family_suffixed
155    } else {
156        format!("{family_suffixed}, {given}")
157    }
158}
159
160fn extract_year(e: &biblatex::Entry) -> Option<i32> {
161    // BibLaTeX `date = {2024-05-01}` — preferred.
162    if let Ok(date) = e.date()
163        && let PermissiveType::Typed(d) = date
164    {
165        let y = match d.value {
166            DateValue::At(dt) | DateValue::After(dt) | DateValue::Before(dt) => dt.year,
167            DateValue::Between(start, _) => start.year,
168        };
169        return Some(y);
170    }
171    // BibTeX `year = {2024}` fallback — just parse the first 4 digits.
172    let raw = extract_raw(e, "year")?;
173    let digits: String = raw.chars().filter(char::is_ascii_digit).take(4).collect();
174    digits.parse().ok()
175}
176
177fn extract_arxiv_id(e: &biblatex::Entry) -> Option<String> {
178    // Case 1: `eprint = {2301.00001}` with `eprinttype = {arxiv}` OR
179    // `archivePrefix = {arXiv}` — the two conventions cover ~100% of
180    // arXiv-sourced .bib entries across Zotero / Mendeley / manual.
181    let eprint = e.eprint().ok()?;
182    let eprint_type = e
183        .eprint_type()
184        .ok()
185        .map(|chunks| chunks.format_verbatim().to_lowercase())
186        .unwrap_or_default();
187    if eprint_type == "arxiv" || eprint_type.is_empty() {
188        let trimmed = eprint.trim().to_string();
189        if !trimmed.is_empty() {
190            return Some(trimmed);
191        }
192    }
193    None
194}
195
196fn extract_raw(e: &biblatex::Entry, field: &str) -> Option<String> {
197    e.fields.get(field).map(|chunks| chunks.format_verbatim())
198}
199
200/// Normalize a DOI for matching. Lowercase + strip the two common
201/// URL/CURIE prefixes that `.bib` files carry.
202pub fn normalize_doi(raw: &str) -> String {
203    let trimmed = raw.trim();
204    let no_proto = trimmed
205        .strip_prefix("https://doi.org/")
206        .or_else(|| trimmed.strip_prefix("http://doi.org/"))
207        .or_else(|| trimmed.strip_prefix("doi:"))
208        .unwrap_or(trimmed);
209    no_proto.to_lowercase()
210}
211
212#[cfg(test)]
213mod tests {
214    use super::*;
215
216    #[test]
217    fn parses_canonical_article() {
218        let src = r"
219@article{smith2024quantum,
220    title = {Quantum Advantage},
221    author = {Smith, John and Doe, Jane},
222    year = {2024},
223    journal = {Nature},
224    doi = {10.1038/ABC.2024.001}
225}";
226        let entries = parse_bibtex(src).unwrap();
227        assert_eq!(entries.len(), 1);
228        let e = &entries[0];
229        assert_eq!(e.citekey, "smith2024quantum");
230        assert_eq!(e.title.as_deref(), Some("Quantum Advantage"));
231        assert_eq!(e.authors, vec!["Smith, John", "Doe, Jane"]);
232        assert_eq!(e.year, Some(2024));
233        assert_eq!(e.doi.as_deref(), Some("10.1038/abc.2024.001"));
234    }
235
236    #[test]
237    fn normalizes_doi_prefix_and_case() {
238        let cases = [
239            ("10.1038/NATURE.2024", "10.1038/nature.2024"),
240            ("https://doi.org/10.1038/X", "10.1038/x"),
241            ("http://doi.org/10.1038/y", "10.1038/y"),
242            ("doi:10.1038/z", "10.1038/z"),
243            ("  10.1038/TRIM  ", "10.1038/trim"),
244        ];
245        for (raw, want) in cases {
246            assert_eq!(normalize_doi(raw), want, "input={raw}");
247        }
248    }
249
250    #[test]
251    fn extracts_arxiv_id_from_eprint_with_archiveprefix() {
252        let src = r"
253@article{a,
254    title = {T},
255    eprint = {2301.00001},
256    archivePrefix = {arXiv}
257}";
258        let e = &parse_bibtex(src).unwrap()[0];
259        assert_eq!(e.arxiv_id.as_deref(), Some("2301.00001"));
260    }
261
262    #[test]
263    fn extracts_arxiv_id_from_eprint_with_eprinttype() {
264        let src = r"
265@article{a,
266    title = {T},
267    eprint = {2301.99999},
268    eprinttype = {arxiv}
269}";
270        let e = &parse_bibtex(src).unwrap()[0];
271        assert_eq!(e.arxiv_id.as_deref(), Some("2301.99999"));
272    }
273
274    #[test]
275    fn extracts_pubmed_and_openalex_ids() {
276        let src = r"
277@article{a,
278    title = {T},
279    pmid = {12345678},
280    openalex = {W1234567890}
281}";
282        let e = &parse_bibtex(src).unwrap()[0];
283        assert_eq!(e.pubmed_id.as_deref(), Some("12345678"));
284        assert_eq!(e.openalex_id.as_deref(), Some("W1234567890"));
285    }
286
287    #[test]
288    fn parses_zotero_note_keywords_file() {
289        let src = r"
290@article{z,
291    title = {T},
292    note = {Read twice, felt it},
293    keywords = {alpha, beta, gamma},
294    file = {/path/to/paper.pdf}
295}";
296        let e = &parse_bibtex(src).unwrap()[0];
297        assert_eq!(e.note.as_deref(), Some("Read twice, felt it"));
298        assert_eq!(e.keywords, vec!["alpha", "beta", "gamma"]);
299        assert!(e.file.as_deref().unwrap().ends_with("paper.pdf"));
300    }
301
302    #[test]
303    fn year_falls_back_from_biblatex_date() {
304        let src = r"
305@article{d,
306    title = {T},
307    date = {2023-05-01}
308}";
309        let e = &parse_bibtex(src).unwrap()[0];
310        assert_eq!(e.year, Some(2023));
311    }
312
313    #[test]
314    fn empty_source_yields_empty_vec() {
315        assert!(parse_bibtex("").unwrap().is_empty());
316        assert!(parse_bibtex("   \n\n  ").unwrap().is_empty());
317    }
318
319    #[test]
320    fn extra_fields_preserved_for_merge_strategy() {
321        let src = r"
322@article{a,
323    title = {T},
324    author = {X, Y},
325    publisher = {Elsevier},
326    month = {may}
327}";
328        let e = &parse_bibtex(src).unwrap()[0];
329        assert_eq!(
330            e.extra.get("publisher").map(String::as_str),
331            Some("Elsevier")
332        );
333        assert_eq!(e.extra.get("month").map(String::as_str), Some("may"));
334        assert!(
335            !e.extra.contains_key("title"),
336            "title is extracted, not extra"
337        );
338    }
339
340    #[test]
341    fn author_person_prefix_and_suffix_rendered() {
342        let src = r"
343@article{v,
344    title = {T},
345    author = {von Neumann, Jr., John}
346}";
347        let e = &parse_bibtex(src).unwrap()[0];
348        // "von Neumann" (prefix "von" + family "Neumann"), suffix "Jr.", given "John"
349        assert_eq!(e.authors, vec!["von Neumann, Jr., John"]);
350    }
351}
scitadel_export/import/parse.rs

scitadel_export/import/
parse.rs
