prepyrus 0.3.0

Parsing and writing to MDX files for Chicago author-date citaitons using BibTeX
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
use crate::BiblatexUtils;
use biblatex::Entry;
use regex::Regex;
use serde::Deserialize;
use std::fs;
use std::io::{self, BufReader, Error, Read};

#[derive(Debug, Deserialize, Clone)]
pub struct Metadata {
    pub title: String,
    #[serde(rename = "indexTitle")]
    pub index_title: String,
    pub description: String,
    #[serde(rename = "isArticle")]
    pub is_article: bool,
    pub authors: Option<String>,
    pub editors: Option<String>,
    pub contributors: Option<String>,
}

#[derive(Debug, Clone)]
pub struct ArticleFileData {
    /// Path to the file whose contents were extracted.
    pub path: String,
    /// Metadata enclosed at the top of the file.
    pub metadata: Metadata,
    /// Contents of the file.
    pub markdown_content: String,
    /// A set of citations that exist in the source `.bib` file.
    pub matched_citations: Vec<Entry>,
    /// Original contents of the file, includes metadata.
    pub full_file_content: String,
}

/// Verifies the integrity of MDX files.
/// The function reads the MDX files, extracts metadata and markdown content,
/// verifies the citations format, and matches the citations to the bibliography.
/// The function returns a list of ArticleFileData structs containing the metadata,
/// markdown content, matched citations, and full file content.
pub fn verify_mdx_files(
    mdx_paths: Vec<String>,
    all_entries: &Vec<Entry>,
) -> Result<Vec<ArticleFileData>, Error> {
    let mut article_count = 0;
    let mut all_articles: Vec<ArticleFileData> = Vec::new();
    for mdx_path in &mdx_paths {
        let (metadata, markdown_content, full_file_content) = match read_mdx_file(&mdx_path) {
            Ok(data) => data,
            Err(err) => {
                if err.kind() == io::ErrorKind::InvalidData {
                    eprintln!("Invalid MDX data format: {}", err);
                    std::process::exit(1);
                } else {
                    eprintln!("Unexpected error reading MDX file: {}", err);
                    std::process::exit(1);
                }
            }
        };
        if !metadata.is_article {
            continue;
        }
        if !check_parentheses_balance(&markdown_content) {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!("Unbalanced parentheses in {}", mdx_path),
            ));
        }
        let citations = extract_citations_from_markdown(&markdown_content);
        match verify_citations_format(&citations) {
            Ok(_) => {}
            Err(err) => {
                eprintln!("Error verifying citations: {} in {}", err, mdx_path);
                std::process::exit(1);
            }
        };
        let citations_set = create_citations_set(citations);
        let matched_citations = match match_citations_to_bibliography(citations_set, &all_entries) {
            Ok(data) => data,
            Err(err) => {
                eprintln!(
                    "Error matching citations to bibliography: {} in {}",
                    err, mdx_path
                );
                std::process::exit(1);
            }
        };
        all_articles.push(ArticleFileData {
            path: mdx_path.clone(),
            metadata,
            markdown_content,
            matched_citations,
            full_file_content,
        });
        article_count += 1;
    }
    println!(
        "✓ Integrity verification OK: {} files verified, including {} articles",
        mdx_paths.len(),
        article_count
    );
    Ok(all_articles)
}

/// Reads an MDX file and extracts metadata and markdown content.
/// The function returns a tuple containing the metadata, markdown content, and full file content.
/// The metadata is expected to be enclosed in `---` at the start of the file.
fn read_mdx_file(path: &str) -> io::Result<(Metadata, String, String)> {
    let file = fs::File::open(path)?;
    let mut reader = BufReader::new(file);
    let mut content = String::new();
    reader.read_to_string(&mut content)?;

    // Extract metadata enclosed in `---` at the start of the file
    let parts: Vec<&str> = content.splitn(3, "---").collect();
    if parts.len() != 3 {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
            format!("Unable to extract metadata in {}", path),
        ));
    }

    let metadata_str = parts[1];
    let metadata: Metadata = match serde_yaml::from_str(metadata_str) {
        Ok(data) => data,
        Err(err) => {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!("{} in {}", err, path),
            ))
        }
    };
    let markdown_content = parts[2].to_string();
    let full_file_content = content.clone();

    Ok((metadata, markdown_content, full_file_content))
}

/// Checks if the parentheses in a markdown string are balanced.
/// No odd number of parentheses is allowed.
fn check_parentheses_balance(markdown: &String) -> bool {
    let mut balance = 0;

    for ch in markdown.chars() {
        if ch == '(' {
            balance += 1;
        } else if ch == ')' {
            balance -= 1;
        }

        if balance < 0 {
            return false;
        }
    }

    balance == 0
}

/// Extract citations from a markdown string
/// The citations are assumed to be Chicago author-date style
/// and in the format (Author_last_name 2021) or (Author_last_name 2021, 123)
///
/// ### Example
///
/// (Hegel 2021) or (Hegel 2021, 123)
fn extract_citations_from_markdown(markdown: &String) -> Vec<String> {
    //      Regex explanation
    //
    //      \(      Match an opening parenthesis
    //     (see\s)? Optionally match the word "see" followed by a whitespace
    //      ([A-Z]  Match a capital letter
    //      [^()]*? Match any character except opening and closing parenthesis
    //      \d+     Match one or more digits
    //      (?:     Start a non-capturing group
    //      ,       Match a comma
    //      [^)]*   Match any character except closing parenthesis
    //      )?      End the non-capturing group and make it optional
    //      \)      Match a closing parenthesis
    //
    // The regex will match citations in the format (Author_last_name 2021) or (Author_last_name 2021, 123)
    //
    let citation_regex = Regex::new(r"\((see\s)?([A-Z][^()]*?\d+(?:,[^)]*)?)\)").unwrap();
    let mut citations = Vec::new();

    for line in markdown.lines() {
        for captures in citation_regex.captures_iter(line) {
            match captures.len() {
                2 => {
                    let citation = captures.get(1).unwrap().as_str().trim();
                    citations.push(citation.to_string());
                }
                3 => {
                    let citation = captures.get(2).unwrap().as_str().trim();
                    citations.push(citation.to_string());
                }
                _ => {} // Ignore unexpected capture group lengths
            }
        }
    }
    citations
}

/// Verifies the format of the citations extracted from the markdown.
/// The citations are expected to be in the format (Author_last_name 2021)
/// or (Author_last_name 2021, 123)
fn verify_citations_format(citations: &Vec<String>) -> Result<(), io::Error> {
    for citation in citations {
        let citation_split = citation.splitn(2, ',').collect::<Vec<&str>>();
        let first_part = citation_split[0].trim();
        let has_year = first_part.split_whitespace().any(|word| {
            if let Ok(num) = word.parse::<u32>() {
                num >= 1000 && num <= 9999
            } else {
                false
            }
        });
        if !has_year {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!("Citation is malformed or is missing year: ({})", citation),
            ));
        }
    }
    Ok(())
}

/// Creates a set of unique citations from a list of citations.
fn create_citations_set(citations: Vec<String>) -> Vec<String> {
    let mut citations_set = Vec::new();
    for citation in citations {
        let prepared_citation = citation
            .splitn(2, ',')
            .next()
            .unwrap_or(&citation)
            .to_string();
        if !citations_set.contains(&prepared_citation) {
            citations_set.push(prepared_citation);
        }
    }
    citations_set
}

/// Matches citations to the inputted bibliography
/// the matched list is returned with full bibliographical details.
/// If any citation is not found in the bibliography, an error is returned.
fn match_citations_to_bibliography(
    citations: Vec<String>,
    bibliography: &Vec<Entry>,
) -> Result<Vec<Entry>, io::Error> {
    let mut unmatched_citations = citations.clone();
    let mut matched_citations = Vec::new();

    for citation in citations {
        for entry in bibliography {
            let author = entry.author().unwrap();
            let author_last_name = author[0].name.clone();

            let date: biblatex::PermissiveType<biblatex::Date> = entry.date().unwrap();
            let year = BiblatexUtils::extract_year_from_date(&date, citation.clone()).unwrap();

            let author_year = format!("{} {:?}", author_last_name, year);

            if citation == author_year {
                unmatched_citations.retain(|x| x != &citation);
                matched_citations.push(entry.clone());
            }
        }
    }

    if unmatched_citations.len() > 0 {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
            format!(
                "Citations not found in the library: ({:?})",
                unmatched_citations
            ),
        ));
    }

    Ok(matched_citations)
}

#[cfg(test)]
mod tests_balanced_parentheses {
    use super::*;

    #[test]
    fn balanced_parentheses() {
        let markdown = String::from("This is a balanced citation (Spinoza 2021).");
        assert!(check_parentheses_balance(&markdown));
    }
    #[test]
    fn unbalanced_parentheses_more_open() {
        let markdown = String::from("This is an unbalanced citation (Spinoza 2021.");
        assert!(!check_parentheses_balance(&markdown));
    }
    #[test]
    fn unbalanced_parentheses_more_close() {
        let markdown = String::from("This is an unbalanced citation Spinoza 2021).");
        assert!(!check_parentheses_balance(&markdown));
    }
}

#[cfg(test)]
mod tests_citation_extraction {
    use super::*;

    #[test]
    fn single_citation() {
        let markdown = String::from("This is a citation (Hegel 2021) in the text.");
        let citations = extract_citations_from_markdown(&markdown);
        assert_eq!(citations, vec!["Hegel 2021"]);
    }
    #[test]
    fn single_citation_prefixed_see() {
        let markdown = String::from("This is a citation (see Hegel 2021) in the text.");
        let citations = extract_citations_from_markdown(&markdown);
        assert_eq!(citations, vec!["Hegel 2021"]);
    }
    #[test]
    fn multiple_citations() {
        let markdown =
            String::from("This is a citation (Spinoza 2021) and another one (Kant 2020, 123).");
        let citations = extract_citations_from_markdown(&markdown);
        assert_eq!(citations, vec!["Spinoza 2021", "Kant 2020, 123"]);
    }
    #[test]
    fn multiple_citations_prefixed_see() {
        let markdown = String::from(
            "This is a citation (see Spinoza 2021) and another one (see Kant 2020, 123).",
        );
        let citations = extract_citations_from_markdown(&markdown);
        assert_eq!(citations, vec!["Spinoza 2021", "Kant 2020, 123"]);
    }
    #[test]
    fn no_citation() {
        let markdown = String::from("This text has no citations.");
        let citations = extract_citations_from_markdown(&markdown);
        assert_eq!(citations, Vec::<String>::new());
    }
    #[test]
    fn citation_with_additional_text() {
        let markdown = String::from("This citation (Plato 2019) has additional text.");
        let citations = extract_citations_from_markdown(&markdown);
        assert_eq!(citations, vec!["Plato 2019"]);
    }
    #[test]
    fn multiple_lines() {
        let markdown = String::from(
            "First citation (Aristotle 2020).\n\
            Second citation on a new line (Hume 2018).\n\
            No citation here.",
        );
        let citations = extract_citations_from_markdown(&markdown);
        assert_eq!(citations, vec!["Aristotle 2020", "Hume 2018"]);
    }
    #[test]
    fn incomplete_citation_opening_parenthesis_only() {
        let markdown = String::from("This is an incomplete citation (Spinoza 2021.");
        let valid_citations = extract_citations_from_markdown(&markdown);
        assert!(valid_citations.is_empty());
    }
    #[test]
    fn incomplete_citation_closing_parenthesis_only() {
        let markdown = String::from("This is an incomplete citation Descartes 2021).");
        let valid_citations = extract_citations_from_markdown(&markdown);
        assert!(valid_citations.is_empty());
    }
    #[test]
    fn mixed_valid_and_invalid_citations() {
        let markdown =
            String::from("Valid citation (Sartre 2021). Incomplete citation Derrida 2021).");
        let valid_citations = extract_citations_from_markdown(&markdown);
        assert_eq!(valid_citations, vec!["Sartre 2021"]);
    }
}

#[cfg(test)]
mod tests_validate_citations {
    use super::*;

    #[test]
    fn valid_citations() {
        let citations = vec!["Hegel 2021".to_string(), "Kant 2020, 123".to_string()];
        assert!(verify_citations_format(&citations).is_ok());
    }
    #[test]
    fn missing_year() {
        let citations = vec!["Hegel".to_string(), "Kant 2020, 123".to_string()];
        assert!(verify_citations_format(&citations).is_err());
    }
    #[test]
    fn invalid_citation_extra_comma() {
        let citations = vec![
            "Hegel 2021".to_string(),
            "Kant 2020, 123".to_string(),
            "Hume, 2020".to_string(),
        ];
        assert!(verify_citations_format(&citations).is_err());
    }
    #[test]
    fn valid_citations_set() {
        let citations = vec![
            "Hegel 2021".to_string(),
            "Kant 2020, 123".to_string(),
            "Hegel 2021".to_string(),
            "Hegel 2021, 1234".to_string(),
            "Hegel 2021, 99".to_string(),
        ];
        let citations_set = create_citations_set(citations);
        assert_eq!(citations_set, vec!["Hegel 2021", "Kant 2020"]);
    }
    #[test]
    fn empty_citations_set() {
        let citations = Vec::<String>::new();
        let citations_set = create_citations_set(citations);
        assert!(citations_set.is_empty());
    }
    #[test]
    fn invalid_citations_set() {
        let citations = vec!["Hegel 2021".to_string(), "Kant, 2020, 123".to_string()];
        let citations_set = create_citations_set(citations);
        assert_eq!(citations_set, vec!["Hegel 2021", "Kant"]);
    }
    // TODO what happened here? investigate
    // #[test]
    // fn test_match_citations_to_bibliography() {
    //     let bibliography = vec![
    //         Entry::new("book", "Hegel 2021"),
    //         Entry::new("book", "Kant 2020"),
    //     ];
    //     let citations = vec!["Hegel 2021".to_string(), "Kant 2020".to_string()];
    //     let matched_citations = match_citations_to_bibliography(citations, &bibliography).unwrap();
    //     assert_eq!(matched_citations, bibliography);
    // }
}