gedcomkit 0.1.0

//! Converting a document between GEDCOM 5.5.1 and 7.0.
//!
//! Reading spans 5.5 to 7.1 and a file should be written in the version it
//! declares, so nothing here happens because a file was opened: conversion is
//! an explicit operation, it can be previewed before it runs, and it reports
//! every construct it touched and every one it could not carry.
//!
//! The report is the deliverable as much as the document is. A conversion that
//! silently drops what the target version cannot hold is how a tree loses
//! decades of work one release at a time, so anything unsupported is named,
//! counted, and — wherever the alternative is losing it — left in the document
//! as an extension rather than deleted.
//!
//! An application with extension tags of its own passes them to
//! [`to_version_7_with`], because version 7 wants every extension declared and
//! only the application knows the URIs its own tags are documented at.

use crate::dates::{DateKind, GedcomDate};
use crate::{Document, Node, schema, tags};

/// What happened to one construct.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
pub enum ConversionKind {
    /// Rewritten into the target version's own spelling, losing nothing.
    Converted,
    /// Removed because the target version has no place for it and the
    /// information is carried elsewhere — a character set declaration in a
    /// format that is always UTF-8.
    Dropped,
    /// Kept as written because dropping it would lose something, even though
    /// the target version does not define it.
    Kept,
}

impl ConversionKind {
    /// The word the interface uses.
    #[must_use]
    pub const fn label(self) -> &'static str {
        match self {
            Self::Converted => "converted",
            Self::Dropped => "dropped",
            Self::Kept => "kept as written",
        }
    }
}

/// One construct the conversion touched.
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))]
pub struct ConversionNote {
    /// The record it sat in, when that record has an identifier.
    pub record: Option<String>,
    /// The tag as it was before the conversion.
    pub tag: String,
    /// What happened to it.
    pub kind: ConversionKind,
    /// Why, in the words a person would use.
    pub detail: String,
}

/// A conversion: the document it would produce, and everything it did.
#[derive(Clone, Debug)]
#[non_exhaustive]
pub struct Conversion {
    /// The converted document. Nothing is written anywhere by this module.
    pub document: Document,
    /// The version the document declared.
    pub from: String,
    /// The version it now declares.
    pub to: String,
    /// Every construct touched, in document order.
    pub notes: Vec<ConversionNote>,
}

impl Conversion {
    /// How many notes of each kind, for a summary line.
    #[must_use]
    pub fn counts(&self) -> (usize, usize, usize) {
        let mut converted = 0;
        let mut dropped = 0;
        let mut kept = 0;
        for note in &self.notes {
            match note.kind {
                ConversionKind::Converted => converted += 1,
                ConversionKind::Dropped => dropped += 1,
                ConversionKind::Kept => kept += 1,
            }
        }
        (converted, dropped, kept)
    }
}

/// The URI an extension nobody documented is declared against.
///
/// Version 7 requires every extension tag to be declared, and a file full of
/// somebody else's undeclared extensions cannot be made conformant without
/// saying *something*. This says exactly what is true: the tag came from
/// another application and the converting one does not know what it means.
/// The domain is RFC 2606's `.example`, which can never be registered and so
/// can never be mistaken for real documentation.
const UNDOCUMENTED: &str = "https://gedcom.example/undocumented";

struct Pass {
    notes: Vec<ConversionNote>,
    record: Option<String>,
}

impl Pass {
    fn note(&mut self, tag: &str, kind: ConversionKind, detail: impl Into<String>) {
        self.notes.push(ConversionNote {
            record: self.record.clone(),
            tag: tag.to_owned(),
            kind,
            detail: detail.into(),
        });
    }
}

/// Converts a document to GEDCOM 7.0.
///
/// Every extension found is declared as undocumented; an application with
/// extension tags of its own should use [`to_version_7_with`] so they are
/// declared against their real URIs.
#[must_use]
pub fn to_version_7(document: &Document) -> Conversion {
    to_version_7_with(document, &[])
}

/// Converts a document to GEDCOM 7.0.
///
/// `owned` names the extension tags the caller writes itself, each with the
/// URI it is documented at. Those are declared by name; any other extension in
/// the file is declared as undocumented rather than dropped, because it is
/// somebody's data either way.
#[must_use]
pub fn to_version_7_with(document: &Document, owned: &[(&str, &str)]) -> Conversion {
    let from = document.version().unwrap_or_else(|| "unstated".to_owned());
    let mut converted = document.clone();
    let mut pass = Pass {
        notes: Vec::new(),
        record: None,
    };

    // A note record becomes a shared note record, and every pointer at one has
    // to follow — a `NOTE @N1@` in version 7 is an inline note whose payload
    // happens to look like a pointer, which is a different structure entirely.
    let shared: Vec<String> = converted
        .records_of("NOTE")
        .filter_map(|record| record.xref.clone())
        .collect();

    for record in &mut converted.records {
        pass.record = record.xref.clone();
        if record.tag == "HEAD" {
            head_to_7(record, &mut pass);
        }
        if record.tag == "NOTE" && record.xref.is_some() {
            "SNOTE".clone_into(&mut record.tag);
            record.forget_verbatim();
            pass.note(
                "NOTE",
                ConversionKind::Converted,
                "a note record is a shared note record in version 7",
            );
        }
        // The record itself, not only its substructures: a media object is a
        // record, and its `FORM` is the one that has to move.
        structures_to_7_node(record, &shared, &mut pass);
    }

    // Every extension the file carries has to be declared, ours by name and
    // anybody else's as what it is.
    let extension_tags = extension_tags(&converted);
    if let Some(header) = converted.records.iter_mut().find(|node| node.tag == "HEAD") {
        pass.record = None;
        for tag in extension_tags {
            if schema::is_declared(header, &tag) {
                continue;
            }
            if let Some((_, uri)) = owned.iter().find(|(own, _)| *own == tag) {
                schema::declare(header, &tag, uri);
                header.forget_verbatim();
                pass.note(
                    &tag,
                    ConversionKind::Converted,
                    "declared in the header, which version 7 requires of every extension",
                );
            } else {
                schema::declare(header, &tag, UNDOCUMENTED);
                header.forget_verbatim();
                pass.note(
                    &tag,
                    ConversionKind::Kept,
                    "another application's extension, declared as undocumented rather than dropped",
                );
            }
        }
    }

    Conversion {
        document: converted,
        from,
        to: "7.0".to_owned(),
        notes: pass.notes,
    }
}

fn head_to_7(header: &mut Node, pass: &mut Pass) {
    header.forget_verbatim();
    if let Some(gedc) = header.first_mut("GEDC") {
        gedc.forget_verbatim();
        if let Some(vers) = gedc.first_mut("VERS") {
            vers.set_value(Some("7.0".to_owned()));
        } else {
            gedc.push(Node::with_value("VERS", "7.0"));
        }
        // `FORM` under `GEDC` said the file was lineage-linked, which is the
        // only thing version 7 is.
        if remove_child(gedc, "FORM") {
            pass.note(
                "GEDC.FORM",
                ConversionKind::Dropped,
                "version 7 has one form and does not name it",
            );
        }
    } else {
        // A header with no `GEDC` at all — a minimal or careless 5.x file.
        // Version 7 requires the declaration, so it is created whole.
        header
            .children
            .insert(0, Node::new("GEDC").child(Node::with_value("VERS", "7.0")));
        pass.note(
            "GEDC",
            ConversionKind::Converted,
            "the header declared no GEDCOM version, and version 7 requires it",
        );
    }
    if remove_child(header, "CHAR") {
        pass.note(
            "CHAR",
            ConversionKind::Dropped,
            "version 7 is always UTF-8, so a character set declaration says nothing",
        );
    }
}

fn structures_to_7_node(node: &mut Node, shared: &[String], pass: &mut Pass) -> bool {
    let mut changed = false;

    if node.tag == "_UID" {
        "UID".clone_into(&mut node.tag);
        changed = true;
        pass.note(
            "_UID",
            ConversionKind::Converted,
            "version 7 defines UID, so the extension is no longer needed",
        );
    }

    // A pointer at what is now a shared note record has to say so.
    if node.tag == "NOTE"
        && let Some(pointer) = node.pointer()
        && shared.iter().any(|xref| xref.eq_ignore_ascii_case(pointer))
    {
        "SNOTE".clone_into(&mut node.tag);
        changed = true;
        pass.note(
            "NOTE",
            ConversionKind::Converted,
            "a pointer to a note record is a shared note pointer in version 7",
        );
    }

    if node.tag == "DATE" {
        changed |= date_to_7(node, pass);
    }

    // `FORM` under `OBJE` moves to each `FILE`, which is where version 7 puts
    // it and where a record with two files of different types needs it.
    if node.tag == "OBJE" && node.first("FILE").is_some() {
        changed |= media_form_to_7(node, pass);
    }

    for child in &mut node.children {
        changed |= structures_to_7_node(child, shared, pass);
    }
    if changed {
        node.forget_verbatim();
    }
    changed
}

fn date_to_7(node: &mut Node, pass: &mut Pass) -> bool {
    let payload = node.logical_value();
    let date = GedcomDate::parse(&payload);
    match date.kind {
        // `INT 1 JAN 1900 (a reading of the original)` becomes the date it
        // interpreted plus a `PHRASE` holding the prose, which is exactly what
        // version 7 provides `PHRASE` for.
        DateKind::Interpreted => {
            let Some(phrase) = date.phrase else {
                return false;
            };
            // Taken out of the original text rather than rebuilt from the
            // parsed pieces: the file's own spelling of a date is never
            // rewritten to normalize it, here least of all.
            let Some(plain) = interpreted_date(&payload) else {
                return false;
            };
            node.set_value(Some(plain));
            node.push(Node::with_value("PHRASE", phrase));
            pass.note(
                "DATE",
                ConversionKind::Converted,
                "an interpreted date became the date it interpreted, with the reading in PHRASE",
            );
            true
        }
        DateKind::Unparsed => {
            pass.note(
                "DATE",
                ConversionKind::Kept,
                format!("version 7 will not accept {payload:?}, and it is kept rather than lost"),
            );
            false
        }
        _ => false,
    }
}

/// The date inside `INT <date> (<phrase>)`, exactly as the file wrote it.
fn interpreted_date(payload: &str) -> Option<String> {
    let rest = payload.trim().strip_prefix("INT ")?;
    let plain = rest.find('(').map_or(rest, |at| &rest[..at]);
    let plain = plain.trim();
    (!plain.is_empty()).then(|| plain.to_owned())
}

fn media_form_to_7(object: &mut Node, pass: &mut Pass) -> bool {
    let Some(form) = object.value_of("FORM") else {
        return false;
    };
    let medium = object.first("FORM").and_then(|node| node.value_of("MEDI"));
    remove_child(object, "FORM");
    for file in object.children.iter_mut().filter(|node| node.tag == "FILE") {
        if file.first("FORM").is_none() {
            let mut node = Node::with_value("FORM", &form);
            if let Some(medium) = &medium {
                node.push(Node::with_value("MEDI", medium));
            }
            file.push(node);
            file.forget_verbatim();
        }
    }
    pass.note(
        "OBJE.FORM",
        ConversionKind::Converted,
        "version 7 records a file's format on the file, not on the object",
    );
    true
}

/// Converts a document to GEDCOM 5.5.1.
#[must_use]
pub fn to_version_5_5_1(document: &Document) -> Conversion {
    let from = document.version().unwrap_or_else(|| "unstated".to_owned());
    let mut converted = document.clone();
    let mut pass = Pass {
        notes: Vec::new(),
        record: None,
    };

    for record in &mut converted.records {
        pass.record = record.xref.clone();
        if record.tag == "HEAD" {
            head_to_5(record, &mut pass);
        }
        if record.tag == "SNOTE" {
            "NOTE".clone_into(&mut record.tag);
            record.forget_verbatim();
            pass.note(
                "SNOTE",
                ConversionKind::Converted,
                "5.5.1 has one kind of note record",
            );
        }
        structures_to_5_node(record, &mut pass);
    }

    Conversion {
        document: converted,
        from,
        to: "5.5.1".to_owned(),
        notes: pass.notes,
    }
}

fn head_to_5(header: &mut Node, pass: &mut Pass) {
    header.forget_verbatim();
    if let Some(gedc) = header.first_mut("GEDC") {
        gedc.forget_verbatim();
        if let Some(vers) = gedc.first_mut("VERS") {
            vers.set_value(Some("5.5.1".to_owned()));
        } else {
            gedc.push(Node::with_value("VERS", "5.5.1"));
        }
        if gedc.first("FORM").is_none() {
            gedc.push(Node::with_value("FORM", "LINEAGE-LINKED"));
        }
    } else {
        // 5.5.1 requires the declaration too, `FORM` included.
        header.children.insert(
            0,
            Node::new("GEDC")
                .child(Node::with_value("VERS", "5.5.1"))
                .child(Node::with_value("FORM", "LINEAGE-LINKED")),
        );
        pass.note(
            "GEDC",
            ConversionKind::Converted,
            "the header declared no GEDCOM version, and 5.5.1 requires it",
        );
    }
    if header.first("CHAR").is_none() {
        header.push(Node::with_value("CHAR", "UTF-8"));
        pass.note(
            "CHAR",
            ConversionKind::Converted,
            "5.5.1 requires a character set, and the file is written as UTF-8",
        );
    }
    // `SCHMA` declares extensions, which 5.5.1 has no mechanism for. The
    // extensions themselves stay; only the declaration goes.
    if remove_child(header, "SCHMA") {
        pass.note(
            "SCHMA",
            ConversionKind::Dropped,
            "5.5.1 has no way to declare an extension; the extensions themselves are kept",
        );
    }
}

fn structures_to_5_node(node: &mut Node, pass: &mut Pass) -> bool {
    let mut changed = false;

    if node.tag == "UID" {
        "_UID".clone_into(&mut node.tag);
        changed = true;
        pass.note(
            "UID",
            ConversionKind::Converted,
            "5.5.1 has no UID, and every producer writes _UID instead",
        );
    }

    if node.tag == "SNOTE" && node.pointer().is_some() {
        "NOTE".clone_into(&mut node.tag);
        changed = true;
        pass.note(
            "SNOTE",
            ConversionKind::Converted,
            "a shared note pointer is an ordinary note pointer in 5.5.1",
        );
    }

    // `DATE` with a `PHRASE` folds back into 5.5.1's interpreted form, which
    // is the same information in the older spelling.
    if node.tag == "DATE"
        && let Some(phrase) = node.value_of("PHRASE")
    {
        let payload = node.logical_value();
        remove_child(node, "PHRASE");
        node.set_value(Some(format!("INT {payload} ({phrase})")));
        changed = true;
        pass.note(
            "PHRASE",
            ConversionKind::Converted,
            "a dated phrase became 5.5.1's interpreted form",
        );
    }

    // Everything else version 7 added and 5.5.1 does not define is kept and
    // named. Deleting a translation or a creation date to satisfy a version
    // number would be the worst trade this application could make.
    if matches!(node.tag.as_str(), "TRAN" | "CREA" | "EXID" | "SDATE") {
        pass.note(
            &node.tag,
            ConversionKind::Kept,
            "version 7 defines this and 5.5.1 does not; it is kept rather than lost",
        );
    }

    for child in &mut node.children {
        changed |= structures_to_5_node(child, pass);
    }
    if changed {
        node.forget_verbatim();
    }
    changed
}

fn remove_child(node: &mut Node, tag: &str) -> bool {
    let before = node.children.len();
    node.children.retain(|child| child.tag != tag);
    if node.children.len() != before {
        node.forget_verbatim();
        return true;
    }
    false
}

fn extension_tags(document: &Document) -> Vec<String> {
    let mut tags: Vec<String> = Vec::new();
    for record in &document.records {
        if record.tag == "HEAD" {
            continue;
        }
        for node in record.walk() {
            if tags::is_extension(&node.tag) && !tags.contains(&node.tag) {
                tags.push(node.tag.clone());
            }
        }
    }
    tags.sort();
    tags
}

#[cfg(test)]
mod tests {
    use super::*;

    const OLD: &str = "0 HEAD\n\
1 GEDC\n\
2 VERS 5.5.1\n\
2 FORM LINEAGE-LINKED\n\
1 CHAR UTF-8\n\
0 @I1@ INDI\n\
1 NAME Rian /Testwright/\n\
1 _UID 4E7C9F2A\n\
1 BIRT\n\
2 DATE INT 1 JAN 1900 (from the family bible)\n\
1 NOTE @N1@\n\
1 _SOMEBODY _ELSE\n\
0 @N1@ NOTE\n\
1 CONC Written by hand.\n\
0 @M1@ OBJE\n\
1 FILE media/photo.jpg\n\
1 FORM jpeg\n\
2 MEDI photo\n\
0 TRLR\n";

    fn parse(text: &str) -> Document {
        Document::parse(text).expect("parse")
    }

    #[test]
    fn five_five_one_becomes_version_seven() {
        let outcome = to_version_7(&parse(OLD));
        let text = outcome.document.to_text();

        assert_eq!(outcome.from, "5.5.1");
        assert_eq!(outcome.to, "7.0");
        assert!(text.contains("2 VERS 7.0"), "{text}");
        assert!(!text.contains("CHAR"), "the character set went: {text}");
        assert!(!text.contains("2 FORM LINEAGE-LINKED"), "{text}");

        assert!(text.contains("0 @N1@ SNOTE"), "the record: {text}");
        assert!(
            text.contains("1 SNOTE @N1@"),
            "and the pointer at it: {text}"
        );

        assert!(text.contains("1 UID 4E7C9F2A"), "{text}");
        assert!(text.contains("2 DATE 1 JAN 1900"), "{text}");
        assert!(
            text.contains("3 PHRASE from the family bible"),
            "the reading is kept as a phrase: {text}"
        );

        // The media format moved onto the file.
        assert!(text.contains("2 FORM jpeg"), "{text}");
        assert!(text.contains("3 MEDI photo"), "{text}");

        // Somebody else's extension is declared rather than dropped.
        assert!(text.contains("2 TAG _SOMEBODY "), "{text}");
        assert!(text.contains("undocumented"), "{text}");
        assert!(text.contains("1 _SOMEBODY _ELSE"), "and kept: {text}");
    }

    #[test]
    fn version_seven_becomes_five_five_one() {
        let seven = to_version_7(&parse(OLD)).document;
        let outcome = to_version_5_5_1(&seven);
        let text = outcome.document.to_text();

        assert!(text.contains("2 VERS 5.5.1"), "{text}");
        assert!(text.contains("2 FORM LINEAGE-LINKED"), "{text}");
        assert!(text.contains("1 CHAR UTF-8"), "{text}");
        assert!(!text.contains("SCHMA"), "the declarations went: {text}");
        assert!(text.contains("0 @N1@ NOTE"), "{text}");
        assert!(text.contains("1 NOTE @N1@"), "{text}");
        assert!(text.contains("1 _UID 4E7C9F2A"), "{text}");
        assert!(
            text.contains("2 DATE INT 1 JAN 1900 (from the family bible)"),
            "the phrase folds back into the interpreted form: {text}"
        );
        // The extension itself survived both directions.
        assert!(text.contains("1 _SOMEBODY _ELSE"), "{text}");
    }

    #[test]
    fn the_report_names_every_construct_it_touched() {
        let outcome = to_version_7(&parse(OLD));
        let names: Vec<&str> = outcome.notes.iter().map(|note| note.tag.as_str()).collect();

        for expected in ["CHAR", "GEDC.FORM", "_UID", "NOTE", "DATE", "OBJE.FORM"] {
            assert!(
                names.contains(&expected),
                "{expected} missing from {names:?}"
            );
        }
        let (converted, dropped, kept) = outcome.counts();
        assert!(converted > 0 && dropped > 0, "{:?}", outcome.notes);
        assert_eq!(
            kept, 1,
            "only the undeclared extension: {:?}",
            outcome.notes
        );

        // A note names the record it sat in, so a report can be read against
        // the file rather than only counted.
        let uid = outcome
            .notes
            .iter()
            .find(|note| note.tag == "_UID")
            .expect("_UID note");
        assert_eq!(uid.record.as_deref(), Some("@I1@"));
    }

    #[test]
    fn a_date_version_seven_cannot_hold_is_named_and_kept() {
        let document =
            parse("0 HEAD\n1 GEDC\n2 VERS 5.5.1\n0 @I1@ INDI\n1 BIRT\n2 DATE Infant\n0 TRLR\n");
        let outcome = to_version_7(&document);
        assert!(
            outcome.document.to_text().contains("2 DATE Infant"),
            "kept: {}",
            outcome.document.to_text()
        );
        let note = outcome
            .notes
            .iter()
            .find(|note| note.tag == "DATE")
            .expect("a note about it");
        assert_eq!(note.kind, ConversionKind::Kept);
        assert!(note.detail.contains("Infant"), "{}", note.detail);
    }

    #[test]
    fn an_extension_the_caller_owns_is_declared_against_its_own_uri() {
        let document = parse(
            "0 HEAD\n1 GEDC\n2 VERS 5.5.1\n0 @I1@ INDI\n1 _MINE value\n1 _THEIRS value\n0 TRLR\n",
        );
        let outcome = to_version_7_with(&document, &[("_MINE", "https://mine.example/MINE")]);
        let text = outcome.document.to_text();

        assert!(
            text.contains("2 TAG _MINE https://mine.example/MINE"),
            "{text}"
        );
        assert!(
            text.contains("2 TAG _THEIRS https://gedcom.example/undocumented"),
            "{text}"
        );
        let mine = outcome
            .notes
            .iter()
            .find(|note| note.tag == "_MINE")
            .expect("a note about the owned tag");
        assert_eq!(mine.kind, ConversionKind::Converted);
    }

    #[test]
    fn converting_to_the_version_a_file_already_declares_changes_nothing_it_holds() {
        let seven = to_version_7(&parse(OLD)).document;
        let again = to_version_7(&seven);
        assert_eq!(
            again.document.to_text(),
            seven.to_text(),
            "a second conversion is a no-op"
        );
    }
}