defter 0.8.2

//! The language and license of a work, from what Zotero holds as free text
//! (its `language` and `rights` fields), as an ISO 639-1 code and an SPDX
//! license ID, which the public profile's facets count and narrow by, as
//! dragoman's do. Anything that is neither is left out.

/// ISO 639-2 bibliographic codes, which library catalogues use and Zotero
/// imports, where they differ from ISO 639-3.
const BIBLIOGRAPHIC: &[(&str, &str)] = &[
    ("alb", "sq"),
    ("arm", "hy"),
    ("baq", "eu"),
    ("bur", "my"),
    ("chi", "zh"),
    ("cze", "cs"),
    ("dut", "nl"),
    ("fre", "fr"),
    ("geo", "ka"),
    ("ger", "de"),
    ("gre", "el"),
    ("ice", "is"),
    ("mac", "mk"),
    ("mao", "mi"),
    ("may", "ms"),
    ("per", "fa"),
    ("rum", "ro"),
    ("slo", "sk"),
    ("tib", "bo"),
    ("wel", "cy"),
];

/// German names of the languages works are most often in; the rest are named
/// in English, as dragoman names a language its browser cannot.
const GERMAN: &[(&str, &str)] = &[
    ("af", "Afrikaans"),
    ("ar", "Arabisch"),
    ("be", "Belarussisch"),
    ("bg", "Bulgarisch"),
    ("bn", "Bengalisch"),
    ("bs", "Bosnisch"),
    ("ca", "Katalanisch"),
    ("cs", "Tschechisch"),
    ("cy", "Walisisch"),
    ("da", "Dänisch"),
    ("de", "Deutsch"),
    ("el", "Griechisch"),
    ("en", "Englisch"),
    ("eo", "Esperanto"),
    ("es", "Spanisch"),
    ("et", "Estnisch"),
    ("eu", "Baskisch"),
    ("fa", "Persisch"),
    ("fi", "Finnisch"),
    ("fr", "Französisch"),
    ("ga", "Irisch"),
    ("gd", "Schottisch-Gälisch"),
    ("gl", "Galicisch"),
    ("he", "Hebräisch"),
    ("hi", "Hindi"),
    ("hr", "Kroatisch"),
    ("hu", "Ungarisch"),
    ("hy", "Armenisch"),
    ("id", "Indonesisch"),
    ("is", "Isländisch"),
    ("it", "Italienisch"),
    ("ja", "Japanisch"),
    ("ka", "Georgisch"),
    ("ko", "Koreanisch"),
    ("la", "Latein"),
    ("lb", "Luxemburgisch"),
    ("lt", "Litauisch"),
    ("lv", "Lettisch"),
    ("mk", "Mazedonisch"),
    ("ms", "Malaiisch"),
    ("mt", "Maltesisch"),
    ("nb", "Norwegisch (Bokmål)"),
    ("nl", "Niederländisch"),
    ("nn", "Norwegisch (Nynorsk)"),
    ("no", "Norwegisch"),
    ("pl", "Polnisch"),
    ("pt", "Portugiesisch"),
    ("ro", "Rumänisch"),
    ("ru", "Russisch"),
    ("sk", "Slowakisch"),
    ("sl", "Slowenisch"),
    ("sq", "Albanisch"),
    ("sr", "Serbisch"),
    ("sv", "Schwedisch"),
    ("sw", "Suaheli"),
    ("th", "Thailändisch"),
    ("tr", "Türkisch"),
    ("uk", "Ukrainisch"),
    ("ur", "Urdu"),
    ("vi", "Vietnamesisch"),
    ("zh", "Chinesisch"),
];

/// Names and codes a language goes by that isolang does not know: the common
/// English names, old ISO 639-1 codes, Chinese varieties, and German variants.
const ALIASES: &[(&str, &str)] = &[
    ("american english", "en"),
    ("british english", "en"),
    ("cantonese", "zh"),
    ("castellano", "es"),
    ("castilian", "es"),
    ("chichewa", "ny"),
    ("cmn", "zh"),
    ("divehi", "dv"),
    ("dt", "de"),
    ("dtsch", "de"),
    ("engl", "en"),
    ("farsi", "fa"),
    ("flemish", "nl"),
    ("galizisch", "gl"),
    ("greek", "el"),
    ("haitian creole", "ht"),
    ("in", "id"),
    ("iw", "he"),
    ("ji", "yi"),
    ("jw", "jv"),
    ("kyrgyz", "ky"),
    ("lateinisch", "la"),
    ("mandarin", "zh"),
    ("mo", "ro"),
    ("moldovan", "ro"),
    ("neugriechisch", "el"),
    ("pashto", "ps"),
    ("punjabi", "pa"),
    ("sinhalese", "si"),
    ("slovene", "sl"),
    ("uyghur", "ug"),
    ("valencian", "ca"),
    ("weißrussisch", "be"),
    ("yue", "zh"),
];

/// English names where isolang's are not the ones readers know, as browsers
/// name these languages, and dragoman with them.
const ENGLISH: &[(&str, &str)] = &[
    ("dv", "Divehi"),
    ("el", "Greek"),
    ("ht", "Haitian Creole"),
    ("ky", "Kyrgyz"),
    ("nb", "Norwegian Bokmål"),
    ("nn", "Norwegian Nynorsk"),
    ("ny", "Nyanja"),
    ("pa", "Punjabi"),
    ("ps", "Pashto"),
    ("si", "Sinhala"),
    ("ug", "Uyghur"),
];

/// Every language's own name, lowercased, with it.
static AUTONYMS: std::sync::LazyLock<std::collections::HashMap<String, isolang::Language>> =
    std::sync::LazyLock::new(|| {
        isolang::languages()
            .filter_map(|l| Some((l.to_autonym()?.to_lowercase(), l)))
            .collect()
    });

/// The ISO 639-1 code of the language `text` names: a code (`en`, `eng`,
/// `ger`, `en-US`, `iw`), or a name in English, German or the language itself
/// (`English`, `Greek`, `Englisch`, `Deutsch`), also with a region (`English
/// (US)`). Of several (`en; de`), the first.
pub fn language(text: &str) -> Option<String> {
    text.split([';', ',', '/'])
        .find_map(one_language)
        .map(str::to_string)
}

fn one_language(text: &str) -> Option<&'static str> {
    use isolang::Language;
    // Without a region or script in brackets, and a closing full stop.
    let text = text.split('(').next().unwrap_or(text);
    let text = text.trim().trim_end_matches('.').trim();
    if text.is_empty() {
        return None;
    }
    let lower = text.to_lowercase();
    if let Some((_, code)) = ALIASES.iter().find(|(alias, _)| *alias == lower) {
        return Some(code);
    }
    // A tag's primary subtag: `en` of `en-US` and `en_GB`.
    let primary = lower.split(['-', '_']).next().unwrap_or(&lower);
    let code = |l: Language| l.to_639_1();
    if let Some((_, code)) = ALIASES.iter().find(|(alias, _)| *alias == primary) {
        return Some(code);
    }
    let found = match primary.len() {
        2 => Language::from_639_1(primary).and_then(code),
        3 => BIBLIOGRAPHIC
            .iter()
            .find(|(b, _)| *b == primary)
            .map(|(_, c)| *c)
            .or_else(|| Language::from_639_3(primary).and_then(code)),
        _ => None,
    };
    found
        .or_else(|| Language::from_name(text).and_then(code))
        .or_else(|| Language::from_name_lowercase(&lower).and_then(code))
        .or_else(|| {
            GERMAN
                .iter()
                .find(|(_, name)| name.to_lowercase() == lower)
                .map(|(c, _)| *c)
        })
        .or_else(|| AUTONYMS.get(&lower).copied().and_then(code))
}

/// The name of language `code` in German with `german`, else in English.
pub fn language_name(code: &str, german: bool) -> String {
    if german && let Some((_, name)) = GERMAN.iter().find(|(c, _)| *c == code) {
        return name.to_string();
    }
    if let Some((_, name)) = ENGLISH.iter().find(|(c, _)| *c == code) {
        return name.to_string();
    }
    isolang::Language::from_639_1(code)
        .map(|l| l.to_name().to_string())
        .unwrap_or_else(|| code.to_string())
}

/// Words that say nothing of which license it is.
const FILLER: &[&str] = &[
    "license",
    "licence",
    "lizenz",
    "international",
    "unported",
    "universal",
    "deed",
    "generic",
];

/// The SPDX ID of the license `text` names: an SPDX ID or its name, a license
/// URL (`https://creativecommons.org/licenses/by/4.0/deed.en`), or the way
/// Creative Commons licenses are written (`CC BY 4.0`, `CC BY-NC-SA 4.0
/// International`, `CC0`, `Creative Commons Attribution-ShareAlike 4.0
/// International License`, `Attribution 4.0 International (CC BY 4.0)`).
pub fn license(text: &str) -> Option<String> {
    let text = text.trim();
    if text.is_empty() {
        return None;
    }
    let id = |candidate: &str| commonmeta::spdx::search(candidate).map(|l| l.license_id.clone());
    if let Some(found) = id(text).or_else(|| cc_url(text).and_then(|c| id(&c))) {
        return Some(found);
    }
    // What brackets hold may be the short form: "Attribution 4.0 International
    // (CC BY 4.0)".
    let mut candidates = vec![text.split('(').next().unwrap_or(text).to_string()];
    if let Some((_, inner)) = text.split_once('(') {
        candidates.push(inner.trim_end_matches(')').to_string());
    }
    for candidate in &candidates {
        let words: Vec<String> = candidate
            .to_uppercase()
            .split(|c: char| c.is_whitespace() || c == '_')
            .map(|w| w.trim_matches(',').to_string())
            .filter(|w| !w.is_empty() && !FILLER.contains(&w.to_lowercase().as_str()))
            .collect();
        let mut normalized = words.join("-");
        if normalized == "CC0" || normalized == "CC-ZERO" {
            normalized = "CC0-1.0".to_string();
        }
        if let Some(found) = id(&normalized) {
            return Some(found);
        }
        if let Some((_, found)) = LICENSE_NAMES
            .iter()
            .find(|(name, _)| *name == name_key(candidate))
        {
            return Some(found.clone());
        }
    }
    None
}

/// A license name reduced to what tells licenses apart: lowercase letters and
/// digits, without filler words, with Creative Commons' element names spelled
/// as SPDX spells them.
fn name_key(name: &str) -> String {
    let lower = name
        .to_lowercase()
        .replace("sharealike", "share alike")
        .replace("noncommercial", "non commercial")
        .replace("noderivatives", "no derivatives")
        .replace("noderivs", "no derivatives");
    lower
        .split(|c: char| !c.is_alphanumeric() && c != '.')
        .map(|w| w.trim_matches('.'))
        .filter(|w| !w.is_empty() && !FILLER.contains(w))
        .collect::<Vec<_>>()
        .join("")
}

/// The SPDX ID a creativecommons.org license URL stands for, as a candidate:
/// `…/licenses/by-nc/4.0/deed.en` is `CC-BY-NC-4.0`, `…/licenses/by/3.0/de/`
/// `CC-BY-3.0-DE`, `…/publicdomain/zero/1.0/` `CC0-1.0`.
fn cc_url(text: &str) -> Option<String> {
    let lower = text.to_lowercase();
    let (_, path) = lower.split_once("creativecommons.org/")?;
    let path = path.split(['?', '#']).next().unwrap_or(path);
    let parts: Vec<&str> = path
        .split('/')
        .filter(|p| !p.is_empty() && !p.starts_with("deed") && !p.starts_with("legalcode"))
        .collect();
    match parts.as_slice() {
        ["licenses", kind, version, rest @ ..] => {
            let mut id = format!("CC-{}-{version}", kind.to_uppercase());
            if let [jurisdiction] = rest {
                id = format!("{id}-{}", jurisdiction.to_uppercase());
            }
            Some(id)
        }
        ["publicdomain", "zero", version, ..] => Some(format!("CC0-{version}")),
        _ => None,
    }
}

/// The SPDX licenses' names, reduced by [`name_key`], with their IDs, from the
/// list commonmeta bundles.
static LICENSE_NAMES: std::sync::LazyLock<Vec<(String, String)>> = std::sync::LazyLock::new(|| {
    let list: serde_json::Value = commonmeta::vocabularies::load_vocabulary("SPDX.Licenses")
        .ok()
        .and_then(|raw| serde_json::from_str(raw).ok())
        .unwrap_or_default();
    list["licenses"]
        .as_array()
        .into_iter()
        .flatten()
        // A deprecated ID shares its name with the one that replaced it.
        .filter(|l| l["isDeprecatedLicenseId"] != serde_json::Value::Bool(true))
        .filter_map(|l| {
            Some((
                name_key(l["name"].as_str()?),
                l["licenseId"].as_str()?.to_string(),
            ))
        })
        .collect()
});

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn reads_languages_as_zotero_holds_them() {
        for (text, code) in [
            ("en", "en"),
            ("EN", "en"),
            ("en-US", "en"),
            ("en_GB", "en"),
            ("eng", "en"),
            ("ger", "de"),
            ("deu", "de"),
            ("English", "en"),
            ("english", "en"),
            ("Englisch", "en"),
            ("Deutsch", "de"),
            ("français", "fr"),
            (" French ", "fr"),
        ] {
            assert_eq!(language(text).as_deref(), Some(code), "{text}");
        }
        // Not a language, or one without an ISO 639-1 code: left out.
        for text in ["", "  ", "xx", "Klingon", "zzz", "multiple"] {
            assert_eq!(language(text), None, "{text}");
        }
        for (text, code) in [
            ("Greek", "el"),
            ("Farsi", "fa"),
            ("Punjabi", "pa"),
            ("Slovene", "sl"),
            ("English (US)", "en"),
            ("Portuguese (Brazil)", "pt"),
            ("Engl.", "en"),
            ("iw", "he"),
            ("cmn", "zh"),
            ("Mandarin", "zh"),
            ("en; de", "en"),
            ("English, German", "en"),
            ("Greek, Modern (1453-)", "el"),
            ("Luxemburgisch", "lb"),
        ] {
            assert_eq!(language(text).as_deref(), Some(code), "{text}");
        }
        assert_eq!(language_name("el", false), "Greek");
        assert_eq!(language_name("de", false), "German");
        assert_eq!(language_name("de", true), "Deutsch");
        // Named in English where there is no German name here.
        assert_eq!(language_name("ng", true), "Ndonga");
    }

    #[test]
    fn reads_licenses_as_zotero_holds_them() {
        for (text, id) in [
            ("CC-BY-4.0", "CC-BY-4.0"),
            ("cc-by-4.0", "CC-BY-4.0"),
            ("CC BY 4.0", "CC-BY-4.0"),
            ("CC BY-NC-SA 4.0", "CC-BY-NC-SA-4.0"),
            ("https://creativecommons.org/licenses/by/4.0/", "CC-BY-4.0"),
            (
                "http://creativecommons.org/licenses/by-nc/4.0",
                "CC-BY-NC-4.0",
            ),
            ("CC0", "CC0-1.0"),
            ("CC0 1.0", "CC0-1.0"),
            ("MIT", "MIT"),
            ("MIT License", "MIT"),
            ("Apache 2.0", "Apache-2.0"),
            (
                "Creative Commons Attribution 4.0 International",
                "CC-BY-4.0",
            ),
            (
                "Creative Commons Attribution 4.0 International License",
                "CC-BY-4.0",
            ),
            (
                "Creative Commons Attribution-ShareAlike 4.0 International",
                "CC-BY-SA-4.0",
            ),
            (
                "Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International",
                "CC-BY-NC-ND-4.0",
            ),
            ("Attribution 4.0 International (CC BY 4.0)", "CC-BY-4.0"),
            ("CC BY 4.0 International", "CC-BY-4.0"),
            ("CC BY-NC-ND 4.0 International", "CC-BY-NC-ND-4.0"),
            ("CC BY 3.0 Unported", "CC-BY-3.0"),
            ("CC0 1.0 Universal", "CC0-1.0"),
            ("MIT Licence", "MIT"),
            (
                "https://creativecommons.org/licenses/by/4.0/deed.en",
                "CC-BY-4.0",
            ),
            (
                "https://creativecommons.org/licenses/by-sa/4.0/legalcode.de",
                "CC-BY-SA-4.0",
            ),
            (
                "creativecommons.org/licenses/by-nc-nd/4.0",
                "CC-BY-NC-ND-4.0",
            ),
            (
                "https://creativecommons.org/licenses/by/3.0/de/",
                "CC-BY-3.0-DE",
            ),
            (
                "https://creativecommons.org/publicdomain/zero/1.0/",
                "CC0-1.0",
            ),
        ] {
            assert_eq!(license(text).as_deref(), Some(id), "{text}");
        }
        for text in ["", "All rights reserved", "© 2024 The Authors", "CC BY"] {
            assert_eq!(license(text), None, "{text}");
        }
    }
}