mla-titlecase-cli 0.3.0

CLI for building and inspecting mla-titlecase lexicons
use std::collections::BTreeSet;

use reqwest::header::ACCEPT;
use serde::Deserialize;

use crate::{
    cli::PreparePayloadKind,
    error::Result,
    normalize::{
        canonical_map_payload, multiword_map_payload, protected_spellings_payload,
        NormalizedPayload,
    },
    sources::{
        is_multiword, validate_payload_kind, FetchOptions, PrepareOptions, ResolvedSource,
        SourceDefinition, SourceId,
    },
};

const ENDPOINT: &str = "https://lobid.org/gnd/search";
const DEFAULT_QUERY: &str = "*";
const DEFAULT_FILTER: &str = "type:Person";
const DEFAULT_LIMIT: usize = 250;

pub(crate) fn definition() -> SourceDefinition {
    SourceDefinition {
        id: SourceId::Gnd,
        description: "Optional CC0 authority-style names from the GND / lobid search API",
        license_summary: "GND authority data served by lobid is available under CC0 1.0",
        notice: "The default GND flow queries lobid.org with type:Person filtering and preserves the full request URL in the fetch manifest.",
        default_url: ENDPOINT,
        recommended: false,
        requires_acknowledgement: false,
    }
}

pub(crate) fn fetch(
    client: &reqwest::blocking::Client,
    options: &FetchOptions,
) -> Result<ResolvedSource> {
    let query = options.query.as_deref().unwrap_or(DEFAULT_QUERY);
    let limit = options.limit.unwrap_or(DEFAULT_LIMIT);
    let response = client
        .get(ENDPOINT)
        .query(&[
            ("q", query),
            ("filter", DEFAULT_FILTER),
            ("size", &limit.to_string()),
            ("format", "json"),
        ])
        .header(ACCEPT, "application/json")
        .send()?
        .error_for_status()?;
    let source_url = response.url().to_string();
    let bytes = response.bytes()?.to_vec();

    Ok(ResolvedSource {
        bytes,
        source_url,
        source_version: None,
        license_summary: definition().license_summary.to_string(),
        notice: Some(definition().notice.to_string()),
    })
}

pub(crate) fn prepare(raw: &[u8], options: PrepareOptions) -> Result<NormalizedPayload> {
    let requested_kind = options.payload_kind.unwrap_or(PreparePayloadKind::MultiwordMap);
    validate_payload_kind(
        SourceId::Gnd,
        requested_kind,
        &[
            PreparePayloadKind::CanonicalMap,
            PreparePayloadKind::MultiwordMap,
            PreparePayloadKind::ProtectedSpellings,
        ],
    )?;

    let parsed = parse_records(raw)?;
    let mut payload = match requested_kind {
        PreparePayloadKind::CanonicalMap => canonical_map_payload(parsed.single_word_entries),
        PreparePayloadKind::MultiwordMap => multiword_map_payload(parsed.multiword_entries),
        PreparePayloadKind::ProtectedSpellings => {
            protected_spellings_payload(parsed.single_word_entries)
        }
        _ => unreachable!("validated above"),
    };
    payload.report.ignored_records += parsed.ignored_records;
    Ok(payload)
}

struct ParsedRecords {
    single_word_entries: Vec<(String, String)>,
    multiword_entries: Vec<(String, String)>,
    ignored_records: usize,
}

fn parse_records(raw: &[u8]) -> Result<ParsedRecords> {
    let response: SearchResponse = serde_json::from_slice(raw)?;
    let mut single_word_entries = Vec::new();
    let mut multiword_entries = Vec::new();
    let mut ignored_records = 0_usize;

    for member in response.member {
        let surfaces = member.surfaces();
        if surfaces.is_empty() {
            ignored_records += 1;
            continue;
        }

        for surface in surfaces {
            if is_multiword(&surface) {
                multiword_entries.push((surface.clone(), surface));
            } else {
                single_word_entries.push((surface.clone(), surface));
            }
        }
    }

    Ok(ParsedRecords { single_word_entries, multiword_entries, ignored_records })
}

#[derive(Debug, Deserialize)]
struct SearchResponse {
    #[serde(default)]
    member: Vec<GndMember>,
}

#[derive(Debug, Deserialize)]
struct GndMember {
    #[serde(rename = "preferredName")]
    preferred_name: Option<String>,
    #[serde(rename = "variantName", default)]
    variant_name: Vec<String>,
    #[serde(rename = "preferredNameEntityForThePerson")]
    preferred_person_name: Option<PersonName>,
    #[serde(rename = "variantNameEntityForThePerson", default)]
    variant_person_names: Vec<PersonName>,
}

impl GndMember {
    fn surfaces(&self) -> Vec<String> {
        let mut surfaces = BTreeSet::new();

        if let Some(name) =
            self.preferred_name.as_deref().map(str::trim).filter(|value| !value.is_empty())
        {
            surfaces.insert(name.to_string());
        }

        for name in &self.variant_name {
            let trimmed = name.trim();
            if !trimmed.is_empty() {
                surfaces.insert(trimmed.to_string());
            }
        }

        if let Some(person) = self.preferred_person_name.as_ref().and_then(PersonName::display_name)
        {
            surfaces.insert(person);
        }

        for person in &self.variant_person_names {
            if let Some(name) = person.display_name() {
                surfaces.insert(name);
            }
        }

        surfaces.into_iter().collect()
    }
}

#[derive(Debug, Deserialize)]
struct PersonName {
    #[serde(default)]
    forename: Vec<String>,
    #[serde(default)]
    prefix: Vec<String>,
    #[serde(default)]
    surname: Vec<String>,
}

impl PersonName {
    fn display_name(&self) -> Option<String> {
        let mut parts = Vec::new();
        parts.extend(self.forename.iter().map(String::as_str).filter(|value| !value.is_empty()));
        parts.extend(self.prefix.iter().map(String::as_str).filter(|value| !value.is_empty()));
        parts.extend(self.surname.iter().map(String::as_str).filter(|value| !value.is_empty()));

        (!parts.is_empty()).then(|| parts.join(" "))
    }
}

#[cfg(test)]
mod tests {
    use mla_titlecase::{MapEntry, PluginPayload};

    use crate::cli::PreparePayloadKind;
    use crate::sources::PrepareOptions;

    use super::prepare;

    const FIXTURE: &[u8] = include_bytes!("../../../../testdata/fixtures/gnd-sample.json");

    #[test]
    fn prepares_multiword_payloads() {
        let payload = prepare(
            FIXTURE,
            PrepareOptions { payload_kind: Some(PreparePayloadKind::MultiwordMap) },
        )
        .unwrap();

        assert_eq!(
            payload.payload,
            PluginPayload::MultiwordMap {
                entries: vec![
                    MapEntry {
                        key: "beethoven, ludwig van".to_string(),
                        value: "Beethoven, Ludwig van".to_string(),
                    },
                    MapEntry {
                        key: "beethoven, maria josepha van".to_string(),
                        value: "Beethoven, Maria Josepha van".to_string(),
                    },
                    MapEntry {
                        key: "ludwig van beethoven".to_string(),
                        value: "Ludwig van Beethoven".to_string(),
                    },
                    MapEntry {
                        key: "maria josepha poll".to_string(),
                        value: "Maria Josepha Poll".to_string(),
                    },
                    MapEntry {
                        key: "maria josepha van beethoven".to_string(),
                        value: "Maria Josepha van Beethoven".to_string(),
                    },
                    MapEntry {
                        key: "poll, maria josepha".to_string(),
                        value: "Poll, Maria Josepha".to_string(),
                    },
                ],
            }
        );
    }

    #[test]
    fn prepares_single_word_payloads() {
        let payload = prepare(
            FIXTURE,
            PrepareOptions { payload_kind: Some(PreparePayloadKind::CanonicalMap) },
        )
        .unwrap();

        assert_eq!(
            payload.payload,
            PluginPayload::CanonicalMap {
                entries: vec![MapEntry {
                    key: "bayreuth".to_string(),
                    value: "Bayreuth".to_string()
                }],
            }
        );
    }
}