mig-bo4e 0.9.0

//! Does each code name a rule writes mean what the rulebook says the code means?
//!
//! A rule translates the codes of the element it reads into names, through an
//! inline `enum_map` or a shared `code_list` (#165). The names are meant to be
//! the AHB meaning of the code *at that element*. The FV2510–FV2610 tables
//! were partly built from a message-wide code → name dictionary, so a code got
//! the name it has in some other element: RFF+Z13 "Prüfidentifikator" became
//! `anteilC`, CCI Z14 "Smartmeter-Gateway" became `erfolgreich` (#168).
//!
//! [`audit_variant`] lists, for every rule field with a table, the codes its
//! element permits in each PID the rule serves, with their AHB meanings, and
//! the name the table gives each. [`related`] judges whether a name belongs to
//! a meaning; [`derived_name`] is the name the convention gives a meaning.

use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};

use crate::code_lists::CodeLists;
use crate::code_lookup::CodeLookup;
use crate::definition::{FieldMapping, MappingDefinition};
use crate::engine::{parse_tag_qualifier, MappingEngine};
use crate::path_resolver::PathResolver;
use crate::pid_schema_index::PidSchemaIndex;

/// Where a rule's table lives.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub enum TableRef {
    /// An inline `enum_map` (it wins over a `code_list` given alongside).
    Inline,
    /// A shared list in `mappings/code_lists.toml`.
    Named(String),
}

/// One field of one mapping file that translates codes.
#[derive(Debug, Clone)]
pub struct RuleCodes {
    /// The TOML file.
    pub file: PathBuf,
    /// The field's key in `[fields]`, as written in the file.
    pub field: String,
    /// The BO4E target.
    pub target: String,
    pub table: TableRef,
    /// The table's entries: code → name.
    pub names: BTreeMap<String, String>,
    /// Codes the element permits, over every PID the rule serves:
    /// code → (AHB meaning, curated enum key).
    pub codes: BTreeMap<String, (String, Option<String>)>,
    /// PIDs the rule serves.
    pub pids: BTreeSet<String>,
    /// PIDs in which the element is a data element — the AHB enumerates no
    /// codes there, e.g. STS+E01 D_9013 "Code des Prüfschritts", whose codes
    /// come from the decision-tree code list D_1131 names. A table on such an
    /// element names codes the rulebook never listed there.
    pub data_pids: BTreeSet<String>,
    /// PIDs for which the element could not be found in the PID schema.
    pub unresolved_pids: BTreeSet<String>,
}

impl RuleCodes {
    /// Codes whose table name does not belong to their meaning, with
    /// `(code, meaning, name)`. `accepted` holds curated synonyms as
    /// `(meaning, name)` pairs.
    pub fn mismatches(
        &self,
        accepted: &BTreeSet<(String, String)>,
    ) -> Vec<(String, String, String)> {
        self.codes
            .iter()
            .filter_map(|(code, (meaning, enum_key))| {
                let name = self.names.get(code)?;
                let ok = related(meaning, enum_key.as_deref(), name)
                    || accepted.contains(&(meaning.clone(), name.clone()));
                (!ok).then(|| (code.clone(), meaning.clone(), name.clone()))
            })
            .collect()
    }

    /// Codes the element permits that the table has no name for: the forward
    /// mapping writes them raw next to names for the others.
    pub fn unnamed(&self) -> Vec<(String, String)> {
        self.codes
            .iter()
            .filter(|(code, _)| !self.names.contains_key(*code))
            .map(|(code, (meaning, _))| (code.clone(), meaning.clone()))
            .collect()
    }
}

/// Stand-in code given to data components in [`data_lookup`]'s schema copy.
const DATA_MARK: &str = "\u{0}data";

/// A lookup that answers "is this a data element" through the same path
/// resolution the engine uses for codes: a copy of the schema in which every
/// data component carries [`DATA_MARK`] as its only code. Element 0
/// component 0 is left alone — it is where the lookup reads a segment's
/// qualifier, and marking it would invent one.
fn data_lookup(schema: &serde_json::Value) -> CodeLookup {
    fn mark(v: &mut serde_json::Value, leading: bool) {
        let is_data = v.get("type").and_then(|t| t.as_str()) == Some("data");
        if is_data && !leading {
            v["type"] = "code".into();
            v["codes"] = serde_json::json!([{ "value": DATA_MARK, "name": "" }]);
        }
    }
    fn walk(v: &mut serde_json::Value) {
        match v {
            serde_json::Value::Object(o) => {
                if let Some(serde_json::Value::Array(elements)) = o.get_mut("elements") {
                    for el in elements.iter_mut() {
                        let first = el.get("index").and_then(|i| i.as_u64()) == Some(0);
                        match el.get_mut("components") {
                            Some(serde_json::Value::Array(comps)) => {
                                for c in comps.iter_mut() {
                                    let sub0 =
                                        c.get("sub_index").and_then(|i| i.as_u64()) == Some(0);
                                    mark(c, first && sub0);
                                }
                            }
                            _ => mark(el, first),
                        }
                    }
                }
                for (_, child) in o.iter_mut() {
                    walk(child);
                }
            }
            serde_json::Value::Array(a) => a.iter_mut().for_each(walk),
            _ => {}
        }
    }
    let mut copy = schema.clone();
    walk(&mut copy);
    CodeLookup::from_schema_value(&copy)
}

/// Every table-translated field of one variant directory
/// (`mappings/FV2604/UTILMD_Strom`), judged against the PID schemas in
/// `schema_dir` for the PIDs in `pids`.
pub fn audit_variant(
    variant_dir: &Path,
    schema_dir: &Path,
    pids: &BTreeSet<String>,
    code_lists: &CodeLists,
) -> Result<Vec<RuleCodes>, String> {
    let read = |dir: &Path| -> Result<Vec<(PathBuf, MappingDefinition)>, String> {
        let mut out = Vec::new();
        let Ok(entries) = std::fs::read_dir(dir) else {
            return Ok(out);
        };
        let mut paths: Vec<PathBuf> = entries
            .flatten()
            .map(|e| e.path())
            .filter(|p| p.extension().is_some_and(|e| e == "toml"))
            .collect();
        paths.sort();
        for p in paths {
            let text = std::fs::read_to_string(&p).map_err(|e| format!("{}: {e}", p.display()))?;
            let def = MappingDefinition::from_toml_str(&text)
                .map_err(|e| format!("{}: {e}", p.display()))?;
            out.push((p, def));
        }
        Ok(out)
    };
    let common = read(&variant_dir.join("common"))?;
    let message = read(&variant_dir.join("message"))?;

    let mut rules: BTreeMap<(PathBuf, String), RuleCodes> = BTreeMap::new();
    let mut unresolved: BTreeMap<(PathBuf, String), BTreeSet<String>> = BTreeMap::new();
    for pid in pids {
        let schema_path = schema_dir.join(format!("pid_{pid}_schema.json"));
        let Ok(text) = std::fs::read_to_string(&schema_path) else {
            continue;
        };
        let schema: serde_json::Value =
            serde_json::from_str(&text).map_err(|e| format!("{}: {e}", schema_path.display()))?;
        let lookup = CodeLookup::from_schema_value(&schema);
        let data = data_lookup(&schema);
        let resolver = PathResolver::from_schema(&schema);
        let index = PidSchemaIndex::from_json(&schema);

        let own = read(&variant_dir.join(format!("pid_{pid}")))?;
        let overridden: BTreeSet<(String, Option<String>)> =
            own.iter().map(|(_, d)| override_key(d)).collect();
        let inherited = common.iter().filter(|(_, d)| {
            d.meta
                .source_path
                .as_deref()
                .map_or(true, |sp| index.has_group(sp))
                && !overridden.contains(&override_key(d))
        });

        for (file, def) in message.iter().chain(inherited).chain(own.iter()) {
            for (field, mapping) in &def.fields {
                let FieldMapping::Structured(s) = mapping else {
                    continue;
                };
                let (table, names) = match (&s.enum_map, &s.code_list) {
                    (Some(m), _) => (TableRef::Inline, m.clone()),
                    (None, Some(name)) => match code_lists.get(name) {
                        Some(m) => (TableRef::Named(name.clone()), m.clone()),
                        None => continue,
                    },
                    (None, None) => continue,
                };
                if s.target.is_empty() {
                    continue;
                }
                let Some(sp) = def.meta.source_path.as_deref() else {
                    continue;
                };
                let resolved = resolver.resolve_path(field);
                let parts: Vec<&str> = resolved.split('.').collect();
                let (tag, path_qualifier, _) = parse_tag_qualifier(parts[0]);
                let (element, component) = MappingEngine::parse_element_component(&parts[1..]);
                let mut resolved_def = def.clone();
                resolved_def.meta.discriminator = def
                    .meta
                    .discriminator
                    .as_deref()
                    .map(|d| resolver.resolve_discriminator(d));
                let disc = MappingEngine::discriminator_qualifier_for_tag(&resolved_def, &tag);
                let at = |l: &CodeLookup| {
                    l.field_codes(
                        sp,
                        &tag,
                        path_qualifier,
                        disc.as_deref(),
                        element,
                        component,
                    )
                };
                let is_data = at(&data).is_some_and(|c| c.contains_key(DATA_MARK));
                let codes = match at(&lookup) {
                    Some(codes) => codes,
                    None if is_data => BTreeMap::new(),
                    None => {
                        unresolved
                            .entry((file.clone(), field.clone()))
                            .or_default()
                            .insert(pid.clone());
                        continue;
                    }
                };
                let entry = rules
                    .entry((file.clone(), field.clone()))
                    .or_insert_with(|| RuleCodes {
                        file: file.clone(),
                        field: field.clone(),
                        target: s.target.clone(),
                        table: table.clone(),
                        names: names.clone(),
                        codes: BTreeMap::new(),
                        pids: BTreeSet::new(),
                        data_pids: BTreeSet::new(),
                        unresolved_pids: BTreeSet::new(),
                    });
                for (code, e) in codes {
                    entry
                        .codes
                        .entry(code)
                        .or_insert_with(|| (e.meaning.clone(), e.enum_key.clone()));
                }
                entry.pids.insert(pid.clone());
                if is_data {
                    entry.data_pids.insert(pid.clone());
                }
            }
        }
    }
    for (key, pids) in unresolved {
        if let Some(rule) = rules.get_mut(&key) {
            rule.unresolved_pids = pids;
        }
    }
    Ok(rules.into_values().collect())
}

/// The key under which a PID file overrides a common one (`load_with_common`).
fn override_key(d: &MappingDefinition) -> (String, Option<String>) {
    let sg = d
        .meta
        .source_group
        .split('.')
        .map(|p| p.split(':').next().unwrap_or(p))
        .collect::<Vec<_>>()
        .join(".");
    let disc = d
        .meta
        .discriminator
        .as_deref()
        .map(|d| {
            d.rsplit_once('#')
                .filter(|(_, n)| n.chars().all(|c| c.is_ascii_digit()))
                .map_or(d, |(b, _)| b)
        })
        .map(str::to_string);
    (sg, disc)
}

fn fold(s: &str) -> String {
    s.to_lowercase()
        .replace('ä', "ae")
        .replace('ö', "oe")
        .replace('ü', "ue")
        .replace('ß', "ss")
}

fn words(s: &str) -> Vec<String> {
    fold(s)
        .split(|c: char| !c.is_ascii_alphanumeric())
        .filter(|w| !w.is_empty())
        .map(str::to_string)
        .collect()
}

/// The words of a camelCase name (`kundeDesLf` → kunde, des, lf).
fn name_words(name: &str) -> Vec<String> {
    let mut out = Vec::new();
    let mut cur = String::new();
    for c in name.chars() {
        if c.is_uppercase() && !cur.is_empty() {
            out.push(std::mem::take(&mut cur));
        }
        cur.push(c);
    }
    if !cur.is_empty() {
        out.push(cur);
    }
    out.iter().map(|w| fold(w)).collect()
}

/// Whether `name` plausibly names a code meaning `meaning` (or carrying the
/// curated `enum_key`): the two share a word, a four-letter stem, or the name
/// contains a longer word of the meaning. Loose on purpose — it has to accept
/// abbreviations (`Kunde des LF` → `kundeDesLf`) and still reject a name taken
/// from another element (`Prüfidentifikator` → `anteilC`).
pub fn related(meaning: &str, enum_key: Option<&str>, name: &str) -> bool {
    // Function words carry no meaning: "Der NB darf den LF …" and
    // `strukturDerFirmenbezeichnung` share "der" and nothing else.
    const STOP: &[&str] = &[
        "der", "die", "das", "des", "den", "dem", "ein", "eine", "einer", "eines", "einem", "und",
        "oder", "von", "vom", "mit", "fuer", "auf", "bei", "nach", "aus", "zur", "zum", "ist",
        "sind", "wird", "wenn", "nicht", "kein", "keine", "als", "auch", "nur", "dass", "sich",
        "noch", "bzw", "the", "and", "for",
    ];
    let long = |w: &String| w.len() >= 3 && !STOP.contains(&w.as_str());
    let mut m: BTreeSet<String> = words(meaning).into_iter().filter(long).collect();
    if let Some(k) = enum_key {
        m.extend(words(k).into_iter().filter(long));
    }
    let n: BTreeSet<String> = name_words(name).into_iter().filter(long).collect();
    if m.is_empty() || n.is_empty() {
        return true;
    }
    if !m.is_disjoint(&n) {
        return true;
    }
    let stem = |a: &str, b: &str| {
        a.len() >= 4 && b.len() >= 4 && (a.starts_with(&b[..4]) || b.starts_with(&a[..4]))
    };
    if m.iter().any(|a| n.iter().any(|b| stem(a, b))) {
        return true;
    }
    let folded = fold(name);
    m.iter()
        .any(|a| a.len() >= 5 && folded.contains(a.as_str()))
}

/// The name the convention gives a code meaning: its words in camelCase, cut
/// at a word boundary to at most 80 characters (`Kunde des LF` → `kundeDesLf`,
/// `Struktur von Personennamen` → `strukturVonPersonennamen`).
pub fn derived_name(meaning: &str) -> String {
    let mut out = String::new();
    for (i, w) in words(meaning).iter().enumerate() {
        let piece = if i == 0 {
            w.clone()
        } else {
            let mut c = w.chars();
            c.next()
                .map(|f| f.to_uppercase().chain(c).collect())
                .unwrap_or_default()
        };
        if !out.is_empty() && out.len() + piece.len() > 80 {
            break;
        }
        out.push_str(&piece);
    }
    if out.chars().next().is_some_and(|c| c.is_ascii_digit()) {
        out.insert(0, 'c');
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn related_accepts_abbreviations_and_rejects_foreign_names() {
        assert!(related("Kunde des LF", None, "kundeDesLf"));
        assert!(related(
            "Struktur von Personennamen",
            None,
            "strukturVonPersonennamen"
        ));
        assert!(related("Prüfidentifikator", None, "pruefidentifikator"));
        assert!(!related("Prüfidentifikator", None, "anteilC"));
        assert!(!related("Smartmeter-Gateway", None, "erfolgreich"));
        assert!(!related("Liste", None, "strukturVonPersonennamen"));
        assert!(!related(
            "Der NB darf den LF der Marktlokation bzw. Tranche nur dann mit diesem Produktpaket zuordnen",
            None,
            "strukturDerFirmenbezeichnung"
        ));
    }

    #[test]
    fn derived_name_follows_the_convention() {
        assert_eq!(derived_name("Kunde des LF"), "kundeDesLf");
        assert_eq!(
            derived_name("Struktur von Personennamen"),
            "strukturVonPersonennamen"
        );
        assert_eq!(derived_name("Prüfidentifikator"), "pruefidentifikator");
        assert_eq!(derived_name("Smartmeter-Gateway"), "smartmeterGateway");
        assert_eq!(derived_name("Liste"), "liste");
    }
}