edifact-mapper 0.2.0

//! Typed-PID gate: the generated per-PID typed modules
//! (`bo4e_edifact_types::generated::<fv>::<msg>::typed_pids::pid_*`) must accept
//! exactly what the mapping engine emits, and lose nothing.
//!
//! For every generated fixture `fixtures/generated/<fv>/<msg>/*.edi` it runs the
//! real forward pipeline (`Mapper::from_edifact`, code enrichment ON, and the same
//! pipeline with enrichment OFF) and, for the message and each transaction:
//!
//! a) deserializes the engine JSON into the PID's typed `Message`/`Transaktion`;
//! b) compares the typed re-serialization with the engine JSON. The only
//!    normalizations: enriched code objects `{code, meaning, enum}` compare as
//!    their `code`; `null` compares as absent; a single object compares as a
//!    one-element list; an entity collection (array, or map keyed by qualifier)
//!    compares as the unordered list of its elements. No field may be dropped or
//!    changed;
//! c) renders the typed re-serialization with `Mapper::to_edifact` (which clears
//!    `nesting_info`) and requires the same EDIFACT as rendering the engine JSON.
//!    BO4E carries no ordering, so map keys and BTreeMap order must not matter.
//!
//! Compiling the typed modules of all format versions at once needs too much
//! memory, so each format version is its own feature:
//!
//! ```text
//! cargo test -p edifact-mapper --features typed-pid-gate-fv2604 --test typed_pid_gate
//! ```
#![cfg(any(
    feature = "typed-pid-gate-fv2504",
    feature = "typed-pid-gate-fv2510",
    feature = "typed-pid-gate-fv2604",
    feature = "typed-pid-gate-fv2610"
))]

use std::collections::BTreeMap;
use std::path::PathBuf;

use edifact_mapper::{DataDir, Mapper};
use mig_bo4e::engine::DataBundle;
use mig_bo4e::MappingEngine;
use serde_json::Value;

type HopFn = fn(&str, &str, Value) -> Option<Result<Value, serde_json::Error>>;

/// Fixtures that cannot pass for a reason unrelated to typed enums / map keys.
/// `(format version, fixture "<msg dir>/<file stem>", reason)`. Keep minimal.
const KNOWN_FAILURES: &[(&str, &str, &str)] = &[(
    "FV2604",
    "mscons/13024",
    "the FV2604 data bundle has no AHB segment numbers for PID 13024, so \
     `from_edifact` fails before any typed code runs (\"No MIG schema\")",
)];

fn repo_root() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .and_then(|p| p.parent())
        .expect("crate is two levels under the workspace root")
        .to_path_buf()
}

/// Enriched code object `{code, meaning[, enum]}` → its `code`; `null` fields dropped.
fn normalize(v: &Value) -> Value {
    match v {
        Value::Object(map) => {
            if map.contains_key("code")
                && map.contains_key("meaning")
                && map
                    .keys()
                    .all(|k| k == "code" || k == "meaning" || k == "enum")
            {
                return map["code"].clone();
            }
            Value::Object(
                map.iter()
                    .filter(|(_, v)| !v.is_null())
                    .map(|(k, v)| (k.clone(), normalize(v)))
                    .collect(),
            )
        }
        Value::Array(items) => Value::Array(items.iter().map(normalize).collect()),
        other => other.clone(),
    }
}

/// Same heuristic as the engine's reverse mapping (`is_map_keyed_object`).
fn is_map_keyed_object(v: &Value) -> bool {
    let Some(obj) = v.as_object() else {
        return false;
    };
    !obj.is_empty()
        && obj.iter().all(|(k, v)| {
            k.len() <= 5
                && k.chars()
                    .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit())
                && v.is_object()
        })
}

fn as_items(v: &Value) -> Vec<Value> {
    match v {
        Value::Array(items) => items.clone(),
        v if is_map_keyed_object(v) => v.as_object().unwrap().values().cloned().collect(),
        v => vec![v.clone()],
    }
}

/// Collect differences between normalized engine JSON `e` and typed JSON `t`.
///
/// `entity_level` marks the values of a scope object (the entity collections):
/// there a single object equals a one-element list, and lists compare unordered
/// (a map-keyed collection serializes in key order).
fn diff(path: &str, e: &Value, t: &Value, entity_level: bool, out: &mut Vec<String>) {
    if out.len() > 20 {
        return;
    }
    if entity_level && (e.is_array() || t.is_array() || is_map_keyed_object(t)) {
        let mut ei = as_items(e);
        let mut ti = as_items(t);
        if ei.len() != ti.len() {
            out.push(format!(
                "{path}: {} engine element(s) vs {} typed",
                ei.len(),
                ti.len()
            ));
            return;
        }
        ei.sort_by_key(|v| v.to_string());
        ti.sort_by_key(|v| v.to_string());
        for (i, (a, b)) in ei.iter().zip(&ti).enumerate() {
            diff(&format!("{path}{{{i}}}"), a, b, false, out);
        }
        return;
    }
    match (e, t) {
        (Value::Object(em), Value::Object(tm)) => {
            for (k, ev) in em {
                match tm.get(k) {
                    Some(tv) => diff(&format!("{path}.{k}"), ev, tv, false, out),
                    None => out.push(format!("{path}.{k}: dropped ({ev})")),
                }
            }
            for (k, tv) in tm {
                if !em.contains_key(k) {
                    out.push(format!("{path}.{k}: added ({tv})"));
                }
            }
        }
        (Value::Array(ea), Value::Array(ta)) => {
            if ea.len() != ta.len() {
                out.push(format!("{path}: {} vs {} elements", ea.len(), ta.len()));
                return;
            }
            for (i, (a, b)) in ea.iter().zip(ta).enumerate() {
                diff(&format!("{path}[{i}]"), a, b, false, out);
            }
        }
        // A nested child entity may be one object or a list.
        (Value::Object(_), Value::Array(items)) if items.len() == 1 => {
            diff(&format!("{path}[0]"), e, &items[0], false, out)
        }
        _ if e == t => {}
        _ => out.push(format!("{path}: engine {e} vs typed {t}")),
    }
}

/// Compare two scope objects entity by entity.
fn diff_scope(label: &str, e: &Value, t: &Value, out: &mut Vec<String>) {
    match (e.as_object(), t.as_object()) {
        (Some(em), Some(tm)) => {
            for (k, ev) in em {
                match tm.get(k) {
                    Some(tv) => diff(&format!("{label}.{k}"), ev, tv, true, out),
                    None => out.push(format!("{label}.{k}: dropped ({ev})")),
                }
            }
            for (k, tv) in tm {
                if !em.contains_key(k) {
                    out.push(format!("{label}.{k}: added ({tv})"));
                }
            }
        }
        _ => diff(label, e, t, false, out),
    }
}

struct Forward {
    msg: Value,
    txs: Vec<Value>,
}

/// The forward pass splits `Nachricht` into `nachricht_meta` and `Prozessdaten`
/// into `transaktionsdaten`; the reverse resolves definitions against one flat
/// entity map and the typed PID modules model the metadata entities as ordinary
/// fields. So both halves go back together here, and this gate works on the
/// recombined map throughout — including where it compares itself against
/// `Mapper::from_edifact`, whose two halves are recombined the same way.
fn recombine(stammdaten: &Value, key: &str, meta: &Value) -> Value {
    let mut out = stammdaten.clone();
    mig_bo4e::model::restore_entity(&mut out, key, meta);
    out
}

/// `Mapper::from_edifact` with the enrichment flag exposed: the same bundle
/// engines, assembler config and `map_interchange` call.
fn forward(bundle: &DataBundle, edifact: &str, variant: &str, pid: &str, enrich: bool) -> Forward {
    let vc = bundle.variant(variant).unwrap();
    let tx_group = vc.tx_group(pid).unwrap();
    let msg_engine = vc.msg_engine(pid);
    let tx_engine = vc.tx_engine(pid).unwrap();
    let mig = vc.filtered_mig(pid).unwrap();
    let svc = mig_assembly::ConversionService::from_mig(mig);
    let (_, trees, _) = svc
        .convert_interchange_to_trees_with_diagnostics(
            edifact,
            mig_assembly::assembler::AssemblerConfig {
                strict_code_matching: true,
                skip_unknown_segments: true,
                ..Default::default()
            },
        )
        .unwrap();
    let mapped =
        MappingEngine::map_interchange(&msg_engine, &tx_engine, &trees[0], tx_group, enrich);
    Forward {
        msg: recombine(
            &mapped.stammdaten,
            mig_bo4e::model::MSG_METADATA_ENTITY,
            &mapped.nachricht_meta,
        ),
        txs: mapped
            .transaktionen
            .iter()
            .map(|t| {
                recombine(
                    &t.stammdaten,
                    mig_bo4e::model::TX_METADATA_ENTITY,
                    &t.transaktionsdaten,
                )
            })
            .collect(),
    }
}

struct Outcome {
    kind: &'static str,
    detail: String,
}

fn check_fixture(
    mapper: &Mapper,
    bundle: &DataBundle,
    fv: &str,
    variant: &str,
    pid: &str,
    edifact: &str,
    hop_tx: HopFn,
    hop_msg: HopFn,
) -> Vec<Outcome> {
    let enriched = match mapper.from_edifact::<Value, Value>(edifact, fv, variant, pid) {
        Ok(ic) => ic,
        Err(e) => {
            return vec![Outcome {
                kind: "forward-error",
                detail: e.to_string(),
            }]
        }
    };
    let on = forward(bundle, edifact, variant, pid, true);
    let n = &enriched.nachrichten[0];
    let mut facade_msg = n.stammdaten.clone();
    mig_bo4e::model::restore_message_metadata(&mut facade_msg, &n.nachrichtendaten);
    let facade_txs: Vec<Value> = n
        .transaktionen
        .iter()
        .map(|t| match t.get("stammdaten") {
            Some(st) => recombine(
                st,
                mig_bo4e::model::TX_METADATA_ENTITY,
                t.get("transaktionsdaten").unwrap_or(&Value::Null),
            ),
            None => t.clone(),
        })
        .collect();
    assert_eq!(
        (&facade_msg, &facade_txs),
        (&on.msg, &on.txs),
        "{fv} {variant} {pid}: gate forward replica diverges from Mapper::from_edifact"
    );
    let off = forward(bundle, edifact, variant, pid, false);

    // Debug aid: `TYPED_PID_GATE_DUMP=<dir>` writes the plain engine JSON per fixture.
    if let Some(dir) = std::env::var_os("TYPED_PID_GATE_DUMP") {
        let path = PathBuf::from(dir).join(format!("{fv}_{variant}_{pid}.json"));
        let json = serde_json::json!({ "message": off.msg, "transaktionen": off.txs });
        std::fs::write(path, serde_json::to_string_pretty(&json).unwrap()).unwrap();
    }

    [("enriched", &on), ("plain", &off)]
        .into_iter()
        .filter_map(|(mode, fwd)| {
            check_mode(mapper, fv, variant, pid, fwd, hop_tx, hop_msg).map(|o| Outcome {
                kind: o.kind,
                detail: format!("{mode}: {}", o.detail),
            })
        })
        .collect()
}

/// First failure of one forward mode (enriched or plain), if any.
fn check_mode(
    mapper: &Mapper,
    fv: &str,
    variant: &str,
    pid: &str,
    fwd: &Forward,
    hop_tx: HopFn,
    hop_msg: HopFn,
) -> Option<Outcome> {
    let fail = |kind, detail: String| Some(Outcome { kind, detail });
    let mut typed_txs = Vec::new();
    let mut parts: Vec<(String, &Value, HopFn)> = vec![("message".into(), &fwd.msg, hop_msg)];
    for (i, tx) in fwd.txs.iter().enumerate() {
        parts.push((format!("transaktion[{i}]"), tx, hop_tx));
    }
    let mut typed_msg = Value::Null;
    for (label, value, hop) in parts {
        let typed = match hop(variant, pid, value.clone()) {
            None => return fail("deserialize", format!("{label}: no typed module")),
            Some(Err(e)) => return fail("deserialize", format!("{label}: {e}")),
            Some(Ok(t)) => t,
        };
        let mut diffs = Vec::new();
        diff_scope(&label, &normalize(value), &normalize(&typed), &mut diffs);
        if !diffs.is_empty() {
            return fail("lossy", diffs.join("; "));
        }
        if label == "message" {
            typed_msg = typed;
        } else {
            typed_txs.push(typed);
        }
    }

    let original = match mapper.to_edifact(&fwd.msg, &fwd.txs, fv, variant, pid) {
        Ok(s) => s,
        Err(e) => return fail("render-original-error", e.to_string()),
    };
    let typed = match mapper.to_edifact(&typed_msg, &typed_txs, fv, variant, pid) {
        Ok(s) => s,
        Err(e) => return fail("render-typed-error", e.to_string()),
    };
    if original != typed {
        let first = original
            .split('\'')
            .zip(typed.split('\''))
            .find(|(a, b)| a != b)
            .map(|(a, b)| format!("{a}' vs {b}'"))
            .unwrap_or_else(|| "segment count differs".into());
        return fail("edifact-differs", first);
    }
    None
}

fn run_gate(fv: &str, hop_tx: HopFn, hop_msg: HopFn) {
    let root = repo_root();
    let dist = root.join("dist");
    let fixtures = root.join("fixtures/generated").join(fv.to_lowercase());
    let bundle_path = dist.join(format!("edifact-data-{fv}.bin"));
    assert!(
        bundle_path.exists(),
        "missing bundle {}",
        bundle_path.display()
    );
    assert!(fixtures.is_dir(), "missing fixtures {}", fixtures.display());

    let mapper = Mapper::from_data_dir(DataDir::path(&dist).eager(&[fv])).expect("load bundle");
    let bundle = DataBundle::load(&bundle_path).expect("load bundle");
    let mut variants: Vec<&String> = bundle.variants.keys().collect();
    variants.sort();

    // msg dir -> kind -> count; plus totals
    let mut report: BTreeMap<String, BTreeMap<String, usize>> = BTreeMap::new();
    let mut failures: Vec<(String, &'static str, String)> = Vec::new();

    let mut dirs: Vec<PathBuf> = std::fs::read_dir(&fixtures)
        .unwrap()
        .flatten()
        .map(|e| e.path())
        .filter(|p| p.is_dir())
        .collect();
    dirs.sort();
    for dir in dirs {
        let msg = dir.file_name().unwrap().to_string_lossy().to_string();
        let mut files: Vec<PathBuf> = std::fs::read_dir(&dir)
            .unwrap()
            .flatten()
            .map(|e| e.path())
            .filter(|p| p.extension().is_some_and(|x| x == "edi"))
            .collect();
        files.sort();
        for file in files {
            let stem = file.file_stem().unwrap().to_string_lossy().to_string();
            let key = format!("{msg}/{stem}");
            let edifact = std::fs::read_to_string(&file).unwrap();
            let Some((variant, pid)) =
                resolve(&mapper, &bundle, &variants, &msg, &stem, &edifact, hop_tx)
            else {
                *report
                    .entry(msg.clone())
                    .or_default()
                    .entry("no-typed-module".to_string())
                    .or_default() += 1;
                continue;
            };
            let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
                check_fixture(
                    &mapper, &bundle, fv, &variant, &pid, &edifact, hop_tx, hop_msg,
                )
            }))
            .unwrap_or_else(|p| {
                let detail = p
                    .downcast_ref::<String>()
                    .cloned()
                    .or_else(|| p.downcast_ref::<&str>().map(|s| s.to_string()))
                    .unwrap_or_default();
                vec![Outcome {
                    kind: "forward-error",
                    detail: format!("panic: {detail}"),
                }]
            });
            let entry = report.entry(msg.clone()).or_default();
            if outcome.is_empty() {
                *entry.entry("pass".to_string()).or_default() += 1;
            }
            for o in outcome {
                let mode = o.detail.split(':').next().unwrap_or_default();
                let label = if o.kind == "forward-error" {
                    o.kind.to_string()
                } else {
                    format!("{mode}:{}", o.kind)
                };
                *entry.entry(label).or_default() += 1;
                failures.push((key.clone(), o.kind, o.detail));
            }
        }
    }

    eprintln!("\n=== typed-PID gate {fv} ===");
    for (msg, counts) in &report {
        let line: Vec<String> = counts.iter().map(|(k, c)| format!("{k}={c}")).collect();
        eprintln!("{msg:>14}: {}", line.join(" "));
    }
    let mut totals: BTreeMap<&String, usize> = BTreeMap::new();
    for counts in report.values() {
        for (k, c) in counts {
            *totals.entry(k).or_default() += c;
        }
    }
    eprintln!("{:>14}: {totals:?}", "TOTAL");

    let known: Vec<&str> = KNOWN_FAILURES
        .iter()
        .filter(|(f, _, _)| *f == fv)
        .map(|(_, k, _)| *k)
        .collect();
    let unexpected: Vec<_> = failures
        .iter()
        .filter(|(k, _, _)| !known.contains(&k.as_str()))
        .collect();
    for (key, kind, detail) in &failures {
        let marker = if known.contains(&key.as_str()) {
            "known"
        } else {
            "FAIL"
        };
        eprintln!("[{marker}] {key} {kind}: {}", truncate(detail, 600));
    }
    let stale: Vec<&&str> = known
        .iter()
        .filter(|k| !failures.iter().any(|(key, _, _)| key == **k))
        .collect();
    assert!(
        unexpected.is_empty() && stale.is_empty(),
        "{fv}: {} unexpected failure(s), stale KNOWN_FAILURES entries: {stale:?}",
        unexpected.len()
    );
}

fn truncate(s: &str, max: usize) -> String {
    if s.chars().count() <= max {
        s.to_string()
    } else {
        format!("{}…", s.chars().take(max).collect::<String>())
    }
}

/// Find the variant and PID of a fixture: the file stem is the PID for all but
/// the message-only types, whose PID comes from `detect_pid` or the variant's
/// single typed module.
fn resolve(
    mapper: &Mapper,
    bundle: &DataBundle,
    variants: &[&String],
    msg: &str,
    stem: &str,
    edifact: &str,
    hop_tx: HopFn,
) -> Option<(String, String)> {
    let detected = mapper.detect_pid(edifact).ok();
    for variant in variants {
        let lower = variant.to_lowercase();
        if lower != msg && !lower.starts_with(&format!("{msg}_")) {
            continue;
        }
        let vc = bundle.variant(variant).unwrap();
        let mut candidates: Vec<String> = vec![stem.to_string()];
        candidates.extend(detected.clone());
        let typed: Vec<&String> = vc.pid_requirements.keys().filter(|k| k.len() > 4).collect();
        if typed.len() == 1 {
            candidates.push(typed[0].trim_start_matches("pid_").to_string());
        }
        for pid in candidates {
            if vc.tx_group(&pid).is_some()
                && hop_tx(variant, &pid, Value::Object(Default::default())).is_some()
            {
                return Some((variant.to_string(), pid));
            }
        }
    }
    None
}

#[cfg(feature = "typed-pid-gate-fv2504")]
#[test]
fn typed_pid_gate_fv2504() {
    use bo4e_edifact_types::generated::fv2504 as g;
    run_gate("FV2504", g::typed_hop_transaktion, g::typed_hop_message);
}

#[cfg(feature = "typed-pid-gate-fv2510")]
#[test]
fn typed_pid_gate_fv2510() {
    use bo4e_edifact_types::generated::fv2510 as g;
    run_gate("FV2510", g::typed_hop_transaktion, g::typed_hop_message);
}

#[cfg(feature = "typed-pid-gate-fv2604")]
#[test]
fn typed_pid_gate_fv2604() {
    use bo4e_edifact_types::generated::fv2604 as g;
    run_gate("FV2604", g::typed_hop_transaktion, g::typed_hop_message);
}

#[cfg(feature = "typed-pid-gate-fv2610")]
#[test]
fn typed_pid_gate_fv2610() {
    use bo4e_edifact_types::generated::fv2610 as g;
    run_gate("FV2610", g::typed_hop_transaktion, g::typed_hop_message);
}