edifact-mapper 0.8.0

//! Gate: a generated sample message must satisfy the rulebook it was generated for.
//!
//! The samples under `fixtures/generated/` are produced **through the mapping**.
//! A field no rule writes therefore never appears in a sample, and the sample
//! then agrees that the field was not needed — the corpus cannot find a gap, it
//! can only confirm whatever the mapping already does. That is not theoretical:
//! in #126 a rule was added for a qualifier the rulebook marks mandatory with a
//! single permitted code, and the roundtrip failed because the sample lacked
//! the qualifier it had been generated without.
//!
//! This points the validator the web interface already uses at the samples, so
//! the loop is broken from outside: the rulebook judges the sample, not the
//! mapping.
//!
//! The baseline is a ratchet. A sample that gains a violation fails the gate,
//! and one that loses its last violation fails too, so the list cannot rot.
//!
//! ## Why 56 samples carry a `CodeNotAllowedForPid` at `SG4/SG8/SG10/CAV`
//!
//! Those entries are not a mapping defect and cannot be generated away. The
//! AHB leaves `CAV.C889.D_7111` unenumerated there and delegates the admissible
//! values to a list it names but does not contain — "die vom BDEW
//! veröffentlichte Codeliste der TUM-Profile". That list is not in this repo,
//! so the generator has no legal value to put at that position.
//!
//! It used to render the MIG's `<Code Name="Beispielcode">XYZ` and the gate saw
//! nothing, because the PID schema carried that same placeholder as the sole
//! permitted code — the sample and the rulebook agreed on a value that is not
//! real (issue #154). With the schema no longer imposing it, the position is
//! unconstrained for validation and the sample fills it with a data filler,
//! which the rulebook then rejects as a code.
//!
//! So these 56 record a true limitation rather than a regression: the samples
//! were already wrong at that position, and the placeholder was what kept the
//! gate from saying so. They clear when the BDEW list is available to seed a
//! real value from, not before.
//!
//! Regenerate: `UPDATE_SAMPLE_CONFORMANCE=1 cargo test -p edifact-mapper --test sample_conformance_gate`

use edifact_mapper::{DataDir, Mapper, ValidationLevel};
use std::collections::BTreeSet;
use std::path::{Path, PathBuf};

fn repo_root() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .and_then(|p| p.parent())
        .expect("the crate sits two levels below the workspace root")
        .to_path_buf()
}

fn baseline_path() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/data/known_sample_violations.txt")
}

/// `DTM/C507/2380` reports "condition not satisfied" on nearly every message,
/// with no value found, while the message plainly carries the date — the
/// validator matches a qualifier-specific rule against a path that does not
/// carry the qualifier. That is a defect in the validator, not in the sample,
/// and it accounts for 3502 of 6881 findings. Baselining it would bury the real
/// ones; see #146.
fn is_the_known_date_defect(path: Option<&str>) -> bool {
    path.map(|p| p.ends_with("C507/2380")).unwrap_or(false)
}

/// `<format version> <message type> <identifier> <kind> <path>`, deduplicated.
fn violations() -> BTreeSet<String> {
    let root = repo_root();
    let mapper = match Mapper::from_data_dir(DataDir::path(root.join("dist"))) {
        Ok(m) => m,
        Err(e) => panic!("the data bundles are a build artefact — run `bundle-data` first: {e}"),
    };
    let mut out = BTreeSet::new();
    let Ok(fvs) = std::fs::read_dir(root.join("fixtures/generated")) else {
        return out;
    };
    for fv_dir in fvs.flatten() {
        if !fv_dir.path().is_dir() {
            continue;
        }
        let fv = fv_dir.file_name().to_string_lossy().to_uppercase();
        let Ok(msgs) = std::fs::read_dir(fv_dir.path()) else {
            continue;
        };
        for msg_dir in msgs.flatten() {
            if !msg_dir.path().is_dir() {
                continue;
            }
            let msg = msg_dir.file_name().to_string_lossy().to_uppercase();
            let Ok(files) = std::fs::read_dir(msg_dir.path()) else {
                continue;
            };
            for f in files.flatten() {
                let p = f.path();
                if p.extension().and_then(|e| e.to_str()) != Some("edi") {
                    continue;
                }
                let pid = p
                    .file_stem()
                    .unwrap_or_default()
                    .to_string_lossy()
                    .into_owned();
                let Ok(edi) = std::fs::read_to_string(&p) else {
                    continue;
                };
                // UTILMD is the one message type split by division; try both.
                let variants: Vec<String> = if msg == "UTILMD" {
                    vec![format!("{msg}_Strom"), format!("{msg}_Gas")]
                } else {
                    vec![msg.clone()]
                };
                let mut report = None;
                for v in &variants {
                    if let Ok(r) =
                        mapper.validate_edifact_for_pid(&edi, &fv, v, &pid, ValidationLevel::Full)
                    {
                        report = Some(r);
                        break;
                    }
                }
                let Some(report) = report else { continue };
                for issue in &report.issues {
                    if format!("{:?}", issue.severity) != "Error" {
                        continue;
                    }
                    if is_the_known_date_defect(issue.field_path.as_deref()) {
                        continue;
                    }
                    let kind = format!("{:?}", issue.kind);
                    let kind = kind.split(['{', ' ']).next().unwrap_or("?");
                    let path = issue.field_path.as_deref().unwrap_or("-");
                    out.insert(format!("{fv} {msg} {pid} {kind} {path}"));
                }
            }
        }
    }
    out
}

#[test]
fn every_generated_sample_satisfies_its_rulebook() {
    let found = violations();

    if std::env::var("UPDATE_SAMPLE_CONFORMANCE").is_ok() {
        let body: String = found.iter().map(|l| format!("{l}\n")).collect();
        std::fs::write(baseline_path(), body).expect("write baseline");
        eprintln!("wrote {} sample violation(s)", found.len());
        return;
    }

    let baseline: BTreeSet<String> = std::fs::read_to_string(baseline_path())
        .unwrap_or_default()
        .lines()
        .map(str::trim)
        .filter(|l| !l.is_empty() && !l.starts_with('#'))
        .map(str::to_owned)
        .collect();

    let new: Vec<&String> = found.difference(&baseline).collect();
    let fixed: Vec<&String> = baseline.difference(&found).collect();

    assert!(
        new.is_empty(),
        "{} generated sample message(s) gained a violation of their own rulebook. \
         A sample is produced through the mapping, so this usually means a rule \
         started writing something the rulebook does not allow there:\n{}",
        new.len(),
        new.iter()
            .take(30)
            .map(|l| format!("  {l}"))
            .collect::<Vec<_>>()
            .join("\n")
    );
    assert!(
        fixed.is_empty(),
        "{} listed violation(s) are gone — drop them with \
         UPDATE_SAMPLE_CONFORMANCE=1 so the list cannot rot:\n{}",
        fixed.len(),
        fixed
            .iter()
            .take(30)
            .map(|l| format!("  {l}"))
            .collect::<Vec<_>>()
            .join("\n")
    );
}