pleme-doc-gen 0.1.54

//! Fidelity — typed measurement of how perfectly the substrate
//! consumes a behavior into a caixa.
//!
//! Compares an original repo's manifest fields against the substrate's
//! reverse → render output. Reports which typed fields round-trip
//! cleanly (perfect), which round-trip with partial info (lossy),
//! and which don't round-trip at all (gap). The operator's
//! quality compass for the consumption pipeline.
//!
//! Per the operator's "turn behaviors into perfect tested caixas"
//! directive: this module makes "perfect" a measurable property,
//! not a qualitative claim.
//!
//! Per-field semantics:
//!   Perfect — original value present + rendered value matches exactly
//!   Lossy   — original value present + rendered value present but differs
//!   Gap     — original value present + rendered value absent
//!   NA      — original value absent (nothing to round-trip)

use anyhow::{anyhow, Result};
use std::path::{Path, PathBuf};

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FieldStatus {
    Perfect,
    Lossy,
    Gap,
    Na,
}

impl FieldStatus {
    fn as_str(&self) -> &'static str {
        match self {
            Self::Perfect => "perfect",
            Self::Lossy   => "lossy",
            Self::Gap     => "gap",
            Self::Na      => "na",
        }
    }
}

#[derive(Debug, Clone)]
pub struct FieldFidelity {
    pub field: String,
    pub status: FieldStatus,
    pub original: Option<String>,
    pub rendered: Option<String>,
}

#[derive(Debug, Clone)]
pub struct FidelityReport {
    pub original_path: PathBuf,
    pub rendered_path: PathBuf,
    pub ecosystem: Option<String>,
    pub fields: Vec<FieldFidelity>,
    pub perfect_count: usize,
    pub lossy_count: usize,
    pub gap_count: usize,
    pub na_count: usize,
}

impl FidelityReport {
    /// Score = perfect / (perfect + lossy + gap). Range 0.0–1.0.
    /// 1.0 = lossless round-trip; 0.0 = nothing round-tripped.
    pub fn score(&self) -> f64 {
        let total = self.perfect_count + self.lossy_count + self.gap_count;
        if total == 0 { return 1.0; }
        self.perfect_count as f64 / total as f64
    }

    pub fn to_json(&self) -> String {
        use crate::json_ast::Value;
        let mut root = Value::obj();
        root.insert("original", Value::s(self.original_path.to_string_lossy().to_string()));
        root.insert("rendered", Value::s(self.rendered_path.to_string_lossy().to_string()));
        if let Some(eco) = &self.ecosystem { root.insert("ecosystem", Value::s(eco)); }
        root.insert("perfect", Value::i(self.perfect_count as i64));
        root.insert("lossy", Value::i(self.lossy_count as i64));
        root.insert("gap", Value::i(self.gap_count as i64));
        root.insert("na", Value::i(self.na_count as i64));
        // Format score with 3 decimal places via integer trick — avoid
        // format!() on the f64 since it's borderline data-not-code.
        let score_pct = (self.score() * 1000.0).round() as i64;
        root.insert("score-permille", Value::i(score_pct));
        let fields: Vec<Value> = self.fields.iter().map(|f| {
            let mut o = Value::obj();
            o.insert("field", Value::s(&f.field));
            o.insert("status", Value::s(f.status.as_str()));
            if let Some(v) = &f.original { o.insert("original", Value::s(v)); }
            if let Some(v) = &f.rendered { o.insert("rendered", Value::s(v)); }
            o
        }).collect();
        root.insert("fields", Value::Array(fields));
        crate::json_ast::render(&root)
    }
}

/// Field extraction for the comparison. Mirrors the substrate's
/// existing reverse + manifest-reader logic but for raw lookup.
fn extract_field(path: &Path, ecosystem: &str, field: &str) -> Option<String> {
    use std::fs;
    let read = |rel: &str| -> Option<String> { fs::read_to_string(path.join(rel)).ok() };
    match (ecosystem, field) {
        ("rust-single-crate" | "rust-workspace", "name")
            => read("Cargo.toml").and_then(|t| cargo_field(&t, "name")),
        ("rust-single-crate" | "rust-workspace", "version")
            => read("Cargo.toml").and_then(|t| cargo_field(&t, "version")),
        ("rust-single-crate" | "rust-workspace", "description")
            => read("Cargo.toml").and_then(|t| cargo_field(&t, "description")),
        ("rust-single-crate" | "rust-workspace", "license")
            => read("Cargo.toml").and_then(|t| cargo_field(&t, "license")),
        ("rust-single-crate" | "rust-workspace", "keywords")
            => read("Cargo.toml").map(|t| cargo_string_array_joined(&t, "keywords")),
        ("rust-single-crate" | "rust-workspace", "categories")
            => read("Cargo.toml").map(|t| cargo_string_array_joined(&t, "categories")),
        ("npm" | "js-pnpm", "keywords")
            => read("package.json").map(|t| npm_string_array_joined(&t, "keywords")),
        ("npm" | "js-pnpm", f)
            => read("package.json").and_then(|t| json_top_string(&t, f)),
        ("python" | "python-pdm", f)
            => read("pyproject.toml").and_then(|t| toml_top_field(&t, "[project]", f)),
        ("helm", f)
            => read("Chart.yaml").and_then(|t| yaml_top_string(&t, f)),
        ("github-action", "description")
            => read("action.yml").or_else(|| read("action.yaml"))
                .and_then(|t| yaml_top_string(&t, "description")),
        ("github-action", "branding-icon")
            => read("action.yml").or_else(|| read("action.yaml"))
                .and_then(|t| crate::yaml::parse(&t).branding_icon),
        ("github-action", "branding-color")
            => read("action.yml").or_else(|| read("action.yaml"))
                .and_then(|t| crate::yaml::parse(&t).branding_color),
        ("go", "module-path")
            => read("go.mod").and_then(|t| go_mod_directive(&t, "module")),
        ("nix-flake", "description")
            => read("flake.nix").and_then(|t| nix_string_field(&t, "description")),
        ("go", "go-version")
            => read("go.mod").and_then(|t| go_mod_directive(&t, "go")),
        // `name` for github-action is INTENTIONALLY ADOPTED by the
        // substrate (prefixed "pleme-io · <dirname>") so consumed
        // actions become typed pleme-io actions. Excluded from the
        // preserved-fields comparison; the substrate-side renderer
        // is the source of truth for the adopted form.
        ("github-action", "name") => None,
        _ => None,
    }
}

/// Read an npm package.json string array (keywords), normalize to
/// sorted comma-joined string. Uses serde_json for robust parsing.
fn npm_string_array_joined(text: &str, key: &str) -> String {
    let Ok(v) = serde_json::from_str::<serde_json::Value>(text) else {
        return String::new();
    };
    let Some(arr) = v.get(key).and_then(|x| x.as_array()) else {
        return String::new();
    };
    let mut items: Vec<String> = arr.iter()
        .filter_map(|x| x.as_str().map(String::from))
        .collect();
    items.sort();
    items.join(",")
}

/// Read a Cargo TOML string-array (keywords / categories) from either
/// [package] or [workspace.package], normalize to a sorted comma-
/// joined string. Set-equality round-trip via canonical text form.
fn cargo_string_array_joined(text: &str, key: &str) -> String {
    use crate::manifest_io::read_toml_string_array;
    let mut items = read_toml_string_array(text, "[package]", key);
    if items.is_empty() {
        items = read_toml_string_array(text, "[workspace.package]", key);
    }
    items.sort();
    items.join(",")
}


/// Wrap manifest_io::read_cargo_field — substrate-wide single source
/// of truth for Cargo manifest field resolution.
fn cargo_field(text: &str, key: &str) -> Option<String> {
    crate::manifest_io::read_cargo_field(text, key)
}

fn toml_top_field(text: &str, header: &str, key: &str) -> Option<String> {
    let mut in_section = false;
    for line in text.lines() {
        let t = line.trim();
        if t.starts_with('[') { in_section = t == header; continue; }
        if !in_section { continue; }
        if let Some(after) = t.strip_prefix(key) {
            let after = after.trim_start().strip_prefix('=')?.trim_start();
            let val = after.trim_matches('"').trim_matches('\'');
            let val = match val.find('#') { Some(i) => &val[..i], None => val };
            return Some(val.trim().trim_matches('"').to_string());
        }
    }
    None
}

fn json_top_string(text: &str, key: &str) -> Option<String> {
    let needle = format!("\"{key}\":");
    let pos = text.find(&needle)?;
    let after = text[pos + needle.len()..].trim_start();
    let after = after.strip_prefix('"')?;
    let end = after.find('"')?;
    Some(after[..end].to_string())
}

/// Read a top-level `<key> = "..."` from a Nix expression file.
/// Honors `\"` and `\\` escapes in the quoted value.
fn nix_string_field(text: &str, key: &str) -> Option<String> {
    let needle = format!("{key} = \"");
    let pos = text.find(&needle)?;
    let after = &text[pos + needle.len()..];
    let bytes = after.as_bytes();
    let mut end = 0;
    while end < bytes.len() {
        if bytes[end] == b'\\' && end + 1 < bytes.len() {
            end += 2;
            continue;
        }
        if bytes[end] == b'"' { break; }
        end += 1;
    }
    let raw = &after[..end];
    let mut out = String::with_capacity(raw.len());
    let mut it = raw.chars().peekable();
    while let Some(c) = it.next() {
        if c == '\\' {
            if let Some(&n) = it.peek() {
                if n == '"' || n == '\\' || n == '$' { out.push(it.next().unwrap()); continue; }
                if n == 'n' { it.next(); out.push('\n'); continue; }
            }
        }
        out.push(c);
    }
    Some(out)
}

/// Parse a top-level go.mod directive. `module github.com/x/y` → "github.com/x/y";
/// `go 1.22` → "1.22". Both are space-separated single-value directives.
fn go_mod_directive(text: &str, directive: &str) -> Option<String> {
    for line in text.lines() {
        let t = line.trim();
        if let Some(rest) = t.strip_prefix(directive) {
            let v = rest.trim();
            if !v.is_empty() && !v.starts_with('(') {
                return Some(v.to_string());
            }
        }
    }
    None
}

fn yaml_top_string(text: &str, key: &str) -> Option<String> {
    let needle = format!("{key}:");
    for line in text.lines() {
        if let Some(rest) = line.trim_start().strip_prefix(&needle) {
            let val = rest.trim();
            // YAML scalar forms supported here:
            //   "..."   double-quoted; unescape \" and \\ per spec
            //   '...'   single-quoted; '' → '
            //   bare    no escapes
            if let Some(inner) = val.strip_prefix('"').and_then(|s| s.strip_suffix('"')) {
                let mut out = String::with_capacity(inner.len());
                let mut it = inner.chars().peekable();
                while let Some(c) = it.next() {
                    if c == '\\' {
                        if let Some(&n) = it.peek() {
                            if n == '"' || n == '\\' { out.push(it.next().unwrap()); continue; }
                            if n == 'n' { it.next(); out.push('\n'); continue; }
                            if n == 't' { it.next(); out.push('\t'); continue; }
                        }
                    }
                    out.push(c);
                }
                return Some(out);
            }
            if let Some(inner) = val.strip_prefix('\'').and_then(|s| s.strip_suffix('\'')) {
                return Some(inner.replace("''", "'"));
            }
            return Some(val.to_string());
        }
    }
    None
}

/// Per-ecosystem preserved-field catalog — typed self-description of
/// which manifest fields the substrate captures + preserves on round-
/// trip. github-action gets branding-icon + branding-color in addition
/// to the four shared fields.
///
/// Public so `ecosystems::query()` can surface it to operators per the
/// ★★ CATALOG REFLECTION directive.
pub fn compared_fields_for(ecosystem: &str) -> &'static [&'static str] {
    match ecosystem {
        "github-action" => &["name", "version", "description", "license",
                              "branding-icon", "branding-color"],
        // go.mod has no name/version/description/license — its typed
        // fields are module-path + go-version. Right-sized so the
        // fidelity score reflects what's actually preserved.
        "go" => &["module-path", "go-version"],
        // nix-flake's typed top-level field is description (the only
        // standardized single-string slot in the flake spec).
        "nix-flake" => &["description"],
        // Rust: 4 standard + crates.io discoverability surfaces
        // (keywords + categories). Round-trip means set equality.
        "rust-single-crate" | "rust-workspace" =>
            &["name", "version", "description", "license", "keywords", "categories"],
        // npm: 4 standard + keywords array (npmjs discoverability).
        "npm" | "js-pnpm" =>
            &["name", "version", "description", "license", "keywords"],
        _ => &["name", "version", "description", "license"],
    }
}

/// Compare an original repo's manifest fields against a rendered repo's
/// (from `reverse + caixa::render`). Returns the typed FidelityReport.
pub fn measure(original: &Path, rendered: &Path) -> Result<FidelityReport> {
    let detected = crate::discover::detect(original)
        .ok_or_else(|| anyhow!("no ecosystem detected in original: {}", original.display()))?;
    let ecosystem = detected.ecosystem.to_string();

    let mut report = FidelityReport {
        original_path: original.to_path_buf(),
        rendered_path: rendered.to_path_buf(),
        ecosystem: Some(ecosystem.clone()),
        fields: Vec::new(),
        perfect_count: 0, lossy_count: 0, gap_count: 0, na_count: 0,
    };

    for field in compared_fields_for(&ecosystem) {
        let orig = extract_field(original, &ecosystem, field);
        let rend = extract_field(rendered, &ecosystem, field);
        let status = match (&orig, &rend) {
            (None, _) => FieldStatus::Na,
            (Some(o), Some(r)) if o == r => FieldStatus::Perfect,
            (Some(_), Some(_)) => FieldStatus::Lossy,
            (Some(_), None) => FieldStatus::Gap,
        };
        match status {
            FieldStatus::Perfect => report.perfect_count += 1,
            FieldStatus::Lossy   => report.lossy_count += 1,
            FieldStatus::Gap     => report.gap_count += 1,
            FieldStatus::Na      => report.na_count += 1,
        }
        report.fields.push(FieldFidelity {
            field: (*field).to_string(), status, original: orig, rendered: rend,
        });
    }
    Ok(report)
}

/// Run reverse → render → measure in one shot. Returns the typed
/// FidelityReport against a fresh tempdir-rendered copy.
pub fn measure_via_reverse_render(original: &Path) -> Result<FidelityReport> {
    use crate::ast::Render;
    let forms = crate::reverse::reverse_from_path(original)?;
    let src = forms.render();
    let tmp = tempdir::TempDir::new("fidelity").map_err(|e| anyhow!("tempdir: {e}"))?;
    crate::caixa::render(&src, tmp.path(), true)?;
    measure(original, tmp.path())
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::fs;

    fn mk(files: &[(&str, &str)]) -> tempdir::TempDir {
        let tmp = tempdir::TempDir::new("fid").unwrap();
        for (n, b) in files {
            if let Some(p) = std::path::Path::new(n).parent() {
                let _ = fs::create_dir_all(tmp.path().join(p));
            }
            fs::write(tmp.path().join(n), b).unwrap();
        }
        tmp
    }

    #[test]
    fn perfect_round_trip_scores_1_0() {
        // Synthetic: same content in both → all fields Perfect.
        // Rust now compares 6 fields incl. keywords + categories (NA
        // when absent — don't count toward perfect/lossy/gap).
        let dir = mk(&[("Cargo.toml",
            "[package]\nname = \"x\"\nversion = \"1.0\"\ndescription = \"d\"\nlicense = \"MIT\"\n")]);
        let r = measure(dir.path(), dir.path()).unwrap();
        // 6 typed fields for rust-single-crate; all match → all Perfect.
        assert!(r.perfect_count >= 4, "expected ≥4 perfect, got {}", r.perfect_count);
        assert!((r.score() - 1.0).abs() < 0.001);
    }

    #[test]
    fn gap_when_rendered_drops_field() {
        let orig = mk(&[("Cargo.toml",
            "[package]\nname = \"x\"\nversion = \"1.0\"\ndescription = \"d\"\nlicense = \"MIT\"\n")]);
        let rend = mk(&[("Cargo.toml",
            "[package]\nname = \"x\"\nversion = \"1.0\"\n")]);
        let r = measure(orig.path(), rend.path()).unwrap();
        // 2 perfect (name + version); 2 gap (description + license).
        // keywords + categories: both empty strings → perfect.
        assert!(r.perfect_count >= 2, "expected ≥2 perfect, got {}", r.perfect_count);
        assert_eq!(r.gap_count, 2); // description + license dropped
    }

    #[test]
    fn lossy_when_value_differs() {
        let orig = mk(&[("Cargo.toml",
            "[package]\nname = \"x\"\nversion = \"1.0\"\n")]);
        let rend = mk(&[("Cargo.toml",
            "[package]\nname = \"x\"\nversion = \"2.0\"\n")]);
        let r = measure(orig.path(), rend.path()).unwrap();
        assert!(r.perfect_count >= 1, "expected ≥1 perfect, got {}", r.perfect_count);
        assert_eq!(r.lossy_count, 1);
    }

    #[test]
    fn end_to_end_rust_self_round_trip_hits_perfect() {
        let dir = mk(&[("Cargo.toml",
            "[package]\nname = \"end-to-end\"\nversion = \"3.0.0\"\ndescription = \"e2e\"\nlicense = \"Apache-2.0\"\n")]);
        let r = measure_via_reverse_render(dir.path()).unwrap();
        // Rust extractor + renderer round-trip all 4 fields cleanly.
        assert!(r.score() > 0.99,
            "expected perfect round-trip; got score={} fields={:?}",
            r.score(), r.fields.iter().map(|f| (f.field.as_str(), f.status.as_str())).collect::<Vec<_>>());
    }

    #[test]
    fn go_mod_directive_extracts_module_and_version() {
        let go_mod = "module github.com/x/y\n\ngo 1.22\n\nrequire foo v1.0.0\n";
        assert_eq!(go_mod_directive(go_mod, "module").as_deref(),
            Some("github.com/x/y"));
        assert_eq!(go_mod_directive(go_mod, "go").as_deref(), Some("1.22"));
    }

    #[test]
    fn go_mod_directive_ignores_block_form_require() {
        // `require ( ... )` is block-form — should NOT match as a flat
        // directive. The first non-block `require foo v1.0.0` would
        // match the heuristic if we weren't careful.
        let go_mod = "module github.com/x/y\nrequire (\n  foo v1.0.0\n)\n";
        assert_eq!(go_mod_directive(go_mod, "module").as_deref(),
            Some("github.com/x/y"));
        // require block opens with `(` — skipped.
        assert_eq!(go_mod_directive(go_mod, "require"), None);
    }

    #[test]
    fn compared_fields_for_go_returns_right_sized_set() {
        let fields = compared_fields_for("go");
        assert_eq!(fields, &["module-path", "go-version"]);
    }

    #[test]
    fn compared_fields_for_unknown_returns_default_4() {
        let fields = compared_fields_for("totally-unknown-ecosystem");
        assert_eq!(fields, &["name", "version", "description", "license"]);
    }

    #[test]
    fn compared_fields_for_rust_includes_crates_io_surfaces() {
        let fields = compared_fields_for("rust-single-crate");
        assert!(fields.contains(&"keywords"));
        assert!(fields.contains(&"categories"));
    }

    #[test]
    fn cargo_field_resolves_workspace_inherit_inline_table() {
        let toml = "[workspace.package]\nversion = \"0.1.5\"\n\n[package]\nname = \"x\"\nversion = { workspace = true }\n";
        assert_eq!(cargo_field(toml, "name").as_deref(), Some("x"));
        assert_eq!(cargo_field(toml, "version").as_deref(), Some("0.1.5"));
    }

    #[test]
    fn cargo_field_resolves_workspace_shorthand() {
        let toml = "[workspace.package]\nlicense = \"MIT\"\n\n[package]\nname = \"x\"\nlicense.workspace = true\n";
        assert_eq!(cargo_field(toml, "license").as_deref(), Some("MIT"));
    }

    #[test]
    fn cargo_field_workspace_with_root_pleme_sui_pattern() {
        // Regression: surfaced by the fleet fidelity scan on pleme-io/sui.
        // Workspace with a root crate inheriting all common fields.
        let toml = r#"[workspace]
members = ["a", "b"]
resolver = "3"

[workspace.package]
version = "0.1.5"
edition = "2024"
license = "MIT"
repository = "https://github.com/x/sui"

[package]
name = "sui"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
description = "the description"
"#;
        assert_eq!(cargo_field(toml, "name").as_deref(), Some("sui"));
        assert_eq!(cargo_field(toml, "version").as_deref(), Some("0.1.5"));
        assert_eq!(cargo_field(toml, "license").as_deref(), Some("MIT"));
        assert_eq!(cargo_field(toml, "description").as_deref(), Some("the description"));
    }

    #[test]
    fn report_to_json_includes_typed_status() {
        let dir = mk(&[("Cargo.toml",
            "[package]\nname = \"x\"\n")]);
        let r = measure_via_reverse_render(dir.path()).unwrap();
        let j = r.to_json();
        assert!(j.contains("\"perfect\":"));
        assert!(j.contains("\"score-permille\":"));
        assert!(j.contains("\"field\":"));
    }
}