kowalski-core 2.3.1

//! Markdown-staged **apps**: a manifest (`app.md` or `horde.md`) plus `agents/*.md` drive
//! multi-step runs. This module is **layout-agnostic** — no wiki/Obsidian-specific repair or
//! index generation. Each stage declares prompt paths, optional context files (relative to the
//! workdir), optional `@artifact@` / `@step:name@` tokens, and optional markdown normalization.

use crate::error::KowalskiError;
use crate::horde_graph::HordeEdge;
use crate::horde_trigger::HordeTrigger;
use crate::operator_input::OperatorInputField;
use serde::Deserialize;
use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};

/// Top-level manifest (TOML frontmatter in `app.md` or `horde.md`).
#[derive(Debug, Deserialize, Clone)]
pub struct AppManifestMeta {
    pub id: String,
    #[serde(default)]
    pub display_name: Option<String>,
    #[serde(default)]
    pub description: Option<String>,
    /// Semver workflow version (used by manifest export; optional in horde.md).
    #[serde(default)]
    pub version: Option<String>,
    #[serde(default)]
    pub capability_prefix: Option<String>,
    pub pipeline: Vec<String>,
    /// Optional scheduling edges; absent or empty → implicit chain along `pipeline` order.
    #[serde(default)]
    pub edges: Vec<HordeEdge>,
    /// Optional event-driven run declarations (`[[triggers]]`): cron / watch / webhook.
    #[serde(default)]
    pub triggers: Vec<HordeTrigger>,
    #[serde(default)]
    pub default_question: Option<String>,
    #[serde(default)]
    pub default_topic: Option<String>,
    #[serde(default)]
    pub workdir: Option<String>,
    #[serde(default)]
    pub delivery_title: Option<String>,
    #[serde(default)]
    pub delivery_note: Option<String>,
    #[serde(default)]
    pub delivery_root_rel: Option<String>,
    #[serde(default)]
    pub delivery_summary_note: Option<String>,
    #[serde(default)]
    pub prompt_tip: Option<String>,
}

/// One pipeline stage (`agents/<name>.md` frontmatter).
#[derive(Debug, Default, Deserialize, Clone)]
pub struct StageAgentMeta {
    pub name: String,
    pub kind: String,
    #[serde(default)]
    pub display_name: Option<String>,
    #[serde(default)]
    pub description: Option<String>,
    #[serde(default)]
    pub capability: Option<String>,
    #[serde(default)]
    pub default_agent_id: Option<String>,
    /// UI avatar id (maps to `ui/src/assets/pinguins/<id>.png`).
    #[serde(default)]
    pub avatar: Option<String>,
    /// Pinned model (omitted = deployment default).
    #[serde(default)]
    pub model_id: Option<String>,
    /// Cap on the attached context in characters, shared between ingested sources (each keeps its head);
    /// keeps small local models inside their window. `None` attaches everything.
    #[serde(default)]
    pub context_max_chars: Option<usize>,
    /// `in_process` (default) or `process` step isolation.
    #[serde(default)]
    pub isolation: Option<String>,
    #[serde(default)]
    pub prompt_file: Option<String>,
    /// Relative to **workdir** (e.g. `debug/reports/latest.md`). Required for LLM stages.
    #[serde(default)]
    pub output: Option<String>,
    /// Paths relative to **workdir**, and/or:
    /// - `@artifact@` — previous stage artifact path (federation / chained runs),
    /// - `@step:<pipeline_step_name>@` — output path recorded for that step (local runner).
    #[serde(default)]
    pub context_paths: Vec<String>,
    #[serde(default)]
    pub normalize_doc_title: Option<String>,
    #[serde(default)]
    pub normalize_sections: Vec<String>,
    #[serde(default)]
    pub normalize_fallback: Option<String>,
    /// Section headings (## name) that receive `normalize_fallback` body when synthesized.
    #[serde(default)]
    pub normalize_fallback_sections: Vec<String>,
    /// Tool names allowed for this stage (empty = no tools / markdown-only).
    #[serde(default)]
    pub tool_ids: Vec<String>,
    /// Shell command for `kind = "verify"` (e.g. `cargo test`).
    #[serde(default)]
    pub verify_command: Option<String>,
    /// Optional working directory relative to operator `project_path`.
    #[serde(default)]
    pub verify_cwd: Option<String>,
    /// `dry-run` (default) or `execute` for `kind = "apply"` (execute requires env gate).
    #[serde(default)]
    pub apply_mode: Option<String>,
    /// Pre-run operator form (`[[inputs]]` in frontmatter).
    #[serde(default)]
    pub inputs: Vec<OperatorInputField>,
}

/// Prefer `app.md`, then legacy `horde.md`.
pub fn resolve_manifest_path(app_root: &Path) -> PathBuf {
    let app_md = app_root.join("app.md");
    if app_md.is_file() {
        return app_md;
    }
    app_root.join("horde.md")
}

pub fn parse_app_manifest(path: &Path) -> Result<AppManifestMeta, KowalskiError> {
    parse_md_frontmatter(path)
}

pub fn parse_stage_agent(path: &Path) -> Result<StageAgentMeta, KowalskiError> {
    parse_md_frontmatter(path)
}

pub fn load_stage_agents(agents_dir: &Path) -> Result<BTreeMap<String, StageAgentMeta>, KowalskiError> {
    let mut map = BTreeMap::new();
    let rd = fs::read_dir(agents_dir)
        .map_err(|e| KowalskiError::Validation(format!("read agents dir {}: {}", agents_dir.display(), e)))?;
    for e in rd.flatten() {
        let p = e.path();
        if p.extension().and_then(|x| x.to_str()) != Some("md") {
            continue;
        }
        let doc: StageAgentMeta = parse_stage_agent(&p)?;
        map.insert(doc.name.clone(), doc);
    }
    Ok(map)
}

fn parse_md_frontmatter<T: for<'de> Deserialize<'de>>(path: &Path) -> Result<T, KowalskiError> {
    let raw = fs::read_to_string(path)
        .map_err(|e| KowalskiError::Validation(format!("read {}: {}", path.display(), e)))?;
    let mut lines = raw.lines();
    if lines.next().map(|s| s.trim()) != Some("---") {
        return Err(KowalskiError::Validation(format!(
            "missing frontmatter start in {}",
            path.display()
        )));
    }
    let mut fm = String::new();
    let mut in_fm = true;
    for line in raw.lines().skip(1) {
        if in_fm && line.trim() == "---" {
            in_fm = false;
            break;
        }
        if in_fm {
            fm.push_str(line);
            fm.push('\n');
        }
    }
    if in_fm {
        return Err(KowalskiError::Validation(format!(
            "missing frontmatter end in {}",
            path.display()
        )));
    }
    toml::from_str::<T>(&fm).map_err(|e| {
        KowalskiError::Validation(format!("toml in {}: {}", path.display(), e))
    })
}

fn resolve_path_token(
    workdir: &Path,
    token: &str,
    step_paths: &BTreeMap<String, PathBuf>,
    previous_artifact: Option<&Path>,
) -> Result<PathBuf, KowalskiError> {
    let t = token.trim();
    if t == "@artifact@" {
        return previous_artifact
            .map(|p| p.to_path_buf())
            .ok_or_else(|| KowalskiError::Validation("@artifact@ used but no previous artifact".into()));
    }
    if let Some(inner) = t.strip_prefix("@step:").and_then(|s| s.strip_suffix("@")) {
        let name = inner.trim();
        return step_paths.get(name).cloned().ok_or_else(|| {
            KowalskiError::Validation(format!("@step:{name}@ not available (no output for that stage)"))
        });
    }
    let p = workdir.join(t);
    if p.is_file() {
        Ok(p)
    } else {
        Err(KowalskiError::Validation(format!(
            "context path not found: {}",
            p.display()
        )))
    }
}

/// Concatenate resolved context files as markdown sections for LLM consumption.
pub fn render_context_attachments(
    workdir: &Path,
    tokens: &[String],
    step_paths: &BTreeMap<String, PathBuf>,
    previous_artifact: Option<&Path>,
) -> Result<String, KowalskiError> {
    let mut out = String::new();
    for token in tokens {
        let path = resolve_path_token(workdir, token, step_paths, previous_artifact)?;
        for file in context_files(&path) {
            let label = file.strip_prefix(workdir).unwrap_or(&file).display().to_string();
            let body = fs::read_to_string(&file).unwrap_or_default();
            out.push_str(&format!("## Context file: `{label}`\n\n{body}\n\n---\n\n"));
        }
    }
    Ok(out)
}

/// `text` cut to at most `max` characters (on a char boundary), with a note saying so.
pub fn cap_chars(text: &str, max: usize) -> String {
    match text.char_indices().nth(max) {
        None => text.to_string(),
        Some((cut, _)) => format!(
            "{}\n\n[… {} more characters left out to fit the model's context]\n",
            &text[..cut],
            text[cut..].chars().count()
        ),
    }
}

/// Context capped at `max` characters, shared evenly between the sources of an ingest collection
/// (`<!-- source:N:…:begin -->` sections), so one long page cannot crowd out the others.
pub fn cap_context(text: &str, max: usize) -> String {
    const BEGIN: &str = "<!-- source:";
    let starts: Vec<usize> = text
        .match_indices(BEGIN)
        .map(|(i, _)| i)
        .filter(|&i| text[i..].lines().next().is_some_and(|l| l.contains(":begin")))
        .collect();
    if starts.len() < 2 {
        return cap_chars(text, max);
    }
    let head = &text[..starts[0]];
    let share = max.saturating_sub(head.chars().count()) / starts.len();
    let mut out = head.to_string();
    for (n, &start) in starts.iter().enumerate() {
        let end = starts.get(n + 1).copied().unwrap_or(text.len());
        out.push_str(&cap_chars(&text[start..end], share.max(500)));
    }
    out
}

/// A context path as files: itself, or for a folder output (such as `ingest`'s `debug/raw/`,
/// which gains one timestamped file per run) its newest Markdown file, i.e. this run's.
fn context_files(path: &Path) -> Vec<PathBuf> {
    if !path.is_dir() {
        return vec![path.to_path_buf()];
    }
    fs::read_dir(path)
        .map(|rd| {
            rd.flatten()
                .map(|e| e.path())
                .filter(|p| p.is_file() && p.extension().is_some_and(|x| x == "md"))
                .max_by_key(|p| (fs::metadata(p).and_then(|m| m.modified()).ok(), p.clone()))
        })
        .ok()
        .flatten()
        .into_iter()
        .collect()
}

/// Optional markdown normalization (H1 + required ## sections).
pub fn maybe_normalize_markdown(agent: &StageAgentMeta, raw: &str) -> String {
    let Some(ref title) = agent.normalize_doc_title else {
        return raw.trim().to_string();
    };
    if agent.normalize_sections.is_empty() {
        return raw.trim().to_string();
    }
    let sections: Vec<String> = agent.normalize_sections.clone();
    let sec_refs: Vec<&str> = sections.iter().map(String::as_str).collect();
    let fallback = agent
        .normalize_fallback
        .as_deref()
        .unwrap_or("Fallback model output was empty or unusable.");
    let fallback_for: Vec<&str> = if agent.normalize_fallback_sections.is_empty() {
        vec!["Summary", "Response", "Issues", "Snapshot"]
    } else {
        agent
            .normalize_fallback_sections
            .iter()
            .map(String::as_str)
            .collect()
    };
    normalize_markdown_sections(raw, title, &sec_refs, fallback, &fallback_for)
}

/// True when the note already has a Markdown H1, including after optional YAML frontmatter
/// (`---` … `---`) at the very start (Obsidian handoff files).
fn markdown_body_has_h1(raw: &str) -> bool {
    let t = raw.trim_start();
    if t.starts_with("# ") {
        return true;
    }
    if !t.starts_with("---") {
        return false;
    }
    let mut lines = t.lines();
    lines.next(); // opening ---
    let mut in_fm = true;
    for line in lines {
        if in_fm {
            if line.trim() == "---" {
                in_fm = false;
            }
            continue;
        }
        let s = line.trim_start();
        if s.is_empty() {
            continue;
        }
        return s.starts_with("# ");
    }
    false
}

fn normalize_markdown_sections(
    raw: &str,
    title: &str,
    required_sections: &[&str],
    fallback_body: &str,
    fallback_for: &[&str],
) -> String {
    let trimmed = raw.trim();
    let mut out = String::new();
    if trimmed.is_empty() || trimmed == "{}" || trimmed == "null" {
        out.push_str(&format!("# {}\n\n", title));
        for s in required_sections {
            out.push_str(&format!("## {}\n", s));
            if fallback_for.contains(s) {
                out.push_str(fallback_body);
                out.push('\n');
            }
            out.push('\n');
        }
        return out;
    }

    let mut body = trimmed.to_string();
    // If there is no top-level H1 (`# `…), prefix the manifest title. Do **not** append
    // synthetic `##` sections here: models often use emoji or alternate headings (e.g.
    // `## 📝 TL;DR` instead of `## TL;DR`), and exact `contains("## TL;DR")` checks falsely
    // inject duplicate sections plus `normalize_fallback` text into otherwise valid output.
    let start = body.trim_start();
    if start.starts_with("---") {
        // YAML frontmatter first (e.g. Obsidian `tags:`); never prepend `# title` before `---`.
    } else if !markdown_body_has_h1(start) {
        body = format!("# {}\n\n{}", title, start);
    }
    body.push('\n');
    body
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn cap_chars_keeps_the_head_and_says_what_was_cut() {
        assert_eq!(cap_chars("short", 10), "short");
        let capped = cap_chars("héllo world", 5);
        assert!(capped.starts_with("héllo") && capped.contains("6 more characters"), "{capped}");
    }

    #[test]
    fn cap_context_shares_the_budget_between_sources() {
        let long = "x".repeat(20_000);
        let text = format!("# Raw\n<!-- source:1:url:begin -->\n{long}\n<!-- source:1:url:end -->\n<!-- source:2:url:begin -->\nsecond page\n<!-- source:2:url:end -->\n");
        let capped = cap_context(&text, 4_000);
        assert!(capped.contains("second page"), "the short source survives");
        assert!(capped.chars().count() < 4_500, "{}", capped.chars().count());
    }

    #[test]
    fn a_folder_step_output_attaches_its_newest_markdown_file() {
        let dir = tempfile::tempdir().unwrap();
        let raw = dir.path().join("debug/raw");
        fs::create_dir_all(&raw).unwrap();
        fs::write(raw.join("20260101-000000-inputs-1.md"), "an older run's question").unwrap();
        std::thread::sleep(std::time::Duration::from_millis(20));
        fs::write(raw.join("20260102-000000-inputs-1.md"), "Who spent the most?").unwrap();
        fs::write(raw.join("notes.txt"), "skipped").unwrap();
        let steps = BTreeMap::from([("ingest".to_string(), raw.clone())]);
        let ctx = render_context_attachments(dir.path(), &["@step:ingest@".into()], &steps, None).unwrap();
        assert!(ctx.contains("Who spent the most?"), "{ctx}");
        assert!(!ctx.contains("older run") && !ctx.contains("skipped"), "{ctx}");
    }

    fn lint_like_agent() -> StageAgentMeta {
        StageAgentMeta {
            name: "lint".into(),
            kind: "lint".into(),
            normalize_doc_title: Some("Vault paste pack".into()),
            normalize_sections: vec!["TL;DR".into(), "Suggested notes".into()],
            normalize_fallback: Some("SHOULD_NOT_APPEAR".into()),
            normalize_fallback_sections: vec!["TL;DR".into()],
            ..Default::default()
        }
    }

    #[test]
    fn maybe_normalize_non_empty_does_not_inject_fallback_for_emoji_headings() {
        let agent = lint_like_agent();
        let raw = "## 📝 TL;DR\n\nReal content.\n\n## 🔗 Links\n\nMore.\n";
        let out = maybe_normalize_markdown(&agent, raw);
        assert!(
            !out.contains("SHOULD_NOT_APPEAR"),
            "unexpected fallback injection: {}",
            out
        );
        assert!(out.contains("Real content."));
        assert!(out.contains("Vault paste pack"));
    }

    #[test]
    fn maybe_normalize_empty_still_synthesizes_with_fallback() {
        let agent = lint_like_agent();
        let out = maybe_normalize_markdown(&agent, "  ");
        assert!(out.contains("SHOULD_NOT_APPEAR"));
        assert!(out.contains("# Vault paste pack"));
    }

    #[test]
    fn maybe_normalize_does_not_prepend_h1_before_yaml_frontmatter() {
        let agent = lint_like_agent();
        let raw = "---\ntags:\n  - cuda\n  - hpc\n---\n\n# YarentY Profile\n\nbody\n";
        let out = maybe_normalize_markdown(&agent, raw);
        assert!(out.trim_start().starts_with("---"));
        assert!(!out.contains("# Vault paste pack\n\n---"));
        assert!(out.contains("# YarentY Profile"));
    }
}