eidos_kernel/eval/

workflow.rs

use std::collections::BTreeSet;

use serde::Serialize;

use crate::graph_index::GraphIndex;
use crate::retrieval::ground_subgraph;
use crate::schema::Graph;
use crate::workflow::{
    TaskContextItem, TaskOperator, TaskRun, TaskState, draft_task_dag, initial_run, ready_tasks,
    validate_dag,
};

use super::{GoldenCase, kind_label};

/// Per-case brief/decomposition outcome.
#[derive(Serialize, Clone, Debug)]
pub struct WorkflowCaseResult {
    pub query: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub expected_profile: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub profile: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub profile_ok: Option<bool>,
    pub valid: bool,
    pub bounded: bool,
    pub required_chain: bool,
    pub initial_ready_context_only: bool,
    /// Fraction of `context_must` ids represented by inspection nodes.
    pub inspection_recall: Option<f64>,
    /// Fraction of `expected_required_checks` represented by brief/decomposition nodes.
    pub required_check_recall: Option<f64>,
    /// Fraction of `expected_risk_hints` represented by medium/high-risk brief/decomposition nodes.
    pub risk_hint_recall: Option<f64>,
    pub node_count: usize,
    pub ready: Vec<String>,
    pub missing_inspections: Vec<String>,
    pub missing_required_checks: Vec<String>,
    pub missing_risk_hints: Vec<String>,
    pub error: Option<String>,
}

/// Aggregate brief/decomposition metrics over non-garbage cases.
#[derive(Serialize, Clone, Debug)]
pub struct WorkflowReport {
    #[serde(rename = "brief_cases")]
    pub workflow_cases: usize,
    pub valid_rate: f64,
    pub bounded_rate: f64,
    pub required_chain_rate: f64,
    pub initial_ready_rate: f64,
    /// Mean inspection recall over cases with `context_must`; `None` means no inspection judgment.
    pub inspection_recall: Option<f64>,
    /// Match rate over cases with `expected_profile`; `None` means no profile judgments exist.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub profile_match_rate: Option<f64>,
    /// Mean required-check recall over cases with `expected_required_checks`.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub required_check_recall: Option<f64>,
    /// Mean risk-hint recall over cases with `expected_risk_hints`.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub risk_hint_recall: Option<f64>,
    pub mean_node_count: f64,
    pub cases: Vec<WorkflowCaseResult>,
}

#[derive(Clone, Debug)]
pub struct WorkflowInspectionJudgment {
    pub recall: Option<f64>,
    pub missing: Vec<String>,
}

/// Score the internal decomposition artifact produced from a focused context pack.
///
/// This evaluates structural usefulness, not task execution: valid acyclic decomposition, bounded size,
/// expected operator chain, deterministic initial ready task, and inspection coverage for judged
/// context nodes.
pub fn evaluate_workflow(
    graph: &Graph,
    cases: &[GoldenCase],
    limit: usize,
    depth: usize,
    width: usize,
    max_nodes: usize,
) -> WorkflowReport {
    let mut results = Vec::new();
    let index = GraphIndex::build(graph);

    for c in cases.iter().filter(|case| !case.garbage) {
        let sg = ground_subgraph(graph, &c.query, limit, depth, width);
        let context = sg
            .context_order
            .iter()
            .filter_map(|id| {
                let node = index.node(id)?;
                Some(TaskContextItem {
                    id: node.id.clone(),
                    title: node.title.clone(),
                    kind: kind_label(node.kind),
                    source: index.source_of(&node.id).unwrap_or_default(),
                })
            })
            .collect::<Vec<_>>();
        let dag = draft_task_dag(&c.query, &context);
        let node_count = dag.nodes.len();
        let valid = validate_dag(&dag).is_ok();
        let bounded = node_count <= max_nodes;
        let required_chain = has_required_operator_chain(&dag.nodes);
        let (ready, initial_ready_context_only, error) = match initial_run(&dag) {
            Ok(run) => match ready_tasks(&dag, &run) {
                Ok(ready) => {
                    let ok = workflow_initial_ready_context_only(&run, &ready);
                    (ready, ok, None)
                }
                Err(err) => (Vec::new(), false, Some(format!("{err:?}"))),
            },
            Err(err) => (Vec::new(), false, Some(format!("{err:?}"))),
        };

        let inspected = dag
            .nodes
            .iter()
            .flat_map(|node| node.inputs.iter().map(String::as_str))
            .collect::<BTreeSet<_>>();
        let inspection = judge_workflow_inspections(c, &inspected);
        let required_checks = dag
            .nodes
            .iter()
            .flat_map(|node| node.required_checks.iter().map(String::as_str))
            .collect::<BTreeSet<_>>();
        let risk_hints = dag
            .nodes
            .iter()
            .filter(|node| node.risk != crate::workflow::TaskRisk::Low)
            .map(|node| node.title.as_str())
            .collect::<BTreeSet<_>>();
        let required_check_judgment = judge_required_checks(c, &required_checks);
        let risk_hint_judgment = judge_risk_hints(c, &risk_hints);

        results.push(WorkflowCaseResult {
            query: c.query.clone(),
            expected_profile: None,
            profile: None,
            profile_ok: None,
            valid,
            bounded,
            required_chain,
            initial_ready_context_only,
            inspection_recall: inspection.recall,
            required_check_recall: required_check_judgment.recall,
            risk_hint_recall: risk_hint_judgment.recall,
            node_count,
            ready,
            missing_inspections: inspection.missing,
            missing_required_checks: required_check_judgment.missing,
            missing_risk_hints: risk_hint_judgment.missing,
            error,
        });
    }

    summarize_workflow(results)
}

pub fn summarize_workflow(results: Vec<WorkflowCaseResult>) -> WorkflowReport {
    let workflow_cases = results.len();
    let frac = |n: usize| {
        if workflow_cases == 0 {
            0.0
        } else {
            n as f64 / workflow_cases as f64
        }
    };
    let valid_n = results.iter().filter(|case| case.valid).count();
    let bounded_n = results.iter().filter(|case| case.bounded).count();
    let required_chain_n = results.iter().filter(|case| case.required_chain).count();
    let initial_ready_n = results
        .iter()
        .filter(|case| case.initial_ready_context_only)
        .count();
    let inspection_recall = mean_optional(results.iter().map(|case| case.inspection_recall));
    let profile_match_rate = mean_bool(results.iter().filter_map(|case| case.profile_ok));
    let required_check_recall =
        mean_optional(results.iter().map(|case| case.required_check_recall));
    let risk_hint_recall = mean_optional(results.iter().map(|case| case.risk_hint_recall));
    let node_count_sum = results.iter().map(|case| case.node_count).sum::<usize>();

    WorkflowReport {
        workflow_cases,
        valid_rate: frac(valid_n),
        bounded_rate: frac(bounded_n),
        required_chain_rate: frac(required_chain_n),
        initial_ready_rate: frac(initial_ready_n),
        inspection_recall,
        profile_match_rate,
        required_check_recall,
        risk_hint_recall,
        mean_node_count: if workflow_cases == 0 {
            0.0
        } else {
            node_count_sum as f64 / workflow_cases as f64
        },
        cases: results,
    }
}

pub fn judge_workflow_inspections(
    case: &GoldenCase,
    inspected: &BTreeSet<&str>,
) -> WorkflowInspectionJudgment {
    let missing = case
        .context_must
        .iter()
        .filter(|id| !inspected.contains(id.as_str()))
        .cloned()
        .collect::<Vec<_>>();
    let recall = if case.context_must.is_empty() {
        None
    } else {
        Some((case.context_must.len() - missing.len()) as f64 / case.context_must.len() as f64)
    };
    WorkflowInspectionJudgment { recall, missing }
}

pub fn judge_required_checks(
    case: &GoldenCase,
    checks: &BTreeSet<&str>,
) -> WorkflowInspectionJudgment {
    let missing = case
        .expected_required_checks
        .iter()
        .filter(|check| !checks.contains(check.as_str()))
        .cloned()
        .collect::<Vec<_>>();
    let recall = if case.expected_required_checks.is_empty() {
        None
    } else {
        Some(
            (case.expected_required_checks.len() - missing.len()) as f64
                / case.expected_required_checks.len() as f64,
        )
    };
    WorkflowInspectionJudgment { recall, missing }
}

pub fn judge_risk_hints(case: &GoldenCase, hints: &BTreeSet<&str>) -> WorkflowInspectionJudgment {
    let missing = case
        .expected_risk_hints
        .iter()
        .filter(|hint| !hints.contains(hint.as_str()))
        .cloned()
        .collect::<Vec<_>>();
    let recall = if case.expected_risk_hints.is_empty() {
        None
    } else {
        Some(
            (case.expected_risk_hints.len() - missing.len()) as f64
                / case.expected_risk_hints.len() as f64,
        )
    };
    WorkflowInspectionJudgment { recall, missing }
}

pub fn workflow_initial_ready_context_only(run: &TaskRun, ready: &[String]) -> bool {
    ready == ["context"] && run.states.get("context") == Some(&TaskState::Ready)
}

pub fn has_required_operator_chain(nodes: &[crate::workflow::TaskNode]) -> bool {
    let has = |operator| nodes.iter().any(|node| node.operator == operator);
    has(TaskOperator::ReadContext)
        && has(TaskOperator::Decompose)
        && (has(TaskOperator::Edit) || has(TaskOperator::ProposeDoc))
        && has(TaskOperator::Check)
        && has(TaskOperator::Review)
        && has(TaskOperator::HumanApproval)
}

fn mean_optional(values: impl Iterator<Item = Option<f64>>) -> Option<f64> {
    let mut sum = 0.0;
    let mut count = 0usize;
    for value in values.flatten() {
        sum += value;
        count += 1;
    }
    (count > 0).then_some(sum / count as f64)
}

fn mean_bool(values: impl Iterator<Item = bool>) -> Option<f64> {
    let mut ok = 0usize;
    let mut count = 0usize;
    for value in values {
        ok += value as usize;
        count += 1;
    }
    (count > 0).then_some(ok as f64 / count as f64)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn summarize_workflow_reports_profile_match_rate_for_judged_cases() {
        let report = summarize_workflow(vec![
            workflow_case(Some(true), Some(1.0), 10),
            workflow_case(Some(false), Some(0.5), 20),
            workflow_case(None, None, 30),
        ]);

        assert_eq!(report.workflow_cases, 3);
        assert_eq!(report.profile_match_rate, Some(0.5));
        assert_eq!(report.inspection_recall, Some(0.75));
        assert_eq!(report.mean_node_count, 20.0);
    }

    #[test]
    fn summarize_workflow_keeps_unjudged_inspection_recall_unmeasured() {
        let report = summarize_workflow(vec![workflow_case(None, None, 10)]);

        assert_eq!(report.workflow_cases, 1);
        assert_eq!(report.inspection_recall, None);
    }

    fn workflow_case(
        profile_ok: Option<bool>,
        inspection_recall: Option<f64>,
        node_count: usize,
    ) -> WorkflowCaseResult {
        WorkflowCaseResult {
            query: "task".to_string(),
            expected_profile: profile_ok.map(|_| "safe-change".to_string()),
            profile: profile_ok.map(|ok| {
                if ok {
                    "safe-change".to_string()
                } else {
                    "other".to_string()
                }
            }),
            profile_ok,
            valid: true,
            bounded: true,
            required_chain: true,
            initial_ready_context_only: true,
            inspection_recall,
            required_check_recall: None,
            risk_hint_recall: None,
            node_count,
            ready: vec!["context".to_string()],
            missing_inspections: Vec::new(),
            missing_required_checks: Vec::new(),
            missing_risk_hints: Vec::new(),
            error: None,
        }
    }
}