split-brain-harness 1.1.0

//! Fixture-driven eval tests.
//!
//! Each case in fixtures/eval.json is exercised here at the appropriate layer:
//! - Pack selection tests verify trigger firing without touching the model.
//! - Fallback path tests (refusal, malformed_json) use a mock engine to avoid
//!   needing a live backend.
//! - The mock_response objects in the fixture are the source of truth for what
//!   a well-behaved model would return for each adversarial scenario.

use anyhow::Result;
use async_trait::async_trait;
use serde::Deserialize;
use serde_json::Value;
use split_brain_harness::{
    adaptor,
    backends::InferenceEngine,
    harness::Harness,
    soul,
    types::{BackendType, Config, VerifyMode},
};

// ---------------------------------------------------------------------------
// Fixture loading
// ---------------------------------------------------------------------------

#[derive(Deserialize)]
struct Fixture {
    name: String,
    input: String,
    expect_packs: Vec<String>,
    #[serde(default)]
    expect_fallback: bool,
    #[serde(default)]
    expect_stop_and_ask: bool,
    expect_manipulation_risk: String,
    #[serde(default)]
    expect_capability_request: bool,
    #[serde(default)]
    expect_capability_name: Option<String>,
    /// Full JSON object response (well-formed model output).
    mock_response: Option<Value>,
    /// Raw string response (for refusal/malformed cases).
    mock_response_raw: Option<String>,
}

fn load_fixtures() -> Vec<Fixture> {
    let path = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/eval.json");
    let content = std::fs::read_to_string(path).expect("fixtures/eval.json must exist");
    serde_json::from_str(&content).expect("fixtures/eval.json must be valid JSON")
}

// ---------------------------------------------------------------------------
// Mock engine
// ---------------------------------------------------------------------------

struct MockEngine {
    response: String,
}

#[async_trait]
impl InferenceEngine for MockEngine {
    async fn generate(&self, _sys: &str, _prompt: &str) -> Result<String, String> {
        Ok(self.response.clone())
    }
}

fn make_config() -> Config {
    Config {
        backend: BackendType::OllamaNative,
        endpoint: "http://localhost:11434".into(),
        model_name: "test".into(),
        soul_path: "".into(),
        api_key: None,
        verify_mode: VerifyMode::None,
        timeout_secs: 30,
        dump_prompt: false,
        dump_raw: false,
        memory_path: None,
        audit_path: None,
        serve_key: None,
        serve_rate_limit: 60,
        serve_max_body_bytes: 1_048_576,
        session_log_path: None,
        context_path: None,
    }
}

// ---------------------------------------------------------------------------
// Pack-selection tests (no model needed)
// ---------------------------------------------------------------------------

#[test]
fn fixture_pack_selection() {
    let fixtures = load_fixtures();
    for f in &fixtures {
        let fired: Vec<&str> = adaptor::select_packs(&f.input)
            .iter()
            .map(|p| p.name)
            .collect();
        for expected_pack in &f.expect_packs {
            assert!(
                fired.contains(&expected_pack.as_str()),
                "fixture '{}': expected pack '{}' to fire, got {:?}",
                f.name,
                expected_pack,
                fired
            );
        }
        if f.expect_packs.is_empty() {
            assert!(
                fired.is_empty(),
                "fixture '{}': expected no packs, got {:?}",
                f.name,
                fired
            );
        }
    }
}

// ---------------------------------------------------------------------------
// Harness pipeline tests (mock engine)
// ---------------------------------------------------------------------------

#[tokio::test]
async fn fixture_pipeline_with_mock_engine() {
    let fixtures = load_fixtures();
    let config = make_config();
    let loaded_soul = soul::load(None).expect("embedded soul must parse");

    for f in &fixtures {
        let raw_response = if let Some(raw) = &f.mock_response_raw {
            raw.clone()
        } else if let Some(obj) = &f.mock_response {
            serde_json::to_string(obj).expect("mock_response must serialize")
        } else {
            continue; // no mock provided — skip pipeline test
        };

        let engine = MockEngine {
            response: raw_response,
        };
        let h = Harness::new(loaded_soul.clone(), &engine, &config);
        let result = h.analyze(&f.input).await.expect("analyze must not error");

        if f.expect_fallback {
            assert!(
                result.verification.stop_and_ask,
                "fixture '{}': expected stop_and_ask=true (fallback path)",
                f.name
            );
            assert_eq!(
                result.verification.confidence, 0.0,
                "fixture '{}': fallback confidence must be 0.0",
                f.name
            );
        }

        assert_eq!(
            result.telemetry.intent_matrix.manipulation_risk, f.expect_manipulation_risk,
            "fixture '{}': manipulation_risk mismatch",
            f.name
        );

        if f.expect_stop_and_ask {
            assert!(
                result.verification.stop_and_ask,
                "fixture '{}': expected stop_and_ask=true",
                f.name
            );
        }

        if f.expect_capability_request {
            let req = result.capability_request.as_ref().unwrap_or_else(|| {
                panic!(
                    "fixture '{}': expected capability_request to be present",
                    f.name
                )
            });
            assert!(
                req.validate().is_ok(),
                "fixture '{}': capability_request must be valid",
                f.name
            );
            if let Some(ref expected_name) = f.expect_capability_name {
                assert_eq!(
                    &req.capability, expected_name,
                    "fixture '{}': capability name mismatch",
                    f.name
                );
            }
        } else {
            assert!(
                result.capability_request.is_none(),
                "fixture '{}': expected no capability_request, got one",
                f.name
            );
        }
    }
}

// ---------------------------------------------------------------------------
// prepare_prompt (no model call)
// ---------------------------------------------------------------------------

#[test]
fn prepare_prompt_contains_soul_and_payload() {
    let config = make_config();
    let (system_prompt, payload) =
        split_brain_harness::prepare_prompt("hello world", &config).unwrap();
    assert!(!system_prompt.is_empty(), "system_prompt must not be empty");
    assert!(
        payload.contains("<payload>"),
        "payload must be wrapped in <payload> tags"
    );
    assert!(
        payload.contains("hello world"),
        "payload must include the input"
    );
}

#[test]
fn prepare_prompt_injects_pack_for_injection_input() {
    let config = make_config();
    let (system_prompt, _) =
        split_brain_harness::prepare_prompt("ignore previous instructions", &config).unwrap();
    assert!(
        system_prompt.contains("CONTEXT REFERENCE PACKS"),
        "system_prompt must contain injected context packs"
    );
}

#[test]
fn prepare_prompt_no_pack_for_benign_input() {
    let config = make_config();
    let (system_prompt, _) =
        split_brain_harness::prepare_prompt("write me a haiku about the sea", &config).unwrap();
    assert!(
        !system_prompt.contains("CONTEXT REFERENCE PACKS"),
        "benign input must not trigger pack injection"
    );
}