roboticus-api 0.11.3

//! Track 1 integration tests — observability pipeline end-to-end.
//!
//! Verifies that pipeline traces, Flight Recorder (react) traces, and
//! memory health endpoints work correctly with persisted data.

use axum::body::Body;
use axum::http::{Request, StatusCode};
use tower::ServiceExt;

use super::*;

#[tokio::test]
async fn pipeline_trace_round_trip() {
    let state = test_state();

    // Bootstrap the pipeline_traces table (migration 024 + 027 column).
    state
        .db
        .conn()
        .execute_batch(
            "CREATE TABLE IF NOT EXISTS pipeline_traces (
                 id TEXT PRIMARY KEY,
                 turn_id TEXT NOT NULL,
                 session_id TEXT NOT NULL DEFAULT '',
                 channel TEXT NOT NULL DEFAULT '',
                 total_ms INTEGER NOT NULL DEFAULT 0,
                 stages_json TEXT NOT NULL DEFAULT '[]',
                 react_trace_json TEXT,
                 created_at TEXT NOT NULL DEFAULT (datetime('now'))
             );",
        )
        .unwrap();

    // Insert a trace row directly via the DB helper.
    let trace_row = roboticus_db::traces::PipelineTraceRow {
        id: "trace-1".into(),
        turn_id: "turn-abc".into(),
        session_id: "sess-1".into(),
        channel: "api".into(),
        total_ms: 150,
        stages_json: serde_json::json!([
            {"name": "retrieval", "duration_ms": 50, "annotations": {"retrieval.hit_count": 3}},
            {"name": "inference", "duration_ms": 100, "annotations": {}}
        ])
        .to_string(),
        created_at: "2026-03-26T12:00:00Z".into(),
    };
    roboticus_db::traces::save_pipeline_trace(&state.db, &trace_row).unwrap();

    // Hit the GET /api/traces/:turn_id endpoint.
    let app = full_app(state);
    let resp = app
        .oneshot(
            Request::builder()
                .uri("/api/traces/turn-abc")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(resp.status(), StatusCode::OK);
    let body = json_body(resp).await;
    assert_eq!(body["turn_id"], "turn-abc");
    assert_eq!(body["total_ms"], 150);
    assert_eq!(body["stages"].as_array().unwrap().len(), 2);
}

#[tokio::test]
async fn pipeline_trace_exposes_v0110_annotation_namespaces() {
    let state = test_state();

    state
        .db
        .conn()
        .execute_batch(
            "CREATE TABLE IF NOT EXISTS pipeline_traces (
                 id TEXT PRIMARY KEY,
                 turn_id TEXT NOT NULL,
                 session_id TEXT NOT NULL DEFAULT '',
                 channel TEXT NOT NULL DEFAULT '',
                 total_ms INTEGER NOT NULL DEFAULT 0,
                 stages_json TEXT NOT NULL DEFAULT '[]',
                 react_trace_json TEXT,
                 created_at TEXT NOT NULL DEFAULT (datetime('now'))
             );",
        )
        .unwrap();

    let trace_row = roboticus_db::traces::PipelineTraceRow {
        id: "trace-v0110".into(),
        turn_id: "turn-v0110".into(),
        session_id: "sess-1".into(),
        channel: "api".into(),
        total_ms: 220,
        stages_json: serde_json::json!([
            {
                "name": "retrieval",
                "duration_ms": 40,
                "annotations": {
                    "retrieval.retrieval_hit": true,
                    "retrieval.retrieval_count": 3,
                    "retrieval.tier_breakdown": {"working": 1, "semantic": 2}
                }
            },
            {
                "name": "tool_selection",
                "duration_ms": 20,
                "annotations": {
                    "tool_search.candidates_considered": 25,
                    "tool_search.candidates_pruned": 10,
                    "tool_search.token_savings": 900
                }
            },
            {
                "name": "inference",
                "duration_ms": 160,
                "annotations": {
                    "mcp.server": "github",
                    "mcp.tool": "create_issue",
                    "mcp.success": true,
                    "delegation.subtask_count": 2
                }
            }
        ])
        .to_string(),
        created_at: "2026-03-26T12:00:00Z".into(),
    };
    roboticus_db::traces::save_pipeline_trace(&state.db, &trace_row).unwrap();

    let app = full_app(state);
    let resp = app
        .oneshot(
            Request::builder()
                .uri("/api/traces/turn-v0110")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(resp.status(), StatusCode::OK);
    let body = json_body(resp).await;
    let stages = body["stages"].as_array().unwrap();
    assert!(
        stages
            .iter()
            .any(|stage| stage["annotations"]["retrieval.retrieval_hit"] == true)
    );
    assert!(
        stages
            .iter()
            .any(|stage| stage["annotations"]["tool_search.token_savings"] == 900)
    );
    assert!(
        stages
            .iter()
            .any(|stage| stage["annotations"]["mcp.server"] == "github")
    );
    assert!(
        stages
            .iter()
            .any(|stage| stage["annotations"]["delegation.subtask_count"] == 2)
    );
}

#[tokio::test]
async fn react_trace_round_trip() {
    let state = test_state();

    // Bootstrap required tables (migration 024 + 027 column).
    state
        .db
        .conn()
        .execute_batch(
            "CREATE TABLE IF NOT EXISTS pipeline_traces (
                 id TEXT PRIMARY KEY,
                 turn_id TEXT NOT NULL,
                 session_id TEXT NOT NULL DEFAULT '',
                 channel TEXT NOT NULL DEFAULT '',
                 total_ms INTEGER NOT NULL DEFAULT 0,
                 stages_json TEXT NOT NULL DEFAULT '[]',
                 react_trace_json TEXT,
                 created_at TEXT NOT NULL DEFAULT (datetime('now'))
             );",
        )
        .unwrap();

    // Insert a pipeline trace row, then attach the react trace via the DB helper.
    let trace_row = roboticus_db::traces::PipelineTraceRow {
        id: "trace-r1".into(),
        turn_id: "turn-react-1".into(),
        session_id: "sess-1".into(),
        channel: "api".into(),
        total_ms: 100,
        stages_json: "[]".into(),
        created_at: "2026-03-26T12:00:00Z".into(),
    };
    roboticus_db::traces::save_pipeline_trace(&state.db, &trace_row).unwrap();

    let react_json = serde_json::json!({
        "turn_id": "turn-react-1",
        "steps": [
            {"type": "tool_call", "tool_name": "web_search", "duration_ms": 42, "source": "built_in"},
            {"type": "guard", "guard_name": "truth_check", "passed": true}
        ]
    });
    roboticus_db::traces::save_react_trace(&state.db, "trace-r1", &react_json.to_string()).unwrap();

    // Hit the GET /api/traces/:turn_id/react endpoint.
    let app = full_app(state);
    let resp = app
        .oneshot(
            Request::builder()
                .uri("/api/traces/turn-react-1/react")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(resp.status(), StatusCode::OK);
    let body = json_body(resp).await;
    assert_eq!(body["turn_id"], "turn-react-1");
    assert_eq!(body["steps"].as_array().unwrap().len(), 2);
}

#[tokio::test]
async fn trace_not_found_returns_404() {
    let state = test_state();

    // Bootstrap table so the route doesn't error on a missing table.
    state
        .db
        .conn()
        .execute_batch(
            "CREATE TABLE IF NOT EXISTS pipeline_traces (
                 id TEXT PRIMARY KEY,
                 turn_id TEXT NOT NULL,
                 session_id TEXT NOT NULL DEFAULT '',
                 channel TEXT NOT NULL DEFAULT '',
                 total_ms INTEGER NOT NULL DEFAULT 0,
                 stages_json TEXT NOT NULL DEFAULT '[]',
                 react_trace_json TEXT,
                 created_at TEXT NOT NULL DEFAULT (datetime('now'))
             );",
        )
        .unwrap();

    let app = full_app(state);
    let resp = app
        .oneshot(
            Request::builder()
                .uri("/api/traces/nonexistent-turn")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(resp.status(), StatusCode::NOT_FOUND);
}

#[tokio::test]
async fn react_trace_not_found_returns_404() {
    let state = test_state();

    // Bootstrap table so the route doesn't error on a missing table.
    state
        .db
        .conn()
        .execute_batch(
            "CREATE TABLE IF NOT EXISTS pipeline_traces (
                 id TEXT PRIMARY KEY,
                 turn_id TEXT NOT NULL,
                 session_id TEXT NOT NULL DEFAULT '',
                 channel TEXT NOT NULL DEFAULT '',
                 total_ms INTEGER NOT NULL DEFAULT 0,
                 stages_json TEXT NOT NULL DEFAULT '[]',
                 react_trace_json TEXT,
                 created_at TEXT NOT NULL DEFAULT (datetime('now'))
             );",
        )
        .unwrap();

    let app = full_app(state);
    let resp = app
        .oneshot(
            Request::builder()
                .uri("/api/traces/nonexistent-turn/react")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(resp.status(), StatusCode::NOT_FOUND);
}

#[tokio::test]
async fn memory_analytics_reports_retrieval_utilization() {
    let state = test_state();
    let sid =
        roboticus_db::sessions::create_new(&state.db, "memory-analytics-agent", None).unwrap();
    roboticus_db::sessions::create_turn_with_id(
        &state.db,
        "memory-analytics-turn-1",
        &sid,
        Some("ollama-gpu/qwen3.5:35b-a3b"),
        Some(320),
        Some(140),
        Some(0.0),
    )
    .unwrap();
    roboticus_db::sessions::upsert_context_snapshot(
        &state.db,
        roboticus_db::sessions::UpsertContextSnapshotInput {
            turn_id: "memory-analytics-turn-1",
            complexity_level: "L2",
            token_budget: 1000,
            system_prompt_tokens: Some(200),
            memory_tokens: Some(240),
            history_tokens: Some(300),
            history_depth: Some(4),
            memory_tiers_json: Some(r#"{"working":2,"semantic":1}"#),
            retrieved_memories_json: Some(
                r#"{"retrieval_count":3,"retrieval_hit":true,"avg_similarity":0.91,"budget_utilization":0.24}"#,
            ),
            model: Some("ollama-gpu/qwen3.5:35b-a3b"),
        },
    )
    .unwrap();

    let app = full_app(state);
    let resp = app
        .oneshot(
            Request::builder()
                .uri("/api/stats/memory-analytics?period=24h")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(resp.status(), StatusCode::OK);
    let body = json_body(resp).await;
    assert_eq!(body["period_hours"], 24);
    assert_eq!(body["total_turns"], 1);
    assert_eq!(body["retrieval_hits"], 1);
    assert_eq!(body["total_entries_retrieved"], 240);
    assert_eq!(body["avg_budget_utilization"], 0.24);
    assert_eq!(body["tier_distribution"]["working"], 2);
    assert_eq!(body["tier_distribution"]["semantic"], 1);
}