trusty-console 0.12.2

//! Axum HTTP server for the trusty-console.
//!
//! Why: The console needs a lightweight HTTP server that serves the embedded
//! SPA, a JSON API route for service status, and a reverse-proxy layer for
//! all daemon sub-paths.
//! What: Builds an axum `Router` with:
//!   - `GET /health` — liveness probe.
//!   - `GET /api/console/services` — return cached snapshot (background poll).
//!   - `GET /api/console/metrics/{analyze,memory,search,review,mpm}` — MCP-polled metrics.
//!   - `POST /api/webhooks/{source}` — GitHub webhook ingress: verify once,
//!     spool durably, relay over UDS (#5089 step 3, ADR-0034). Mounted only by
//!     [`build_router_with_webhooks`].
//!   - `GET /api/console/metrics/webhooks` — oldest-pending spool age as a red
//!     health state.
//!   - `DELETE /api/console/memory/palaces/{id}` — delete one palace via
//!     trusty-memory's `palace_delete` on its socket (#6360).
//!   - `DELETE /api/console/search/indexes/{id}` — delete one index via
//!     trusty-search's own `search.index.delete` on its socket (#6360, #6285).
//!   - `GET /api/console/metrics/analyze/indexes` — analyze index list via stdio MCP.
//!   - `GET /api/console/metrics/analyze/visualize?index=<id>` — graph+entities+clusters.
//!   - `…/api/console/sessions/*` — the single HTTP front door for the trusty-mpm
//!     session manager (#1222); handlers live in `crate::routes::sessions`.
//!   - `ANY /api/search/{*path}` — translate the request into a trusty-search
//!     RPC call on its socket (#6285); see `crate::search_uds`.
//!   - `ANY /api/memory/{*path}` — the same for trusty-memory (#6155); see
//!     `crate::memory_uds`.
//!   - `ANY /api/{service}/{*path}` — reverse-proxy to live daemon via clean path
//!     (#1849 Phase 2); `{service}` ∈ {review, mpm, agents}.
//!   - `ANY /proxy/{daemon}/{*path}` — DEPRECATED alias; routes to the same
//!     handler with a trace-level deprecation note.
//!   - `GET /` and `GET /ui/*path` — serve the embedded Svelte SPA; the
//!     handlers live in `crate::console_ui` (#6285, 500-SLOC split).
//!   - `GET /tools/search/*path` — serve the embedded trusty-search SPA
//!     (#6155); see `crate::tools_ui`.
//!   - `GET /tools/memory/*path` — the same for the trusty-memory SPA (#6155).
//!
//! All logs go to stderr; stdout is clean.
//!
//! Test: The `tests` module starts the router in a real axum test client.

use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};

use axum::{
    extract::{Query, State},
    http::StatusCode,
    response::IntoResponse,
};
use serde::Deserialize;
use serde_json::json;

use crate::connector::{ServiceConnector, ServiceInfo, ServiceStatus};
use crate::mcp_handle::{McpHandleError, McpServiceHandle};
use crate::metrics_poller::MetricsCache;
use crate::poller::PollerCache;

// ─── app state ───────────────────────────────────────────────────────────────

/// Shared application state injected into every route handler.
///
/// Why: Connectors, the poller cache, metrics caches, and HTTP client are
/// created once at startup and reused for every request so there is no per-
/// request allocation. A separate `MetricsCache` is maintained for each
/// stdio-MCP-polled service (analyze, memory, search, review) so they can be
/// updated independently and served without coupling. `analyze_handle` is held
/// in Arc so the on-demand visualize/index routes can call the analyze stdio MCP
/// without going through the /proxy path.
/// `mcp_handles` maps each service id to its `McpServiceHandle` so the
/// services route can overlay the connector-reported status with the actual
/// tools/list probe result (Degraded when `console_metrics` is absent).
/// What: Wraps the connector list, poller cache, per-service metrics caches,
/// reqwest client, the analyze MCP handle, and the full handle map in `Arc`s
/// for cheap cloning.
/// Test: Constructed in `build_router`; exercised by the integration tests.
#[derive(Clone)]
pub struct AppState {
    connectors: Arc<Vec<Box<dyn ServiceConnector>>>,
    poller_cache: PollerCache,
    metrics_cache: MetricsCache,
    memory_metrics_cache: MetricsCache,
    search_metrics_cache: MetricsCache,
    review_metrics_cache: MetricsCache,
    /// trusty-mpm `console_metrics` cache (#1222). Populated by the background
    /// poller; served by `GET /api/console/metrics/mpm`.
    mpm_metrics_cache: MetricsCache,
    /// Whole-machine host-metrics cache (#6517). Populated by the background
    /// host sampler (`crate::host_status`); served by
    /// `GET /api/console/machine-status`.
    host_metrics_cache: crate::host_status::HostMetricsCache,
    /// Bounded 10-minute sample window + per-service transition log (#6641).
    /// Written by `machine_history::sampler`; served by
    /// `GET /api/console/machine-status/history` and its SSE stream. In-memory
    /// only — a restart begins an empty window by design (owner ruling, #6516).
    machine_history: crate::machine_history::MachineHistory,
    http_client: Arc<reqwest::Client>,
    /// The client the proxy uses for Server-Sent Events (#6155).
    ///
    /// Why: `http_client` sets a 30-second whole-request timeout, which is
    /// right for a request/response API call and fatal for a stream that is
    /// meant to stay open — the console-served search SPA opens
    /// `/status/stream` on every page load, and under the shared client that
    /// connection would be cut every 30 seconds and reopened by `EventSource`
    /// forever.
    /// What: identical to `http_client` except the total timeout is replaced by
    /// a 60-second read timeout, which bounds an upstream that has gone silent
    /// without bounding one that is still sending. Both search streams stay
    /// well inside it: `/status/stream` pushes every 2 seconds and
    /// `/reindex/stream` heartbeats every 20.
    ///
    /// The absent total deadline is why `proxy_handler` does not hand this
    /// client's body straight to the caller. `Accept` is a caller claim, so a
    /// response the upstream did not label `text/event-stream` is read under
    /// `NON_STREAM_BODY_TIMEOUT` instead — otherwise any proxied GET could opt
    /// into an unbounded connection by naming that Accept type.
    /// Test: `proxy::routes::tests::test_wants_event_stream_*` covers which
    /// client a request gets; the timeout itself is construction, exercised by
    /// the live `/status/stream` smoke run recorded on #6155.
    stream_client: Arc<reqwest::Client>,
    /// Analyze stdio MCP handle — shared with the metrics poller so both the
    /// background poll and on-demand route calls reuse the same child process.
    analyze_handle: Arc<McpServiceHandle>,
    /// All per-service MCP handles keyed by service id.
    ///
    /// Why: The services route reads each handle's degraded state to override
    /// the connector-reported status when a reachable service is missing
    /// `console_metrics`. Using a HashMap avoids adding individual Arc fields
    /// for every future service.
    /// What: Populated by `AppState::new`; read by `apply_handle_overrides`.
    mcp_handles: Arc<HashMap<String, Arc<McpServiceHandle>>>,
    /// Override for the trusty-search socket path (#6285).
    ///
    /// Why: `search_uds` and `detect::SearchConnector` resolve that path through
    /// `trusty_common::daemon_socket_path`, which reads the process-global
    /// `TRUSTY_DATA_DIR_OVERRIDE`. A test that redirected it would redirect five
    /// sibling connectors running in the same binary at the same time, so the
    /// override is carried here instead — the same argument
    /// `detect::AnalyzeConnector::with_socket` records.
    /// What: `None` in production, which resolves the real path.
    /// Test: `tests/search_uds_bridge.rs` sets it on every case.
    pub(crate) search_socket: Option<Arc<PathBuf>>,
    /// Override for the trusty-memory socket path (#6155).
    ///
    /// Why: the same argument [`AppState::search_socket`] records — resolution
    /// runs through `trusty_common::daemon_socket_path`, which reads the
    /// process-global `TRUSTY_DATA_DIR_OVERRIDE`, and a test that redirected it
    /// would redirect every sibling connector in the same binary at once.
    /// What: `None` in production, which resolves the real path.
    /// Test: `tests/memory_uds_bridge.rs` sets it on every case.
    pub(crate) memory_socket: Option<Arc<PathBuf>>,
    /// Override for the trusty-analyze socket path (#6155).
    ///
    /// Why: the same argument [`AppState::search_socket`] records — resolution
    /// runs through `trusty_common::daemon_socket_path`, which reads the
    /// process-global `TRUSTY_DATA_DIR_OVERRIDE`, and a test that redirected it
    /// would redirect every sibling connector in the same binary at once.
    /// What: `None` in production, which resolves the real path.
    /// Test: `tests/analyze_uds_bridge.rs` sets it on every case.
    pub(crate) analyze_socket: Option<Arc<PathBuf>>,
    /// When this server's state was built, which is when the console started
    /// serving (#6908).
    ///
    /// Why an `Instant` on the state rather than a process-global: the console's
    /// own Services row now opens a details pane that reports how long the
    /// console has been up, and `/health` is the route that already answers
    /// "which console is this". A `OnceLock` would be the global state this
    /// workspace does not keep, and reading the process start time back out of
    /// the OS would be a second answer to a question construction already
    /// settled.
    /// What: set once in [`AppState::new`]; read only by [`health_handler`].
    /// Test: `test_health_route`, `health_uptime_is_monotonic`.
    started_at: Instant,
}

impl AppState {
    /// Create a new `AppState` from a list of connectors.
    ///
    /// Why: Lets tests inject a custom connector list and fresh caches.
    /// What: Wraps `connectors` in `Arc`; initialises empty `PollerCache`,
    /// three `MetricsCache` instances (analyze / memory / search), and a
    /// `reqwest::Client` with idle-connection pooling disabled (#1984 — see the
    /// builder comment below). Creates the analyze stdio MCP handle that is
    /// shared between the background metrics poller and on-demand routes.
    /// Populates `mcp_handles` with all three per-service handles so the
    /// services route can read their degraded state.
    /// Test: Used in `build_router` and directly in `tests`.
    pub fn new(connectors: Vec<Box<dyn ServiceConnector>>) -> Self {
        // Why pool_max_idle_per_host(0): the proxy client must survive an upstream
        // daemon restart (#1984). With the default keep-alive pool, the FIRST
        // proxied request after an upstream restart reuses a stale idle connection
        // to the now-dead process and fails — an instant RST → 502, a half-open
        // hang → 30s-timeout → 502, or a partial write the restarted daemon
        // rejects → 500 — even though a direct curl (which never pools across
        // invocations) always opens a fresh connection and succeeds. reqwest does
        // NOT retry a non-idempotent POST on a broken pooled connection, so the
        // failure is surfaced to the caller (e.g. `tm session new`). Disabling
        // idle-connection reuse forces every proxied request to open a fresh
        // connection to whatever process currently owns the port, eliminating the
        // stale-reuse failure at the root. Loopback connect cost is negligible.
        // #6360: reqwest follows up to 10 redirects by default, and every
        // loopback check in this crate — the proxy's `is_local_upstream`, the
        // delete routes' reuse of it — validates only the URL it was handed. A
        // 3xx from an upstream would re-issue the request, body and method
        // intact, at whatever host the `Location` names, which for a DELETE
        // means a destructive call to an address nothing checked. Refusing to
        // follow leaves the 3xx as the response: the proxy hands it to the
        // browser (which is what a reverse proxy should do with a redirect the
        // upstream chose) and the delete routes read it as a non-2xx refusal.
        // Nothing in this crate relied on following one.
        let client = reqwest::Client::builder()
            .timeout(Duration::from_secs(30))
            .pool_max_idle_per_host(0)
            .redirect(reqwest::redirect::Policy::none())
            .build()
            .expect("reqwest client init");
        // #6155: same connection policy, no whole-request deadline — see the
        // `stream_client` field doc.
        let stream_client = reqwest::Client::builder()
            .read_timeout(Duration::from_secs(60))
            .pool_max_idle_per_host(0)
            .redirect(reqwest::redirect::Policy::none())
            .build()
            .expect("reqwest stream client init");
        let analyze_handle = Arc::new(McpServiceHandle::new(
            "trusty-analyze",
            vec!["mcp".to_string()],
        ));
        let memory_handle = Arc::new(McpServiceHandle::new(
            "trusty-memory",
            vec!["serve".to_string(), "--stdio".to_string()],
        ));
        let search_handle = Arc::new(McpServiceHandle::new(
            "trusty-search",
            vec!["serve".to_string()],
        ));
        // Why: trusty-review's stdio MCP mode is `serve --stdio` (see ServeArgs
        // in commands/serve.rs). This is the canonical command the console spawns
        // to poll `console_metrics` without requiring the HTTP daemon to be running.
        let review_handle = Arc::new(McpServiceHandle::new(
            "trusty-review",
            vec!["serve".to_string(), "--stdio".to_string()],
        ));
        // Why: trusty-mpm's stdio MCP mode is `serve --stdio` (the #1221 bridge
        // that auto-starts the durable daemon and forwards JSON-RPC to its
        // loopback POST /rpc). The console spawns this to render the Sessions tab
        // natively (#1222) without ever touching the daemon's HTTP port (#1104).
        let mpm_handle = Arc::new(McpServiceHandle::new(
            "trusty-mpm",
            vec!["serve".to_string(), "--stdio".to_string()],
        ));
        let mut handles: HashMap<String, Arc<McpServiceHandle>> = HashMap::new();
        handles.insert("trusty-analyze".to_string(), Arc::clone(&analyze_handle));
        handles.insert("trusty-memory".to_string(), Arc::clone(&memory_handle));
        handles.insert("trusty-search".to_string(), Arc::clone(&search_handle));
        handles.insert("trusty-review".to_string(), Arc::clone(&review_handle));
        handles.insert("trusty-mpm".to_string(), Arc::clone(&mpm_handle));
        Self {
            connectors: Arc::new(connectors),
            poller_cache: PollerCache::new(),
            metrics_cache: MetricsCache::new(),
            memory_metrics_cache: MetricsCache::new(),
            search_metrics_cache: MetricsCache::new(),
            review_metrics_cache: MetricsCache::new(),
            mpm_metrics_cache: MetricsCache::new(),
            host_metrics_cache: crate::host_status::HostMetricsCache::new(),
            machine_history: crate::machine_history::MachineHistory::new(),
            http_client: Arc::new(client),
            stream_client: Arc::new(stream_client),
            analyze_handle,
            mcp_handles: Arc::new(handles),
            search_socket: None,
            memory_socket: None,
            analyze_socket: None,
            started_at: Instant::now(),
        }
    }

    /// Access the per-service MCP handle map.
    ///
    /// Why: The services route reads handles from this map to overlay connector
    /// statuses with the tools/list probe result.
    /// What: Returns a clone of the `Arc<HashMap>` (cheap).
    /// Test: Used by `apply_handle_overrides` and the services handler.
    pub fn mcp_handles(&self) -> Arc<HashMap<String, Arc<McpServiceHandle>>> {
        Arc::clone(&self.mcp_handles)
    }

    /// Access the shared analyze MCP handle.
    ///
    /// Why: On-demand routes (`/api/console/metrics/analyze/indexes`,
    /// `/api/console/metrics/analyze/visualize`) call the analyze stdio MCP
    /// without touching the analyze daemon HTTP directly (architecture: console
    /// is a stdio MCP client only, per #1104).
    /// What: Returns a clone of the `Arc<McpServiceHandle>` (cheap).
    /// Test: Exercised by the analyze index and visualize route tests.
    pub fn analyze_handle(&self) -> Arc<McpServiceHandle> {
        Arc::clone(&self.analyze_handle)
    }

    /// Access the shared connector list.
    ///
    /// Why: The background poller and the fallback `spawn_blocking` path both
    /// need the connector list.
    /// What: Returns a clone of the `Arc` (cheap).
    /// Test: Used by `run_serve` in `main.rs`.
    pub fn connectors(&self) -> Arc<Vec<Box<dyn ServiceConnector>>> {
        Arc::clone(&self.connectors)
    }

    /// Access the background poll cache.
    ///
    /// Why: Routes read from the cache; the background task writes to it.
    /// What: Returns a clone of the `PollerCache` handle (cheap — it's an Arc).
    /// Test: Used by `services_handler` and `proxy_handler`.
    pub fn poller_cache(&self) -> &PollerCache {
        &self.poller_cache
    }

    /// Access the metrics cache for the trusty-analyze stdio MCP poller.
    ///
    /// Why: The metrics poller writes `ConsoleMetricsReport`s here; the
    /// `/api/console/metrics/analyze` route reads from it.
    /// What: Returns a reference to the `MetricsCache` handle.
    /// Test: `test_metrics_analyze_route_cold_cache_returns_503`.
    pub fn metrics_cache(&self) -> &MetricsCache {
        &self.metrics_cache
    }

    /// Access the metrics cache for the trusty-memory stdio MCP poller.
    ///
    /// Why: Separate cache per service so memory and analyze reports can be
    /// updated and served independently.
    /// What: Returns a reference to the `MetricsCache` handle for memory.
    /// Test: `test_metrics_memory_route_cold_cache_returns_503`.
    pub fn memory_metrics_cache(&self) -> &MetricsCache {
        &self.memory_metrics_cache
    }

    /// Access the metrics cache for the trusty-search stdio MCP poller.
    ///
    /// Why: Separate cache per service so search and analyze reports can be
    /// updated and served independently.
    /// What: Returns a reference to the `MetricsCache` handle for search.
    /// Test: `test_metrics_search_route_cold_cache_returns_503`.
    pub fn search_metrics_cache(&self) -> &MetricsCache {
        &self.search_metrics_cache
    }

    /// Access the metrics cache for the trusty-review stdio MCP poller.
    ///
    /// Why: Separate cache per service so review reports can be updated and
    /// served independently from the other service caches.
    /// What: Returns a reference to the `MetricsCache` handle for review.
    /// Test: `test_metrics_review_route_cold_cache_returns_503`.
    pub fn review_metrics_cache(&self) -> &MetricsCache {
        &self.review_metrics_cache
    }

    /// Access the metrics cache for the trusty-mpm stdio MCP poller (#1222).
    ///
    /// Why: separate cache per service so the mpm session/supervisor report can
    /// be updated and served independently from the other service caches.
    /// What: returns a reference to the `MetricsCache` handle for mpm.
    /// Test: `test_metrics_mpm_route_cold_cache_returns_503`.
    pub fn mpm_metrics_cache(&self) -> &MetricsCache {
        &self.mpm_metrics_cache
    }

    /// Access the whole-machine host-metrics cache (#6517).
    ///
    /// Why: the background host sampler writes here; the machine-status route
    /// reads it. `run_serve` clones it to start the sampler.
    /// What: returns a reference to the `HostMetricsCache` handle.
    /// Test: `machine_status_route_cold_cache_returns_503`.
    pub fn host_metrics_cache(&self) -> &crate::host_status::HostMetricsCache {
        &self.host_metrics_cache
    }

    /// Access the bounded machine-status history (#6641).
    ///
    /// Why: `machine_history::sampler` writes every sample and every service
    /// transition here; the history endpoint and the SSE stream read it. One
    /// handle in the state means an open stream and a fresh page load are
    /// looking at the same buffer.
    /// What: returns a reference to the `MachineHistory` handle (cheap to clone).
    /// Test: `machine_history_route_cold_cache_returns_empty_200`.
    pub fn machine_history(&self) -> &crate::machine_history::MachineHistory {
        &self.machine_history
    }

    /// Gather whichever per-service `ConsoleMetricsReport`s are currently cached
    /// (#6517).
    ///
    /// Why: the machine-status rollup counts and lists every service that has a
    /// warm report. A service whose cache is still `None` (binary absent or not
    /// yet polled) is simply omitted — the rollup describes what is reachable.
    /// What: reads all five per-service metrics caches and collects the `Some`
    /// reports into a `Vec` in a stable service order.
    /// Test: `machine_status_route_warm_cache_returns_json`.
    pub async fn collect_service_reports(
        &self,
    ) -> Vec<trusty_common::console_metrics::ConsoleMetricsReport> {
        let caches = [
            &self.metrics_cache,
            &self.memory_metrics_cache,
            &self.search_metrics_cache,
            &self.review_metrics_cache,
            &self.mpm_metrics_cache,
        ];
        let mut reports = Vec::with_capacity(caches.len());
        for cache in caches {
            if let Some(report) = cache.get().await {
                reports.push(report);
            }
        }
        reports
    }

    /// Access the shared `reqwest::Client`.
    ///
    /// Why: Re-using one client enables connection pooling across proxy requests.
    /// What: Returns a clone of the `Arc<reqwest::Client>` (cheap).
    /// Test: Used by `proxy_handler`.
    pub fn http_client(&self) -> Arc<reqwest::Client> {
        Arc::clone(&self.http_client)
    }

    /// The client to proxy a Server-Sent Events request with (#6155).
    pub fn stream_client(&self) -> Arc<reqwest::Client> {
        Arc::clone(&self.stream_client)
    }
}

// ─── router ─────────────────────────────────────────────────────────────────
//
// #6155: the router moved to `router.rs` when the memory dashboard's five
// routes pushed this file over the 500-SLOC cap. The re-export keeps every
// call site — `main`, `lib::run_serve`, and every test — unchanged.
mod router;
pub use router::{build_router, build_router_with_self_origins, build_router_with_webhooks};

// ─── handlers ────────────────────────────────────────────────────────────────

/// `GET /health` — liveness probe.
///
/// Why: Required by process monitors and the `trusty-console status` CLI
/// subcommand. Returns a minimal JSON body so callers can confirm the server
/// is up and which version is running. #6908 added `uptime_secs` because the
/// console's own details pane reports how long the console has been up, and
/// this is the route the SPA already reads for the version beside it.
/// What: Returns `{"status":"ok","version":"<v>","uptime_secs":<n>}`, where the
/// uptime is whole seconds since [`AppState::new`] built this state. Additive:
/// a caller that only reads `status` and `version` is unaffected.
/// Test: Tested by `test_health_route` and `health_uptime_is_monotonic` below.
async fn health_handler(State(state): State<AppState>) -> impl IntoResponse {
    axum::Json(json!({
        "status": "ok",
        "version": env!("CARGO_PKG_VERSION"),
        "uptime_secs": state.started_at.elapsed().as_secs(),
    }))
}

/// Apply per-service MCP handle state on top of connector-reported statuses.
///
/// Why: The connector `detect()` path (TCP probe / `which`) can only report
/// `Running`, `Available`, or `Absent`. It has no knowledge of the MCP
/// `tools/list` probe result.  When a service is reachable but the
/// `console_metrics` tool is absent (`HandleState::Degraded`), the connector
/// still reports `Running` or `Available` — the UI incorrectly shows a healthy
/// badge. This function overlays the handle's known state: if a handle is
/// Degraded, the corresponding `ServiceInfo` is updated in-place to
/// `status = Degraded` and `hint = DEGRADED_HINT`. If a handle is Connected, the
/// daemon version from the `initialize` response is surfaced (unless the connector
/// already reported a version from the HTTP `/health` endpoint).
/// What: Iterates `infos` in place; for each entry looks up the matching handle
/// by `id`. If `handle.degraded_hint()` returns `Some(hint)` and the current
/// status is NOT already `Absent`, sets `status = Degraded` and `hint = Some`.
/// If `info.version` is `None` and `handle.daemon_version()` returns `Some`,
/// sets `info.version` from the MCP `serverInfo.version`.
/// A process-down (`Absent`) service is never overridden — only reachable ones.
/// Skipping only `Absent` is safe: `Available` handles always return `None`
/// from `degraded_hint` (no tools/list probe runs until the first poll), so
/// they pass through unchanged.
/// Test: `test_services_route_handle_degraded_overlay` and
/// `test_services_route_daemon_version_overlay` below.
async fn apply_handle_overrides(
    infos: &mut [ServiceInfo],
    handles: &HashMap<String, Arc<McpServiceHandle>>,
) {
    for info in infos.iter_mut() {
        if info.status == ServiceStatus::Absent {
            continue;
        }
        if let Some(handle) = handles.get(&info.id) {
            if let Some(hint) = handle.degraded_hint().await {
                info.status = ServiceStatus::Degraded;
                info.hint = Some(hint);
            }
            // Surface the MCP daemon version when the connector hasn't
            // already provided one (e.g. when the HTTP daemon isn't running
            // but the stdio MCP process is up and has responded to initialize).
            if info.version.is_none()
                && let Some(ver) = handle.daemon_version().await
            {
                info.version = Some(ver);
            }
        }
    }
}

/// `GET /api/console/services` — return cached snapshot of all services.
///
/// Why: The Svelte SPA fetches this endpoint on load to render service cards.
///      With the background poller in place the response is instant (no per-
///      request TCP probes).
/// What: Reads the latest `CachedSnapshot` from the `PollerCache`. If the first
/// poll has not completed yet, falls back to a synchronous on-demand detection
/// so the UI always gets data (the first-boot latency is acceptable; after that
/// every response is cache-backed).  A panic in the fallback blocking task
/// surfaces as HTTP 500 rather than an empty 200.
/// After obtaining the base service list (from cache or fallback), applies
/// per-service handle degraded overrides via `apply_handle_overrides` so
/// reachable services missing `console_metrics` surface as `status: degraded`,
/// then sorts the list with `detect::order_for_display` so the Overview grid
/// leads with the services that are actually live (#6370).
/// Test: `test_services_route_returns_json`,
/// `test_services_handler_returns_500_on_panic`,
/// `test_services_route_handle_degraded_overlay`, and
/// `test_services_route_orders_running_before_absent` below.
async fn services_handler(State(state): State<AppState>) -> axum::response::Response {
    let handles = state.mcp_handles();

    if let Some(snap) = state.poller_cache().snapshot().await {
        let mut services = snap.services;
        apply_handle_overrides(&mut services, &handles).await;
        crate::service_metrics::apply_metrics_overlay(&mut services, state.machine_history()).await; // #6642
        // #6370: sort AFTER the overrides so a service demoted to Degraded here
        // ranks as degraded, not as the Running the poller recorded.
        crate::detect::order_for_display(&mut services);
        return axum::Json(services).into_response();
    }

    // First-boot fallback: run a one-shot detection synchronously.
    let connectors = state.connectors();
    match tokio::task::spawn_blocking(move || {
        connectors.iter().map(|c| c.detect()).collect::<Vec<_>>()
    })
    .await
    {
        Ok(mut infos) => {
            apply_handle_overrides(&mut infos, &handles).await;
            crate::service_metrics::apply_metrics_overlay(&mut infos, state.machine_history())
                .await; // #6642
            crate::detect::order_for_display(&mut infos); // #6370
            axum::Json(infos).into_response()
        }
        Err(e) => {
            tracing::error!("service detection task panicked: {e}");
            StatusCode::INTERNAL_SERVER_ERROR.into_response()
        }
    }
}

/// Query params for the analyze visualize route.
///
/// Why: The index id must be a query param so the Svelte component can change
/// the selected index without a page navigation.
/// What: `index` is the analyze index id (string). Optional: no default —
/// returns 400 when absent.
/// Test: `test_analyze_visualize_handler_no_index_returns_400` below.
#[derive(Deserialize)]
struct VisualizeQuery {
    index: Option<String>,
}

/// `GET /api/console/metrics/analyze/indexes` — list analyze indexes via stdio.
///
/// Why: The Analyze tab needs a list of indexes to populate the dropdown.
/// This route calls the analyze stdio MCP (via `McpServiceHandle::call_tool_checked`)
/// instead of the browser hitting the analyze daemon HTTP directly, honouring
/// the #1104 architecture principle: the console is a stdio MCP client only.
/// Using `call_tool_checked` instead of `call_tool_raw` prevents a raw -32601
/// JSON-RPC error from reaching the browser as a 502 when the stale daemon lacks
/// the `list_analyze_indexes` tool — the capability-gate returns `ToolUnavailable`
/// which maps to a clean 503 with an actionable hint.
/// What: Calls the `list_analyze_indexes` MCP tool (which proxies `GET /indexes`
/// on the daemon). Returns the JSON array on 200, 503+hint when the analyze binary
/// is absent, in backoff, degraded, or the tool is not in the cached tool set;
/// 502 on any other error.
/// Test: `test_analyze_indexes_absent_binary_returns_503` and
/// `test_analyze_indexes_tool_unavailable_returns_degraded_hint` below.
async fn analyze_indexes_handler(State(state): State<AppState>) -> axum::response::Response {
    match state
        .analyze_handle()
        .call_tool_checked("list_analyze_indexes", serde_json::json!({}))
        .await
    {
        Ok(val) => axum::Json(val).into_response(),
        Err(McpHandleError::ToolUnavailable { tool, hint }) => {
            tracing::warn!(
                tool = %tool,
                hint = %hint,
                "analyze_indexes_handler: tool not available — capability-gate triggered"
            );
            (
                StatusCode::SERVICE_UNAVAILABLE,
                axum::Json(serde_json::json!({
                    "status": "degraded",
                    "hint": hint,
                })),
            )
                .into_response()
        }
        Err(
            McpHandleError::Absent
            | McpHandleError::Backoff { .. }
            | McpHandleError::Degraded { .. },
        ) => StatusCode::SERVICE_UNAVAILABLE.into_response(),
        Err(e) => {
            tracing::warn!("analyze_indexes_handler error: {e:#}");
            StatusCode::BAD_GATEWAY.into_response()
        }
    }
}

/// `GET /api/console/metrics/analyze/visualize?index=<id>` — combined viz data.
///
/// Why: The Analyze tab needs graph nodes, entities, and clusters in one round
/// trip. This route calls the analyze stdio MCP for all three without the
/// browser hitting the analyze daemon HTTP directly (#1104 architecture).
/// Using `call_tool_checked` prevents a raw -32601 from reaching the browser
/// as a 502 when a stale daemon lacks `extract_graph`/`list_entities`/
/// `cluster_concepts` — the capability-gate returns `ToolUnavailable` which maps
/// to a clean 503+hint response.
/// What: Calls `extract_graph`, `list_entities`, and `cluster_concepts` (k=8)
/// via `McpServiceHandle::call_tool_checked` and returns a combined JSON object:
/// `{"graph": ..., "entities": ..., "clusters": ...}`. Missing index param
/// returns 400 (BAD_REQUEST). Absent binary, backoff, degraded, or tool
/// unavailable returns 503 (SERVICE_UNAVAILABLE) with optional hint JSON.
/// A hard graph error (non-absent/backoff/tool-unavailable) returns 502 (BAD_GATEWAY).
/// Test: `test_analyze_visualize_handler_no_index_returns_400` and
/// `test_analyze_visualize_handler_absent_binary_returns_503` below.
async fn analyze_visualize_handler(
    State(state): State<AppState>,
    Query(params): Query<VisualizeQuery>,
) -> axum::response::Response {
    let index_id = match params.index {
        Some(id) if !id.is_empty() => id,
        _ => {
            return (
                StatusCode::BAD_REQUEST,
                axum::Json(json!({"error": "missing required query param: index"})),
            )
                .into_response();
        }
    };

    let handle = state.analyze_handle();
    let args = serde_json::json!({ "index_id": index_id });

    // NOTE: although `tokio::join!` normally drives all three futures
    // concurrently, these three `call_tool_checked` calls share a single stdio
    // child process behind `McpServiceHandle`'s inner `Arc<Mutex<StdioMcpClient>>`.
    // Each call acquires that inner mutex for the full duration of its
    // JSON-RPC round trip, so the three futures effectively serialize behind
    // the lock — `join!` does not provide real I/O parallelism here. The
    // `join!` form is retained for code readability (all three results
    // collected symmetrically) and because the serialization is transparent
    // to callers. If the analyze MCP child ever supports multiplexed requests
    // (separate stdin/stdout framing per call), this join would gain true
    // concurrency automatically without changing the call sites.
    let (graph_res, entities_res, clusters_res) = tokio::join!(
        handle.call_tool_checked("extract_graph", args.clone()),
        handle.call_tool_checked("list_entities", args.clone()),
        handle.call_tool_checked("cluster_concepts", {
            let mut a = args.clone();
            if let Some(m) = a.as_object_mut() {
                m.insert("k".to_string(), serde_json::json!(8));
            }
            a
        }),
    );

    // Classify the graph result: tool unavailable → 503+hint, absent/backoff/degraded → 503,
    // hard error → 502, success → combine with best-effort entities and clusters.
    match &graph_res {
        Err(McpHandleError::ToolUnavailable { tool, hint }) => {
            tracing::warn!(
                tool = %tool,
                hint = %hint,
                "analyze_visualize_handler: tool not available — capability-gate triggered"
            );
            return (
                StatusCode::SERVICE_UNAVAILABLE,
                axum::Json(serde_json::json!({
                    "status": "degraded",
                    "hint": hint,
                })),
            )
                .into_response();
        }
        Err(
            McpHandleError::Absent
            | McpHandleError::Backoff { .. }
            | McpHandleError::Degraded { .. },
        ) => {
            return StatusCode::SERVICE_UNAVAILABLE.into_response();
        }
        Err(e) => {
            tracing::warn!("analyze_visualize_handler graph error: {e:#}");
            return StatusCode::BAD_GATEWAY.into_response();
        }
        Ok(_) => {}
    }

    // Log a warning when a best-effort tool is missing (e.g. stale daemon that
    // predates list_entities or cluster_concepts).  We do NOT return 503 here —
    // these two are genuinely best-effort and the route still returns a useful
    // partial payload.  The primary `extract_graph` gate above is the hard 503
    // path; these are only observable degradation signals.
    if let Err(McpHandleError::ToolUnavailable { tool, .. }) = &entities_res {
        tracing::warn!(
            tool = %tool,
            "analyze_visualize_handler: list_entities tool unavailable — returning partial payload"
        );
    }
    if let Err(McpHandleError::ToolUnavailable { tool, .. }) = &clusters_res {
        tracing::warn!(
            tool = %tool,
            "analyze_visualize_handler: cluster_concepts tool unavailable — returning partial payload"
        );
    }

    let combined = json!({
        "graph":    graph_res.unwrap_or(serde_json::Value::Null),
        "entities": entities_res.unwrap_or(serde_json::Value::Null),
        "clusters": clusters_res.unwrap_or(serde_json::Value::Null),
    });
    axum::Json(combined).into_response()
}

// ─── tests ───────────────────────────────────────────────────────────────────

// ─── tests ───────────────────────────────────────────────────────────────────

#[cfg(test)]
mod tests;