subc-daemon 0.41.0

Embeddable subc daemon: bootstrap, module supervision, and opaque-byte splice routing.
Documentation
struct BusyRouterRig {
    router: Arc<Router>,
    client: RouteCtx,
    client_rx: mpsc::Receiver<OutboundFrame>,
    module: RouteCtx,
    module_rx: mpsc::Receiver<OutboundFrame>,
    pending: crate::forwarding::PendingRouteBindRelay,
    _module_egress_rx: mpsc::Receiver<OutboundFrame>,
}

fn busy_router_rig(opted_in: bool) -> BusyRouterRig {
    let (router, _forwarding, client, client_rx, module, module_egress_rx, module_rx, pending) =
        dynamic_route_fixture_with_busy(true, opted_in);
    BusyRouterRig {
        router: Arc::new(router),
        client,
        client_rx,
        module,
        module_rx,
        pending,
        _module_egress_rx: module_egress_rx,
    }
}

fn busy_router_request(rig: &BusyRouterRig, corr: u64, watch: bool) -> Frame {
    let mut frame = route_frame(
        FrameType::Request,
        rig.pending.client_channel,
        rig.pending.client_epoch,
        corr,
    );
    if watch {
        frame.header.flags.0 |= subc_protocol::FLAG_SUBSCRIPTION;
    }
    frame
}

async fn busy_router_receive(rx: &mut mpsc::Receiver<OutboundFrame>) -> Frame {
    tokio::time::timeout(Duration::from_secs(1), rx.recv())
        .await
        .expect("expected frame was not delivered promptly")
        .expect("connection closed")
        .frame
}

#[tokio::test]
async fn route_busy_refusal_has_detail_origin_and_returns_ok_without_server_warning() {
    let logs = Arc::new(std::sync::Mutex::new(Vec::new()));
    let writer = BusyLogWriter(Arc::clone(&logs));
    let subscriber = tracing_subscriber::fmt()
        .without_time()
        .with_ansi(false)
        .with_writer(move || writer.clone())
        .finish();
    let _subscriber = tracing::subscriber::set_default(subscriber);
    let mut rig = busy_router_rig(true);
    busy_router_receive(&mut rig.client_rx).await;
    for corr in 1..=32 {
        rig.router
            .route_for_connection(&rig.client, busy_router_request(&rig, corr, false))
            .await
            .unwrap();
        assert_eq!(
            busy_router_receive(&mut rig.module_rx).await.header.corr,
            corr
        );
    }
    for corr in 33..=34 {
        tokio::time::timeout(
            Duration::from_millis(100),
            rig.router
                .route_for_connection(&rig.client, busy_router_request(&rig, corr, false)),
        )
        .await
        .expect("full opted-in route must not wait")
        .expect("busy refusal is handled, not a server routing error");
        let error = busy_router_receive(&mut rig.client_rx).await;
        assert_eq!(
            (error.header.channel, error.header.epoch, error.header.corr),
            (rig.pending.client_channel, rig.pending.client_epoch, corr)
        );
        assert!(error.header.flags.is_daemon_origin());
        let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
        assert_eq!(body.code, error_codes::ROUTE_BUSY);
        assert_eq!(
            body.detail.unwrap(),
            serde_json::json!({"reason":"no_credit", "request_in_flight":32, "watch_in_flight":0, "request_window":32, "watch_window":24})
        );
    }
    assert!(
        rig.module_rx.try_recv().is_err(),
        "refused requests were never forwarded"
    );
    let logs = String::from_utf8(logs.lock().unwrap().clone()).unwrap();
    assert_eq!(
        logs.matches("request refused: route_busy").count(),
        1,
        "busy WARN is rate-limited per route: {logs}"
    );
    assert!(
        !logs.contains("routing failure recovered with ERROR frame"),
        "{logs}"
    );
    assert_eq!(
        rig.router.counters.snapshot()["route_busy_by_module_principal"]["epoch-router"]["direct"],
        2
    );
    rig.router
        .route_for_connection(
            &rig.client,
            route_frame(
                FrameType::Cancel,
                rig.pending.client_channel,
                rig.pending.client_epoch,
                33,
            ),
        )
        .await
        .unwrap();
    assert_eq!(
        busy_router_receive(&mut rig.module_rx).await.header.ty,
        FrameType::Cancel
    );
}

#[tokio::test]
async fn route_busy_watch_cancel_keeps_credit_until_terminal_and_releases_once() {
    let mut rig = busy_router_rig(true);
    busy_router_receive(&mut rig.client_rx).await;
    for corr in 1..=24 {
        rig.router
            .route_for_connection(&rig.client, busy_router_request(&rig, corr, true))
            .await
            .unwrap();
        busy_router_receive(&mut rig.module_rx).await;
    }
    rig.router
        .route_for_connection(
            &rig.client,
            route_frame(
                FrameType::Cancel,
                rig.pending.client_channel,
                rig.pending.client_epoch,
                1,
            ),
        )
        .await
        .unwrap();
    assert_eq!(
        busy_router_receive(&mut rig.module_rx).await.header.ty,
        FrameType::Cancel
    );
    rig.router
        .route_for_connection(&rig.client, busy_router_request(&rig, 25, true))
        .await
        .unwrap();
    let refused: ErrorBody =
        serde_json::from_slice(&busy_router_receive(&mut rig.client_rx).await.body).unwrap();
    assert_eq!(refused.detail.unwrap()["watch_in_flight"], 24);
    let terminal = route_frame(
        FrameType::Error,
        rig.pending.module_channel,
        rig.pending.module_epoch,
        1,
    );
    rig.router
        .route_for_connection(&rig.module, terminal.clone())
        .await
        .unwrap();
    busy_router_receive(&mut rig.client_rx).await;
    rig.router
        .route_for_connection(&rig.client, busy_router_request(&rig, 26, true))
        .await
        .unwrap();
    assert_eq!(
        busy_router_receive(&mut rig.module_rx).await.header.corr,
        26
    );
    rig.router
        .route_for_connection(&rig.module, terminal)
        .await
        .unwrap();
    busy_router_receive(&mut rig.client_rx).await;
    rig.router
        .route_for_connection(&rig.client, busy_router_request(&rig, 27, true))
        .await
        .unwrap();
    let refused: ErrorBody =
        serde_json::from_slice(&busy_router_receive(&mut rig.client_rx).await.body).unwrap();
    assert_eq!(refused.detail.unwrap()["watch_in_flight"], 24);
    rig.router
        .route_for_connection(&rig.client, busy_router_request(&rig, 28, false))
        .await
        .unwrap();
    assert_eq!(
        busy_router_receive(&mut rig.module_rx).await.header.corr,
        28
    );
}

#[derive(Clone)]
struct BusyLogWriter(Arc<std::sync::Mutex<Vec<u8>>>);

impl std::io::Write for BusyLogWriter {
    fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
        self.0.lock().unwrap().extend_from_slice(bytes);
        Ok(bytes.len())
    }
    fn flush(&mut self) -> std::io::Result<()> {
        Ok(())
    }
}

#[tokio::test(start_paused = true)]
async fn route_busy_legacy_credit_wait_stays_pending_and_is_logged_and_counted() {
    let logs = Arc::new(std::sync::Mutex::new(Vec::new()));
    let writer = BusyLogWriter(Arc::clone(&logs));
    let subscriber = tracing_subscriber::fmt()
        .without_time()
        .with_ansi(false)
        .with_writer(move || writer.clone())
        .finish();
    let _subscriber = tracing::subscriber::set_default(subscriber);
    let mut rig = busy_router_rig(false);
    let open: serde_json::Value =
        serde_json::from_slice(&busy_router_receive(&mut rig.client_rx).await.body).unwrap();
    assert!(open.get("request_window").is_none());
    assert!(open.get("watch_window").is_none());
    for corr in 1..=32 {
        rig.router
            .route_for_connection(&rig.client, busy_router_request(&rig, corr, true))
            .await
            .unwrap();
        busy_router_receive(&mut rig.module_rx).await;
    }
    let request = busy_router_request(&rig, 33, false);
    let router = Arc::clone(&rig.router);
    let client = rig.client.clone();
    let pending = tokio::spawn(async move { router.route_for_connection(&client, request).await });
    tokio::task::yield_now().await;
    tokio::time::advance(Duration::from_secs(6)).await;
    tokio::task::yield_now().await;
    assert!(
        !pending.is_finished(),
        "legacy request must keep waiting, not be refused"
    );
    assert!(rig.client_rx.try_recv().is_err());
    assert_eq!(
        rig.router.counters.snapshot()["reader_wait_over_5s_by_module_principal"]["epoch-router"]
            ["direct"]["legacy_credit"],
        1
    );
    let logs = String::from_utf8(logs.lock().unwrap().clone()).unwrap();
    assert!(
        logs.contains("connection reader wait exceeded 5 seconds"),
        "{logs}"
    );
    assert!(logs.contains("legacy_credit"), "{logs}");
    rig.router
        .route_for_connection(
            &rig.module,
            route_frame(
                FrameType::StreamEnd,
                rig.pending.module_channel,
                rig.pending.module_epoch,
                1,
            ),
        )
        .await
        .unwrap();
    pending.await.unwrap().unwrap();
    assert_eq!(
        busy_router_receive(&mut rig.module_rx).await.header.corr,
        33
    );
}

#[tokio::test]
async fn route_busy_pending_bind_installs_policy_and_reply_windows_atomically() {
    let (router, forwarding, client, mut client_rx, module, _module_egress_rx, _module_rx, pending) =
        dynamic_route_fixture_with_busy(false, true);
    assert!(matches!(
        forwarding
            .lookup_data_route(
                client.connection_id,
                pending.client_channel,
                pending.client_epoch
            )
            .unwrap(),
        DataRoute::Client(DataRouteState::Reserved)
    ));
    assert!(client_rx.try_recv().is_err());
    forwarding
        .complete_pending_relay(
            module.connection_id,
            pending.corr,
            RouteBindRelayOutcome::Accepted,
        )
        .unwrap();
    let open: subc_control::ClientControlResponse =
        serde_json::from_slice(&busy_router_receive(&mut client_rx).await.body).unwrap();
    assert!(matches!(
        open,
        subc_control::ClientControlResponse::RouteOpen {
            request_window: Some(32),
            watch_window: Some(24),
            ..
        }
    ));
    let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
        .lookup_data_route(
            client.connection_id,
            pending.client_channel,
            pending.client_epoch,
        )
        .unwrap()
    else {
        panic!("route must be bound");
    };
    assert!(route.flow.busy_refusals());
    assert_eq!(
        router
            .counters
            .snapshot()
            .get("route_busy_by_module_principal"),
        None
    );
}

#[tokio::test]
async fn route_busy_closed_full_binding_returns_reloading_or_stale_not_backend_or_busy() {
    for draining in [false, true] {
        let mut rig = busy_router_rig(true);
        busy_router_receive(&mut rig.client_rx).await;
        for corr in 1..=32 {
            rig.router
                .route_for_connection(&rig.client, busy_router_request(&rig, corr, false))
                .await
                .unwrap();
            busy_router_receive(&mut rig.module_rx).await;
        }
        let DataRoute::Client(DataRouteState::Bound(route)) = rig
            .router
            .forwarding
            .lookup_data_route(
                rig.client.connection_id,
                rig.pending.client_channel,
                rig.pending.client_epoch,
            )
            .unwrap()
        else {
            panic!("expected live route");
        };
        if draining {
            rig.router
                .forwarding
                .begin_module_drain("epoch-router", subc_protocol::RouteCloseReason::Reload)
                .unwrap();
        } else {
            rig.router
                .forwarding
                .release_client_route(
                    rig.client.connection_id,
                    rig.pending.client_channel,
                    rig.pending.client_epoch,
                )
                .unwrap();
        }
        let error = rig
            .router
            .forward_backend
            .handle_bound(busy_router_request(&rig, 33, false), route)
            .await
            .unwrap_err();
        if draining {
            assert!(
                matches!(error, RouterError::RouteError { ref code, .. } if code == error_codes::MODULE_RELOADING),
                "{error:?}"
            );
        } else {
            assert!(
                matches!(error, RouterError::StaleRouteEpoch { .. }),
                "{error:?}"
            );
        }
    }
}