Skip to main content

orchestral_cli/remote/
api.rs

1use std::collections::{BTreeMap, BTreeSet};
2use std::convert::Infallible;
3use std::path::PathBuf;
4use std::sync::{Arc, Mutex};
5use std::time::{Duration, Instant};
6
7use axum::extract::{DefaultBodyLimit, MatchedPath};
8use axum::extract::{Extension, Path, Query, State};
9use axum::http::{header, HeaderMap, HeaderValue, Request, StatusCode};
10use axum::middleware::{self, Next};
11use axum::response::sse::{Event, KeepAlive};
12use axum::response::{IntoResponse, Response, Sse};
13use axum::routing::{delete, get, post};
14use axum::{Json, Router};
15use orchestral_core::agent_connector::{
16    AgentConnectorId, AgentSessionActionId, AgentSessionActionOutcome, AgentSessionActivityId,
17    AgentSessionChange, AgentSessionHistoryAnchor, AgentSessionListQuery, AgentSessionPage,
18    AgentSessionReadQuery, AgentSessionRequestResolution, AgentSessionSummary,
19    CreateAgentSessionRequest, InvokeAgentSessionActionRequest, ResolveAgentSessionRequest,
20};
21use orchestral_core::agent_protocol::spi::AgentStartError;
22use orchestral_core::agent_protocol::wire::{
23    AgentCommand, AgentCommandEnvelope, AgentEvent, AgentRejectionCode, AgentRunView,
24    AgentSessionId, ApprovalDecision, ArtifactRef, ArtifactRefWithDigest, CommandAck,
25    CommandAckState, CommandId, Content, ContentBody, Digest, Extensions, PendingRequest,
26    PendingRequestPayload, RequestId, RequestResolution, RunId,
27};
28use orchestral_core::io::{ArtifactResolver, BlobStore};
29use orchestral_runtime::api::AgentApi;
30use orchestral_runtime::{
31    AgentControlError, AgentControlEvent, AgentDirectory, AgentDirectoryError, AgentSdkError,
32    ApprovalBridgeError, InMemoryHostApprovalBroker,
33};
34use serde::{Deserialize, Serialize};
35use serde_json::{json, Value};
36use tokio::sync::broadcast;
37use tracing::Instrument;
38
39use super::auth::{GatewayAuthenticator, GatewayPrincipal};
40use super::session_coordinator::AgentSessionCoordinatorRegistry;
41use super::state::{
42    DevicePrincipal, DeviceView, NativeSessionDefaults, PairingClaim, RemoteRegistry, SessionView,
43};
44
45const APPROVAL_GRANT_TTL_MS: i64 = 5 * 60 * 1_000;
46const RUN_SUPERVISOR_POLL_INTERVAL: Duration = Duration::from_secs(15);
47const RUN_SUPERVISOR_INITIAL_BACKOFF: Duration = Duration::from_millis(100);
48const RUN_SUPERVISOR_MAX_BACKOFF: Duration = Duration::from_secs(5);
49const DEFAULT_RUN_INACTIVITY_TIMEOUT: Duration = Duration::from_secs(10 * 60);
50const DEFAULT_RUN_STOP_GRACE: Duration = Duration::from_secs(30);
51/// Host-controlled Runs are a durable live-history overlay while a Provider's
52/// native transcript index catches up. Keep a bounded causal suffix instead
53/// of only the newest Run: one native turn can legitimately remain active
54/// across many Host Runs, and dropping the intervening mirrors creates a
55/// visible hole after a browser reload.
56const CONTROLLED_SESSION_RUN_LIMIT: usize = 100;
57const REQUEST_ID_HEADER: &str = "x-request-id";
58
59#[derive(Debug, Clone, Copy)]
60struct RunSupervisionPolicy {
61    inactivity_timeout: Duration,
62    stop_grace: Duration,
63}
64
65impl Default for RunSupervisionPolicy {
66    fn default() -> Self {
67        Self {
68            inactivity_timeout: duration_from_env(
69                "ORCHESTRAL_AGENT_RUN_INACTIVITY_TIMEOUT_SECS",
70                DEFAULT_RUN_INACTIVITY_TIMEOUT,
71            ),
72            stop_grace: duration_from_env(
73                "ORCHESTRAL_AGENT_RUN_STOP_GRACE_SECS",
74                DEFAULT_RUN_STOP_GRACE,
75            ),
76        }
77    }
78}
79
80fn duration_from_env(name: &str, fallback: Duration) -> Duration {
81    std::env::var(name)
82        .ok()
83        .and_then(|value| value.parse::<u64>().ok())
84        .filter(|seconds| *seconds > 0)
85        .map(Duration::from_secs)
86        .unwrap_or(fallback)
87}
88
89#[derive(Debug, Clone, Serialize)]
90struct RemoteRunSupervisionView {
91    state: &'static str,
92    reason: String,
93    detected_at_unix_ms: i64,
94}
95
96#[derive(Debug, Clone)]
97struct RunSupervisionIssue {
98    state: &'static str,
99    reason: String,
100    detected_at_unix_ms: i64,
101}
102
103#[derive(Default)]
104pub struct RunSupervisorRegistry {
105    active: Mutex<BTreeSet<String>>,
106    manual_recovery: Mutex<BTreeMap<String, String>>,
107    issues: Mutex<BTreeMap<String, RunSupervisionIssue>>,
108    policy: RunSupervisionPolicy,
109}
110
111impl RunSupervisorRegistry {
112    pub(super) fn key(connector_id: Option<&AgentConnectorId>, run_id: &RunId) -> String {
113        format!(
114            "{}\0{}",
115            connector_id.map_or("orchestral", AgentConnectorId::as_str),
116            run_id.as_str()
117        )
118    }
119
120    fn begin(&self, key: &str) -> bool {
121        if self
122            .manual_recovery
123            .lock()
124            .expect("Run recovery registry lock poisoned")
125            .contains_key(key)
126        {
127            return false;
128        }
129        self.active
130            .lock()
131            .expect("Run supervisor registry lock poisoned")
132            .insert(key.to_owned())
133    }
134
135    fn finish(&self, key: &str) {
136        self.active
137            .lock()
138            .expect("Run supervisor registry lock poisoned")
139            .remove(key);
140    }
141
142    fn mark_issue(&self, key: String, state: &'static str, reason: String) {
143        let mut issues = self
144            .issues
145            .lock()
146            .expect("Run supervision registry lock poisoned");
147        let detected_at_unix_ms = issues
148            .get(&key)
149            .map(|issue| issue.detected_at_unix_ms)
150            .unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
151        issues.insert(
152            key,
153            RunSupervisionIssue {
154                state,
155                reason,
156                detected_at_unix_ms,
157            },
158        );
159    }
160
161    fn clear_issue(&self, key: &str) {
162        self.issues
163            .lock()
164            .expect("Run supervision registry lock poisoned")
165            .remove(key);
166    }
167
168    fn issue(&self, key: &str) -> Option<RemoteRunSupervisionView> {
169        self.issues
170            .lock()
171            .expect("Run supervision registry lock poisoned")
172            .get(key)
173            .cloned()
174            .map(|issue| RemoteRunSupervisionView {
175                state: issue.state,
176                reason: issue.reason,
177                detected_at_unix_ms: issue.detected_at_unix_ms,
178            })
179    }
180
181    pub(super) fn mark_manual(&self, key: String, reason: String) {
182        self.manual_recovery
183            .lock()
184            .expect("Run recovery registry lock poisoned")
185            .insert(key, reason);
186    }
187
188    fn clear_manual(&self, key: &str) {
189        self.manual_recovery
190            .lock()
191            .expect("Run recovery registry lock poisoned")
192            .remove(key);
193    }
194
195    fn manual_reason(&self, key: &str) -> Option<String> {
196        self.manual_recovery
197            .lock()
198            .expect("Run recovery registry lock poisoned")
199            .get(key)
200            .cloned()
201    }
202}
203
204#[derive(Clone)]
205pub struct RemoteApiState {
206    pub agent: AgentApi,
207    pub agent_directory: Arc<AgentDirectory>,
208    pub native_session_defaults: NativeSessionDefaults,
209    pub approvals: Arc<InMemoryHostApprovalBroker>,
210    pub registry: RemoteRegistry,
211    pub gateway_authenticator: Option<Arc<dyn GatewayAuthenticator>>,
212    pub run_supervisors: Arc<RunSupervisorRegistry>,
213    pub(super) session_coordinators: Arc<AgentSessionCoordinatorRegistry>,
214    pub artifact_resolver: Option<Arc<dyn ArtifactResolver>>,
215    pub artifact_blob_store: Option<Arc<dyn BlobStore>>,
216}
217
218#[derive(Debug, Clone)]
219enum RemotePrincipal {
220    Device(DevicePrincipal),
221    Gateway(GatewayPrincipal),
222}
223
224#[derive(Debug, Clone)]
225struct RequestLogContext {
226    request_id: String,
227}
228
229#[derive(Debug, Clone)]
230struct ApiErrorLogCode(String);
231
232struct SseLifecycleLog {
233    request_id: String,
234    stream_id: String,
235    stream_kind: &'static str,
236    connector_id: String,
237    session_id: Option<String>,
238    run_id: Option<String>,
239    opened_at: Instant,
240    close_reason: Option<&'static str>,
241}
242
243impl SseLifecycleLog {
244    fn open_agent_session(
245        request: &RequestLogContext,
246        connector_id: &AgentConnectorId,
247        session_id: &AgentSessionId,
248    ) -> Self {
249        Self::open(
250            request,
251            "agent_session",
252            connector_id.as_str(),
253            Some(session_id.as_str()),
254            None,
255        )
256    }
257
258    fn open_run(
259        request: &RequestLogContext,
260        connector_id: Option<&AgentConnectorId>,
261        run_id: &RunId,
262    ) -> Self {
263        Self::open(
264            request,
265            "run",
266            connector_id.map_or("orchestral", AgentConnectorId::as_str),
267            None,
268            Some(run_id.as_str()),
269        )
270    }
271
272    fn open(
273        request: &RequestLogContext,
274        stream_kind: &'static str,
275        connector_id: &str,
276        session_id: Option<&str>,
277        run_id: Option<&str>,
278    ) -> Self {
279        let lifecycle = Self {
280            request_id: request.request_id.clone(),
281            stream_id: uuid::Uuid::new_v4().to_string(),
282            stream_kind,
283            connector_id: connector_id.to_owned(),
284            session_id: session_id.map(str::to_owned),
285            run_id: run_id.map(str::to_owned),
286            opened_at: Instant::now(),
287            close_reason: None,
288        };
289        tracing::info!(
290            request_id = %lifecycle.request_id,
291            stream_id = %lifecycle.stream_id,
292            stream_kind = lifecycle.stream_kind,
293            connector_id = %lifecycle.connector_id,
294            session_id = lifecycle.session_id.as_deref().unwrap_or("-"),
295            run_id = lifecycle.run_id.as_deref().unwrap_or("-"),
296            "SSE stream opened"
297        );
298        lifecycle
299    }
300
301    fn close_as(&mut self, reason: &'static str) {
302        self.close_reason = Some(reason);
303    }
304
305    fn lagged(&self, skipped: u64) {
306        tracing::warn!(
307            request_id = %self.request_id,
308            stream_id = %self.stream_id,
309            stream_kind = self.stream_kind,
310            connector_id = %self.connector_id,
311            session_id = self.session_id.as_deref().unwrap_or("-"),
312            run_id = self.run_id.as_deref().unwrap_or("-"),
313            skipped,
314            "SSE subscriber lagged"
315        );
316    }
317}
318
319impl Drop for SseLifecycleLog {
320    fn drop(&mut self) {
321        let lifetime_ms = u64::try_from(self.opened_at.elapsed().as_millis()).unwrap_or(u64::MAX);
322        tracing::info!(
323            request_id = %self.request_id,
324            stream_id = %self.stream_id,
325            stream_kind = self.stream_kind,
326            connector_id = %self.connector_id,
327            session_id = self.session_id.as_deref().unwrap_or("-"),
328            run_id = self.run_id.as_deref().unwrap_or("-"),
329            close_reason = self.close_reason.unwrap_or("client_disconnected"),
330            lifetime_ms,
331            "SSE stream closed"
332        );
333    }
334}
335
336impl RemotePrincipal {
337    fn current_device_id(&self) -> Option<&str> {
338        match self {
339            Self::Device(principal) => Some(&principal.device_id),
340            Self::Gateway(_) => None,
341        }
342    }
343}
344
345pub fn router(state: RemoteApiState) -> Router {
346    let protected = Router::new()
347        .route("/me", get(me))
348        .route("/devices", get(list_devices))
349        .route("/devices/{device_id}", delete(revoke_device))
350        .route("/sessions", get(list_sessions).post(create_session))
351        .route("/sessions/{session_id}", get(get_session))
352        .route("/sessions/{session_id}/runs", post(start_run))
353        .route("/agent-connectors", get(list_agent_connectors))
354        .route(
355            "/agent-sessions",
356            get(list_agent_sessions).post(create_agent_session),
357        )
358        .route("/agent-session", get(get_agent_session))
359        .route("/agent-session/stream", get(agent_session_stream))
360        .route("/agent-session/actions", post(invoke_agent_session_action))
361        .route(
362            "/agent-session/requests/{request_id}/input",
363            post(resolve_agent_session_input),
364        )
365        .route(
366            "/agent-session/requests/{request_id}/approval",
367            post(resolve_agent_session_approval),
368        )
369        .route("/agent-runs", post(start_agent_run))
370        .route("/runs/{run_id}", get(inspect_run))
371        .route("/runs/{run_id}/events", get(run_events))
372        .route("/runs/{run_id}/stream", get(run_stream))
373        .route("/runs/{run_id}/recover", post(recover_run))
374        .route("/runs/{run_id}/steer", post(steer_run))
375        .route("/runs/{run_id}/cancel", post(cancel_run))
376        .route(
377            "/runs/{run_id}/requests/{request_id}/input",
378            post(resolve_input),
379        )
380        .route(
381            "/runs/{run_id}/requests/{request_id}/approval",
382            post(resolve_approval),
383        )
384        .layer(middleware::from_fn_with_state(state.clone(), authenticate));
385
386    Router::new()
387        .route("/health", get(health))
388        .route("/pairing/claim", post(claim_pairing))
389        .merge(protected)
390        .layer(DefaultBodyLimit::max(256 * 1_024))
391        .layer(middleware::from_fn(no_store))
392        .layer(middleware::from_fn(log_request))
393        .with_state(state)
394}
395
396#[derive(Debug, Serialize)]
397struct HealthResponse {
398    status: &'static str,
399    protocol: &'static str,
400}
401
402async fn health() -> Json<HealthResponse> {
403    Json(HealthResponse {
404        status: "ok",
405        protocol: "orchestral-remote-v1",
406    })
407}
408
409#[derive(Debug, Deserialize)]
410#[serde(deny_unknown_fields)]
411struct PairingClaimRequest {
412    secret: String,
413    device_name: String,
414}
415
416async fn claim_pairing(
417    State(state): State<RemoteApiState>,
418    Json(request): Json<PairingClaimRequest>,
419) -> Result<Json<PairingClaim>, ApiError> {
420    let claim = state
421        .registry
422        .claim_pairing(&request.secret, &request.device_name)
423        .await
424        .map_err(|error| ApiError::unauthorized("pairing_failed", error.to_string()))?;
425    Ok(Json(claim))
426}
427
428#[derive(Debug, Serialize)]
429struct MeResponse {
430    auth_mode: &'static str,
431    #[serde(skip_serializing_if = "Option::is_none")]
432    device_id: Option<String>,
433    #[serde(skip_serializing_if = "Option::is_none")]
434    subject: Option<String>,
435    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
436    attributes: BTreeMap<String, String>,
437}
438
439async fn me(Extension(principal): Extension<RemotePrincipal>) -> Json<MeResponse> {
440    Json(match principal {
441        RemotePrincipal::Device(principal) => MeResponse {
442            auth_mode: "device_token",
443            device_id: Some(principal.device_id),
444            subject: None,
445            attributes: BTreeMap::new(),
446        },
447        RemotePrincipal::Gateway(principal) => MeResponse {
448            auth_mode: "gateway_jwt",
449            device_id: None,
450            subject: principal.subject,
451            attributes: principal.attributes,
452        },
453    })
454}
455
456async fn list_devices(
457    State(state): State<RemoteApiState>,
458    Extension(principal): Extension<RemotePrincipal>,
459) -> Json<Vec<DeviceView>> {
460    Json(
461        state
462            .registry
463            .devices(principal.current_device_id().unwrap_or_default())
464            .await,
465    )
466}
467
468async fn revoke_device(
469    State(state): State<RemoteApiState>,
470    Path(device_id): Path<String>,
471) -> Result<StatusCode, ApiError> {
472    state
473        .registry
474        .revoke_device(&device_id)
475        .await
476        .map_err(|error| ApiError::not_found("device_not_found", error.to_string()))?;
477    Ok(StatusCode::NO_CONTENT)
478}
479
480async fn list_sessions(
481    State(state): State<RemoteApiState>,
482) -> Result<Json<Vec<SessionView>>, ApiError> {
483    Ok(Json(session_views(&state).await?))
484}
485
486#[derive(Debug, Deserialize)]
487#[serde(deny_unknown_fields)]
488struct CreateSessionRequest {
489    #[serde(default)]
490    session_id: Option<String>,
491}
492
493async fn create_session(
494    State(state): State<RemoteApiState>,
495    Json(request): Json<CreateSessionRequest>,
496) -> Result<(StatusCode, Json<SessionView>), ApiError> {
497    let preferred = request.session_id.map(AgentSessionId::new);
498    let session_id = state.agent.create_session(preferred).await?;
499    let timestamp = chrono::Utc::now().timestamp_millis();
500    let session = SessionView {
501        id: session_id.as_str().to_owned(),
502        created_at_unix_ms: timestamp,
503        updated_at_unix_ms: timestamp,
504        run_ids: Vec::new(),
505        cwd: state.native_session_defaults.cwd.clone(),
506        execution_profile: state.native_session_defaults.execution_profile.clone(),
507    };
508    Ok((StatusCode::CREATED, Json(session)))
509}
510
511async fn get_session(
512    State(state): State<RemoteApiState>,
513    Path(session_id): Path<String>,
514) -> Result<Json<SessionView>, ApiError> {
515    session_views(&state)
516        .await?
517        .into_iter()
518        .find(|session| session.id == session_id)
519        .map(Json)
520        .ok_or_else(|| ApiError::not_found("session_not_found", "session was not found"))
521}
522
523async fn list_agent_connectors(
524    State(state): State<RemoteApiState>,
525) -> Json<Vec<orchestral_core::agent_connector::AgentConnectorDescriptor>> {
526    Json(state.agent_directory.connectors().await)
527}
528
529#[derive(Debug, Deserialize)]
530#[serde(deny_unknown_fields)]
531struct AgentSessionsQuery {
532    connector_id: String,
533    #[serde(default)]
534    cursor: Option<String>,
535    #[serde(default = "default_agent_session_limit")]
536    limit: u32,
537    #[serde(default)]
538    cwd: Option<String>,
539    #[serde(default)]
540    search: Option<String>,
541}
542
543const fn default_agent_session_limit() -> u32 {
544    50
545}
546
547async fn list_agent_sessions(
548    State(state): State<RemoteApiState>,
549    Query(query): Query<AgentSessionsQuery>,
550) -> Result<Json<AgentSessionPage>, ApiError> {
551    let connector_id = AgentConnectorId::new(query.connector_id);
552    Ok(Json(
553        state
554            .agent_directory
555            .list_sessions(
556                &connector_id,
557                AgentSessionListQuery {
558                    cursor: query.cursor,
559                    limit: query.limit,
560                    cwd: query.cwd,
561                    search: query.search,
562                },
563            )
564            .await?,
565    ))
566}
567
568#[derive(Debug, Deserialize)]
569#[serde(deny_unknown_fields)]
570struct CreateExternalAgentSessionRequest {
571    connector_id: String,
572    #[serde(default)]
573    cwd: Option<String>,
574    #[serde(default)]
575    title: Option<String>,
576    #[serde(default)]
577    options: serde_json::Value,
578    #[serde(default)]
579    extensions: BTreeMap<String, serde_json::Value>,
580}
581
582async fn create_agent_session(
583    State(state): State<RemoteApiState>,
584    Json(request): Json<CreateExternalAgentSessionRequest>,
585) -> Result<(StatusCode, Json<AgentSessionSummary>), ApiError> {
586    let connector_id = AgentConnectorId::new(request.connector_id);
587    let cwd = expand_host_home(request.cwd)?;
588    let summary = state
589        .agent_directory
590        .create_session(
591            &connector_id,
592            CreateAgentSessionRequest {
593                cwd,
594                title: request.title,
595                options: request.options,
596                extensions: request.extensions,
597            },
598        )
599        .await?;
600    Ok((StatusCode::CREATED, Json(summary)))
601}
602
603fn expand_host_home(cwd: Option<String>) -> Result<Option<String>, ApiError> {
604    let Some(cwd) = cwd else {
605        return Ok(None);
606    };
607    let expanded = if cwd == "~" || cwd.starts_with("~/") {
608        #[cfg(target_os = "windows")]
609        let home_variable = "USERPROFILE";
610        #[cfg(not(target_os = "windows"))]
611        let home_variable = "HOME";
612        let home = std::env::var_os(home_variable)
613            .filter(|value| !value.is_empty())
614            .map(PathBuf::from)
615            .ok_or_else(|| {
616                ApiError::new(
617                    StatusCode::BAD_REQUEST,
618                    "host_home_unavailable",
619                    "Host cannot expand ~ because its home directory is unavailable",
620                )
621            })?;
622        if cwd == "~" {
623            home
624        } else {
625            home.join(cwd.trim_start_matches("~/"))
626        }
627        .to_string_lossy()
628        .into_owned()
629    } else {
630        cwd
631    };
632    Ok(Some(expanded))
633}
634
635#[derive(Debug, Deserialize)]
636#[serde(deny_unknown_fields)]
637struct AgentSessionQuery {
638    connector_id: String,
639    session_id: String,
640    #[serde(default)]
641    cursor: Option<String>,
642    #[serde(default)]
643    limit: Option<u32>,
644}
645
646async fn get_agent_session(
647    State(state): State<RemoteApiState>,
648    Query(query): Query<AgentSessionQuery>,
649    headers: HeaderMap,
650) -> Result<Response, ApiError> {
651    let connector_id = AgentConnectorId::new(query.connector_id);
652    let session_id = AgentSessionId::new(query.session_id);
653    // Establish the Host-owned native subscription before reading the
654    // snapshot. Events committed while the snapshot is in flight are then
655    // replayable after `stream_cursor`, closing the read/subscribe race without
656    // forcing every browser to perform a second full read.
657    let stream_cursor = match state
658        .session_coordinators
659        .get(&connector_id, &session_id)
660        .ensure_hub(state.agent_directory.clone(), &connector_id, &session_id)
661        .await
662    {
663        Ok(hub) => Some(hub.cursor()),
664        Err(error) => {
665            tracing::debug!(
666                connector_id = %connector_id.as_str(),
667                session_id = %session_id.as_str(),
668                %error,
669                "Agent session has no live Hub; bounded polling remains available"
670            );
671            None
672        }
673    };
674    let detail = state
675        .agent_directory
676        .read_session_page(
677            &connector_id,
678            &session_id,
679            AgentSessionReadQuery {
680                cursor: query.cursor,
681                limit: query.limit.unwrap_or(100),
682            },
683        )
684        .await?;
685    let controlled_runs = controlled_session_runs(&state, &connector_id, &session_id).await?;
686    let mut payload = serde_json::to_value(RemoteAgentSessionDetail {
687        detail,
688        controlled_runs,
689        stream_cursor,
690    })
691    .map_err(|error| ApiError::internal("agent_session_encode_failed", error.to_string()))?;
692    enrich_artifact_access(&mut payload, state.artifact_resolver.as_deref()).await;
693    let body = serde_json::to_vec(&payload)
694        .map_err(|error| ApiError::internal("agent_session_encode_failed", error.to_string()))?;
695    let etag = format!(
696        "\"{}\"",
697        orchestral_core::agent_protocol::wire::Digest::sha256(&body)
698    );
699    let mut response = if request_etag_matches(&headers, &etag) {
700        StatusCode::NOT_MODIFIED.into_response()
701    } else {
702        (
703            StatusCode::OK,
704            [(header::CONTENT_TYPE, "application/json")],
705            body,
706        )
707            .into_response()
708    };
709    response.headers_mut().insert(
710        header::ETAG,
711        header::HeaderValue::from_str(&etag).expect("SHA-256 ETag is a valid header value"),
712    );
713    Ok(response)
714}
715
716#[derive(Debug, Serialize)]
717struct RemoteAgentSessionDetail {
718    #[serde(flatten)]
719    detail: orchestral_core::agent_connector::AgentSessionDetail,
720    /// Bounded causal suffix of Host-controlled Runs for this native session.
721    /// A Provider transcript can lag an active native turn, so these durable
722    /// mirrors preserve the intervening conversation as well as the identity
723    /// of the latest Run that can still be controlled.
724    controlled_runs: Vec<ControlledRemoteRunView>,
725    /// Canonical Host-side event cursor captured before the native snapshot
726    /// read. Clients resume from this point and receive any concurrent changes
727    /// from the shared session Hub.
728    #[serde(skip_serializing_if = "Option::is_none")]
729    stream_cursor: Option<u64>,
730}
731
732#[derive(Debug, Serialize)]
733struct ControlledRemoteRunView {
734    #[serde(flatten)]
735    run: RemoteRunView,
736    /// Stable wall-clock anchor used to merge this Host-side mirror into the
737    /// bounded native transcript. The native page can already contain the
738    /// response while the correlated user item lives on an older page.
739    created_at_unix_ms: i64,
740    /// Recency anchor used by clients to decide whether this Host mirror or a
741    /// later provider-owned turn describes the current Session status.
742    updated_at_unix_ms: i64,
743    /// Causal boundary captured when the Run was submitted. Unlike a browser
744    /// timestamp, this connector-issued identity survives refreshes, bounded
745    /// history windows and clocks on different devices.
746    #[serde(skip_serializing_if = "Option::is_none")]
747    after_activity_id: Option<AgentSessionActivityId>,
748}
749
750async fn controlled_session_runs(
751    state: &RemoteApiState,
752    connector_id: &AgentConnectorId,
753    session_id: &AgentSessionId,
754) -> Result<Vec<ControlledRemoteRunView>, ApiError> {
755    let agent = state.agent_directory.agent_api(connector_id).await?;
756    let mut catalog = agent
757        .catalog_runs()
758        .await?
759        .into_iter()
760        .filter(|entry| entry.session_id == *session_id)
761        .collect::<Vec<_>>();
762    catalog.sort_by(|left, right| {
763        left.created_at_unix_ms
764            .cmp(&right.created_at_unix_ms)
765            .then_with(|| left.run_id.cmp(&right.run_id))
766    });
767    let suffix_start = catalog.len().saturating_sub(CONTROLLED_SESSION_RUN_LIMIT);
768    let mut controlled = Vec::with_capacity(catalog.len() - suffix_start);
769    for entry in catalog.into_iter().skip(suffix_start) {
770        // Native session history is authoritative and must remain readable
771        // after a connector capability upgrade. A controlled Run registered
772        // against an older descriptor is supplementary history; the current
773        // controller cannot safely rehydrate it, so omit only that Run instead
774        // of failing or discarding the rest of the causal suffix.
775        if !agent.can_control_run(&entry.run_id).await? {
776            continue;
777        }
778        let view = agent.inspect(&entry.run_id).await?;
779        let history_anchor =
780            AgentSessionHistoryAnchor::from_extensions(&agent.run_extensions(&entry.run_id).await?)
781                .map_err(|error| {
782                    ApiError::internal("invalid_session_history_anchor", error.to_string())
783                })?;
784        let remote = RemoteRunView::new(
785            state,
786            Some(connector_id),
787            view,
788            agent.initial_input(&entry.run_id).await?,
789        );
790        if !remote.view.state.is_terminal() {
791            spawn_run_supervisor(
792                state.clone(),
793                agent.clone(),
794                Some(connector_id.clone()),
795                entry.run_id,
796            );
797        }
798        controlled.push(ControlledRemoteRunView {
799            run: remote,
800            created_at_unix_ms: entry.created_at_unix_ms,
801            updated_at_unix_ms: entry.updated_at_unix_ms,
802            after_activity_id: history_anchor.map(|anchor| anchor.after_activity_id),
803        });
804    }
805    Ok(controlled)
806}
807
808#[derive(Debug, Deserialize)]
809#[serde(deny_unknown_fields)]
810struct AgentSessionStreamQuery {
811    connector_id: String,
812    session_id: String,
813    #[serde(default)]
814    after: Option<u64>,
815}
816
817async fn agent_session_stream(
818    State(state): State<RemoteApiState>,
819    Extension(request_context): Extension<RequestLogContext>,
820    Query(query): Query<AgentSessionStreamQuery>,
821    headers: HeaderMap,
822) -> Result<Sse<impl futures_util::Stream<Item = Result<Event, Infallible>>>, ApiError> {
823    let connector_id = AgentConnectorId::new(query.connector_id);
824    let session_id = AgentSessionId::new(query.session_id);
825    let header_cursor = headers
826        .get("last-event-id")
827        .and_then(|value| value.to_str().ok())
828        .and_then(|value| value.parse::<u64>().ok())
829        .unwrap_or_default();
830    let initial_cursor = query.after.unwrap_or_default().max(header_cursor);
831    let hub = state
832        .session_coordinators
833        .get(&connector_id, &session_id)
834        .ensure_hub(state.agent_directory.clone(), &connector_id, &session_id)
835        .await?;
836    let subscription = hub.subscribe(initial_cursor);
837    let replay = subscription.replay;
838    let mut live = subscription.live;
839    let lifecycle =
840        SseLifecycleLog::open_agent_session(&request_context, &connector_id, &session_id);
841    let artifact_resolver = state.artifact_resolver.clone();
842    let stream = async_stream::stream! {
843        let mut lifecycle = lifecycle;
844        let mut cursor = initial_cursor;
845        for change in replay {
846            cursor = change.sequence;
847            match agent_session_change_event(&change, artifact_resolver.as_deref()).await {
848                Ok(event) => yield Ok(event),
849                Err(error) => {
850                    lifecycle.close_as("replay_event_encode_failed");
851                    tracing::error!(
852                        request_id = %lifecycle.request_id,
853                        stream_id = %lifecycle.stream_id,
854                        %error,
855                        "could not encode replayed Agent session SSE event"
856                    );
857                    yield Ok(api_error_event(&ApiError::internal(
858                        "session_stream_encode_failed",
859                        error.to_string(),
860                    )));
861                    return;
862                }
863            }
864        }
865        loop {
866            match live.recv().await {
867                Ok(change) => {
868                    if change.sequence != 0 && change.sequence <= cursor {
869                        continue;
870                    }
871                    // Every native mutation is significant. In particular,
872                    // item/completed is commonly followed immediately by
873                    // turn/completed; retaining only the latter drops the
874                    // actual message until the next full snapshot.
875                    match agent_session_change_event(&change, artifact_resolver.as_deref()).await {
876                        Ok(event) => {
877                            cursor = change.sequence;
878                            yield Ok(event)
879                        },
880                        Err(error) => {
881                            lifecycle.close_as("event_encode_failed");
882                            tracing::error!(
883                                request_id = %lifecycle.request_id,
884                                stream_id = %lifecycle.stream_id,
885                                %error,
886                                "could not encode Agent session SSE event"
887                            );
888                            yield Ok(api_error_event(&ApiError::internal(
889                                "session_stream_encode_failed",
890                                error.to_string(),
891                            )));
892                            return;
893                        }
894                    }
895                }
896                Err(broadcast::error::RecvError::Lagged(skipped)) => {
897                    lifecycle.lagged(skipped);
898                    // The broadcast ring is only the fast path. Recover from
899                    // the Hub replay before declaring a snapshot gap.
900                    let replacement = hub.subscribe(cursor);
901                    live = replacement.live;
902                    for change in replacement.replay {
903                        match agent_session_change_event(&change, artifact_resolver.as_deref()).await {
904                            Ok(event) => {
905                                cursor = change.sequence;
906                                yield Ok(event)
907                            },
908                            Err(error) => {
909                                lifecycle.close_as("gap_replay_encode_failed");
910                                tracing::error!(
911                                    request_id = %lifecycle.request_id,
912                                    stream_id = %lifecycle.stream_id,
913                                    %error,
914                                    "could not encode Agent session gap replay"
915                                );
916                                yield Ok(api_error_event(&ApiError::internal(
917                                    "session_stream_encode_failed",
918                                    error.to_string(),
919                                )));
920                                return;
921                            }
922                        }
923                    }
924                }
925                Err(broadcast::error::RecvError::Closed) => {
926                    lifecycle.close_as("source_closed");
927                    return;
928                }
929            }
930        }
931    };
932    Ok(Sse::new(stream).keep_alive(
933        KeepAlive::new()
934            .interval(Duration::from_secs(10))
935            .text("keep-alive"),
936    ))
937}
938
939async fn agent_session_change_event(
940    change: &AgentSessionChange,
941    artifact_resolver: Option<&dyn ArtifactResolver>,
942) -> Result<Event, axum::Error> {
943    let mut payload = serde_json::to_value(change).map_err(axum::Error::new)?;
944    enrich_artifact_access(&mut payload, artifact_resolver).await;
945    Event::default()
946        .event("session_changed")
947        .id(change.sequence.to_string())
948        .json_data(payload)
949}
950
951/// Adds non-durable, storage-resolved access data to API views while keeping
952/// the Agent Protocol's permanent Artifact identity free of expiring URLs.
953/// The same projection is used by bounded snapshots and incremental SSE
954/// events, so every client renders the same object-store address.
955async fn enrich_artifact_access(
956    payload: &mut serde_json::Value,
957    artifact_resolver: Option<&dyn ArtifactResolver>,
958) {
959    let Some(artifact_resolver) = artifact_resolver else {
960        return;
961    };
962    let mut artifacts = BTreeMap::<String, ArtifactRefWithDigest>::new();
963    collect_artifact_references(payload, &mut artifacts);
964    let mut access = BTreeMap::<String, serde_json::Value>::new();
965    for (reference, artifact) in artifacts {
966        match artifact_resolver.resolve(&artifact).await {
967            Ok(resolved) => {
968                access.insert(
969                    reference,
970                    serde_json::json!({
971                        "uri": resolved.uri,
972                        "file_name": resolved.file_name,
973                        "media_type": resolved.media_type,
974                        "byte_size": resolved.byte_size,
975                        "expires_at": resolved.expires_at,
976                    }),
977                );
978            }
979            Err(error) => {
980                tracing::warn!(
981                    artifact_ref = %artifact.artifact_ref,
982                    %error,
983                    "could not resolve Artifact access for remote session view"
984                );
985            }
986        }
987    }
988    inject_artifact_access(payload, &access);
989}
990
991fn collect_artifact_references(
992    value: &serde_json::Value,
993    artifacts: &mut BTreeMap<String, ArtifactRefWithDigest>,
994) {
995    match value {
996        serde_json::Value::Array(values) => {
997            for value in values {
998                collect_artifact_references(value, artifacts);
999            }
1000        }
1001        serde_json::Value::Object(object) => {
1002            if object
1003                .get("body")
1004                .and_then(|body| body.get("kind"))
1005                .and_then(serde_json::Value::as_str)
1006                == Some("artifact")
1007            {
1008                if let Some(artifact) = object
1009                    .get("body")
1010                    .and_then(|body| body.get("value"))
1011                    .cloned()
1012                    .and_then(|value| serde_json::from_value::<ArtifactRefWithDigest>(value).ok())
1013                    .filter(|artifact| artifact.validate_integrity().is_ok())
1014                {
1015                    artifacts.insert(artifact.artifact_ref.to_string(), artifact);
1016                }
1017            }
1018            for value in object.values() {
1019                collect_artifact_references(value, artifacts);
1020            }
1021        }
1022        _ => {}
1023    }
1024}
1025
1026fn inject_artifact_access(
1027    value: &mut serde_json::Value,
1028    access: &BTreeMap<String, serde_json::Value>,
1029) {
1030    match value {
1031        serde_json::Value::Array(values) => {
1032            for value in values {
1033                inject_artifact_access(value, access);
1034            }
1035        }
1036        serde_json::Value::Object(object) => {
1037            let reference = object
1038                .get("body")
1039                .and_then(|body| body.get("value"))
1040                .and_then(|value| value.get("artifact_ref"))
1041                .and_then(serde_json::Value::as_str)
1042                .map(str::to_owned);
1043            if let Some(resolved) = reference.and_then(|reference| access.get(&reference)) {
1044                object.insert("access".to_owned(), resolved.clone());
1045            }
1046            for value in object.values_mut() {
1047                inject_artifact_access(value, access);
1048            }
1049        }
1050        _ => {}
1051    }
1052}
1053
1054fn request_etag_matches(headers: &HeaderMap, etag: &str) -> bool {
1055    headers
1056        .get(header::IF_NONE_MATCH)
1057        .and_then(|value| value.to_str().ok())
1058        .is_some_and(|values| {
1059            values
1060                .split(',')
1061                .map(str::trim)
1062                .any(|candidate| candidate == "*" || candidate == etag)
1063        })
1064}
1065
1066#[derive(Debug, Deserialize)]
1067#[serde(deny_unknown_fields)]
1068struct InvokeExternalAgentSessionActionRequest {
1069    connector_id: String,
1070    session_id: String,
1071    action_id: String,
1072    #[serde(default)]
1073    arguments: serde_json::Value,
1074    #[serde(default)]
1075    run_id: Option<String>,
1076}
1077
1078async fn invoke_agent_session_action(
1079    State(state): State<RemoteApiState>,
1080    Json(request): Json<InvokeExternalAgentSessionActionRequest>,
1081) -> Result<Json<AgentSessionActionOutcome>, ApiError> {
1082    Ok(Json(
1083        state
1084            .agent_directory
1085            .invoke_action(
1086                &AgentConnectorId::new(request.connector_id),
1087                InvokeAgentSessionActionRequest {
1088                    session_id: AgentSessionId::new(request.session_id),
1089                    action_id: AgentSessionActionId::new(request.action_id),
1090                    arguments: request.arguments,
1091                    run_id: request.run_id.map(RunId::new),
1092                },
1093            )
1094            .await?,
1095    ))
1096}
1097
1098#[derive(Debug, Deserialize)]
1099#[serde(deny_unknown_fields)]
1100struct StartRunRequest {
1101    run_id: String,
1102    input: String,
1103    #[serde(default)]
1104    attachments: Vec<RemoteArtifactInput>,
1105}
1106
1107#[derive(Debug, Serialize)]
1108struct StartRunResponse {
1109    run_id: RunId,
1110    view: RemoteRunView,
1111}
1112
1113#[derive(Debug, Deserialize)]
1114#[serde(deny_unknown_fields)]
1115struct StartAgentRunRequest {
1116    connector_id: String,
1117    session_id: String,
1118    run_id: String,
1119    input: String,
1120    #[serde(default)]
1121    after_activity_id: Option<String>,
1122    #[serde(default)]
1123    attachments: Vec<RemoteArtifactInput>,
1124}
1125
1126#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1127#[serde(deny_unknown_fields)]
1128struct RemoteArtifactInput {
1129    artifact_ref: String,
1130    digest: String,
1131    file_name: String,
1132    media_type: String,
1133    byte_size: u64,
1134}
1135
1136#[derive(Debug, Serialize)]
1137struct StartAgentRunResponse {
1138    connector_id: AgentConnectorId,
1139    run_id: RunId,
1140    operation: &'static str,
1141    #[serde(skip_serializing_if = "Option::is_none")]
1142    command_id: Option<CommandId>,
1143    view: RemoteRunView,
1144}
1145
1146async fn start_agent_run(
1147    State(state): State<RemoteApiState>,
1148    Json(request): Json<StartAgentRunRequest>,
1149) -> Result<(StatusCode, Json<StartAgentRunResponse>), ApiError> {
1150    let input = message_content(&state, &request.input, &request.attachments).await?;
1151    let connector_id = AgentConnectorId::new(request.connector_id);
1152    let session_id = AgentSessionId::new(request.session_id);
1153    let run_id = RunId::new(request.run_id);
1154    let history_anchor =
1155        request
1156            .after_activity_id
1157            .map(|after_activity_id| AgentSessionHistoryAnchor {
1158                after_activity_id: AgentSessionActivityId::new(after_activity_id),
1159            });
1160    let history_extensions = session_history_anchor_extensions(history_anchor.as_ref())?;
1161    let coordinator = state.session_coordinators.get(&connector_id, &session_id);
1162    let _operation_guard = coordinator.operation().lock().await;
1163    let agent = state.agent_directory.agent_api(&connector_id).await?;
1164
1165    // A retry of the same browser operation is a pure read. This also keeps a
1166    // response lost after start from being reinterpreted as a steer on retry.
1167    if agent.has_run(&run_id).await? {
1168        if agent.inspect(&run_id).await?.execution.session_id != session_id {
1169            return Err(ApiError::conflict(
1170                "run_id_conflict",
1171                "run_id belongs to another session",
1172            ));
1173        }
1174        if agent.initial_input(&run_id).await? != input {
1175            return Err(ApiError::conflict(
1176                "run_id_conflict",
1177                "run_id was already used with different input",
1178            ));
1179        }
1180        let existing_anchor =
1181            AgentSessionHistoryAnchor::from_extensions(&agent.run_extensions(&run_id).await?)
1182                .map_err(|error| {
1183                    ApiError::internal("invalid_session_history_anchor", error.to_string())
1184                })?;
1185        if existing_anchor != history_anchor {
1186            return Err(ApiError::conflict(
1187                "run_id_conflict",
1188                "run_id was already used with a different session history anchor",
1189            ));
1190        }
1191        let view = RemoteRunView::new(
1192            &state,
1193            Some(&connector_id),
1194            agent.inspect(&run_id).await?,
1195            agent.initial_input(&run_id).await?,
1196        );
1197        spawn_run_supervisor(
1198            state.clone(),
1199            agent,
1200            Some(connector_id.clone()),
1201            run_id.clone(),
1202        );
1203        log_agent_input_accepted(&connector_id, &session_id, &run_id, "replayed", None);
1204        return Ok((
1205            StatusCode::OK,
1206            Json(StartAgentRunResponse {
1207                connector_id,
1208                run_id,
1209                operation: "replayed",
1210                command_id: None,
1211                view,
1212            }),
1213        ));
1214    }
1215
1216    // A session submission may have become a command on an earlier Run.
1217    // Resolve that durable identity before consulting the current active Run;
1218    // otherwise a retry after completion would execute the same input again.
1219    let submission_command_id = CommandId::new(format!("agent-submit-{}", run_id.as_str()));
1220    for entry in agent
1221        .catalog_runs()
1222        .await?
1223        .into_iter()
1224        .filter(|entry| entry.session_id == session_id)
1225    {
1226        let Some(recorded) = agent
1227            .recorded_command(&entry.run_id, &submission_command_id)
1228            .await?
1229        else {
1230            continue;
1231        };
1232        let expected = AgentCommandEnvelope::new_with_extensions(
1233            submission_command_id.clone(),
1234            entry.run_id.clone(),
1235            None,
1236            AgentCommand::Steer {
1237                content: input.clone(),
1238            },
1239            history_extensions.clone(),
1240        )?;
1241        if recorded != expected {
1242            return Err(ApiError::conflict(
1243                "run_id_conflict",
1244                "session submission identity was already used with different input or metadata",
1245            ));
1246        }
1247        if !agent.can_control_run(&entry.run_id).await? {
1248            return Err(ApiError::conflict(
1249                "submission_contract_changed",
1250                "this submission belongs to an earlier Agent contract and cannot be resubmitted",
1251            ));
1252        }
1253        let ack = command_run(&agent, &entry.run_id, expected).await?;
1254        if !matches!(
1255            ack.state,
1256            CommandAckState::Accepted { .. } | CommandAckState::Applied { .. }
1257        ) {
1258            if rejected_steer_can_start_run(&agent, &entry.run_id, &ack.state).await? {
1259                continue;
1260            }
1261            return Err(ApiError::conflict(
1262                "agent_session_command_rejected",
1263                rejected_session_message(&ack.state),
1264            ));
1265        }
1266        let view = RemoteRunView::new(
1267            &state,
1268            Some(&connector_id),
1269            agent.inspect(&entry.run_id).await?,
1270            agent.initial_input(&entry.run_id).await?,
1271        );
1272        return Ok((
1273            StatusCode::OK,
1274            Json(StartAgentRunResponse {
1275                connector_id,
1276                run_id: entry.run_id,
1277                operation: "steered",
1278                command_id: Some(submission_command_id),
1279                view,
1280            }),
1281        ));
1282    }
1283
1284    'steer: {
1285        if let Some(entry) = latest_session_run(&agent, &session_id).await? {
1286            let mut current = agent.inspect(&entry.run_id).await?;
1287            if current.state.status()
1288                == orchestral_core::agent_protocol::reference::AgentRunStatus::Unknown
1289            {
1290                current = agent.recover(&entry.run_id).await?;
1291            }
1292            if !current.state.is_terminal() {
1293                let command_id = CommandId::new(format!("agent-submit-{}", run_id.as_str()));
1294                let command = AgentCommandEnvelope::new_with_extensions(
1295                    command_id.clone(),
1296                    entry.run_id.clone(),
1297                    None,
1298                    AgentCommand::Steer {
1299                        content: input.clone(),
1300                    },
1301                    history_extensions.clone(),
1302                )?;
1303                let ack = command_run(&agent, &entry.run_id, command).await?;
1304                if !matches!(
1305                    ack.state,
1306                    CommandAckState::Accepted { .. } | CommandAckState::Applied { .. }
1307                ) {
1308                    // The provider may discover completion while checking this
1309                    // command. Its explicit rejection proves this input was not
1310                    // dispatched; only a reconciled terminal Run permits a new Run.
1311                    if rejected_steer_can_start_run(&agent, &entry.run_id, &ack.state).await? {
1312                        break 'steer;
1313                    }
1314                    return Err(ApiError::conflict(
1315                        "agent_session_command_rejected",
1316                        rejected_session_message(&ack.state),
1317                    ));
1318                }
1319                let view = RemoteRunView::new(
1320                    &state,
1321                    Some(&connector_id),
1322                    agent.inspect(&entry.run_id).await?,
1323                    agent.initial_input(&entry.run_id).await?,
1324                );
1325                spawn_run_supervisor(
1326                    state.clone(),
1327                    agent,
1328                    Some(connector_id.clone()),
1329                    entry.run_id.clone(),
1330                );
1331                log_agent_input_accepted(
1332                    &connector_id,
1333                    &session_id,
1334                    &entry.run_id,
1335                    "steered",
1336                    Some(&command_id),
1337                );
1338                return Ok((
1339                    StatusCode::OK,
1340                    Json(StartAgentRunResponse {
1341                        connector_id,
1342                        run_id: entry.run_id,
1343                        operation: "steered",
1344                        command_id: Some(command_id),
1345                        view,
1346                    }),
1347                ));
1348            }
1349        }
1350    }
1351
1352    let handle = state
1353        .agent_directory
1354        .start_content_with_extensions(
1355            &connector_id,
1356            &session_id,
1357            Some(run_id.clone()),
1358            input,
1359            history_extensions,
1360        )
1361        .await?;
1362    spawn_run_supervisor(
1363        state.clone(),
1364        agent.clone(),
1365        Some(connector_id.clone()),
1366        run_id.clone(),
1367    );
1368    let view = RemoteRunView::new(
1369        &state,
1370        Some(&connector_id),
1371        handle.inspect().await?,
1372        agent.initial_input(&run_id).await?,
1373    );
1374    log_agent_input_accepted(&connector_id, &session_id, &run_id, "started", None);
1375    Ok((
1376        StatusCode::CREATED,
1377        Json(StartAgentRunResponse {
1378            connector_id,
1379            run_id,
1380            operation: "started",
1381            command_id: None,
1382            view,
1383        }),
1384    ))
1385}
1386
1387async fn rejected_steer_can_start_run(
1388    agent: &AgentApi,
1389    run_id: &RunId,
1390    ack: &CommandAckState,
1391) -> Result<bool, ApiError> {
1392    use orchestral_core::agent_protocol::wire::AgentProtocolErrorCode;
1393    if !matches!(
1394        ack,
1395        CommandAckState::Rejected {
1396            code: AgentProtocolErrorCode::TerminalRun | AgentProtocolErrorCode::InvalidTransition,
1397            ..
1398        }
1399    ) {
1400        return Ok(false);
1401    }
1402    Ok(agent.inspect(run_id).await?.state.is_terminal())
1403}
1404
1405fn rejected_session_message(ack: &CommandAckState) -> String {
1406    match ack {
1407        CommandAckState::Rejected { message, .. } => message.clone(),
1408        _ => "the active Agent Run did not accept this session message".to_owned(),
1409    }
1410}
1411
1412fn log_agent_input_accepted(
1413    connector_id: &AgentConnectorId,
1414    session_id: &AgentSessionId,
1415    run_id: &RunId,
1416    operation: &'static str,
1417    command_id: Option<&CommandId>,
1418) {
1419    tracing::info!(
1420        connector_id = %connector_id.as_str(),
1421        session_id = %session_id.as_str(),
1422        run_id = %run_id.as_str(),
1423        operation,
1424        command_id = command_id.map(CommandId::as_str).unwrap_or("-"),
1425        "Agent session input accepted"
1426    );
1427}
1428
1429async fn latest_session_run(
1430    agent: &AgentApi,
1431    session_id: &AgentSessionId,
1432) -> Result<Option<orchestral_core::agent_protocol::spi::AgentRunCatalogEntry>, ApiError> {
1433    let mut catalog = agent
1434        .catalog_runs()
1435        .await?
1436        .into_iter()
1437        .filter(|entry| entry.session_id == *session_id)
1438        .collect::<Vec<_>>();
1439    catalog.sort_by_key(|entry| {
1440        std::cmp::Reverse((entry.updated_at_unix_ms, entry.created_at_unix_ms))
1441    });
1442    let Some(latest) = catalog.into_iter().next() else {
1443        return Ok(None);
1444    };
1445    // A connector upgrade may leave durable journals registered against its
1446    // previous descriptor. They remain history, but must not be inspected,
1447    // recovered, or steered through the new controller. Treat the native
1448    // session as having no current Host Run so a fresh compatible Run can
1449    // attach to it.
1450    if !agent.can_control_run(&latest.run_id).await? {
1451        return Ok(None);
1452    }
1453    Ok(Some(latest))
1454}
1455
1456fn session_history_anchor_extensions(
1457    anchor: Option<&AgentSessionHistoryAnchor>,
1458) -> Result<Extensions, ApiError> {
1459    let mut extensions = Extensions::new();
1460    if let Some(anchor) = anchor {
1461        anchor.insert_into(&mut extensions).map_err(|error| {
1462            ApiError::new(
1463                StatusCode::BAD_REQUEST,
1464                "invalid_session_history_anchor",
1465                error.to_string(),
1466            )
1467        })?;
1468    }
1469    Ok(extensions)
1470}
1471
1472async fn message_content(
1473    state: &RemoteApiState,
1474    text: &str,
1475    attachments: &[RemoteArtifactInput],
1476) -> Result<Vec<Content>, ApiError> {
1477    if text.trim().is_empty() && attachments.is_empty() {
1478        return Err(ApiError::new(
1479            StatusCode::BAD_REQUEST,
1480            "invalid_agent_input",
1481            "message text or at least one attachment is required",
1482        ));
1483    }
1484    if attachments.len() > 10 {
1485        return Err(ApiError::new(
1486            StatusCode::BAD_REQUEST,
1487            "too_many_attachments",
1488            "a message may contain at most 10 attachments",
1489        ));
1490    }
1491    let resolver = if attachments.is_empty() {
1492        None
1493    } else {
1494        Some(state.artifact_resolver.as_ref().ok_or_else(|| {
1495            ApiError::service_unavailable(
1496                "artifact_resolver_unavailable",
1497                "Artifact storage is not configured on this Host",
1498            )
1499        })?)
1500    };
1501
1502    let mut total_bytes = 0_u64;
1503    let mut validated = Vec::with_capacity(attachments.len());
1504    for attachment in attachments {
1505        validate_remote_artifact(attachment)?;
1506        total_bytes = total_bytes
1507            .checked_add(attachment.byte_size)
1508            .ok_or_else(|| {
1509                ApiError::new(
1510                    StatusCode::BAD_REQUEST,
1511                    "attachments_too_large",
1512                    "attachment byte total overflowed",
1513                )
1514            })?;
1515        if total_bytes > 10 * 64 * 1024 * 1024 {
1516            return Err(ApiError::new(
1517                StatusCode::PAYLOAD_TOO_LARGE,
1518                "attachments_too_large",
1519                "attachment total exceeds 640 MiB",
1520            ));
1521        }
1522        let artifact = ArtifactRefWithDigest {
1523            artifact_ref: ArtifactRef::new(&attachment.artifact_ref),
1524            digest: Digest::new(&attachment.digest),
1525        };
1526        let resolved = resolver
1527            .expect("non-empty attachments require a resolver")
1528            .resolve(&artifact)
1529            .await
1530            .map_err(artifact_resolve_error)?;
1531        if resolved.media_type != attachment.media_type
1532            || resolved.byte_size != attachment.byte_size
1533            || resolved.artifact != artifact
1534        {
1535            return Err(ApiError::conflict(
1536                "artifact_metadata_conflict",
1537                "Artifact metadata changed after upload",
1538            ));
1539        }
1540        validated.push((attachment, artifact));
1541    }
1542
1543    let mut description = if text.trim().is_empty() {
1544        "请查看并处理随消息附上的文件。".to_owned()
1545    } else {
1546        text.to_owned()
1547    };
1548    if !validated.is_empty() {
1549        description.push_str("\n\n附件(内容已由 Host 按 SHA-256 校验):");
1550        for (index, (attachment, _)) in validated.iter().enumerate() {
1551            description.push_str(&format!(
1552                "\n{}. {}({},{} bytes,sha256 {})",
1553                index + 1,
1554                attachment.file_name,
1555                attachment.media_type,
1556                attachment.byte_size,
1557                attachment.digest
1558            ));
1559        }
1560    }
1561    let mut content = vec![Content::text(description)];
1562    content.extend(validated.into_iter().map(|(attachment, artifact)| Content {
1563        media_type: attachment.media_type.clone(),
1564        schema_id: None,
1565        body: ContentBody::Artifact(artifact),
1566    }));
1567    Ok(content)
1568}
1569
1570fn validate_remote_artifact(attachment: &RemoteArtifactInput) -> Result<(), ApiError> {
1571    if attachment.artifact_ref != attachment.digest
1572        || attachment.digest.len() != 64
1573        || !attachment
1574            .digest
1575            .bytes()
1576            .all(|byte| byte.is_ascii_hexdigit())
1577    {
1578        return Err(ApiError::new(
1579            StatusCode::BAD_REQUEST,
1580            "invalid_artifact_identity",
1581            "Artifact reference must equal its SHA-256 digest",
1582        ));
1583    }
1584    if attachment.file_name.trim().is_empty()
1585        || attachment.file_name.len() > 255
1586        || attachment.file_name.chars().any(char::is_control)
1587    {
1588        return Err(ApiError::new(
1589            StatusCode::BAD_REQUEST,
1590            "invalid_artifact_file_name",
1591            "Artifact file name is invalid",
1592        ));
1593    }
1594    if attachment.media_type.trim().is_empty()
1595        || attachment.media_type.len() > 160
1596        || attachment.media_type.chars().any(char::is_control)
1597    {
1598        return Err(ApiError::new(
1599            StatusCode::BAD_REQUEST,
1600            "invalid_artifact_media_type",
1601            "Artifact media type is invalid",
1602        ));
1603    }
1604    if attachment.byte_size == 0 || attachment.byte_size > 64 * 1024 * 1024 {
1605        return Err(ApiError::new(
1606            StatusCode::PAYLOAD_TOO_LARGE,
1607            "artifact_too_large",
1608            "Artifact must be between 1 byte and 64 MiB",
1609        ));
1610    }
1611    Ok(())
1612}
1613
1614fn artifact_resolve_error(error: orchestral_core::io::ArtifactResolveError) -> ApiError {
1615    use orchestral_core::io::ArtifactResolveError;
1616    match error {
1617        ArtifactResolveError::Invalid(message) => {
1618            ApiError::new(StatusCode::BAD_REQUEST, "invalid_artifact", message)
1619        }
1620        ArtifactResolveError::NotFound(message) => {
1621            ApiError::not_found("artifact_not_found", message)
1622        }
1623        ArtifactResolveError::Integrity(message) => {
1624            ApiError::conflict("artifact_integrity_failed", message)
1625        }
1626        ArtifactResolveError::Unavailable(message) | ArtifactResolveError::Internal(message) => {
1627            tracing::warn!(%message, "Artifact resolver failed");
1628            ApiError::service_unavailable(
1629                "artifact_resolver_unavailable",
1630                "Artifact storage is temporarily unavailable",
1631            )
1632        }
1633    }
1634}
1635
1636#[derive(Debug, Serialize)]
1637struct RemoteRunView {
1638    #[serde(flatten)]
1639    view: AgentRunView,
1640    /// Immutable initial Run input. This is read from the controller-owned
1641    /// RunSpec rather than copied into the mobile session registry.
1642    input: Vec<Content>,
1643    /// Host control-plane recovery disposition. Protocol `unknown` only says
1644    /// continuity is unproven; it does not tell a client whether waiting can
1645    /// make progress or whether the session must continue with a fresh Run.
1646    #[serde(skip_serializing_if = "Option::is_none")]
1647    recovery: Option<RemoteRunRecoveryView>,
1648    /// Host-side liveness supervision. This is intentionally separate from
1649    /// Provider continuity: an attached stream can remain connected while its
1650    /// native execution has stopped making progress.
1651    #[serde(skip_serializing_if = "Option::is_none")]
1652    supervision: Option<RemoteRunSupervisionView>,
1653}
1654
1655#[derive(Debug, Serialize)]
1656struct RemoteRunRecoveryView {
1657    mode: &'static str,
1658    can_start_new_run: bool,
1659    #[serde(skip_serializing_if = "Option::is_none")]
1660    reason: Option<String>,
1661}
1662
1663impl RemoteRunView {
1664    fn new(
1665        state: &RemoteApiState,
1666        connector_id: Option<&AgentConnectorId>,
1667        view: AgentRunView,
1668        input: Vec<Content>,
1669    ) -> Self {
1670        let recovery = if view.state.status()
1671            == orchestral_core::agent_protocol::reference::AgentRunStatus::Unknown
1672        {
1673            let key = RunSupervisorRegistry::key(connector_id, &view.execution.run_id);
1674            match state.run_supervisors.manual_reason(&key) {
1675                Some(reason) => Some(RemoteRunRecoveryView {
1676                    mode: "manual",
1677                    // A Run in `unknown` still owns its Session until the
1678                    // runtime records a terminal event. Starting around that
1679                    // invariant would create two writers for one Session.
1680                    can_start_new_run: false,
1681                    reason: Some(reason),
1682                }),
1683                None => Some(RemoteRunRecoveryView {
1684                    mode: "automatic",
1685                    can_start_new_run: false,
1686                    reason: None,
1687                }),
1688            }
1689        } else {
1690            None
1691        };
1692        let supervision = state.run_supervisors.issue(&RunSupervisorRegistry::key(
1693            connector_id,
1694            &view.execution.run_id,
1695        ));
1696        Self {
1697            view,
1698            input,
1699            recovery,
1700            supervision,
1701        }
1702    }
1703}
1704
1705async fn start_run(
1706    State(state): State<RemoteApiState>,
1707    Path(session_id): Path<String>,
1708    Json(request): Json<StartRunRequest>,
1709) -> Result<(StatusCode, Json<StartRunResponse>), ApiError> {
1710    if !request.attachments.is_empty() {
1711        return Err(ApiError::new(
1712            StatusCode::NOT_IMPLEMENTED,
1713            "agent_artifacts_unsupported",
1714            "the built-in Agent does not declare Artifact input support",
1715        ));
1716    }
1717    let session_id = AgentSessionId::new(session_id);
1718    state.agent.create_session(Some(session_id.clone())).await?;
1719    let run_id = RunId::new(request.run_id);
1720    let handle = state
1721        .agent
1722        .start_text(&session_id, Some(run_id.clone()), request.input)
1723        .await?;
1724    spawn_remembered_approval_driver(state.clone(), run_id.clone());
1725    let view = RemoteRunView::new(
1726        &state,
1727        None,
1728        handle.inspect().await?,
1729        state.agent.initial_input(&run_id).await?,
1730    );
1731    Ok((StatusCode::CREATED, Json(StartRunResponse { run_id, view })))
1732}
1733
1734pub(super) fn spawn_remembered_approval_driver(state: RemoteApiState, run_id: RunId) {
1735    spawn_run_supervisor(state.clone(), state.agent.clone(), None, run_id);
1736}
1737
1738pub(super) fn spawn_run_supervisor(
1739    state: RemoteApiState,
1740    agent: AgentApi,
1741    connector_id: Option<AgentConnectorId>,
1742    run_id: RunId,
1743) {
1744    let key = RunSupervisorRegistry::key(connector_id.as_ref(), &run_id);
1745    if !state.run_supervisors.begin(&key) {
1746        return;
1747    }
1748    let registry = state.run_supervisors.clone();
1749    tokio::spawn(async move {
1750        supervise_run(state, agent, connector_id, run_id).await;
1751        registry.finish(&key);
1752    });
1753}
1754
1755async fn supervise_run(
1756    state: RemoteApiState,
1757    agent: AgentApi,
1758    connector_id: Option<AgentConnectorId>,
1759    run_id: RunId,
1760) {
1761    let key = RunSupervisorRegistry::key(connector_id.as_ref(), &run_id);
1762    let policy = state.run_supervisors.policy;
1763    let mut failures = 0_u32;
1764    let now = std::time::Instant::now();
1765    let initial_age = agent
1766        .catalog_runs()
1767        .await
1768        .ok()
1769        .and_then(|runs| {
1770            runs.into_iter()
1771                .find(|entry| entry.run_id == run_id)
1772                .map(|entry| {
1773                    chrono::Utc::now()
1774                        .timestamp_millis()
1775                        .saturating_sub(entry.updated_at_unix_ms)
1776                        .max(0) as u64
1777                })
1778        })
1779        .map(Duration::from_millis)
1780        .unwrap_or_default();
1781    let mut last_progress = now
1782        .checked_sub(initial_age.min(policy.inactivity_timeout))
1783        .unwrap_or(now);
1784    let mut watchdog_cancel_command = None;
1785    let mut watchdog_stop_requested_at: Option<std::time::Instant> = None;
1786    loop {
1787        // Subscribe before inspecting so a transition committed between the
1788        // two operations remains observable by this supervisor.
1789        let mut live = match agent.subscribe(&run_id).await {
1790            Ok(live) => live,
1791            Err(error) => {
1792                failures = failures.saturating_add(1);
1793                tracing::warn!(
1794                    connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1795                    run_id = %run_id.as_str(),
1796                    %error,
1797                    "could not subscribe to supervised Agent Run"
1798                );
1799                tokio::time::sleep(run_supervisor_backoff(failures)).await;
1800                continue;
1801            }
1802        };
1803        let view = match agent.inspect(&run_id).await {
1804            Ok(view) => view,
1805            Err(error) => {
1806                failures = failures.saturating_add(1);
1807                tracing::warn!(
1808                    connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1809                    run_id = %run_id.as_str(),
1810                    %error,
1811                    "could not inspect supervised Agent Run"
1812                );
1813                tokio::time::sleep(run_supervisor_backoff(failures)).await;
1814                continue;
1815            }
1816        };
1817        if view.state.is_terminal() {
1818            state.run_supervisors.clear_manual(&key);
1819            state.run_supervisors.clear_issue(&key);
1820            return;
1821        }
1822        if view.state.status()
1823            == orchestral_core::agent_protocol::reference::AgentRunStatus::Unknown
1824        {
1825            state.run_supervisors.clear_issue(&key);
1826            match agent.recover(&run_id).await {
1827                Ok(_) => {
1828                    // Recovery acknowledgement is only the start of restored
1829                    // observation. A Provider stream that fails immediately
1830                    // can put the Run back into Unknown before the next loop;
1831                    // retaining exponential backoff prevents an unbounded
1832                    // continuity_lost/restored journal storm.
1833                    failures = failures.saturating_add(1);
1834                    state.run_supervisors.clear_manual(&key);
1835                    tracing::info!(
1836                        connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1837                        run_id = %run_id.as_str(),
1838                        "restored Agent Run continuity"
1839                    );
1840                    tokio::time::sleep(run_supervisor_backoff(failures)).await;
1841                }
1842                Err(error) => {
1843                    if !is_retryable_agent_error(&error) {
1844                        state
1845                            .run_supervisors
1846                            .mark_manual(key.clone(), error.to_string());
1847                        tracing::info!(
1848                            connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1849                            run_id = %run_id.as_str(),
1850                            %error,
1851                            "Agent Run requires manual recovery; stopped automatic retries"
1852                        );
1853                        return;
1854                    }
1855                    failures = failures.saturating_add(1);
1856                    tracing::warn!(
1857                        connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1858                        run_id = %run_id.as_str(),
1859                        %error,
1860                        retry_after_ms = run_supervisor_backoff(failures).as_millis(),
1861                        "Agent Run recovery attempt failed"
1862                    );
1863                    tokio::time::sleep(run_supervisor_backoff(failures)).await;
1864                }
1865            }
1866            continue;
1867        }
1868
1869        failures = 0;
1870        apply_remembered_approvals(&state, &agent, &run_id, &view.pending_requests).await;
1871
1872        if view.state.status()
1873            == orchestral_core::agent_protocol::reference::AgentRunStatus::Waiting
1874        {
1875            // A blocking input/approval request is healthy quiescence. Its
1876            // lifetime belongs to the user, not the execution watchdog.
1877            last_progress = std::time::Instant::now();
1878            watchdog_cancel_command = None;
1879            watchdog_stop_requested_at = None;
1880            state.run_supervisors.clear_issue(&key);
1881        } else if view.state.status()
1882            == orchestral_core::agent_protocol::reference::AgentRunStatus::Stopping
1883            && watchdog_stop_requested_at.is_none()
1884        {
1885            state.run_supervisors.mark_issue(
1886                key.clone(),
1887                "interrupting",
1888                "Agent stop was accepted; waiting for the Provider to confirm a terminal state"
1889                    .to_owned(),
1890            );
1891            watchdog_stop_requested_at = Some(std::time::Instant::now());
1892        } else if let Some(requested_at) = watchdog_stop_requested_at {
1893            if requested_at.elapsed() >= policy.stop_grace {
1894                state.run_supervisors.mark_issue(
1895                    key.clone(),
1896                    "stalled",
1897                    format!(
1898                        "Agent execution stopped making progress and did not reach a terminal state within {} seconds after the Host requested cancellation",
1899                        policy.stop_grace.as_secs()
1900                    ),
1901                );
1902            }
1903        } else if last_progress.elapsed() >= policy.inactivity_timeout {
1904            let reason = format!(
1905                "Agent execution produced no model, Tool, output, or request progress for {} seconds; the Host watchdog requested a safe stop",
1906                policy.inactivity_timeout.as_secs()
1907            );
1908            state
1909                .run_supervisors
1910                .mark_issue(key.clone(), "interrupting", reason.clone());
1911            if watchdog_cancel_command.is_none() {
1912                watchdog_cancel_command = match AgentCommandEnvelope::new(
1913                    CommandId::new(format!(
1914                        "host-stall-cancel-{}-{}",
1915                        run_id.as_str(),
1916                        view.last_run_seq.unwrap_or(0)
1917                    )),
1918                    run_id.clone(),
1919                    None,
1920                    AgentCommand::Cancel { reason },
1921                ) {
1922                    Ok(command) => Some(command),
1923                    Err(error) => {
1924                        state.run_supervisors.mark_issue(
1925                            key.clone(),
1926                            "stalled",
1927                            format!("could not construct the Agent watchdog cancellation: {error}"),
1928                        );
1929                        return;
1930                    }
1931                };
1932            }
1933            match agent
1934                .command(
1935                    watchdog_cancel_command
1936                        .clone()
1937                        .expect("watchdog cancellation was initialized"),
1938                )
1939                .await
1940            {
1941                Ok(ack)
1942                    if matches!(
1943                        ack.state,
1944                        CommandAckState::Accepted { .. } | CommandAckState::Applied { .. }
1945                    ) =>
1946                {
1947                    watchdog_stop_requested_at = Some(std::time::Instant::now());
1948                    tracing::warn!(
1949                        connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1950                        run_id = %run_id.as_str(),
1951                        inactivity_seconds = policy.inactivity_timeout.as_secs(),
1952                        "Agent Run watchdog requested cancellation after execution inactivity"
1953                    );
1954                }
1955                Ok(ack) => {
1956                    state.run_supervisors.mark_issue(
1957                        key.clone(),
1958                        "stalled",
1959                        format!(
1960                            "Agent execution is stalled and its Provider did not accept the Host watchdog cancellation: {:?}",
1961                            ack.state
1962                        ),
1963                    );
1964                    return;
1965                }
1966                Err(error) => {
1967                    failures = failures.saturating_add(1);
1968                    state.run_supervisors.mark_issue(
1969                        key.clone(),
1970                        "stalled",
1971                        format!(
1972                            "Agent execution is stalled and the Host watchdog could not request cancellation: {error}"
1973                        ),
1974                    );
1975                    tracing::warn!(
1976                        connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1977                        run_id = %run_id.as_str(),
1978                        %error,
1979                        "Agent Run watchdog cancellation failed"
1980                    );
1981                    tokio::time::sleep(run_supervisor_backoff(failures)).await;
1982                    continue;
1983                }
1984            }
1985        }
1986
1987        match tokio::time::timeout(RUN_SUPERVISOR_POLL_INTERVAL, live.recv()).await {
1988            Ok(Ok(event)) => {
1989                if agent_control_event_is_execution_progress(&event) {
1990                    last_progress = std::time::Instant::now();
1991                    if watchdog_stop_requested_at.is_none() {
1992                        state.run_supervisors.clear_issue(&key);
1993                    }
1994                }
1995            }
1996            Ok(Err(broadcast::error::RecvError::Lagged(_))) | Err(_) => {}
1997            Ok(Err(broadcast::error::RecvError::Closed)) => {
1998                failures = failures.saturating_add(1);
1999                tokio::time::sleep(run_supervisor_backoff(failures)).await;
2000            }
2001        }
2002    }
2003}
2004
2005fn agent_control_event_is_execution_progress(event: &AgentControlEvent) -> bool {
2006    match event {
2007        AgentControlEvent::Telemetry(_) => true,
2008        AgentControlEvent::Durable(record) => {
2009            agent_event_is_execution_progress(&record.event.payload)
2010        }
2011        _ => false,
2012    }
2013}
2014
2015fn agent_event_is_execution_progress(event: &AgentEvent) -> bool {
2016    !matches!(
2017        event,
2018        AgentEvent::RunAccepted { .. }
2019            | AgentEvent::ResourceBindingSkipped { .. }
2020            | AgentEvent::CommandReceived { .. }
2021            | AgentEvent::CommandDispositionRecorded { .. }
2022            | AgentEvent::StopRequested { .. }
2023            | AgentEvent::ContinuityLost { .. }
2024            | AgentEvent::ContinuityRestored { .. }
2025    )
2026}
2027
2028/// Distinguishes transient supervision failures from durable contract or
2029/// recovery-boundary rejections. Retrying a non-retryable protocol error can
2030/// never change the outcome and otherwise produces an endless warning loop.
2031pub(super) fn is_retryable_agent_error(error: &AgentSdkError) -> bool {
2032    match error {
2033        AgentSdkError::InvalidInput(_) => false,
2034        AgentSdkError::Protocol(error)
2035        | AgentSdkError::Control(AgentControlError::Protocol(error)) => error.retryable,
2036        AgentSdkError::Control(AgentControlError::Start(AgentStartError::Rejected(rejection))) => {
2037            rejection.retryable
2038        }
2039        AgentSdkError::Control(AgentControlError::Start(AgentStartError::OutcomeUnknown(_)))
2040        | AgentSdkError::Control(AgentControlError::Journal(_))
2041        | AgentSdkError::ControlStreamClosed(_) => true,
2042        AgentSdkError::Control(
2043            AgentControlError::RunNotFound(_)
2044            | AgentControlError::ContinuityUnknown(_)
2045            | AgentControlError::RecoveryMismatch(_),
2046        ) => false,
2047        _ => true,
2048    }
2049}
2050
2051async fn apply_remembered_approvals(
2052    state: &RemoteApiState,
2053    agent: &AgentApi,
2054    run_id: &RunId,
2055    requests: &[PendingRequest],
2056) {
2057    for request in requests {
2058        if !matches!(request.payload, PendingRequestPayload::Approval { .. }) {
2059            continue;
2060        }
2061        let Ok(Some(grant_ref)) = state
2062            .approvals
2063            .approve_if_remembered(&request.request_id, approval_expiry_ms())
2064        else {
2065            continue;
2066        };
2067        let Ok(command) = AgentCommandEnvelope::new(
2068            CommandId::new(format!(
2069                "host-remembered-approval-{}",
2070                request.request_id.as_str()
2071            )),
2072            run_id.clone(),
2073            Some(request.request_id.clone()),
2074            AgentCommand::ResolveRequest {
2075                response: RequestResolution::Approval {
2076                    decision: ApprovalDecision::Allow,
2077                    grant_ref: Some(grant_ref),
2078                },
2079            },
2080        ) else {
2081            continue;
2082        };
2083        if let Err(error) = agent.command(command).await {
2084            tracing::warn!(
2085                run_id = %run_id.as_str(),
2086                request_id = %request.request_id.as_str(),
2087                %error,
2088                "could not apply remembered approval"
2089            );
2090        }
2091    }
2092}
2093
2094fn run_supervisor_backoff(failures: u32) -> Duration {
2095    let exponent = failures.saturating_sub(1).min(8);
2096    RUN_SUPERVISOR_INITIAL_BACKOFF
2097        .saturating_mul(2_u32.saturating_pow(exponent))
2098        .min(RUN_SUPERVISOR_MAX_BACKOFF)
2099}
2100
2101async fn inspect_run(
2102    State(state): State<RemoteApiState>,
2103    Path(run_id): Path<String>,
2104    Query(query): Query<RunTargetQuery>,
2105) -> Result<Json<RemoteRunView>, ApiError> {
2106    let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2107    let (agent, run_id) = require_run(
2108        &state,
2109        connector_id.as_ref().map(AgentConnectorId::as_str),
2110        run_id,
2111    )
2112    .await?;
2113    Ok(Json(RemoteRunView::new(
2114        &state,
2115        connector_id.as_ref(),
2116        agent.inspect(&run_id).await?,
2117        agent.initial_input(&run_id).await?,
2118    )))
2119}
2120
2121async fn recover_run(
2122    State(state): State<RemoteApiState>,
2123    Path(run_id): Path<String>,
2124    Query(query): Query<RunTargetQuery>,
2125) -> Result<Json<RemoteRunView>, ApiError> {
2126    let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2127    let (agent, run_id) = require_run(
2128        &state,
2129        connector_id.as_ref().map(AgentConnectorId::as_str),
2130        run_id,
2131    )
2132    .await?;
2133    let current = agent.inspect(&run_id).await?;
2134    let view = if current.state.status()
2135        == orchestral_core::agent_protocol::reference::AgentRunStatus::Unknown
2136    {
2137        let key = RunSupervisorRegistry::key(connector_id.as_ref(), &run_id);
2138        if state.run_supervisors.manual_reason(&key).is_some() {
2139            current
2140        } else {
2141            match agent.recover(&run_id).await {
2142                Ok(view) => {
2143                    state.run_supervisors.clear_manual(&key);
2144                    view
2145                }
2146                Err(error) if !is_retryable_agent_error(&error) => {
2147                    state.run_supervisors.mark_manual(key, error.to_string());
2148                    current
2149                }
2150                Err(error) => return Err(error.into()),
2151            }
2152        }
2153    } else {
2154        state
2155            .run_supervisors
2156            .clear_manual(&RunSupervisorRegistry::key(connector_id.as_ref(), &run_id));
2157        current
2158    };
2159    Ok(Json(RemoteRunView::new(
2160        &state,
2161        connector_id.as_ref(),
2162        view,
2163        agent.initial_input(&run_id).await?,
2164    )))
2165}
2166
2167#[derive(Debug, Deserialize, Default)]
2168#[serde(deny_unknown_fields)]
2169struct RunTargetQuery {
2170    #[serde(default)]
2171    connector_id: Option<String>,
2172}
2173
2174#[derive(Debug, Deserialize, Default)]
2175#[serde(deny_unknown_fields)]
2176struct EventsQuery {
2177    #[serde(default)]
2178    after: u64,
2179    #[serde(default)]
2180    connector_id: Option<String>,
2181}
2182
2183#[derive(Debug, Serialize)]
2184struct EventsResponse {
2185    after: u64,
2186    next: u64,
2187    records: Vec<orchestral_core::agent_protocol::wire::AgentJournalRecord>,
2188}
2189
2190async fn run_events(
2191    State(state): State<RemoteApiState>,
2192    Path(run_id): Path<String>,
2193    Query(query): Query<EventsQuery>,
2194) -> Result<Json<EventsResponse>, ApiError> {
2195    let (agent, run_id) = require_run(&state, query.connector_id.as_deref(), run_id).await?;
2196    let records = agent.events(&run_id, query.after).await?;
2197    let next = records
2198        .last()
2199        .map_or(query.after, |record| record.event.run_seq);
2200    Ok(Json(EventsResponse {
2201        after: query.after,
2202        next,
2203        records,
2204    }))
2205}
2206
2207async fn run_stream(
2208    State(state): State<RemoteApiState>,
2209    Extension(request_context): Extension<RequestLogContext>,
2210    Path(run_id): Path<String>,
2211    Query(query): Query<EventsQuery>,
2212    headers: HeaderMap,
2213) -> Result<Sse<impl futures_util::Stream<Item = Result<Event, Infallible>>>, ApiError> {
2214    let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2215    let (agent, run_id) = require_run(
2216        &state,
2217        connector_id.as_ref().map(AgentConnectorId::as_str),
2218        run_id,
2219    )
2220    .await?;
2221    let header_cursor = headers
2222        .get("last-event-id")
2223        .and_then(|value| value.to_str().ok())
2224        .and_then(|value| value.parse::<u64>().ok())
2225        .unwrap_or_default();
2226    let initial_cursor = query.after.max(header_cursor);
2227    // Subscribe before replay. Any event committed between these operations is
2228    // either present in replay or remains queued; sequence filtering removes
2229    // the harmless overlap.
2230    let mut live = agent.subscribe(&run_id).await?;
2231    let lifecycle = SseLifecycleLog::open_run(&request_context, connector_id.as_ref(), &run_id);
2232    let stream = async_stream::stream! {
2233        let mut lifecycle = lifecycle;
2234        let mut cursor = initial_cursor;
2235        match replay_events(&agent, &run_id, &mut cursor).await {
2236            Ok(events) => {
2237                for event in events {
2238                    yield Ok(event);
2239                }
2240            }
2241            Err(error) => {
2242                lifecycle.close_as("initial_replay_failed");
2243                tracing::error!(
2244                    request_id = %lifecycle.request_id,
2245                    stream_id = %lifecycle.stream_id,
2246                    status = error.status.as_u16(),
2247                    error_code = %error.body.code,
2248                    "could not replay initial Run SSE events"
2249                );
2250                yield Ok(api_error_event(&error));
2251                return;
2252            }
2253        }
2254
2255        loop {
2256            match live.recv().await {
2257                Ok(AgentControlEvent::Durable(record)) => {
2258                    if record.event.run_seq <= cursor {
2259                        continue;
2260                    }
2261                    cursor = record.event.run_seq;
2262                    yield Ok(durable_event(record.as_ref()));
2263                }
2264                Ok(AgentControlEvent::Telemetry(telemetry)) => {
2265                    match Event::default().event("telemetry").json_data(&telemetry) {
2266                        Ok(event) => yield Ok(event),
2267                        Err(error) => {
2268                            lifecycle.close_as("telemetry_encode_failed");
2269                            tracing::error!(
2270                                request_id = %lifecycle.request_id,
2271                                stream_id = %lifecycle.stream_id,
2272                                %error,
2273                                "could not encode Run telemetry SSE event"
2274                            );
2275                            yield Ok(api_error_event(&ApiError::internal(
2276                                "stream_encode_failed",
2277                                error.to_string(),
2278                            )));
2279                            return;
2280                        }
2281                    }
2282                }
2283                Ok(_) => {}
2284                Err(broadcast::error::RecvError::Lagged(skipped)) => {
2285                    lifecycle.lagged(skipped);
2286                    match replay_events(&agent, &run_id, &mut cursor).await {
2287                        Ok(events) => {
2288                            for event in events {
2289                                yield Ok(event);
2290                            }
2291                        }
2292                        Err(error) => {
2293                            lifecycle.close_as("lag_replay_failed");
2294                            tracing::error!(
2295                                request_id = %lifecycle.request_id,
2296                                stream_id = %lifecycle.stream_id,
2297                                status = error.status.as_u16(),
2298                                error_code = %error.body.code,
2299                                "could not replay Run SSE events after subscriber lag"
2300                            );
2301                            yield Ok(api_error_event(&error));
2302                            return;
2303                        }
2304                    }
2305                }
2306                Err(broadcast::error::RecvError::Closed) => {
2307                    match replay_events(&agent, &run_id, &mut cursor).await {
2308                        Ok(events) => {
2309                            for event in events {
2310                                yield Ok(event);
2311                            }
2312                            lifecycle.close_as("source_closed");
2313                        }
2314                        Err(error) => {
2315                            lifecycle.close_as("source_closed_replay_failed");
2316                            tracing::error!(
2317                                request_id = %lifecycle.request_id,
2318                                stream_id = %lifecycle.stream_id,
2319                                status = error.status.as_u16(),
2320                                error_code = %error.body.code,
2321                                "could not replay final Run SSE events after source closed"
2322                            );
2323                        }
2324                    }
2325                    return;
2326                }
2327            }
2328        }
2329    };
2330    Ok(Sse::new(stream).keep_alive(
2331        KeepAlive::new()
2332            .interval(Duration::from_secs(10))
2333            .text("keep-alive"),
2334    ))
2335}
2336
2337async fn replay_events(
2338    agent: &AgentApi,
2339    run_id: &RunId,
2340    cursor: &mut u64,
2341) -> Result<Vec<Event>, ApiError> {
2342    let records = agent.events(run_id, *cursor).await?;
2343    let mut events = Vec::with_capacity(records.len());
2344    for record in records {
2345        if record.event.run_seq <= *cursor {
2346            continue;
2347        }
2348        *cursor = record.event.run_seq;
2349        events.push(durable_event(&record));
2350    }
2351    Ok(events)
2352}
2353
2354fn durable_event(record: &orchestral_core::agent_protocol::wire::AgentJournalRecord) -> Event {
2355    match Event::default()
2356        .event("durable")
2357        .id(record.event.run_seq.to_string())
2358        .json_data(record)
2359    {
2360        Ok(event) => event,
2361        Err(error) => api_error_event(&ApiError::internal(
2362            "stream_encode_failed",
2363            error.to_string(),
2364        )),
2365    }
2366}
2367
2368fn api_error_event(error: &ApiError) -> Event {
2369    Event::default().event("error").data(
2370        serde_json::to_string(&error.body)
2371            .unwrap_or_else(|_| r#"{"code":"stream_failed","message":"stream failed"}"#.to_owned()),
2372    )
2373}
2374
2375#[derive(Debug, Deserialize)]
2376#[serde(deny_unknown_fields)]
2377struct TextCommandRequest {
2378    command_id: String,
2379    text: String,
2380    #[serde(default)]
2381    after_activity_id: Option<String>,
2382    #[serde(default)]
2383    attachments: Vec<RemoteArtifactInput>,
2384}
2385
2386async fn steer_run(
2387    State(state): State<RemoteApiState>,
2388    Path(run_id): Path<String>,
2389    Query(query): Query<RunTargetQuery>,
2390    Json(request): Json<TextCommandRequest>,
2391) -> Result<Json<CommandAck>, ApiError> {
2392    let content = message_content(&state, &request.text, &request.attachments).await?;
2393    let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2394    let (agent, run_id) =
2395        require_commandable_run(&state, query.connector_id.as_deref(), run_id).await?;
2396    let supervision_key = RunSupervisorRegistry::key(connector_id.as_ref(), &run_id);
2397    if let Some(issue) = state.run_supervisors.issue(&supervision_key) {
2398        return Err(ApiError::conflict(
2399            "agent_run_stalled",
2400            format!(
2401                "{}; wait for the watchdog cancellation to finish or stop the Run explicitly",
2402                issue.reason
2403            ),
2404        ));
2405    }
2406    let history_anchor =
2407        request
2408            .after_activity_id
2409            .map(|after_activity_id| AgentSessionHistoryAnchor {
2410                after_activity_id: AgentSessionActivityId::new(after_activity_id),
2411            });
2412    let command = AgentCommandEnvelope::new_with_extensions(
2413        CommandId::new(request.command_id),
2414        run_id.clone(),
2415        None,
2416        AgentCommand::Steer { content },
2417        session_history_anchor_extensions(history_anchor.as_ref())?,
2418    )?;
2419    Ok(Json(
2420        command_run_for_session(&state, connector_id.as_ref(), &agent, &run_id, command).await?,
2421    ))
2422}
2423
2424#[derive(Debug, Deserialize)]
2425#[serde(deny_unknown_fields)]
2426struct CancelRequest {
2427    command_id: String,
2428    reason: String,
2429}
2430
2431async fn cancel_run(
2432    State(state): State<RemoteApiState>,
2433    Path(run_id): Path<String>,
2434    Query(query): Query<RunTargetQuery>,
2435    Json(request): Json<CancelRequest>,
2436) -> Result<Json<CommandAck>, ApiError> {
2437    let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2438    let (agent, run_id) =
2439        require_commandable_run(&state, query.connector_id.as_deref(), run_id).await?;
2440    let command = AgentCommandEnvelope::new(
2441        CommandId::new(request.command_id),
2442        run_id.clone(),
2443        None,
2444        AgentCommand::Cancel {
2445            reason: request.reason,
2446        },
2447    )?;
2448    Ok(Json(
2449        command_run_for_session(&state, connector_id.as_ref(), &agent, &run_id, command).await?,
2450    ))
2451}
2452
2453async fn resolve_input(
2454    State(state): State<RemoteApiState>,
2455    Path((run_id, request_id)): Path<(String, String)>,
2456    Query(query): Query<RunTargetQuery>,
2457    Json(request): Json<TextCommandRequest>,
2458) -> Result<Json<CommandAck>, ApiError> {
2459    let content = message_content(&state, &request.text, &request.attachments).await?;
2460    let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2461    let (agent, run_id) =
2462        require_commandable_run(&state, query.connector_id.as_deref(), run_id).await?;
2463    let request_id = RequestId::new(request_id);
2464    let pending = require_pending_for(&agent, &run_id, &request_id).await?;
2465    if !matches!(pending.payload, PendingRequestPayload::Input { .. }) {
2466        return Err(ApiError::conflict(
2467            "request_kind_mismatch",
2468            "pending request does not accept text input",
2469        ));
2470    }
2471    let command = AgentCommandEnvelope::new(
2472        CommandId::new(request.command_id),
2473        run_id.clone(),
2474        Some(request_id),
2475        AgentCommand::ResolveRequest {
2476            response: RequestResolution::Input { content },
2477        },
2478    )?;
2479    Ok(Json(
2480        command_run_for_session(&state, connector_id.as_ref(), &agent, &run_id, command).await?,
2481    ))
2482}
2483
2484async fn resolve_agent_session_input(
2485    State(state): State<RemoteApiState>,
2486    Path(request_id): Path<String>,
2487    Query(query): Query<AgentSessionQuery>,
2488    Json(request): Json<AgentSessionTextRequest>,
2489) -> Result<Json<Value>, ApiError> {
2490    let content = message_content(&state, &request.text, &request.attachments).await?;
2491    state
2492        .agent_directory
2493        .resolve_request(
2494            &AgentConnectorId::new(query.connector_id),
2495            ResolveAgentSessionRequest {
2496                session_id: AgentSessionId::new(query.session_id),
2497                request_id: RequestId::new(request_id),
2498                response: AgentSessionRequestResolution::Input { content },
2499            },
2500        )
2501        .await?;
2502    Ok(Json(json!({"resolved": true})))
2503}
2504
2505/// A provider-native session request is not a Host Run command. A redundant
2506/// command_id is tolerated for already-open PWA clients, but has no command
2507/// ledger semantics. Keep this wire contract separate from
2508/// `TextCommandRequest`; reusing the Run DTO made the PWA's valid `{text}`
2509/// response fail JSON extraction before it reached the connector SPI.
2510#[derive(Debug, Deserialize)]
2511#[serde(deny_unknown_fields)]
2512struct AgentSessionTextRequest {
2513    text: String,
2514    #[serde(default, rename = "command_id")]
2515    _legacy_command_id: Option<String>,
2516    #[serde(default)]
2517    attachments: Vec<RemoteArtifactInput>,
2518}
2519
2520#[derive(Debug, Clone, Copy, Deserialize)]
2521#[serde(rename_all = "snake_case")]
2522enum ApprovalChoice {
2523    AllowOnce,
2524    AllowSession,
2525    Deny,
2526}
2527
2528#[derive(Debug, Deserialize)]
2529#[serde(deny_unknown_fields)]
2530struct ApprovalRequest {
2531    command_id: String,
2532    decision: ApprovalChoice,
2533}
2534
2535#[derive(Debug, Deserialize)]
2536#[serde(deny_unknown_fields)]
2537struct AgentSessionApprovalRequest {
2538    decision: ApprovalChoice,
2539    #[serde(default, rename = "command_id")]
2540    _legacy_command_id: Option<String>,
2541}
2542
2543async fn resolve_approval(
2544    State(state): State<RemoteApiState>,
2545    Path((run_id, request_id)): Path<(String, String)>,
2546    Query(query): Query<RunTargetQuery>,
2547    Json(request): Json<ApprovalRequest>,
2548) -> Result<Json<CommandAck>, ApiError> {
2549    let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2550    let (agent, run_id) =
2551        require_commandable_run(&state, query.connector_id.as_deref(), run_id).await?;
2552    let request_id = RequestId::new(request_id);
2553    let pending = require_pending_for(&agent, &run_id, &request_id).await?;
2554    let PendingRequestPayload::Approval {
2555        session_approval_scope,
2556        ..
2557    } = pending.payload
2558    else {
2559        return Err(ApiError::conflict(
2560            "request_kind_mismatch",
2561            "pending request is not an approval",
2562        ));
2563    };
2564    let (decision, grant_ref) = match request.decision {
2565        ApprovalChoice::Deny => (ApprovalDecision::Deny, None),
2566        ApprovalChoice::AllowOnce => (
2567            ApprovalDecision::Allow,
2568            Some(state.approvals.approve(&request_id, approval_expiry_ms())?),
2569        ),
2570        ApprovalChoice::AllowSession => {
2571            if session_approval_scope.is_none() {
2572                return Err(ApiError::conflict(
2573                    "session_approval_unavailable",
2574                    "this operation cannot be approved for the session",
2575                ));
2576            }
2577            (
2578                ApprovalDecision::Allow,
2579                Some(
2580                    state
2581                        .approvals
2582                        .approve_for_session(&request_id, approval_expiry_ms())?,
2583                ),
2584            )
2585        }
2586    };
2587    let command = AgentCommandEnvelope::new(
2588        CommandId::new(request.command_id),
2589        run_id.clone(),
2590        Some(request_id),
2591        AgentCommand::ResolveRequest {
2592            response: RequestResolution::Approval {
2593                decision,
2594                grant_ref,
2595            },
2596        },
2597    )?;
2598    Ok(Json(
2599        command_run_for_session(&state, connector_id.as_ref(), &agent, &run_id, command).await?,
2600    ))
2601}
2602
2603async fn resolve_agent_session_approval(
2604    State(state): State<RemoteApiState>,
2605    Path(request_id): Path<String>,
2606    Query(query): Query<AgentSessionQuery>,
2607    Json(request): Json<AgentSessionApprovalRequest>,
2608) -> Result<Json<Value>, ApiError> {
2609    let decision = match request.decision {
2610        ApprovalChoice::Deny => ApprovalDecision::Deny,
2611        ApprovalChoice::AllowOnce => ApprovalDecision::Allow,
2612        ApprovalChoice::AllowSession => {
2613            return Err(ApiError::conflict(
2614                "session_approval_unavailable",
2615                "provider-native session requests do not declare a remembered approval scope",
2616            ));
2617        }
2618    };
2619    state
2620        .agent_directory
2621        .resolve_request(
2622            &AgentConnectorId::new(query.connector_id),
2623            ResolveAgentSessionRequest {
2624                session_id: AgentSessionId::new(query.session_id),
2625                request_id: RequestId::new(request_id),
2626                response: AgentSessionRequestResolution::Approval { decision },
2627            },
2628        )
2629        .await?;
2630    Ok(Json(json!({"resolved": true})))
2631}
2632
2633async fn require_run(
2634    state: &RemoteApiState,
2635    connector_id: Option<&str>,
2636    run_id: String,
2637) -> Result<(AgentApi, RunId), ApiError> {
2638    // Old PWA builds routed a native history projection to Run control. This
2639    // identity was never an executable Run: require the session endpoint and
2640    // give those clients an actionable message instead of a misleading 404.
2641    if run_id.starts_with("agent-history:") {
2642        return Err(ApiError::conflict(
2643            "client_update_required",
2644            "此页面版本过旧,请刷新页面后重新处理审批或回复;当前操作尚未执行",
2645        ));
2646    }
2647    let agent = match connector_id {
2648        Some(connector_id) => {
2649            state
2650                .agent_directory
2651                .agent_api(&AgentConnectorId::new(connector_id))
2652                .await?
2653        }
2654        None => state.agent.clone(),
2655    };
2656    let run_id = RunId::new(run_id);
2657    if !agent.has_run(&run_id).await? {
2658        tracing::warn!(%run_id, connector_id, "requested Agent Run was not found");
2659        return Err(ApiError::not_found("run_not_found", "run was not found"));
2660    }
2661    spawn_run_supervisor(
2662        state.clone(),
2663        agent.clone(),
2664        connector_id.map(AgentConnectorId::new),
2665        run_id.clone(),
2666    );
2667    Ok((agent, run_id))
2668}
2669
2670async fn require_commandable_run(
2671    state: &RemoteApiState,
2672    connector_id: Option<&str>,
2673    run_id: String,
2674) -> Result<(AgentApi, RunId), ApiError> {
2675    let (agent, run_id) = require_run(state, connector_id, run_id).await?;
2676    if agent.inspect(&run_id).await?.state.status()
2677        == orchestral_core::agent_protocol::reference::AgentRunStatus::Unknown
2678    {
2679        return Err(ApiError::service_unavailable(
2680            "run_recovery_pending",
2681            "Agent Run continuity is being recovered; retry this command shortly",
2682        ));
2683    }
2684    Ok((agent, run_id))
2685}
2686
2687async fn command_run(
2688    agent: &AgentApi,
2689    run_id: &RunId,
2690    command: AgentCommandEnvelope,
2691) -> Result<CommandAck, ApiError> {
2692    let command_id = command.command_id.clone();
2693    let protocol_request_id = command.request_id.clone();
2694    let command_kind = match &command.payload {
2695        AgentCommand::Steer { .. } => "steer",
2696        AgentCommand::ResolveRequest { response } => match response {
2697            RequestResolution::Input { .. } => "resolve_input",
2698            RequestResolution::Approval { .. } => "resolve_approval",
2699            RequestResolution::ExternalResult { .. } => "resolve_external_result",
2700            _ => "resolve_unknown",
2701        },
2702        AgentCommand::Cancel { .. } => "cancel",
2703        _ => "unknown",
2704    };
2705    let result = agent.command(command).await;
2706    match result {
2707        Ok(ack) => {
2708            let ack_state = match &ack.state {
2709                CommandAckState::Accepted { .. } => "accepted",
2710                CommandAckState::Applied { .. } => "applied",
2711                CommandAckState::Rejected { .. } => "rejected",
2712                CommandAckState::Unsupported { .. } => "unsupported",
2713                _ => "unknown",
2714            };
2715            tracing::info!(
2716                run_id = %run_id.as_str(),
2717                command_id = %command_id.as_str(),
2718                protocol_request_id = protocol_request_id
2719                    .as_ref()
2720                    .map(RequestId::as_str)
2721                    .unwrap_or("-"),
2722                command_kind,
2723                ack_state,
2724                duplicate = ack.duplicate,
2725                "Agent command acknowledged"
2726            );
2727            Ok(ack)
2728        }
2729        Err(_error)
2730            if agent.inspect(run_id).await.is_ok_and(|view| {
2731                view.state.status()
2732                    == orchestral_core::agent_protocol::reference::AgentRunStatus::Unknown
2733            }) =>
2734        {
2735            tracing::warn!(
2736                run_id = %run_id.as_str(),
2737                command_id = %command_id.as_str(),
2738                protocol_request_id = protocol_request_id
2739                    .as_ref()
2740                    .map(RequestId::as_str)
2741                    .unwrap_or("-"),
2742                command_kind,
2743                "Agent command outcome is pending continuity recovery"
2744            );
2745            Err(ApiError::service_unavailable(
2746                "run_recovery_pending",
2747                "Agent Run continuity is being recovered; retry this command shortly",
2748            ))
2749        }
2750        Err(error) => {
2751            tracing::warn!(
2752                run_id = %run_id.as_str(),
2753                command_id = %command_id.as_str(),
2754                protocol_request_id = protocol_request_id
2755                    .as_ref()
2756                    .map(RequestId::as_str)
2757                    .unwrap_or("-"),
2758                command_kind,
2759                "Agent command failed"
2760            );
2761            Err(error.into())
2762        }
2763    }
2764}
2765
2766async fn command_run_for_session(
2767    state: &RemoteApiState,
2768    connector_id: Option<&AgentConnectorId>,
2769    agent: &AgentApi,
2770    run_id: &RunId,
2771    command: AgentCommandEnvelope,
2772) -> Result<CommandAck, ApiError> {
2773    let Some(connector_id) = connector_id else {
2774        return command_run(agent, run_id, command).await;
2775    };
2776    let session_id = agent
2777        .catalog_runs()
2778        .await?
2779        .into_iter()
2780        .find(|entry| entry.run_id == *run_id)
2781        .map(|entry| entry.session_id)
2782        .ok_or_else(|| ApiError::not_found("run_not_found", "run was not found"))?;
2783    let coordinator = state.session_coordinators.get(connector_id, &session_id);
2784    let _operation_guard = coordinator.operation().lock().await;
2785    command_run(agent, run_id, command).await
2786}
2787
2788/// Projects the remote conversation list directly from durable Run
2789/// registrations. The Agent journal is the only Session-to-Run source of
2790/// truth; empty Sessions remain process-local until their first Run starts.
2791async fn session_views(state: &RemoteApiState) -> Result<Vec<SessionView>, AgentSdkError> {
2792    let fallback = chrono::Utc::now().timestamp_millis();
2793    let mut catalog = state.agent.catalog_runs().await?;
2794    catalog.sort_by(|left, right| {
2795        left.created_at_unix_ms
2796            .cmp(&right.created_at_unix_ms)
2797            .then_with(|| left.run_id.cmp(&right.run_id))
2798    });
2799
2800    let mut sessions = BTreeMap::<String, SessionView>::new();
2801    for run in catalog {
2802        let created = if run.created_at_unix_ms > 0 {
2803            run.created_at_unix_ms
2804        } else {
2805            fallback
2806        };
2807        let updated = if run.updated_at_unix_ms > 0 {
2808            run.updated_at_unix_ms
2809        } else {
2810            created
2811        };
2812        let session = sessions
2813            .entry(run.session_id.as_str().to_owned())
2814            .or_insert_with(|| SessionView {
2815                id: run.session_id.as_str().to_owned(),
2816                created_at_unix_ms: created,
2817                updated_at_unix_ms: updated,
2818                run_ids: Vec::new(),
2819                cwd: state.native_session_defaults.cwd.clone(),
2820                execution_profile: state.native_session_defaults.execution_profile.clone(),
2821            });
2822        session.created_at_unix_ms = session.created_at_unix_ms.min(created);
2823        session.updated_at_unix_ms = session.updated_at_unix_ms.max(updated);
2824        session.run_ids.push(run.run_id.as_str().to_owned());
2825    }
2826    let mut sessions = sessions.into_values().collect::<Vec<_>>();
2827    sessions.sort_by_key(|session| std::cmp::Reverse(session.updated_at_unix_ms));
2828    Ok(sessions)
2829}
2830
2831async fn require_pending_for(
2832    agent: &AgentApi,
2833    run_id: &RunId,
2834    request_id: &RequestId,
2835) -> Result<PendingRequest, ApiError> {
2836    let view = agent.inspect(run_id).await?;
2837    view.pending_requests
2838        .into_iter()
2839        .find(|request| request.request_id == *request_id)
2840        .ok_or_else(|| {
2841            ApiError::conflict(
2842                "request_not_pending",
2843                "request is no longer pending for this run",
2844            )
2845        })
2846}
2847
2848async fn authenticate(
2849    State(state): State<RemoteApiState>,
2850    mut request: Request<axum::body::Body>,
2851    next: Next,
2852) -> Result<Response, ApiError> {
2853    if let Some(authenticator) = &state.gateway_authenticator {
2854        let assertion = request
2855            .headers()
2856            .get(authenticator.header_name())
2857            .and_then(|value| value.to_str().ok())
2858            .filter(|value| !value.trim().is_empty())
2859            .ok_or_else(|| {
2860                ApiError::unauthorized(
2861                    "gateway_authentication_required",
2862                    "a signed gateway identity is required",
2863                )
2864            })?;
2865        let principal = authenticator
2866            .authenticate(assertion)
2867            .await
2868            .map_err(|error| {
2869                ApiError::unauthorized("gateway_authentication_failed", error.to_string())
2870            })?;
2871        request
2872            .extensions_mut()
2873            .insert(RemotePrincipal::Gateway(principal));
2874        return Ok(next.run(request).await);
2875    }
2876
2877    let token = bearer_token(request.headers()).ok_or_else(|| {
2878        ApiError::unauthorized(
2879            "authentication_required",
2880            "device authentication is required",
2881        )
2882    })?;
2883    let principal = state.registry.authenticate(token).await.map_err(|_| {
2884        ApiError::unauthorized(
2885            "authentication_failed",
2886            "device authentication is invalid or revoked",
2887        )
2888    })?;
2889    request
2890        .extensions_mut()
2891        .insert(RemotePrincipal::Device(principal));
2892    Ok(next.run(request).await)
2893}
2894
2895async fn log_request(mut request: Request<axum::body::Body>, next: Next) -> Response {
2896    let context = RequestLogContext {
2897        request_id: uuid::Uuid::new_v4().to_string(),
2898    };
2899    let method = request.method().clone();
2900    let route = request
2901        .extensions()
2902        .get::<MatchedPath>()
2903        .map(MatchedPath::as_str)
2904        .unwrap_or("<unmatched>")
2905        .to_owned();
2906    let cf_ray = request
2907        .headers()
2908        .get("cf-ray")
2909        .and_then(|value| value.to_str().ok())
2910        .unwrap_or("-")
2911        .to_owned();
2912    request.extensions_mut().insert(context.clone());
2913
2914    let started_at = Instant::now();
2915    let span = tracing::info_span!(
2916        "http_request",
2917        request_id = %context.request_id,
2918        method = %method,
2919        route = %route,
2920        cf_ray = %cf_ray,
2921    );
2922    let mut response = next.run(request).instrument(span.clone()).await;
2923    if let Ok(value) = HeaderValue::from_str(&context.request_id) {
2924        response.headers_mut().insert(REQUEST_ID_HEADER, value);
2925    }
2926
2927    let status = response.status();
2928    let response_ready_ms = u64::try_from(started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
2929    let error_code = response
2930        .extensions()
2931        .get::<ApiErrorLogCode>()
2932        .map(|code| code.0.as_str())
2933        .unwrap_or("-");
2934    span.in_scope(|| {
2935        if status.is_server_error() {
2936            tracing::error!(
2937                status = status.as_u16(),
2938                response_ready_ms,
2939                error_code,
2940                "HTTP request completed"
2941            );
2942        } else if status.is_client_error() {
2943            tracing::warn!(
2944                status = status.as_u16(),
2945                response_ready_ms,
2946                error_code,
2947                "HTTP request completed"
2948            );
2949        } else {
2950            tracing::info!(
2951                status = status.as_u16(),
2952                response_ready_ms,
2953                "HTTP request completed"
2954            );
2955        }
2956    });
2957    response
2958}
2959
2960async fn no_store(request: Request<axum::body::Body>, next: Next) -> Response {
2961    let mut response = next.run(request).await;
2962    response.headers_mut().insert(
2963        header::CACHE_CONTROL,
2964        header::HeaderValue::from_static("no-store, private"),
2965    );
2966    response.headers_mut().insert(
2967        header::X_CONTENT_TYPE_OPTIONS,
2968        header::HeaderValue::from_static("nosniff"),
2969    );
2970    response
2971}
2972
2973fn bearer_token(headers: &HeaderMap) -> Option<&str> {
2974    headers
2975        .get(header::AUTHORIZATION)?
2976        .to_str()
2977        .ok()?
2978        .strip_prefix("Bearer ")
2979        .filter(|token| !token.trim().is_empty())
2980}
2981
2982fn approval_expiry_ms() -> i64 {
2983    chrono::Utc::now()
2984        .timestamp_millis()
2985        .saturating_add(APPROVAL_GRANT_TTL_MS)
2986}
2987
2988#[derive(Debug, Clone, Serialize)]
2989struct ApiErrorBody {
2990    code: String,
2991    message: String,
2992}
2993
2994#[derive(Debug)]
2995struct ApiError {
2996    status: StatusCode,
2997    body: ApiErrorBody,
2998}
2999
3000impl ApiError {
3001    fn new(status: StatusCode, code: impl Into<String>, message: impl Into<String>) -> Self {
3002        Self {
3003            status,
3004            body: ApiErrorBody {
3005                code: code.into(),
3006                message: message.into(),
3007            },
3008        }
3009    }
3010
3011    fn unauthorized(code: impl Into<String>, message: impl Into<String>) -> Self {
3012        Self::new(StatusCode::UNAUTHORIZED, code, message)
3013    }
3014
3015    fn not_found(code: impl Into<String>, message: impl Into<String>) -> Self {
3016        Self::new(StatusCode::NOT_FOUND, code, message)
3017    }
3018
3019    fn conflict(code: impl Into<String>, message: impl Into<String>) -> Self {
3020        Self::new(StatusCode::CONFLICT, code, message)
3021    }
3022
3023    fn service_unavailable(code: impl Into<String>, message: impl Into<String>) -> Self {
3024        Self::new(StatusCode::SERVICE_UNAVAILABLE, code, message)
3025    }
3026
3027    fn internal(code: impl Into<String>, message: impl Into<String>) -> Self {
3028        Self::new(StatusCode::INTERNAL_SERVER_ERROR, code, message)
3029    }
3030}
3031
3032impl IntoResponse for ApiError {
3033    fn into_response(self) -> Response {
3034        let code = ApiErrorLogCode(self.body.code.clone());
3035        let mut response = (self.status, Json(self.body)).into_response();
3036        response.extensions_mut().insert(code);
3037        response
3038    }
3039}
3040
3041impl From<anyhow::Error> for ApiError {
3042    fn from(error: anyhow::Error) -> Self {
3043        Self::internal("host_state_failed", error.to_string())
3044    }
3045}
3046
3047impl From<AgentSdkError> for ApiError {
3048    fn from(error: AgentSdkError) -> Self {
3049        match &error {
3050            AgentSdkError::InvalidInput(_) => Self::new(
3051                StatusCode::BAD_REQUEST,
3052                "invalid_agent_input",
3053                error.to_string(),
3054            ),
3055            _ => Self::internal("agent_control_failed", error.to_string()),
3056        }
3057    }
3058}
3059
3060impl From<AgentDirectoryError> for ApiError {
3061    fn from(error: AgentDirectoryError) -> Self {
3062        match &error {
3063            AgentDirectoryError::ConnectorNotFound(_) => {
3064                Self::not_found("agent_connector_not_found", error.to_string())
3065            }
3066            AgentDirectoryError::Connector(connector_error) => {
3067                use orchestral_core::agent_connector::AgentConnectorErrorCode;
3068                match connector_error.code {
3069                    AgentConnectorErrorCode::InvalidRequest => Self::new(
3070                        StatusCode::BAD_REQUEST,
3071                        "invalid_agent_connector_request",
3072                        error.to_string(),
3073                    ),
3074                    AgentConnectorErrorCode::NotFound => {
3075                        Self::not_found("agent_session_not_found", error.to_string())
3076                    }
3077                    AgentConnectorErrorCode::Busy | AgentConnectorErrorCode::LeaseConflict => {
3078                        Self::conflict("agent_connector_busy", error.to_string())
3079                    }
3080                    AgentConnectorErrorCode::Unsupported => Self::new(
3081                        StatusCode::NOT_IMPLEMENTED,
3082                        "agent_connector_unsupported",
3083                        error.to_string(),
3084                    ),
3085                    AgentConnectorErrorCode::Unavailable => Self::new(
3086                        StatusCode::SERVICE_UNAVAILABLE,
3087                        "agent_connector_unavailable",
3088                        error.to_string(),
3089                    ),
3090                    AgentConnectorErrorCode::Protocol | AgentConnectorErrorCode::OutcomeUnknown => {
3091                        Self::internal("agent_connector_failed", error.to_string())
3092                    }
3093                    _ => Self::internal("agent_connector_failed", error.to_string()),
3094                }
3095            }
3096            AgentDirectoryError::Agent(AgentSdkError::Control(AgentControlError::Start(
3097                AgentStartError::Rejected(rejection),
3098            ))) if rejection
3099                .details
3100                .get("code")
3101                .and_then(serde_json::Value::as_str)
3102                == Some("live_control_unavailable") =>
3103            {
3104                Self::conflict("live_control_unavailable", rejection.message.clone())
3105            }
3106            AgentDirectoryError::Agent(AgentSdkError::Control(AgentControlError::Start(
3107                AgentStartError::Rejected(rejection),
3108            ))) => match rejection.code {
3109                AgentRejectionCode::SessionConflict | AgentRejectionCode::RunIdConflict => {
3110                    Self::conflict("agent_session_conflict", rejection.message.clone())
3111                }
3112                AgentRejectionCode::InvalidSpec => Self::new(
3113                    StatusCode::BAD_REQUEST,
3114                    "invalid_agent_input",
3115                    rejection.message.clone(),
3116                ),
3117                AgentRejectionCode::UnsupportedProtocol
3118                | AgentRejectionCode::UnsupportedCapability
3119                | AgentRejectionCode::UnsupportedResource => Self::new(
3120                    StatusCode::NOT_IMPLEMENTED,
3121                    "agent_capability_unsupported",
3122                    rejection.message.clone(),
3123                ),
3124                AgentRejectionCode::ProviderUnavailable => Self::new(
3125                    StatusCode::SERVICE_UNAVAILABLE,
3126                    "agent_provider_unavailable",
3127                    rejection.message.clone(),
3128                ),
3129                _ => Self::new(
3130                    StatusCode::BAD_REQUEST,
3131                    "agent_rejected",
3132                    rejection.message.clone(),
3133                ),
3134            },
3135            _ => Self::internal("agent_directory_failed", error.to_string()),
3136        }
3137    }
3138}
3139
3140impl From<orchestral_core::agent_protocol::wire::AgentProtocolError> for ApiError {
3141    fn from(error: orchestral_core::agent_protocol::wire::AgentProtocolError) -> Self {
3142        Self::new(
3143            StatusCode::BAD_REQUEST,
3144            "invalid_command",
3145            error.to_string(),
3146        )
3147    }
3148}
3149
3150impl From<ApprovalBridgeError> for ApiError {
3151    fn from(error: ApprovalBridgeError) -> Self {
3152        match error {
3153            ApprovalBridgeError::RequestNotFound(_) => Self::conflict(
3154                "approval_binding_unavailable",
3155                "此入口缺少该审批的授权信息,本次操作未执行。请刷新会话后重试",
3156            ),
3157            ApprovalBridgeError::SessionScopeUnavailable(_) => Self::conflict(
3158                "session_approval_unavailable",
3159                "此审批不支持本会话允许,请选择允许一次或拒绝",
3160            ),
3161            _ => Self::internal("approval_bridge_failed", error.to_string()),
3162        }
3163    }
3164}
3165
3166#[cfg(test)]
3167mod tests {
3168    use super::*;
3169    use orchestral_core::agent_connector::AgentSessionExecutionProfile;
3170    use std::collections::BTreeSet;
3171    use std::sync::atomic::{AtomicUsize, Ordering};
3172    use std::sync::Arc;
3173
3174    use async_trait::async_trait;
3175    use axum::body::Body;
3176    use axum::http::{HeaderName, Request};
3177    use futures_util::{stream, StreamExt};
3178    use http_body_util::BodyExt;
3179    use orchestral_agent_protocol_testkit::{
3180        ProviderFixtureFactory, ProviderScenario, ScriptedStatelessFactory,
3181        SessionfulRecoverFactory, TestProbes,
3182    };
3183    use orchestral_core::agent_connector::{
3184        AgentConnector, AgentConnectorDescriptor, AgentConnectorError, AgentConnectorHealth,
3185        AgentSessionActionDescriptor, AgentSessionActionExecution, AgentSessionActionOutcome,
3186        AgentSessionActionStatus, AgentSessionActivity, AgentSessionActivityId,
3187        AgentSessionActivityKind, AgentSessionActivityStatus, AgentSessionCapabilities,
3188        AgentSessionDetail, AgentSessionState, AgentSessionSummary, AgentSessionTurn,
3189        AgentSessionTurnId, AgentSessionTurnStatus, CreateAgentSessionRequest,
3190        InvokeAgentSessionActionRequest, SESSION_FORK_ACTION, SESSION_RENAME_ACTION,
3191        SESSION_REVIEW_ACTION,
3192    };
3193    use orchestral_core::agent_protocol::{
3194        spi::{
3195            AgentProvider, AgentRecovery, AgentRecoveryRequest, AgentStart, AgentStartError,
3196            InMemoryAgentJournalStore,
3197        },
3198        wire::{
3199            AgentAdmission, AgentCapabilities, AgentDescriptor, AgentDescriptorEnvelope,
3200            AgentEvent, AgentEventDraft, AgentExecutionRef, AgentId, AgentProtocolError,
3201            AgentProtocolErrorCode, AgentProviderId, AgentProviderStreamItem, AgentRunEnvelope,
3202            EffectMediation, PendingRequestKind, ProviderBindingRef, ProviderCommandDisposition,
3203            ProviderCommandOutcome,
3204        },
3205        AGENT_PROTOCOL_V1,
3206    };
3207    use orchestral_core::io::{ArtifactResolveError, ResolvedArtifact};
3208    use orchestral_core::tool_protocol::{
3209        ApprovalBinding, CapabilityRequest, EffectScope, ToolCallId, ToolId,
3210    };
3211    use orchestral_runtime::{AgentApprovalBridge, AgentController};
3212    use tokio::sync::broadcast;
3213    use tower::ServiceExt;
3214
3215    #[test]
3216    fn provider_native_request_bodies_do_not_require_run_command_identity() {
3217        let input: AgentSessionTextRequest = serde_json::from_value(serde_json::json!({
3218            "text": "continue"
3219        }))
3220        .unwrap();
3221        assert_eq!(input.text, "continue");
3222        assert!(input.attachments.is_empty());
3223
3224        let approval: AgentSessionApprovalRequest = serde_json::from_value(serde_json::json!({
3225            "decision": "allow_once"
3226        }))
3227        .unwrap();
3228        assert!(matches!(approval.decision, ApprovalChoice::AllowOnce));
3229    }
3230
3231    #[tokio::test]
3232    async fn native_request_http_accepts_current_and_already_open_pwa_bodies() {
3233        let (app, token) = test_app().await;
3234        for legacy in [false, true] {
3235            for (kind, mut body) in [
3236                ("approval", json!({"decision": "allow_once"})),
3237                ("approval", json!({"decision": "deny"})),
3238                ("input", json!({"text": "continue"})),
3239            ] {
3240                if legacy {
3241                    body["command_id"] = json!("old-browser-command");
3242                }
3243                let response = app.clone().oneshot(authorized(
3244                    "POST",
3245                    &format!("/agent-session/requests/native-{kind}/{kind}?connector_id=fixture%2Flocal&session_id=fixture-session"),
3246                    &token,
3247                    body,
3248                )).await.unwrap();
3249                assert_eq!(response.status(), StatusCode::OK, "{kind}, legacy={legacy}");
3250                let body = response.into_body().collect().await.unwrap().to_bytes();
3251                assert_eq!(
3252                    serde_json::from_slice::<Value>(&body).unwrap(),
3253                    json!({"resolved": true})
3254                );
3255            }
3256        }
3257        // Compatibility is restricted to redundant command metadata; native
3258        // requests still cannot grant remembered session permissions.
3259        let response = app.clone().oneshot(authorized(
3260            "POST", "/agent-session/requests/native-approval/approval?connector_id=fixture%2Flocal&session_id=fixture-session",
3261            &token, json!({"decision": "allow_session", "command_id": "legacy"}),
3262        )).await.unwrap();
3263        assert_eq!(response.status(), StatusCode::CONFLICT);
3264        let response = app.oneshot(authorized(
3265            "POST", "/agent-session/requests/native-approval/approval?connector_id=fixture%2Flocal&session_id=fixture-session",
3266            &token, json!({"decision": "allow_once", "grant_ref": "unexpected"}),
3267        )).await.unwrap();
3268        assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
3269    }
3270
3271    #[tokio::test]
3272    async fn history_projection_control_reports_client_update_required() {
3273        let (app, token) = test_app().await;
3274        let response = app.oneshot(authorized(
3275            "POST", "/runs/agent-history%3Afixture%2Flocal%3Afixture-session/requests/native-approval/approval?connector_id=fixture%2Flocal",
3276            &token, json!({"command_id": "old-page", "decision": "allow_once"}),
3277        )).await.unwrap();
3278        assert_eq!(response.status(), StatusCode::CONFLICT);
3279        let body = response.into_body().collect().await.unwrap().to_bytes();
3280        assert_eq!(
3281            serde_json::from_slice::<Value>(&body).unwrap()["code"],
3282            "client_update_required"
3283        );
3284    }
3285
3286    #[test]
3287    fn supervision_retries_only_retryable_agent_failures() {
3288        let permanent =
3289            AgentSdkError::Control(AgentControlError::Protocol(AgentProtocolError::new(
3290                AgentProtocolErrorCode::InvalidTransition,
3291                "manual recovery required",
3292            )));
3293        assert!(!is_retryable_agent_error(&permanent));
3294
3295        let transient = AgentSdkError::Control(AgentControlError::Protocol(
3296            AgentProtocolError::new(
3297                AgentProtocolErrorCode::ProviderUnavailable,
3298                "provider restarting",
3299            )
3300            .with_retryable(true),
3301        ));
3302        assert!(is_retryable_agent_error(&transient));
3303
3304        let missing =
3305            AgentSdkError::Control(AgentControlError::RunNotFound(RunId::new("missing-run")));
3306        assert!(!is_retryable_agent_error(&missing));
3307    }
3308
3309    #[test]
3310    fn session_creation_expands_host_home_shortcuts_without_touching_other_paths() {
3311        #[cfg(target_os = "windows")]
3312        let home_variable = "USERPROFILE";
3313        #[cfg(not(target_os = "windows"))]
3314        let home_variable = "HOME";
3315        let home = std::env::var_os(home_variable)
3316            .filter(|value| !value.is_empty())
3317            .map(PathBuf::from)
3318            .expect("test Host has a home directory");
3319        assert_eq!(
3320            expand_host_home(Some("~".to_owned())).unwrap(),
3321            Some(home.to_string_lossy().into_owned())
3322        );
3323        assert_eq!(
3324            expand_host_home(Some("~/".to_owned())).unwrap(),
3325            Some(home.join("").to_string_lossy().into_owned())
3326        );
3327        assert_eq!(
3328            expand_host_home(Some("~/rust_ws/project".to_owned())).unwrap(),
3329            Some(home.join("rust_ws/project").to_string_lossy().into_owned())
3330        );
3331        assert_eq!(
3332            expand_host_home(Some("/srv/project".to_owned())).unwrap(),
3333            Some("/srv/project".to_owned())
3334        );
3335    }
3336
3337    use super::super::auth::GatewayAuthError;
3338
3339    struct StaticGatewayAuthenticator {
3340        header_name: HeaderName,
3341    }
3342
3343    struct StaticAgentConnector;
3344
3345    struct DirectArtifactResolver;
3346
3347    #[async_trait]
3348    impl ArtifactResolver for DirectArtifactResolver {
3349        async fn resolve(
3350            &self,
3351            artifact: &ArtifactRefWithDigest,
3352        ) -> Result<ResolvedArtifact, ArtifactResolveError> {
3353            Ok(ResolvedArtifact {
3354                artifact: artifact.clone(),
3355                uri: format!(
3356                    "https://orchestral-files.example/v1/blobs/{}?capability=signed",
3357                    artifact.artifact_ref
3358                ),
3359                file_name: Some("generated.png".to_owned()),
3360                media_type: "image/png".to_owned(),
3361                byte_size: 123,
3362                expires_at: None,
3363            })
3364        }
3365    }
3366
3367    struct ObservableAgentConnector {
3368        subscriptions: Arc<AtomicUsize>,
3369        changes: broadcast::Sender<AgentSessionChange>,
3370    }
3371
3372    struct DisconnectFirstProvider {
3373        inner: Arc<dyn AgentProvider>,
3374    }
3375
3376    struct HoldingProvider {
3377        inner: Arc<dyn AgentProvider>,
3378        finish: Arc<tokio::sync::Notify>,
3379        rejection: Option<AgentProtocolErrorCode>,
3380    }
3381
3382    struct UnrecoverableDisconnectProvider {
3383        inner: Arc<dyn AgentProvider>,
3384    }
3385
3386    #[test]
3387    fn active_external_agent_writer_is_reported_as_a_conflict() {
3388        let rejection = orchestral_core::agent_protocol::wire::AgentRejection::new(
3389            AgentRejectionCode::SessionConflict,
3390            "thread already has an active writer",
3391        );
3392        let error = AgentDirectoryError::Agent(AgentSdkError::Control(AgentControlError::Start(
3393            AgentStartError::Rejected(rejection),
3394        )));
3395
3396        let response = ApiError::from(error);
3397        assert_eq!(response.status, StatusCode::CONFLICT);
3398        assert_eq!(response.body.code, "agent_session_conflict");
3399    }
3400
3401    #[test]
3402    fn realtime_only_agent_writer_conflict_has_a_stable_api_code() {
3403        let rejection = orchestral_core::agent_protocol::wire::AgentRejection::new(
3404            AgentRejectionCode::UnsupportedCapability,
3405            "live control unavailable",
3406        )
3407        .with_details(serde_json::json!({"code": "live_control_unavailable"}));
3408        let error = AgentDirectoryError::Agent(AgentSdkError::Control(AgentControlError::Start(
3409            AgentStartError::Rejected(rejection),
3410        )));
3411
3412        let response = ApiError::from(error);
3413        assert_eq!(response.status, StatusCode::CONFLICT);
3414        assert_eq!(response.body.code, "live_control_unavailable");
3415    }
3416
3417    #[tokio::test]
3418    async fn session_views_add_direct_storage_access_without_changing_artifact_identity() {
3419        let digest = "a".repeat(64);
3420        let mut payload = serde_json::json!({
3421            "turns": [{
3422                "activities": [{
3423                    "content": [{
3424                        "media_type": "image/png",
3425                        "schema_id": null,
3426                        "body": {
3427                            "kind": "artifact",
3428                            "value": {
3429                                "artifact_ref": digest,
3430                                "digest": "a".repeat(64)
3431                            }
3432                        }
3433                    }]
3434                }]
3435            }]
3436        });
3437
3438        enrich_artifact_access(&mut payload, Some(&DirectArtifactResolver)).await;
3439
3440        let content = payload.pointer("/turns/0/activities/0/content/0").unwrap();
3441        assert_eq!(
3442            content.pointer("/body/value/artifact_ref"),
3443            Some(&serde_json::Value::String("a".repeat(64)))
3444        );
3445        assert_eq!(
3446            content.pointer("/access/uri").and_then(serde_json::Value::as_str),
3447            Some(
3448                "https://orchestral-files.example/v1/blobs/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?capability=signed"
3449            )
3450        );
3451        assert_eq!(
3452            content.pointer("/access/byte_size"),
3453            Some(&serde_json::json!(123))
3454        );
3455    }
3456
3457    #[tokio::test]
3458    async fn latest_session_run_ignores_a_journal_from_an_old_controller_contract() {
3459        let factory = ScriptedStatelessFactory::conformant().expect("fixture descriptor");
3460        let scenario = ProviderScenario::standard(&factory.descriptor()).expect("fixture scenario");
3461        let journal = Arc::new(InMemoryAgentJournalStore::default());
3462        let previous = Arc::new(
3463            AgentController::with_journal_store(
3464                factory.create(scenario.clone(), TestProbes::default()),
3465                ProviderBindingRef::new("previous-binding"),
3466                journal.clone(),
3467            )
3468            .expect("previous controller binds"),
3469        );
3470        let execution = previous
3471            .start(scenario.start_request.run.clone())
3472            .await
3473            .expect("previous Run starts");
3474        previous
3475            .wait_for_terminal(&execution.run_id)
3476            .await
3477            .expect("previous Run completes");
3478        drop(previous);
3479
3480        let upgraded = Arc::new(
3481            AgentController::with_journal_store(
3482                factory.create(scenario.clone(), TestProbes::default()),
3483                ProviderBindingRef::new("upgraded-binding"),
3484                journal,
3485            )
3486            .expect("upgraded controller binds"),
3487        );
3488        let agent = AgentApi::new(upgraded);
3489
3490        assert!(
3491            latest_session_run(&agent, &scenario.start_request.run.spec.session_id)
3492                .await
3493                .expect("catalog remains readable")
3494                .is_none()
3495        );
3496    }
3497
3498    #[async_trait]
3499    impl AgentProvider for DisconnectFirstProvider {
3500        fn describe(&self) -> AgentDescriptorEnvelope {
3501            self.inner.describe()
3502        }
3503
3504        async fn start(
3505            &self,
3506            request: orchestral_core::agent_protocol::wire::AgentStartRequest,
3507        ) -> Result<AgentStart, AgentStartError> {
3508            let started = self.inner.start(request).await?;
3509            Ok(AgentStart {
3510                execution: started.execution,
3511                admission: started.admission,
3512                stream: started.stream.take(1).boxed(),
3513            })
3514        }
3515
3516        async fn command(
3517            &self,
3518            execution: &AgentExecutionRef,
3519            command: AgentCommandEnvelope,
3520        ) -> Result<ProviderCommandDisposition, AgentProtocolError> {
3521            self.inner.command(execution, command).await
3522        }
3523
3524        async fn recover(
3525            &self,
3526            request: AgentRecoveryRequest,
3527        ) -> Result<AgentRecovery, AgentProtocolError> {
3528            self.inner.recover(request).await
3529        }
3530    }
3531
3532    #[async_trait]
3533    impl AgentProvider for HoldingProvider {
3534        fn describe(&self) -> AgentDescriptorEnvelope {
3535            self.inner.describe()
3536        }
3537
3538        async fn start(
3539            &self,
3540            request: orchestral_core::agent_protocol::wire::AgentStartRequest,
3541        ) -> Result<AgentStart, AgentStartError> {
3542            let run_id = request.run.spec.run_id.clone();
3543            let finish = self.finish.clone();
3544            let started = self.inner.start(request).await?;
3545            let terminal = stream::once(async move {
3546                finish.notified().await;
3547                Ok(AgentProviderStreamItem::Event(Box::new(AgentEventDraft {
3548                    event_id: orchestral_core::agent_protocol::wire::AgentEventId::new(format!(
3549                        "fixture-finished-{run_id}"
3550                    )),
3551                    run_id,
3552                    causation_id: None,
3553                    source_fingerprint: None,
3554                    payload: AgentEvent::RunFailed {
3555                        failure: orchestral_core::agent_protocol::wire::AgentFailure {
3556                            code: "fixture_finished".to_owned(),
3557                            message: "controlled fixture completed".to_owned(),
3558                            retryable: false,
3559                            details: Value::Null,
3560                        },
3561                    },
3562                })))
3563            });
3564            Ok(AgentStart {
3565                execution: started.execution,
3566                admission: started.admission,
3567                stream: started.stream.take(1).chain(terminal).boxed(),
3568            })
3569        }
3570
3571        async fn command(
3572            &self,
3573            _execution: &AgentExecutionRef,
3574            command: AgentCommandEnvelope,
3575        ) -> Result<ProviderCommandDisposition, AgentProtocolError> {
3576            Ok(ProviderCommandDisposition {
3577                command_id: command.command_id,
3578                run_id: command.run_id,
3579                outcome: self
3580                    .rejection
3581                    .clone()
3582                    .map_or(ProviderCommandOutcome::Accepted, |code| {
3583                        ProviderCommandOutcome::Rejected {
3584                            code,
3585                            message: "fixture command was not dispatched".to_owned(),
3586                        }
3587                    }),
3588                duplicate: false,
3589            })
3590        }
3591
3592        async fn recover(
3593            &self,
3594            request: AgentRecoveryRequest,
3595        ) -> Result<AgentRecovery, AgentProtocolError> {
3596            self.inner.recover(request).await
3597        }
3598    }
3599
3600    #[async_trait]
3601    impl AgentProvider for UnrecoverableDisconnectProvider {
3602        fn describe(&self) -> AgentDescriptorEnvelope {
3603            self.inner.describe()
3604        }
3605
3606        async fn start(
3607            &self,
3608            request: orchestral_core::agent_protocol::wire::AgentStartRequest,
3609        ) -> Result<AgentStart, AgentStartError> {
3610            let started = self.inner.start(request).await?;
3611            Ok(AgentStart {
3612                execution: started.execution,
3613                admission: started.admission,
3614                stream: started.stream.take(1).boxed(),
3615            })
3616        }
3617
3618        async fn command(
3619            &self,
3620            execution: &AgentExecutionRef,
3621            command: AgentCommandEnvelope,
3622        ) -> Result<ProviderCommandDisposition, AgentProtocolError> {
3623            self.inner.command(execution, command).await
3624        }
3625
3626        async fn recover(
3627            &self,
3628            _request: AgentRecoveryRequest,
3629        ) -> Result<AgentRecovery, AgentProtocolError> {
3630            Err(AgentProtocolError::new(
3631                AgentProtocolErrorCode::ProviderUnavailable,
3632                "fixture recovery remains unavailable",
3633            ))
3634        }
3635    }
3636
3637    impl StaticAgentConnector {
3638        fn summary() -> AgentSessionSummary {
3639            AgentSessionSummary {
3640                connector_id: AgentConnectorId::new("fixture/local"),
3641                session_id: AgentSessionId::new("fixture-session"),
3642                title: Some("Existing fixture session".to_owned()),
3643                preview: Some("resume me".to_owned()),
3644                cwd: Some("/fixture/workspace".to_owned()),
3645                created_at_unix_ms: Some(1_000),
3646                updated_at_unix_ms: Some(2_000),
3647                state: AgentSessionState::Idle,
3648                execution_profile: Default::default(),
3649                extensions: BTreeMap::new(),
3650            }
3651        }
3652
3653        fn created_summary() -> AgentSessionSummary {
3654            AgentSessionSummary {
3655                session_id: AgentSessionId::new("fixture-created"),
3656                title: Some("Created from HTTP".to_owned()),
3657                ..Self::summary()
3658            }
3659        }
3660    }
3661
3662    #[async_trait]
3663    impl AgentConnector for StaticAgentConnector {
3664        fn describe(&self) -> AgentConnectorDescriptor {
3665            AgentConnectorDescriptor {
3666                connector_id: AgentConnectorId::new("fixture/local"),
3667                provider_binding: ProviderBindingRef::new("fixture/external"),
3668                agent_family: "test-agent".to_owned(),
3669                display_name: "Fixture Agent".to_owned(),
3670                capabilities: AgentSessionCapabilities {
3671                    create: true,
3672                    resolve_requests: true,
3673                    ..AgentSessionCapabilities::discoverable()
3674                },
3675                creation: None,
3676                actions: vec![
3677                    AgentSessionActionDescriptor {
3678                        action_id: AgentSessionActionId::new(SESSION_FORK_ACTION),
3679                        title: "Fork".to_owned(),
3680                        description: "Fork a fixture session".to_owned(),
3681                        input_schema: None,
3682                        execution: AgentSessionActionExecution::Immediate,
3683                    },
3684                    AgentSessionActionDescriptor {
3685                        action_id: AgentSessionActionId::new(SESSION_RENAME_ACTION),
3686                        title: "Rename".to_owned(),
3687                        description: "Rename a fixture session".to_owned(),
3688                        input_schema: Some(serde_json::json!({"type": "object"})),
3689                        execution: AgentSessionActionExecution::Immediate,
3690                    },
3691                    AgentSessionActionDescriptor {
3692                        action_id: AgentSessionActionId::new(SESSION_REVIEW_ACTION),
3693                        title: "Review".to_owned(),
3694                        description: "Review fixture changes".to_owned(),
3695                        input_schema: Some(serde_json::json!({"type": "object"})),
3696                        execution: AgentSessionActionExecution::Run,
3697                    },
3698                ],
3699            }
3700        }
3701
3702        async fn health(&self) -> Result<AgentConnectorHealth, AgentConnectorError> {
3703            Ok(AgentConnectorHealth::ready(Some("test".to_owned())))
3704        }
3705
3706        async fn list_sessions(
3707            &self,
3708            _query: AgentSessionListQuery,
3709        ) -> Result<AgentSessionPage, AgentConnectorError> {
3710            Ok(AgentSessionPage {
3711                sessions: vec![Self::summary()],
3712                next_cursor: None,
3713            })
3714        }
3715
3716        async fn read_session(
3717            &self,
3718            session_id: &AgentSessionId,
3719        ) -> Result<AgentSessionDetail, AgentConnectorError> {
3720            if session_id.as_str() == "fixture-created" {
3721                return Ok(AgentSessionDetail {
3722                    summary: Self::created_summary(),
3723                    turns: Vec::new(),
3724                    pending_requests: Vec::new(),
3725                    next_cursor: None,
3726                });
3727            }
3728            if session_id.as_str() != "fixture-session" {
3729                return Err(AgentConnectorError::new(
3730                    orchestral_core::agent_connector::AgentConnectorErrorCode::NotFound,
3731                    "fixture session not found",
3732                    false,
3733                ));
3734            }
3735            Ok(AgentSessionDetail {
3736                summary: Self::summary(),
3737                turns: vec![AgentSessionTurn {
3738                    turn_id: AgentSessionTurnId::new("turn-large"),
3739                    status: AgentSessionTurnStatus::Completed,
3740                    failure: None,
3741                    activities: (0..60)
3742                        .map(|index| AgentSessionActivity {
3743                            activity_id: AgentSessionActivityId::new(format!("activity-{index}")),
3744                            kind: AgentSessionActivityKind::Command,
3745                            status: AgentSessionActivityStatus::Completed,
3746                            title: Some(format!("command-{index}")),
3747                            content: vec![Content::text("x".repeat(10_000))],
3748                            details: serde_json::Value::Null,
3749                        })
3750                        .collect(),
3751                }],
3752                pending_requests: Vec::new(),
3753                next_cursor: None,
3754            })
3755        }
3756
3757        async fn create_session(
3758            &self,
3759            request: CreateAgentSessionRequest,
3760        ) -> Result<AgentSessionSummary, AgentConnectorError> {
3761            assert_eq!(request.cwd.as_deref(), Some("/fixture/new"));
3762            assert_eq!(request.title.as_deref(), Some("Created from HTTP"));
3763            assert!(request.extensions.is_empty());
3764            Ok(Self::created_summary())
3765        }
3766
3767        async fn invoke_action(
3768            &self,
3769            request: InvokeAgentSessionActionRequest,
3770        ) -> Result<AgentSessionActionOutcome, AgentConnectorError> {
3771            assert_eq!(request.session_id.as_str(), "fixture-session");
3772            assert_eq!(request.action_id.as_str(), SESSION_RENAME_ACTION);
3773            assert_eq!(request.arguments["name"], "Renamed over HTTP");
3774            let mut summary = Self::summary();
3775            summary.title = Some("Renamed over HTTP".to_owned());
3776            Ok(AgentSessionActionOutcome {
3777                status: AgentSessionActionStatus::Completed,
3778                session: Some(summary),
3779                content: Vec::new(),
3780                details: serde_json::Value::Null,
3781            })
3782        }
3783
3784        async fn resolve_request(
3785            &self,
3786            request: ResolveAgentSessionRequest,
3787        ) -> Result<(), AgentConnectorError> {
3788            self.read_session(&request.session_id).await?;
3789            request.response.validate()?;
3790            Ok(())
3791        }
3792    }
3793
3794    #[async_trait]
3795    impl AgentConnector for ObservableAgentConnector {
3796        fn describe(&self) -> AgentConnectorDescriptor {
3797            StaticAgentConnector.describe()
3798        }
3799
3800        async fn health(&self) -> Result<AgentConnectorHealth, AgentConnectorError> {
3801            StaticAgentConnector.health().await
3802        }
3803
3804        async fn list_sessions(
3805            &self,
3806            query: AgentSessionListQuery,
3807        ) -> Result<AgentSessionPage, AgentConnectorError> {
3808            StaticAgentConnector.list_sessions(query).await
3809        }
3810
3811        async fn read_session(
3812            &self,
3813            session_id: &AgentSessionId,
3814        ) -> Result<AgentSessionDetail, AgentConnectorError> {
3815            StaticAgentConnector.read_session(session_id).await
3816        }
3817
3818        async fn subscribe_session_changes(
3819            &self,
3820            session_id: &AgentSessionId,
3821        ) -> Result<broadcast::Receiver<AgentSessionChange>, AgentConnectorError> {
3822            if session_id.as_str() != "fixture-session" {
3823                return Err(AgentConnectorError::invalid("unexpected fixture session"));
3824            }
3825            self.subscriptions.fetch_add(1, Ordering::SeqCst);
3826            Ok(self.changes.subscribe())
3827        }
3828    }
3829
3830    #[async_trait]
3831    impl GatewayAuthenticator for StaticGatewayAuthenticator {
3832        fn header_name(&self) -> &HeaderName {
3833            &self.header_name
3834        }
3835
3836        async fn authenticate(&self, token: &str) -> Result<GatewayPrincipal, GatewayAuthError> {
3837            if token != "valid-gateway-assertion" {
3838                return Err(GatewayAuthError::Invalid(
3839                    "test assertion was rejected".to_owned(),
3840                ));
3841            }
3842            Ok(GatewayPrincipal {
3843                subject: Some("gateway-user".to_owned()),
3844                attributes: BTreeMap::from([("email".to_owned(), "person@example.com".to_owned())]),
3845            })
3846        }
3847    }
3848
3849    struct ApprovalProvider {
3850        descriptor: AgentDescriptorEnvelope,
3851        events: broadcast::Sender<AgentEventDraft>,
3852        approvals: Arc<InMemoryHostApprovalBroker>,
3853    }
3854
3855    impl ApprovalProvider {
3856        fn new(approvals: Arc<InMemoryHostApprovalBroker>) -> Self {
3857            let descriptor = AgentDescriptorEnvelope::seal(AgentDescriptor {
3858                provider_id: AgentProviderId::new("test.remote-approval"),
3859                agent_id: AgentId::new("approval-v1"),
3860                supported_protocol_versions: vec![AGENT_PROTOCOL_V1],
3861                accepted_content_types: BTreeSet::from(["text/plain".to_owned()]),
3862                capabilities: AgentCapabilities {
3863                    session_reuse: true,
3864                    pending_request_kinds: BTreeSet::from([PendingRequestKind::Approval]),
3865                    effect_mediation: EffectMediation::HostMediated,
3866                    ..AgentCapabilities::default()
3867                },
3868                extensions: Default::default(),
3869            })
3870            .unwrap();
3871            let (events, _) = broadcast::channel(32);
3872            Self {
3873                descriptor,
3874                events,
3875                approvals,
3876            }
3877        }
3878
3879        fn draft(
3880            run_id: &RunId,
3881            event_id: impl Into<String>,
3882            causation_id: Option<CommandId>,
3883            payload: AgentEvent,
3884        ) -> AgentEventDraft {
3885            AgentEventDraft {
3886                event_id: orchestral_core::agent_protocol::wire::AgentEventId::new(event_id),
3887                run_id: run_id.clone(),
3888                causation_id,
3889                source_fingerprint: None,
3890                payload,
3891            }
3892        }
3893    }
3894
3895    #[async_trait]
3896    impl AgentProvider for ApprovalProvider {
3897        fn describe(&self) -> AgentDescriptorEnvelope {
3898            self.descriptor.clone()
3899        }
3900
3901        async fn start(
3902            &self,
3903            request: orchestral_core::agent_protocol::wire::AgentStartRequest,
3904        ) -> Result<AgentStart, AgentStartError> {
3905            request
3906                .validate_for_descriptor(&self.descriptor)
3907                .map_err(AgentStartError::OutcomeUnknown)?;
3908            let execution = AgentExecutionRef::for_start(&request, &self.descriptor)
3909                .map_err(AgentStartError::OutcomeUnknown)?;
3910            let run_id = execution.run_id.clone();
3911            self.approvals
3912                .stage(
3913                    &RequestId::new("approval-request"),
3914                    ApprovalBinding {
3915                        run_id: run_id.clone(),
3916                        call_id: ToolCallId::new("approval-call"),
3917                        tool_id: ToolId::new("test/write"),
3918                        args_digest: orchestral_core::agent_protocol::wire::Digest::sha256(
3919                            "write args",
3920                        ),
3921                        operation_digest: orchestral_core::agent_protocol::wire::Digest::sha256(
3922                            "write operation",
3923                        ),
3924                        permission_digest: orchestral_core::agent_protocol::wire::Digest::sha256(
3925                            "write permission",
3926                        ),
3927                        requested_capabilities: CapabilityRequest::from_effects(BTreeSet::from([
3928                            EffectScope::FilesystemWrite,
3929                        ])),
3930                        session_approval_scope: None,
3931                        policy_digest: orchestral_core::agent_protocol::wire::Digest::sha256(
3932                            "test policy",
3933                        ),
3934                    },
3935                )
3936                .await
3937                .map_err(|error| {
3938                    AgentStartError::OutcomeUnknown(AgentProtocolError::new(
3939                        AgentProtocolErrorCode::Internal,
3940                        error.to_string(),
3941                    ))
3942                })?;
3943            let mut receiver = self.events.subscribe();
3944            let _ = self.events.send(Self::draft(
3945                &run_id,
3946                "approval-run-started",
3947                None,
3948                AgentEvent::RunStarted,
3949            ));
3950            let _ = self.events.send(Self::draft(
3951                &run_id,
3952                "approval-request-opened",
3953                None,
3954                AgentEvent::RequestOpened {
3955                    request: PendingRequest {
3956                        request_id: RequestId::new("approval-request"),
3957                        blocking: true,
3958                        payload: PendingRequestPayload::Approval {
3959                            operation_digest: orchestral_core::agent_protocol::wire::Digest::sha256(
3960                                "write operation",
3961                            ),
3962                            requested_scope: vec!["filesystem_write".to_owned()],
3963                            session_approval_scope: None,
3964                            reason: "write the requested file".to_owned(),
3965                        },
3966                    },
3967                },
3968            ));
3969            let stream = async_stream::stream! {
3970                loop {
3971                    match receiver.recv().await {
3972                        Ok(event) => yield Ok(AgentProviderStreamItem::Event(Box::new(event))),
3973                        Err(broadcast::error::RecvError::Lagged(_)) => continue,
3974                        Err(broadcast::error::RecvError::Closed) => break,
3975                    }
3976                }
3977            }
3978            .boxed();
3979            Ok(AgentStart {
3980                execution,
3981                admission: AgentAdmission {
3982                    skipped_optional_bindings: Vec::new(),
3983                },
3984                stream,
3985            })
3986        }
3987
3988        async fn command(
3989            &self,
3990            execution: &AgentExecutionRef,
3991            command: AgentCommandEnvelope,
3992        ) -> Result<ProviderCommandDisposition, AgentProtocolError> {
3993            command.verify_digest()?;
3994            let outcome = match (&command.request_id, &command.payload) {
3995                (Some(request_id), AgentCommand::ResolveRequest { response }) => {
3996                    let resolution = response.clone();
3997                    self.events
3998                        .send(Self::draft(
3999                            &execution.run_id,
4000                            format!("approval-resolved-{}", command.command_id.as_str()),
4001                            Some(command.command_id.clone()),
4002                            AgentEvent::RequestResolved {
4003                                request_id: request_id.clone(),
4004                                resolution_digest: resolution.digest()?,
4005                                resolution,
4006                            },
4007                        ))
4008                        .map_err(|_| {
4009                            AgentProtocolError::new(
4010                                AgentProtocolErrorCode::Internal,
4011                                "approval test stream is closed",
4012                            )
4013                        })?;
4014                    ProviderCommandOutcome::Accepted
4015                }
4016                _ => ProviderCommandOutcome::Unsupported {
4017                    feature: "command".to_owned(),
4018                },
4019            };
4020            Ok(ProviderCommandDisposition {
4021                command_id: command.command_id,
4022                run_id: command.run_id,
4023                outcome,
4024                duplicate: false,
4025            })
4026        }
4027
4028        async fn recover(
4029            &self,
4030            _request: AgentRecoveryRequest,
4031        ) -> Result<AgentRecovery, AgentProtocolError> {
4032            Err(AgentProtocolError::new(
4033                AgentProtocolErrorCode::Unsupported,
4034                "approval fixture does not recover",
4035            ))
4036        }
4037    }
4038
4039    async fn test_app() -> (Router, String) {
4040        test_app_with_gateway(None).await
4041    }
4042
4043    async fn holding_agent_app() -> (Router, String) {
4044        let (app, token, _) = completable_agent_app().await;
4045        (app, token)
4046    }
4047
4048    async fn completable_agent_app() -> (Router, String, Arc<tokio::sync::Notify>) {
4049        completable_agent_app_with_rejection(None).await
4050    }
4051
4052    async fn completable_agent_app_with_rejection(
4053        rejection: Option<AgentProtocolErrorCode>,
4054    ) -> (Router, String, Arc<tokio::sync::Notify>) {
4055        let finish = Arc::new(tokio::sync::Notify::new());
4056        let factory = ScriptedStatelessFactory::conformant().unwrap();
4057        let descriptor = factory.descriptor();
4058        let scenario = ProviderScenario::standard(&descriptor).unwrap();
4059        let controller = Arc::new(
4060            AgentController::new(
4061                factory.create(scenario, TestProbes::default()),
4062                ProviderBindingRef::new("remote-test"),
4063            )
4064            .unwrap(),
4065        );
4066        let ticket = super::super::state::PairingTicket::issue(60_000).unwrap();
4067        let secret = ticket.secret().to_owned();
4068        let registry = RemoteRegistry::in_memory(Some(ticket));
4069        let claim = registry.claim_pairing(&secret, "Two tabs").await.unwrap();
4070        let approvals =
4071            Arc::new(InMemoryHostApprovalBroker::new(b"0123456789abcdef0123456789abcdef").unwrap());
4072        let external_factory = SessionfulRecoverFactory::new().unwrap();
4073        let external_scenario = ProviderScenario::standard(&external_factory.descriptor()).unwrap();
4074        let external_provider = Arc::new(HoldingProvider {
4075            finish: finish.clone(),
4076            rejection,
4077            inner: external_factory.create(external_scenario, TestProbes::default()),
4078        });
4079        let agent_directory = Arc::new(AgentDirectory::new());
4080        agent_directory
4081            .register(Arc::new(StaticAgentConnector), external_provider)
4082            .await
4083            .unwrap();
4084        (
4085            router(RemoteApiState {
4086                agent: AgentApi::new(controller),
4087                agent_directory,
4088                native_session_defaults: NativeSessionDefaults::default(),
4089                approvals,
4090                registry,
4091                gateway_authenticator: None,
4092                run_supervisors: Arc::default(),
4093                session_coordinators: Arc::default(),
4094                artifact_resolver: None,
4095                artifact_blob_store: None,
4096            }),
4097            claim.token,
4098            finish,
4099        )
4100    }
4101
4102    async fn test_app_with_gateway(
4103        gateway_authenticator: Option<Arc<dyn GatewayAuthenticator>>,
4104    ) -> (Router, String) {
4105        let factory = ScriptedStatelessFactory::conformant().unwrap();
4106        let descriptor = factory.descriptor();
4107        let scenario = ProviderScenario::standard(&descriptor).unwrap();
4108        let provider = factory.create(scenario, TestProbes::default());
4109        let controller = Arc::new(
4110            AgentController::new(provider, ProviderBindingRef::new("remote-test")).unwrap(),
4111        );
4112        let ticket = super::super::state::PairingTicket::issue(60_000).unwrap();
4113        let secret = ticket.secret().to_owned();
4114        let registry = RemoteRegistry::in_memory(Some(ticket));
4115        let claim = registry.claim_pairing(&secret, "Test phone").await.unwrap();
4116        let approvals =
4117            Arc::new(InMemoryHostApprovalBroker::new(b"0123456789abcdef0123456789abcdef").unwrap());
4118        let external_factory = SessionfulRecoverFactory::new().unwrap();
4119        let external_scenario = ProviderScenario::standard(&external_factory.descriptor()).unwrap();
4120        let external_provider = Arc::new(DisconnectFirstProvider {
4121            inner: external_factory.create(external_scenario, TestProbes::default()),
4122        });
4123        let agent_directory = Arc::new(AgentDirectory::new());
4124        agent_directory
4125            .register(Arc::new(StaticAgentConnector), external_provider)
4126            .await
4127            .unwrap();
4128        (
4129            router(RemoteApiState {
4130                agent: AgentApi::new(controller),
4131                agent_directory,
4132                native_session_defaults: NativeSessionDefaults {
4133                    cwd: Some("/fixture/workspace".to_owned()),
4134                    execution_profile: AgentSessionExecutionProfile {
4135                        model: Some("fixture-model".to_owned()),
4136                        reasoning_effort: Some("default".to_owned()),
4137                        permissions: Default::default(),
4138                    },
4139                },
4140                approvals,
4141                registry,
4142                gateway_authenticator,
4143                run_supervisors: Arc::default(),
4144                session_coordinators: Arc::default(),
4145                artifact_resolver: None,
4146                artifact_blob_store: None,
4147            }),
4148            claim.token,
4149        )
4150    }
4151
4152    async fn stale_external_agent_app() -> (Router, String) {
4153        let journal = Arc::new(InMemoryAgentJournalStore::default());
4154        let previous_factory = ScriptedStatelessFactory::conformant().unwrap();
4155        let previous_scenario = ProviderScenario::standard(&previous_factory.descriptor()).unwrap();
4156        let previous = Arc::new(
4157            AgentController::with_journal_store(
4158                previous_factory.create(previous_scenario.clone(), TestProbes::default()),
4159                ProviderBindingRef::new("fixture/previous"),
4160                journal.clone(),
4161            )
4162            .unwrap(),
4163        );
4164        let mut previous_spec = previous_scenario.start_request.run.spec;
4165        previous_spec.session_id = AgentSessionId::new("fixture-session");
4166        previous_spec.run_id = RunId::new("fixture-stale-run");
4167        let previous_execution = previous
4168            .start(AgentRunEnvelope::seal(previous_spec).unwrap())
4169            .await
4170            .unwrap();
4171        previous
4172            .wait_for_terminal(&previous_execution.run_id)
4173            .await
4174            .unwrap();
4175        drop(previous);
4176
4177        let external_factory = SessionfulRecoverFactory::new().unwrap();
4178        let external_scenario = ProviderScenario::standard(&external_factory.descriptor()).unwrap();
4179        let external_provider = Arc::new(DisconnectFirstProvider {
4180            inner: external_factory.create(external_scenario, TestProbes::default()),
4181        });
4182        let agent_directory = Arc::new(AgentDirectory::new());
4183        agent_directory
4184            .register_with_journal(Arc::new(StaticAgentConnector), external_provider, journal)
4185            .await
4186            .unwrap();
4187
4188        let generic_factory = ScriptedStatelessFactory::conformant().unwrap();
4189        let generic_scenario = ProviderScenario::standard(&generic_factory.descriptor()).unwrap();
4190        let generic_controller = Arc::new(
4191            AgentController::new(
4192                generic_factory.create(generic_scenario, TestProbes::default()),
4193                ProviderBindingRef::new("remote-test"),
4194            )
4195            .unwrap(),
4196        );
4197        let ticket = super::super::state::PairingTicket::issue(60_000).unwrap();
4198        let secret = ticket.secret().to_owned();
4199        let registry = RemoteRegistry::in_memory(Some(ticket));
4200        let claim = registry
4201            .claim_pairing(&secret, "Descriptor upgrade phone")
4202            .await
4203            .unwrap();
4204        let approvals =
4205            Arc::new(InMemoryHostApprovalBroker::new(b"0123456789abcdef0123456789abcdef").unwrap());
4206        (
4207            router(RemoteApiState {
4208                agent: AgentApi::new(generic_controller),
4209                agent_directory,
4210                native_session_defaults: NativeSessionDefaults::default(),
4211                approvals,
4212                registry,
4213                gateway_authenticator: None,
4214                run_supervisors: Arc::default(),
4215                session_coordinators: Arc::default(),
4216                artifact_resolver: None,
4217                artifact_blob_store: None,
4218            }),
4219            claim.token,
4220        )
4221    }
4222
4223    async fn approval_app() -> (Router, String) {
4224        let approvals =
4225            Arc::new(InMemoryHostApprovalBroker::new(b"0123456789abcdef0123456789abcdef").unwrap());
4226        let provider = Arc::new(ApprovalProvider::new(approvals.clone()));
4227        let controller = Arc::new(
4228            AgentController::new(provider, ProviderBindingRef::new("remote-test")).unwrap(),
4229        );
4230        let ticket = super::super::state::PairingTicket::issue(60_000).unwrap();
4231        let secret = ticket.secret().to_owned();
4232        let registry = RemoteRegistry::in_memory(Some(ticket));
4233        let claim = registry
4234            .claim_pairing(&secret, "Approval phone")
4235            .await
4236            .unwrap();
4237        (
4238            router(RemoteApiState {
4239                agent: AgentApi::new(controller),
4240                agent_directory: Arc::new(AgentDirectory::new()),
4241                native_session_defaults: NativeSessionDefaults::default(),
4242                approvals,
4243                registry,
4244                gateway_authenticator: None,
4245                run_supervisors: Arc::default(),
4246                session_coordinators: Arc::default(),
4247                artifact_resolver: None,
4248                artifact_blob_store: None,
4249            }),
4250            claim.token,
4251        )
4252    }
4253
4254    async fn unrecoverable_app() -> (Router, String) {
4255        let approvals =
4256            Arc::new(InMemoryHostApprovalBroker::new(b"0123456789abcdef0123456789abcdef").unwrap());
4257        let provider = Arc::new(UnrecoverableDisconnectProvider {
4258            inner: Arc::new(ApprovalProvider::new(approvals.clone())),
4259        });
4260        let controller = Arc::new(
4261            AgentController::new(provider, ProviderBindingRef::new("remote-test")).unwrap(),
4262        );
4263        let ticket = super::super::state::PairingTicket::issue(60_000).unwrap();
4264        let secret = ticket.secret().to_owned();
4265        let registry = RemoteRegistry::in_memory(Some(ticket));
4266        let claim = registry
4267            .claim_pairing(&secret, "Recovery phone")
4268            .await
4269            .unwrap();
4270        (
4271            router(RemoteApiState {
4272                agent: AgentApi::new(controller),
4273                agent_directory: Arc::new(AgentDirectory::new()),
4274                native_session_defaults: NativeSessionDefaults::default(),
4275                approvals,
4276                registry,
4277                gateway_authenticator: None,
4278                run_supervisors: Arc::default(),
4279                session_coordinators: Arc::default(),
4280                artifact_resolver: None,
4281                artifact_blob_store: None,
4282            }),
4283            claim.token,
4284        )
4285    }
4286
4287    fn authorized(method: &str, uri: &str, token: &str, body: serde_json::Value) -> Request<Body> {
4288        Request::builder()
4289            .method(method)
4290            .uri(uri)
4291            .header(header::AUTHORIZATION, format!("Bearer {token}"))
4292            .header(header::CONTENT_TYPE, "application/json")
4293            .body(Body::from(body.to_string()))
4294            .unwrap()
4295    }
4296
4297    #[tokio::test]
4298    async fn multiple_clients_share_one_connector_subscription_and_sequence() {
4299        let subscriptions = Arc::new(AtomicUsize::new(0));
4300        let (changes, _) = broadcast::channel(16);
4301        let connector = Arc::new(ObservableAgentConnector {
4302            subscriptions: subscriptions.clone(),
4303            changes: changes.clone(),
4304        });
4305        let factory = SessionfulRecoverFactory::new().unwrap();
4306        let scenario = ProviderScenario::standard(&factory.descriptor()).unwrap();
4307        let directory = Arc::new(AgentDirectory::new());
4308        directory
4309            .register(connector, factory.create(scenario, TestProbes::default()))
4310            .await
4311            .unwrap();
4312
4313        let connector_id = AgentConnectorId::new("fixture/local");
4314        let session_id = AgentSessionId::new("fixture-session");
4315        let registry = AgentSessionCoordinatorRegistry::default();
4316        let coordinator = registry.get(&connector_id, &session_id);
4317        let first_hub = coordinator
4318            .ensure_hub(directory.clone(), &connector_id, &session_id)
4319            .await
4320            .unwrap();
4321        let second_hub = registry
4322            .get(&connector_id, &session_id)
4323            .ensure_hub(directory, &connector_id, &session_id)
4324            .await
4325            .unwrap();
4326        assert!(Arc::ptr_eq(&first_hub, &second_hub));
4327        assert_eq!(subscriptions.load(Ordering::SeqCst), 1);
4328
4329        let mut phone = first_hub.subscribe(0).live;
4330        let mut desktop = second_hub.subscribe(0).live;
4331        changes
4332            .send(AgentSessionChange {
4333                connector_id: connector_id.clone(),
4334                session_id: session_id.clone(),
4335                sequence: 900,
4336                change: orchestral_core::agent_connector::AgentSessionChangeKind::RefreshRequired {
4337                    reason: "fixture-change".to_owned(),
4338                },
4339            })
4340            .unwrap();
4341        let phone_change = tokio::time::timeout(Duration::from_secs(1), phone.recv())
4342            .await
4343            .unwrap()
4344            .unwrap();
4345        let desktop_change = tokio::time::timeout(Duration::from_secs(1), desktop.recv())
4346            .await
4347            .unwrap()
4348            .unwrap();
4349        assert_eq!(phone_change, desktop_change);
4350        assert_eq!(phone_change.sequence, 1);
4351    }
4352
4353    #[tokio::test]
4354    async fn protected_routes_require_a_paired_device() {
4355        let (app, _) = test_app().await;
4356        let response = app
4357            .oneshot(
4358                Request::builder()
4359                    .uri("/sessions")
4360                    .body(Body::empty())
4361                    .unwrap(),
4362            )
4363            .await
4364            .unwrap();
4365        assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
4366        let request_id = response
4367            .headers()
4368            .get(REQUEST_ID_HEADER)
4369            .and_then(|value| value.to_str().ok())
4370            .expect("every API response carries its log correlation id");
4371        uuid::Uuid::parse_str(request_id).expect("request id is a UUID");
4372        assert!(response.extensions().get::<ApiErrorLogCode>().is_some());
4373    }
4374
4375    #[tokio::test]
4376    async fn native_sessions_expose_the_composed_host_execution_profile() {
4377        let (app, token) = test_app().await;
4378        let response = app
4379            .clone()
4380            .oneshot(authorized(
4381                "POST",
4382                "/sessions",
4383                &token,
4384                serde_json::json!({"session_id": "profile-session"}),
4385            ))
4386            .await
4387            .unwrap();
4388        assert_eq!(response.status(), StatusCode::CREATED);
4389        let body = response.into_body().collect().await.unwrap().to_bytes();
4390        let created: SessionView = serde_json::from_slice(&body).unwrap();
4391        assert_eq!(created.cwd.as_deref(), Some("/fixture/workspace"));
4392        assert_eq!(
4393            created.execution_profile,
4394            AgentSessionExecutionProfile {
4395                model: Some("fixture-model".to_owned()),
4396                reasoning_effort: Some("default".to_owned()),
4397                permissions: Default::default(),
4398            }
4399        );
4400
4401        let response = app
4402            .clone()
4403            .oneshot(authorized(
4404                "POST",
4405                "/sessions/profile-session/runs",
4406                &token,
4407                serde_json::json!({
4408                    "run_id": "profile-run",
4409                    "input": "complete the deterministic fixture"
4410                }),
4411            ))
4412            .await
4413            .unwrap();
4414        assert_eq!(response.status(), StatusCode::CREATED);
4415
4416        let response = app
4417            .oneshot(authorized(
4418                "GET",
4419                "/sessions",
4420                &token,
4421                serde_json::Value::Null,
4422            ))
4423            .await
4424            .unwrap();
4425        assert_eq!(response.status(), StatusCode::OK);
4426        let body = response.into_body().collect().await.unwrap().to_bytes();
4427        let sessions: Vec<SessionView> = serde_json::from_slice(&body).unwrap();
4428        let listed = sessions
4429            .iter()
4430            .find(|session| session.id == "profile-session")
4431            .unwrap();
4432        assert_eq!(listed.cwd.as_deref(), Some("/fixture/workspace"));
4433        assert_eq!(listed.execution_profile, created.execution_profile);
4434    }
4435
4436    #[tokio::test]
4437    async fn rejected_session_submission_restarts_only_after_confirmed_completion() {
4438        for code in [
4439            AgentProtocolErrorCode::InvalidTransition,
4440            AgentProtocolErrorCode::TerminalRun,
4441            AgentProtocolErrorCode::Unsupported,
4442        ] {
4443            let (app, token, finish) =
4444                completable_agent_app_with_rejection(Some(code.clone())).await;
4445            let request = |id: &str| {
4446                authorized(
4447                    "POST",
4448                    "/agent-runs",
4449                    &token,
4450                    serde_json::json!({
4451                        "connector_id": "fixture/local",
4452                        "session_id": "fixture-session",
4453                        "run_id": id,
4454                        "input": format!("input for {id}")
4455                    }),
4456                )
4457            };
4458            let first = app.clone().oneshot(request("first")).await.unwrap();
4459            assert_eq!(first.status(), StatusCode::CREATED);
4460            let rejected = app.clone().oneshot(request("next")).await.unwrap();
4461            assert_eq!(rejected.status(), StatusCode::CONFLICT);
4462            let body = rejected.into_body().collect().await.unwrap().to_bytes();
4463            let error: Value = serde_json::from_slice(&body).unwrap();
4464            assert_eq!(error["message"], "fixture command was not dispatched");
4465
4466            // Even a TerminalRun rejection is insufficient while the durable
4467            // Run is active. Repeating the operation must not dispatch anew.
4468            let retry = app.clone().oneshot(request("next")).await.unwrap();
4469            assert_eq!(retry.status(), StatusCode::CONFLICT);
4470            finish.notify_one();
4471            tokio::time::timeout(Duration::from_secs(2), async {
4472                loop {
4473                    let response = app
4474                        .clone()
4475                        .oneshot(authorized(
4476                            "GET",
4477                            "/runs/first?connector_id=fixture%2Flocal",
4478                            &token,
4479                            Value::Null,
4480                        ))
4481                        .await
4482                        .unwrap();
4483                    let body = response.into_body().collect().await.unwrap().to_bytes();
4484                    let view: Value = serde_json::from_slice(&body).unwrap();
4485                    if view["state"]["state"] == "terminal" {
4486                        break;
4487                    }
4488                    tokio::task::yield_now().await;
4489                }
4490            })
4491            .await
4492            .expect("fixture must commit its terminal state");
4493
4494            let retry = app.clone().oneshot(request("next")).await.unwrap();
4495            if code == AgentProtocolErrorCode::Unsupported {
4496                assert_eq!(retry.status(), StatusCode::CONFLICT);
4497                continue;
4498            }
4499            assert_eq!(retry.status(), StatusCode::CREATED);
4500            let body = retry.into_body().collect().await.unwrap().to_bytes();
4501            let result: Value = serde_json::from_slice(&body).unwrap();
4502            assert_eq!(result["run_id"], "next");
4503            assert_eq!(result["operation"], "started");
4504            let duplicate = app.clone().oneshot(request("next")).await.unwrap();
4505            assert_eq!(duplicate.status(), StatusCode::OK);
4506            let body = duplicate.into_body().collect().await.unwrap().to_bytes();
4507            let result: Value = serde_json::from_slice(&body).unwrap();
4508            assert_eq!(result["operation"], "replayed");
4509            assert_eq!(result["run_id"], "next");
4510        }
4511    }
4512
4513    #[tokio::test]
4514    async fn concurrent_agent_session_posts_share_one_run_and_retry_one_command() {
4515        let (app, token) = holding_agent_app().await;
4516        let request = |run_id: &str, input: &str| {
4517            authorized(
4518                "POST",
4519                "/agent-runs",
4520                &token,
4521                serde_json::json!({
4522                    "connector_id": "fixture/local",
4523                    "session_id": "fixture-session",
4524                    "run_id": run_id,
4525                    "input": input
4526                }),
4527            )
4528        };
4529
4530        let (left, right) = tokio::join!(
4531            app.clone().oneshot(request("tab-a", "message from tab A")),
4532            app.clone().oneshot(request("tab-b", "message from tab B")),
4533        );
4534        let mut responses = Vec::new();
4535        for response in [left.unwrap(), right.unwrap()] {
4536            assert!(matches!(
4537                response.status(),
4538                StatusCode::CREATED | StatusCode::OK
4539            ));
4540            let body = response.into_body().collect().await.unwrap().to_bytes();
4541            responses.push(serde_json::from_slice::<serde_json::Value>(&body).unwrap());
4542        }
4543        let started = responses
4544            .iter()
4545            .find(|response| response["operation"] == "started")
4546            .expect("one tab starts the session Run");
4547        let steered = responses
4548            .iter()
4549            .find(|response| response["operation"] == "steered")
4550            .expect("the other tab steers that same Run");
4551        assert_eq!(started["run_id"], steered["run_id"]);
4552        let active_run_id = started["run_id"].as_str().unwrap();
4553        let steered_operation_id = if active_run_id == "tab-a" {
4554            "tab-b"
4555        } else {
4556            "tab-a"
4557        };
4558
4559        let retry = app
4560            .clone()
4561            .oneshot(request(
4562                steered_operation_id,
4563                if steered_operation_id == "tab-a" {
4564                    "message from tab A"
4565                } else {
4566                    "message from tab B"
4567                },
4568            ))
4569            .await
4570            .unwrap();
4571        assert_eq!(retry.status(), StatusCode::OK);
4572        let body = retry.into_body().collect().await.unwrap().to_bytes();
4573        let retry: serde_json::Value = serde_json::from_slice(&body).unwrap();
4574        assert_eq!(retry["operation"], "steered");
4575        assert_eq!(retry["run_id"], active_run_id);
4576
4577        let response = app
4578            .oneshot(authorized(
4579                "GET",
4580                &format!("/runs/{active_run_id}/events?connector_id=fixture%2Flocal&after=0"),
4581                &token,
4582                serde_json::Value::Null,
4583            ))
4584            .await
4585            .unwrap();
4586        let body = response.into_body().collect().await.unwrap().to_bytes();
4587        let events: serde_json::Value = serde_json::from_slice(&body).unwrap();
4588        let command_id = format!("agent-submit-{steered_operation_id}");
4589        let command_count = events["records"]
4590            .as_array()
4591            .unwrap()
4592            .iter()
4593            .filter(|record| {
4594                record["event"]["payload"]["type"] == "command_received"
4595                    && record["event"]["payload"]["command"]["command_id"] == command_id
4596            })
4597            .count();
4598        assert_eq!(command_count, 1, "network retry must not steer twice");
4599    }
4600
4601    #[tokio::test]
4602    async fn external_agent_session_can_be_discovered_read_and_started_through_http() {
4603        let (app, token) = test_app().await;
4604
4605        let response = app
4606            .clone()
4607            .oneshot(authorized(
4608                "GET",
4609                "/agent-connectors",
4610                &token,
4611                serde_json::Value::Null,
4612            ))
4613            .await
4614            .unwrap();
4615        assert_eq!(response.status(), StatusCode::OK);
4616        let body = response.into_body().collect().await.unwrap().to_bytes();
4617        let connectors: serde_json::Value = serde_json::from_slice(&body).unwrap();
4618        assert_eq!(connectors[0]["connector_id"], "fixture/local");
4619
4620        let response = app
4621            .clone()
4622            .oneshot(authorized(
4623                "GET",
4624                "/agent-sessions?connector_id=fixture%2Flocal&limit=25",
4625                &token,
4626                serde_json::Value::Null,
4627            ))
4628            .await
4629            .unwrap();
4630        assert_eq!(response.status(), StatusCode::OK);
4631        let body = response.into_body().collect().await.unwrap().to_bytes();
4632        let sessions: serde_json::Value = serde_json::from_slice(&body).unwrap();
4633        assert_eq!(sessions["sessions"][0]["session_id"], "fixture-session");
4634
4635        let response = app
4636            .clone()
4637            .oneshot(authorized(
4638                "GET",
4639                "/agent-session?connector_id=fixture%2Flocal&session_id=fixture-session&limit=100",
4640                &token,
4641                serde_json::Value::Null,
4642            ))
4643            .await
4644            .unwrap();
4645        assert_eq!(response.status(), StatusCode::OK);
4646        let session_etag = response
4647            .headers()
4648            .get(header::ETAG)
4649            .unwrap()
4650            .to_str()
4651            .unwrap()
4652            .to_owned();
4653        let body = response.into_body().collect().await.unwrap().to_bytes();
4654        assert!(body.len() < 512 * 1_024);
4655        let session: serde_json::Value = serde_json::from_slice(&body).unwrap();
4656        assert_eq!(session["summary"]["title"], "Existing fixture session");
4657        assert!(session["next_cursor"].is_string());
4658        assert_eq!(
4659            session["turns"][0]["activities"]
4660                .as_array()
4661                .unwrap()
4662                .last()
4663                .unwrap()["activity_id"],
4664            "activity-59"
4665        );
4666
4667        let mut conditional = authorized(
4668            "GET",
4669            "/agent-session?connector_id=fixture%2Flocal&session_id=fixture-session&limit=100",
4670            &token,
4671            serde_json::Value::Null,
4672        );
4673        conditional.headers_mut().insert(
4674            header::IF_NONE_MATCH,
4675            header::HeaderValue::from_str(&session_etag).unwrap(),
4676        );
4677        let response = app.clone().oneshot(conditional).await.unwrap();
4678        assert_eq!(response.status(), StatusCode::NOT_MODIFIED);
4679        assert_eq!(response.headers()[header::ETAG], session_etag);
4680        assert!(response
4681            .into_body()
4682            .collect()
4683            .await
4684            .unwrap()
4685            .to_bytes()
4686            .is_empty());
4687
4688        let response = app
4689            .clone()
4690            .oneshot(authorized(
4691                "POST",
4692                "/agent-sessions",
4693                &token,
4694                serde_json::json!({
4695                    "connector_id": "fixture/local",
4696                    "cwd": "/fixture/new",
4697                    "title": "Created from HTTP"
4698                }),
4699            ))
4700            .await
4701            .unwrap();
4702        assert_eq!(response.status(), StatusCode::CREATED);
4703        let body = response.into_body().collect().await.unwrap().to_bytes();
4704        let created: serde_json::Value = serde_json::from_slice(&body).unwrap();
4705        assert_eq!(created["session_id"], "fixture-created");
4706
4707        let response = app
4708            .clone()
4709            .oneshot(authorized(
4710                "POST",
4711                "/agent-session/actions",
4712                &token,
4713                serde_json::json!({
4714                    "connector_id": "fixture/local",
4715                    "session_id": "fixture-session",
4716                    "action_id": "session.rename",
4717                    "arguments": {"name": "Renamed over HTTP"}
4718                }),
4719            ))
4720            .await
4721            .unwrap();
4722        assert_eq!(response.status(), StatusCode::OK);
4723        let body = response.into_body().collect().await.unwrap().to_bytes();
4724        let renamed: serde_json::Value = serde_json::from_slice(&body).unwrap();
4725        assert_eq!(renamed["session"]["title"], "Renamed over HTTP");
4726
4727        let response = app
4728            .clone()
4729            .oneshot(authorized(
4730                "POST",
4731                "/agent-session/actions",
4732                &token,
4733                serde_json::json!({
4734                    "connector_id": "fixture/local",
4735                    "session_id": "fixture-session",
4736                    "action_id": "session.review",
4737                    "arguments": {"target": "uncommitted_changes"},
4738                    "run_id": "fixture-review-run"
4739                }),
4740            ))
4741            .await
4742            .unwrap();
4743        assert_eq!(response.status(), StatusCode::OK);
4744        let body = response.into_body().collect().await.unwrap().to_bytes();
4745        let review: serde_json::Value = serde_json::from_slice(&body).unwrap();
4746        assert_eq!(review["status"]["state"], "running");
4747        assert_eq!(review["status"]["run_id"], "fixture-review-run");
4748
4749        let response = app
4750            .clone()
4751            .oneshot(authorized(
4752                "POST",
4753                "/agent-session/actions",
4754                &token,
4755                serde_json::json!({
4756                    "connector_id": "fixture/local",
4757                    "session_id": "fixture-session",
4758                    "action_id": "session.undeclared"
4759                }),
4760            ))
4761            .await
4762            .unwrap();
4763        assert_eq!(response.status(), StatusCode::NOT_IMPLEMENTED);
4764
4765        let response = app
4766            .clone()
4767            .oneshot(authorized(
4768                "POST",
4769                "/agent-runs",
4770                &token,
4771                serde_json::json!({
4772                    "connector_id": "fixture/local",
4773                    "session_id": "fixture-created",
4774                    "run_id": "fixture-external-run",
4775                    "input": "continue the existing session",
4776                    "after_activity_id": "fixture-visible-tail"
4777                }),
4778            ))
4779            .await
4780            .unwrap();
4781        assert_eq!(response.status(), StatusCode::CREATED);
4782        let body = response.into_body().collect().await.unwrap().to_bytes();
4783        let started: serde_json::Value = serde_json::from_slice(&body).unwrap();
4784        assert_eq!(started["connector_id"], "fixture/local");
4785        assert_eq!(started["run_id"], "fixture-external-run");
4786
4787        let response = app
4788            .clone()
4789            .oneshot(authorized(
4790                "GET",
4791                "/agent-session?connector_id=fixture%2Flocal&session_id=fixture-created&limit=100",
4792                &token,
4793                serde_json::Value::Null,
4794            ))
4795            .await
4796            .unwrap();
4797        assert_eq!(response.status(), StatusCode::OK);
4798        let body = response.into_body().collect().await.unwrap().to_bytes();
4799        let refreshed: serde_json::Value = serde_json::from_slice(&body).unwrap();
4800        assert_eq!(
4801            refreshed["controlled_runs"][0]["execution"]["run_id"],
4802            "fixture-external-run"
4803        );
4804        assert!(refreshed["controlled_runs"][0]["created_at_unix_ms"].is_i64());
4805        assert!(refreshed["controlled_runs"][0]["updated_at_unix_ms"].is_i64());
4806        assert_eq!(
4807            refreshed["controlled_runs"][0]["after_activity_id"],
4808            "fixture-visible-tail"
4809        );
4810
4811        let response = app
4812            .clone()
4813            .oneshot(authorized(
4814                "GET",
4815                "/runs/fixture-external-run?connector_id=fixture%2Flocal",
4816                &token,
4817                serde_json::Value::Null,
4818            ))
4819            .await
4820            .unwrap();
4821        assert_eq!(response.status(), StatusCode::OK);
4822        let body = response.into_body().collect().await.unwrap().to_bytes();
4823        let run: serde_json::Value = serde_json::from_slice(&body).unwrap();
4824        assert_eq!(
4825            run["input"][0]["body"]["value"],
4826            "continue the existing session"
4827        );
4828
4829        let response = app
4830            .clone()
4831            .oneshot(authorized(
4832                "GET",
4833                "/runs/fixture-external-run/events?connector_id=fixture%2Flocal&after=0",
4834                &token,
4835                serde_json::Value::Null,
4836            ))
4837            .await
4838            .unwrap();
4839        assert_eq!(response.status(), StatusCode::OK);
4840        let body = response.into_body().collect().await.unwrap().to_bytes();
4841        let events: serde_json::Value = serde_json::from_slice(&body).unwrap();
4842        assert!(events["records"]
4843            .as_array()
4844            .is_some_and(|items| !items.is_empty()));
4845        assert!(events["next"].as_u64().is_some_and(|next| next > 0));
4846
4847        let mut recovered_terminal = false;
4848        for _ in 0..100 {
4849            let response = app
4850                .clone()
4851                .oneshot(authorized(
4852                    "GET",
4853                    "/runs/fixture-external-run?connector_id=fixture%2Flocal",
4854                    &token,
4855                    serde_json::Value::Null,
4856                ))
4857                .await
4858                .unwrap();
4859            let body = response.into_body().collect().await.unwrap().to_bytes();
4860            let view: serde_json::Value = serde_json::from_slice(&body).unwrap();
4861            if view["state"]["state"] == "terminal" {
4862                recovered_terminal = true;
4863                break;
4864            }
4865            tokio::task::yield_now().await;
4866        }
4867        assert!(
4868            recovered_terminal,
4869            "the Host supervisor automatically recovers the disconnected stream"
4870        );
4871
4872        let response = app
4873            .clone()
4874            .oneshot(authorized(
4875                "GET",
4876                "/runs/fixture-external-run/events?connector_id=fixture%2Flocal&after=0",
4877                &token,
4878                serde_json::Value::Null,
4879            ))
4880            .await
4881            .unwrap();
4882        let body = response.into_body().collect().await.unwrap().to_bytes();
4883        let events: serde_json::Value = serde_json::from_slice(&body).unwrap();
4884        let event_types = events["records"]
4885            .as_array()
4886            .unwrap()
4887            .iter()
4888            .filter_map(|record| record["event"]["payload"]["type"].as_str())
4889            .collect::<Vec<_>>();
4890        assert!(event_types.contains(&"continuity_lost"));
4891        assert!(event_types.contains(&"continuity_restored"));
4892
4893        let response = app
4894            .clone()
4895            .oneshot(authorized(
4896                "POST",
4897                "/agent-runs",
4898                &token,
4899                serde_json::json!({
4900                    "connector_id": "fixture/local",
4901                    "session_id": "fixture-created",
4902                    "run_id": "fixture-external-run-2",
4903                    "input": "continue once more",
4904                    "after_activity_id": "fixture-newer-tail"
4905                }),
4906            ))
4907            .await
4908            .unwrap();
4909        assert_eq!(response.status(), StatusCode::CREATED);
4910
4911        let response = app
4912            .oneshot(authorized(
4913                "GET",
4914                "/agent-session?connector_id=fixture%2Flocal&session_id=fixture-created&limit=100",
4915                &token,
4916                serde_json::Value::Null,
4917            ))
4918            .await
4919            .unwrap();
4920        assert_eq!(response.status(), StatusCode::OK);
4921        let body = response.into_body().collect().await.unwrap().to_bytes();
4922        let refreshed: serde_json::Value = serde_json::from_slice(&body).unwrap();
4923        let controlled_run_ids = refreshed["controlled_runs"]
4924            .as_array()
4925            .unwrap()
4926            .iter()
4927            .map(|run| run["execution"]["run_id"].as_str().unwrap())
4928            .collect::<Vec<_>>();
4929        assert_eq!(
4930            controlled_run_ids,
4931            ["fixture-external-run", "fixture-external-run-2"],
4932            "a stale native transcript must not collapse the durable Host suffix to one Run"
4933        );
4934    }
4935
4936    #[tokio::test]
4937    async fn old_connector_contract_does_not_block_session_read_or_new_http_run() {
4938        let (app, token) = stale_external_agent_app().await;
4939
4940        let response = app
4941            .clone()
4942            .oneshot(authorized(
4943                "GET",
4944                "/agent-session?connector_id=fixture%2Flocal&session_id=fixture-session&limit=100",
4945                &token,
4946                serde_json::Value::Null,
4947            ))
4948            .await
4949            .unwrap();
4950        assert_eq!(response.status(), StatusCode::OK);
4951        let body = response.into_body().collect().await.unwrap().to_bytes();
4952        let session: serde_json::Value = serde_json::from_slice(&body).unwrap();
4953        assert_eq!(session["summary"]["session_id"], "fixture-session");
4954        assert!(session["controlled_runs"].as_array().unwrap().is_empty());
4955
4956        let response = app
4957            .clone()
4958            .oneshot(authorized(
4959                "POST",
4960                "/agent-runs",
4961                &token,
4962                serde_json::json!({
4963                    "connector_id": "fixture/local",
4964                    "session_id": "fixture-session",
4965                    "run_id": "fixture-current-run",
4966                    "input": "continue after the connector upgrade"
4967                }),
4968            ))
4969            .await
4970            .unwrap();
4971        assert_eq!(response.status(), StatusCode::CREATED);
4972        let body = response.into_body().collect().await.unwrap().to_bytes();
4973        let started: serde_json::Value = serde_json::from_slice(&body).unwrap();
4974        assert_eq!(started["operation"], "started");
4975        assert_eq!(started["run_id"], "fixture-current-run");
4976
4977        let retry = app
4978            .oneshot(authorized(
4979                "POST",
4980                "/agent-runs",
4981                &token,
4982                serde_json::json!({
4983                    "connector_id": "fixture/local",
4984                    "session_id": "fixture-session",
4985                    "run_id": "fixture-current-run",
4986                    "input": "continue after the connector upgrade"
4987                }),
4988            ))
4989            .await
4990            .unwrap();
4991        assert_eq!(retry.status(), StatusCode::OK);
4992        let body = retry.into_body().collect().await.unwrap().to_bytes();
4993        let replayed: serde_json::Value = serde_json::from_slice(&body).unwrap();
4994        assert_eq!(replayed["operation"], "replayed");
4995        assert_eq!(replayed["run_id"], "fixture-current-run");
4996    }
4997
4998    #[tokio::test]
4999    async fn gateway_mode_accepts_a_verified_identity_without_a_bearer_token() {
5000        let authenticator = Arc::new(StaticGatewayAuthenticator {
5001            header_name: HeaderName::from_static("x-gateway-jwt"),
5002        });
5003        let (app, _) = test_app_with_gateway(Some(authenticator)).await;
5004
5005        let response = app
5006            .clone()
5007            .oneshot(
5008                Request::builder()
5009                    .uri("/me")
5010                    .header("x-gateway-jwt", "valid-gateway-assertion")
5011                    .body(Body::empty())
5012                    .unwrap(),
5013            )
5014            .await
5015            .unwrap();
5016        assert_eq!(response.status(), StatusCode::OK);
5017        let body = response.into_body().collect().await.unwrap().to_bytes();
5018        let me: serde_json::Value = serde_json::from_slice(&body).unwrap();
5019        assert_eq!(me["auth_mode"], "gateway_jwt");
5020        assert_eq!(me["attributes"]["email"], "person@example.com");
5021
5022        for assertion in [None, Some("forged-assertion")] {
5023            let mut request = Request::builder().uri("/sessions");
5024            if let Some(assertion) = assertion {
5025                request = request.header("x-gateway-jwt", assertion);
5026            }
5027            let response = app
5028                .clone()
5029                .oneshot(request.body(Body::empty()).unwrap())
5030                .await
5031                .unwrap();
5032            assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
5033        }
5034    }
5035
5036    #[tokio::test]
5037    async fn commands_return_recovery_pending_while_continuity_is_unknown() {
5038        let (app, token) = unrecoverable_app().await;
5039        let response = app
5040            .clone()
5041            .oneshot(authorized(
5042                "POST",
5043                "/sessions",
5044                &token,
5045                serde_json::json!({"session_id": "recovery-session"}),
5046            ))
5047            .await
5048            .unwrap();
5049        assert_eq!(response.status(), StatusCode::CREATED);
5050
5051        let response = app
5052            .clone()
5053            .oneshot(authorized(
5054                "POST",
5055                "/sessions/recovery-session/runs",
5056                &token,
5057                serde_json::json!({
5058                    "run_id": "recovery-run",
5059                    "input": "wait for recovery"
5060                }),
5061            ))
5062            .await
5063            .unwrap();
5064        assert_eq!(response.status(), StatusCode::CREATED);
5065
5066        let mut unknown = false;
5067        for _ in 0..50 {
5068            let response = app
5069                .clone()
5070                .oneshot(authorized(
5071                    "GET",
5072                    "/runs/recovery-run",
5073                    &token,
5074                    serde_json::Value::Null,
5075                ))
5076                .await
5077                .unwrap();
5078            let body = response.into_body().collect().await.unwrap().to_bytes();
5079            let view: serde_json::Value = serde_json::from_slice(&body).unwrap();
5080            if view["state"]["state"] == "unknown" {
5081                assert_eq!(view["recovery"]["mode"], "manual");
5082                assert_eq!(view["recovery"]["can_start_new_run"], false);
5083                unknown = true;
5084                break;
5085            }
5086            tokio::time::sleep(Duration::from_millis(10)).await;
5087        }
5088        assert!(unknown, "fixture Run did not enter Unknown continuity");
5089
5090        let response = app
5091            .clone()
5092            .oneshot(authorized(
5093                "POST",
5094                "/runs/recovery-run/steer",
5095                &token,
5096                serde_json::json!({
5097                    "command_id": "steer-during-recovery",
5098                    "text": "do not duplicate this"
5099                }),
5100            ))
5101            .await
5102            .unwrap();
5103        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
5104        let body = response.into_body().collect().await.unwrap().to_bytes();
5105        let error: serde_json::Value = serde_json::from_slice(&body).unwrap();
5106        assert_eq!(error["code"], "run_recovery_pending");
5107    }
5108
5109    #[test]
5110    fn manual_recovery_registry_stops_repeat_supervision() {
5111        let registry = RunSupervisorRegistry::default();
5112        let run_id = RunId::new("manual-run");
5113        let key = RunSupervisorRegistry::key(None, &run_id);
5114
5115        assert!(registry.begin(&key));
5116        registry.finish(&key);
5117        registry.mark_manual(key.clone(), "unsafe model boundary".to_owned());
5118
5119        assert!(!registry.begin(&key));
5120        assert_eq!(
5121            registry.manual_reason(&key).as_deref(),
5122            Some("unsafe model boundary")
5123        );
5124
5125        registry.clear_manual(&key);
5126        assert!(registry.begin(&key));
5127        registry.finish(&key);
5128    }
5129
5130    #[test]
5131    fn supervision_registry_exposes_and_clears_stall_state() {
5132        let registry = RunSupervisorRegistry::default();
5133        let key = RunSupervisorRegistry::key(None, &RunId::new("stalled-run"));
5134
5135        registry.mark_issue(
5136            key.clone(),
5137            "interrupting",
5138            "execution lease expired".to_owned(),
5139        );
5140        let interrupting = registry.issue(&key).expect("issue is visible");
5141        assert_eq!(interrupting.state, "interrupting");
5142        assert_eq!(interrupting.reason, "execution lease expired");
5143
5144        registry.mark_issue(key.clone(), "stalled", "stop did not converge".to_owned());
5145        let stalled = registry.issue(&key).expect("updated issue is visible");
5146        assert_eq!(stalled.state, "stalled");
5147        assert_eq!(
5148            stalled.detected_at_unix_ms,
5149            interrupting.detected_at_unix_ms
5150        );
5151
5152        registry.clear_issue(&key);
5153        assert!(registry.issue(&key).is_none());
5154    }
5155
5156    #[test]
5157    fn control_plane_chatter_does_not_renew_the_execution_lease() {
5158        assert!(!agent_event_is_execution_progress(
5159            &AgentEvent::CommandDispositionRecorded {
5160                command_id: CommandId::new("accepted-but-not-applied"),
5161                outcome: ProviderCommandOutcome::Accepted,
5162            }
5163        ));
5164        assert!(!agent_event_is_execution_progress(
5165            &AgentEvent::StopRequested {
5166                reason: "watchdog".to_owned(),
5167            }
5168        ));
5169        assert!(agent_event_is_execution_progress(&AgentEvent::RunStarted));
5170        assert!(agent_event_is_execution_progress(
5171            &AgentEvent::RequestClosed {
5172                request_id: RequestId::new("native-request"),
5173                reason: "Provider completed the request".to_owned(),
5174            }
5175        ));
5176    }
5177
5178    #[tokio::test]
5179    async fn approval_is_host_signed_and_resolves_the_pending_request() {
5180        let (app, token) = approval_app().await;
5181        let response = app
5182            .clone()
5183            .oneshot(authorized(
5184                "POST",
5185                "/sessions",
5186                &token,
5187                serde_json::json!({"session_id": "approval-session"}),
5188            ))
5189            .await
5190            .unwrap();
5191        assert_eq!(response.status(), StatusCode::CREATED);
5192
5193        let response = app
5194            .clone()
5195            .oneshot(authorized(
5196                "POST",
5197                "/sessions/approval-session/runs",
5198                &token,
5199                serde_json::json!({
5200                    "run_id": "approval-run",
5201                    "input": "perform the write"
5202                }),
5203            ))
5204            .await
5205            .unwrap();
5206        assert_eq!(response.status(), StatusCode::CREATED);
5207
5208        let mut pending_seen = false;
5209        for _ in 0..50 {
5210            let response = app
5211                .clone()
5212                .oneshot(authorized(
5213                    "GET",
5214                    "/runs/approval-run",
5215                    &token,
5216                    serde_json::Value::Null,
5217                ))
5218                .await
5219                .unwrap();
5220            let body = response.into_body().collect().await.unwrap().to_bytes();
5221            let view: serde_json::Value = serde_json::from_slice(&body).unwrap();
5222            if view["pending_requests"]
5223                .as_array()
5224                .is_some_and(|items| items.len() == 1)
5225            {
5226                pending_seen = true;
5227                break;
5228            }
5229            tokio::time::sleep(Duration::from_millis(10)).await;
5230        }
5231        assert!(pending_seen, "approval request did not become visible");
5232
5233        let response = app
5234            .clone()
5235            .oneshot(authorized(
5236                "POST",
5237                "/runs/approval-run/requests/approval-request/approval",
5238                &token,
5239                serde_json::json!({
5240                    "command_id": "approve-from-phone",
5241                    "decision": "allow_once"
5242                }),
5243            ))
5244            .await
5245            .unwrap();
5246        assert_eq!(response.status(), StatusCode::OK);
5247        let body = response.into_body().collect().await.unwrap().to_bytes();
5248        let ack: serde_json::Value = serde_json::from_slice(&body).unwrap();
5249        assert!(matches!(
5250            ack["state"]["state"].as_str(),
5251            Some("accepted" | "applied")
5252        ));
5253
5254        let mut resolved = None;
5255        for _ in 0..50 {
5256            let response = app
5257                .clone()
5258                .oneshot(authorized(
5259                    "GET",
5260                    "/runs/approval-run/events?after=0",
5261                    &token,
5262                    serde_json::Value::Null,
5263                ))
5264                .await
5265                .unwrap();
5266            let body = response.into_body().collect().await.unwrap().to_bytes();
5267            let page: serde_json::Value = serde_json::from_slice(&body).unwrap();
5268            resolved = page["records"]
5269                .as_array()
5270                .and_then(|records| {
5271                    records
5272                        .iter()
5273                        .find(|record| record["event"]["payload"]["type"] == "request_resolved")
5274                })
5275                .cloned();
5276            if resolved.is_some() {
5277                break;
5278            }
5279            tokio::time::sleep(Duration::from_millis(10)).await;
5280        }
5281        let resolved = resolved.expect("approval resolution is durable");
5282        let resolution = &resolved["event"]["payload"]["resolution"];
5283        assert_eq!(resolution["type"], "approval");
5284        assert_eq!(resolution["decision"], "allow");
5285        assert!(resolution["grant_ref"]
5286            .as_str()
5287            .is_some_and(|grant| !grant.is_empty()));
5288    }
5289
5290    #[tokio::test]
5291    async fn revoking_the_current_device_invalidates_its_next_request() {
5292        let (app, token) = test_app().await;
5293        let response = app
5294            .clone()
5295            .oneshot(authorized("GET", "/me", &token, serde_json::Value::Null))
5296            .await
5297            .unwrap();
5298        let body = response.into_body().collect().await.unwrap().to_bytes();
5299        let me: serde_json::Value = serde_json::from_slice(&body).unwrap();
5300        let device_id = me["device_id"].as_str().unwrap();
5301
5302        let response = app
5303            .clone()
5304            .oneshot(authorized(
5305                "DELETE",
5306                &format!("/devices/{device_id}"),
5307                &token,
5308                serde_json::Value::Null,
5309            ))
5310            .await
5311            .unwrap();
5312        assert_eq!(response.status(), StatusCode::NO_CONTENT);
5313
5314        let response = app
5315            .oneshot(authorized(
5316                "GET",
5317                "/sessions",
5318                &token,
5319                serde_json::Value::Null,
5320            ))
5321            .await
5322            .unwrap();
5323        assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
5324    }
5325
5326    #[tokio::test]
5327    async fn session_start_inspect_and_cursor_replay_use_real_agent_api() {
5328        let (app, token) = test_app().await;
5329        let response = app
5330            .clone()
5331            .oneshot(authorized(
5332                "POST",
5333                "/sessions",
5334                &token,
5335                serde_json::json!({"session_id": "mobile-session"}),
5336            ))
5337            .await
5338            .unwrap();
5339        assert_eq!(response.status(), StatusCode::CREATED);
5340
5341        let response = app
5342            .clone()
5343            .oneshot(authorized(
5344                "POST",
5345                "/sessions/mobile-session/runs",
5346                &token,
5347                serde_json::json!({
5348                    "run_id": "mobile-run",
5349                    "input": "complete the deterministic fixture"
5350                }),
5351            ))
5352            .await
5353            .unwrap();
5354        assert_eq!(response.status(), StatusCode::CREATED);
5355        let bytes = response.into_body().collect().await.unwrap().to_bytes();
5356        let started: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
5357        assert_eq!(
5358            started["view"]["input"][0]["body"]["value"],
5359            "complete the deterministic fixture"
5360        );
5361
5362        tokio::time::sleep(Duration::from_millis(20)).await;
5363        let response = app
5364            .clone()
5365            .oneshot(authorized(
5366                "GET",
5367                "/runs/mobile-run",
5368                &token,
5369                serde_json::Value::Null,
5370            ))
5371            .await
5372            .unwrap();
5373        assert_eq!(response.status(), StatusCode::OK);
5374        let bytes = response.into_body().collect().await.unwrap().to_bytes();
5375        let inspected: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
5376        assert_eq!(
5377            inspected["input"][0]["body"]["value"],
5378            "complete the deterministic fixture"
5379        );
5380
5381        let response = app
5382            .clone()
5383            .oneshot(authorized(
5384                "GET",
5385                "/sessions",
5386                &token,
5387                serde_json::Value::Null,
5388            ))
5389            .await
5390            .unwrap();
5391        assert_eq!(response.status(), StatusCode::OK);
5392        let bytes = response.into_body().collect().await.unwrap().to_bytes();
5393        let sessions: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
5394        assert_eq!(sessions[0]["id"], "mobile-session");
5395        assert_eq!(sessions[0]["run_ids"], serde_json::json!(["mobile-run"]));
5396
5397        let response = app
5398            .clone()
5399            .oneshot(authorized(
5400                "GET",
5401                "/runs/mobile-run/events?after=0",
5402                &token,
5403                serde_json::Value::Null,
5404            ))
5405            .await
5406            .unwrap();
5407        assert_eq!(response.status(), StatusCode::OK);
5408        let bytes = response.into_body().collect().await.unwrap().to_bytes();
5409        let payload: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
5410        let records = payload["records"].as_array().unwrap();
5411        assert!(!records.is_empty());
5412        let next = payload["next"].as_u64().unwrap();
5413
5414        let response = app
5415            .oneshot(authorized(
5416                "GET",
5417                &format!("/runs/mobile-run/events?after={next}"),
5418                &token,
5419                serde_json::Value::Null,
5420            ))
5421            .await
5422            .unwrap();
5423        let bytes = response.into_body().collect().await.unwrap().to_bytes();
5424        let payload: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
5425        assert_eq!(payload["records"], serde_json::json!([]));
5426    }
5427
5428    #[tokio::test]
5429    async fn command_identity_is_forwarded_for_retry_deduplication() {
5430        let (app, token) = test_app().await;
5431        app.clone()
5432            .oneshot(authorized(
5433                "POST",
5434                "/sessions",
5435                &token,
5436                serde_json::json!({"session_id": "mobile-session"}),
5437            ))
5438            .await
5439            .unwrap();
5440        app.clone()
5441            .oneshot(authorized(
5442                "POST",
5443                "/sessions/mobile-session/runs",
5444                &token,
5445                serde_json::json!({"run_id": "mobile-run", "input": "do it"}),
5446            ))
5447            .await
5448            .unwrap();
5449
5450        let command = || {
5451            authorized(
5452                "POST",
5453                "/runs/mobile-run/cancel",
5454                &token,
5455                serde_json::json!({
5456                    "command_id": "mobile-command-1",
5457                    "reason": "stop from phone"
5458                }),
5459            )
5460        };
5461        let first = app.clone().oneshot(command()).await.unwrap();
5462        let second = app.oneshot(command()).await.unwrap();
5463        assert_eq!(first.status(), StatusCode::OK);
5464        assert_eq!(second.status(), StatusCode::OK);
5465        let body = second.into_body().collect().await.unwrap().to_bytes();
5466        let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
5467        assert_eq!(payload["duplicate"], true);
5468    }
5469
5470    #[tokio::test]
5471    async fn sse_replay_uses_durable_sequence_ids_after_the_requested_cursor() {
5472        let (app, token) = test_app().await;
5473        app.clone()
5474            .oneshot(authorized(
5475                "POST",
5476                "/sessions",
5477                &token,
5478                serde_json::json!({"session_id": "stream-session"}),
5479            ))
5480            .await
5481            .unwrap();
5482        app.clone()
5483            .oneshot(authorized(
5484                "POST",
5485                "/sessions/stream-session/runs",
5486                &token,
5487                serde_json::json!({"run_id": "stream-run", "input": "do it"}),
5488            ))
5489            .await
5490            .unwrap();
5491        tokio::time::sleep(Duration::from_millis(20)).await;
5492
5493        let request = Request::builder()
5494            .uri("/runs/stream-run/stream?after=0")
5495            .header(header::AUTHORIZATION, format!("Bearer {token}"))
5496            .header("last-event-id", "2")
5497            .body(Body::empty())
5498            .unwrap();
5499        let response = app.oneshot(request).await.unwrap();
5500        assert_eq!(response.status(), StatusCode::OK);
5501        assert_eq!(
5502            response.headers()[header::CACHE_CONTROL],
5503            "no-store, private"
5504        );
5505        let request_id = response
5506            .headers()
5507            .get(REQUEST_ID_HEADER)
5508            .and_then(|value| value.to_str().ok())
5509            .expect("SSE response carries its lifecycle correlation id");
5510        uuid::Uuid::parse_str(request_id).expect("request id is a UUID");
5511        let mut stream = response.into_body().into_data_stream();
5512        let frame = tokio::time::timeout(Duration::from_secs(1), stream.next())
5513            .await
5514            .unwrap()
5515            .unwrap()
5516            .unwrap();
5517        let text = String::from_utf8(frame.to_vec()).unwrap();
5518        assert!(text.contains("event: durable"));
5519        assert!(!text.contains("id: 1\n"));
5520        assert!(!text.contains("id: 2\n"));
5521    }
5522
5523    #[tokio::test]
5524    async fn redirected_submission_retry_keeps_its_original_run_after_completion() {
5525        let (app, token, finish) = completable_agent_app().await;
5526        let submit = |id: &str, input: &str, session: &str| {
5527            authorized(
5528                "POST",
5529                "/agent-runs",
5530                &token,
5531                json!({"connector_id": "fixture/local", "session_id": session, "run_id": id, "input": input}),
5532            )
5533        };
5534        assert_eq!(
5535            app.clone()
5536                .oneshot(submit("first", "start", "fixture-session"))
5537                .await
5538                .unwrap()
5539                .status(),
5540            StatusCode::CREATED
5541        );
5542        assert_eq!(
5543            app.clone()
5544                .oneshot(submit("second", "follow up", "fixture-session"))
5545                .await
5546                .unwrap()
5547                .status(),
5548            StatusCode::OK
5549        );
5550        assert_eq!(
5551            app.clone()
5552                .oneshot(submit("second", "changed", "fixture-session"))
5553                .await
5554                .unwrap()
5555                .status(),
5556            StatusCode::CONFLICT
5557        );
5558        assert_eq!(
5559            app.clone()
5560                .oneshot(submit("first", "start", "other-session"))
5561                .await
5562                .unwrap()
5563                .status(),
5564            StatusCode::CONFLICT
5565        );
5566        finish.notify_one();
5567        tokio::time::timeout(Duration::from_secs(2), async {
5568            loop {
5569                let response = app
5570                    .clone()
5571                    .oneshot(authorized(
5572                        "GET",
5573                        "/runs/first?connector_id=fixture%2Flocal",
5574                        &token,
5575                        Value::Null,
5576                    ))
5577                    .await
5578                    .unwrap();
5579                let body = response.into_body().collect().await.unwrap().to_bytes();
5580                let view: Value = serde_json::from_slice(&body).unwrap();
5581                if view["state"]["state"] == "terminal" {
5582                    break;
5583                }
5584                tokio::task::yield_now().await;
5585            }
5586        })
5587        .await
5588        .expect("first Run should reach a terminal state");
5589        // Create a newer Run before retrying the lost response.
5590        assert_eq!(
5591            app.clone()
5592                .oneshot(submit("third", "new turn", "fixture-session"))
5593                .await
5594                .unwrap()
5595                .status(),
5596            StatusCode::CREATED
5597        );
5598        let retry = app
5599            .clone()
5600            .oneshot(submit("second", "follow up", "fixture-session"))
5601            .await
5602            .unwrap();
5603        assert_eq!(retry.status(), StatusCode::OK);
5604        let body = retry.into_body().collect().await.unwrap().to_bytes();
5605        let response: Value = serde_json::from_slice(&body).unwrap();
5606        assert_eq!(response["run_id"], "first");
5607        assert_eq!(response["command_id"], "agent-submit-second");
5608        let events = app
5609            .oneshot(authorized(
5610                "GET",
5611                "/runs/third/events?connector_id=fixture%2Flocal&after=0",
5612                &token,
5613                Value::Null,
5614            ))
5615            .await
5616            .unwrap();
5617        let body = events.into_body().collect().await.unwrap().to_bytes();
5618        let events: Value = serde_json::from_slice(&body).unwrap();
5619        assert!(!events["records"]
5620            .as_array()
5621            .unwrap()
5622            .iter()
5623            .any(|record| record["event"]["payload"]["type"] == "command_received"));
5624    }
5625}