1use std::collections::{BTreeMap, BTreeSet};
2use std::convert::Infallible;
3use std::path::PathBuf;
4use std::sync::{Arc, Mutex};
5use std::time::{Duration, Instant};
6
7use axum::extract::{DefaultBodyLimit, MatchedPath};
8use axum::extract::{Extension, Path, Query, State};
9use axum::http::{header, HeaderMap, HeaderValue, Request, StatusCode};
10use axum::middleware::{self, Next};
11use axum::response::sse::{Event, KeepAlive};
12use axum::response::{IntoResponse, Response, Sse};
13use axum::routing::{delete, get, post};
14use axum::{Json, Router};
15use orchestral_core::agent_connector::{
16 AgentConnectorId, AgentSessionActionId, AgentSessionActionOutcome, AgentSessionActivityId,
17 AgentSessionChange, AgentSessionHistoryAnchor, AgentSessionListQuery, AgentSessionPage,
18 AgentSessionReadQuery, AgentSessionRequestResolution, AgentSessionSummary,
19 CreateAgentSessionRequest, InvokeAgentSessionActionRequest, ResolveAgentSessionRequest,
20};
21use orchestral_core::agent_protocol::spi::AgentStartError;
22use orchestral_core::agent_protocol::wire::{
23 AgentCommand, AgentCommandEnvelope, AgentEvent, AgentRejectionCode, AgentRunView,
24 AgentSessionId, ApprovalDecision, ArtifactRef, ArtifactRefWithDigest, CommandAck,
25 CommandAckState, CommandId, Content, ContentBody, Digest, Extensions, PendingRequest,
26 PendingRequestPayload, RequestId, RequestResolution, RunId,
27};
28use orchestral_core::io::{ArtifactResolver, BlobStore};
29use orchestral_runtime::api::AgentApi;
30use orchestral_runtime::{
31 AgentControlError, AgentControlEvent, AgentDirectory, AgentDirectoryError, AgentSdkError,
32 ApprovalBridgeError, InMemoryHostApprovalBroker,
33};
34use serde::{Deserialize, Serialize};
35use serde_json::{json, Value};
36use tokio::sync::broadcast;
37use tracing::Instrument;
38
39use super::auth::{GatewayAuthenticator, GatewayPrincipal};
40use super::session_coordinator::AgentSessionCoordinatorRegistry;
41use super::state::{
42 DevicePrincipal, DeviceView, NativeSessionDefaults, PairingClaim, RemoteRegistry, SessionView,
43};
44
45const APPROVAL_GRANT_TTL_MS: i64 = 5 * 60 * 1_000;
46const RUN_SUPERVISOR_POLL_INTERVAL: Duration = Duration::from_secs(15);
47const RUN_SUPERVISOR_INITIAL_BACKOFF: Duration = Duration::from_millis(100);
48const RUN_SUPERVISOR_MAX_BACKOFF: Duration = Duration::from_secs(5);
49const DEFAULT_RUN_INACTIVITY_TIMEOUT: Duration = Duration::from_secs(10 * 60);
50const DEFAULT_RUN_STOP_GRACE: Duration = Duration::from_secs(30);
51const CONTROLLED_SESSION_RUN_LIMIT: usize = 100;
57const REQUEST_ID_HEADER: &str = "x-request-id";
58
59#[derive(Debug, Clone, Copy)]
60struct RunSupervisionPolicy {
61 inactivity_timeout: Duration,
62 stop_grace: Duration,
63}
64
65impl Default for RunSupervisionPolicy {
66 fn default() -> Self {
67 Self {
68 inactivity_timeout: duration_from_env(
69 "ORCHESTRAL_AGENT_RUN_INACTIVITY_TIMEOUT_SECS",
70 DEFAULT_RUN_INACTIVITY_TIMEOUT,
71 ),
72 stop_grace: duration_from_env(
73 "ORCHESTRAL_AGENT_RUN_STOP_GRACE_SECS",
74 DEFAULT_RUN_STOP_GRACE,
75 ),
76 }
77 }
78}
79
80fn duration_from_env(name: &str, fallback: Duration) -> Duration {
81 std::env::var(name)
82 .ok()
83 .and_then(|value| value.parse::<u64>().ok())
84 .filter(|seconds| *seconds > 0)
85 .map(Duration::from_secs)
86 .unwrap_or(fallback)
87}
88
89#[derive(Debug, Clone, Serialize)]
90struct RemoteRunSupervisionView {
91 state: &'static str,
92 reason: String,
93 detected_at_unix_ms: i64,
94}
95
96#[derive(Debug, Clone)]
97struct RunSupervisionIssue {
98 state: &'static str,
99 reason: String,
100 detected_at_unix_ms: i64,
101}
102
103#[derive(Default)]
104pub struct RunSupervisorRegistry {
105 active: Mutex<BTreeSet<String>>,
106 manual_recovery: Mutex<BTreeMap<String, String>>,
107 issues: Mutex<BTreeMap<String, RunSupervisionIssue>>,
108 policy: RunSupervisionPolicy,
109}
110
111impl RunSupervisorRegistry {
112 pub(super) fn key(connector_id: Option<&AgentConnectorId>, run_id: &RunId) -> String {
113 format!(
114 "{}\0{}",
115 connector_id.map_or("orchestral", AgentConnectorId::as_str),
116 run_id.as_str()
117 )
118 }
119
120 fn begin(&self, key: &str) -> bool {
121 if self
122 .manual_recovery
123 .lock()
124 .expect("Run recovery registry lock poisoned")
125 .contains_key(key)
126 {
127 return false;
128 }
129 self.active
130 .lock()
131 .expect("Run supervisor registry lock poisoned")
132 .insert(key.to_owned())
133 }
134
135 fn finish(&self, key: &str) {
136 self.active
137 .lock()
138 .expect("Run supervisor registry lock poisoned")
139 .remove(key);
140 }
141
142 fn mark_issue(&self, key: String, state: &'static str, reason: String) {
143 let mut issues = self
144 .issues
145 .lock()
146 .expect("Run supervision registry lock poisoned");
147 let detected_at_unix_ms = issues
148 .get(&key)
149 .map(|issue| issue.detected_at_unix_ms)
150 .unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
151 issues.insert(
152 key,
153 RunSupervisionIssue {
154 state,
155 reason,
156 detected_at_unix_ms,
157 },
158 );
159 }
160
161 fn clear_issue(&self, key: &str) {
162 self.issues
163 .lock()
164 .expect("Run supervision registry lock poisoned")
165 .remove(key);
166 }
167
168 fn issue(&self, key: &str) -> Option<RemoteRunSupervisionView> {
169 self.issues
170 .lock()
171 .expect("Run supervision registry lock poisoned")
172 .get(key)
173 .cloned()
174 .map(|issue| RemoteRunSupervisionView {
175 state: issue.state,
176 reason: issue.reason,
177 detected_at_unix_ms: issue.detected_at_unix_ms,
178 })
179 }
180
181 pub(super) fn mark_manual(&self, key: String, reason: String) {
182 self.manual_recovery
183 .lock()
184 .expect("Run recovery registry lock poisoned")
185 .insert(key, reason);
186 }
187
188 fn clear_manual(&self, key: &str) {
189 self.manual_recovery
190 .lock()
191 .expect("Run recovery registry lock poisoned")
192 .remove(key);
193 }
194
195 fn manual_reason(&self, key: &str) -> Option<String> {
196 self.manual_recovery
197 .lock()
198 .expect("Run recovery registry lock poisoned")
199 .get(key)
200 .cloned()
201 }
202}
203
204#[derive(Clone)]
205pub struct RemoteApiState {
206 pub agent: AgentApi,
207 pub agent_directory: Arc<AgentDirectory>,
208 pub native_session_defaults: NativeSessionDefaults,
209 pub approvals: Arc<InMemoryHostApprovalBroker>,
210 pub registry: RemoteRegistry,
211 pub gateway_authenticator: Option<Arc<dyn GatewayAuthenticator>>,
212 pub run_supervisors: Arc<RunSupervisorRegistry>,
213 pub(super) session_coordinators: Arc<AgentSessionCoordinatorRegistry>,
214 pub artifact_resolver: Option<Arc<dyn ArtifactResolver>>,
215 pub artifact_blob_store: Option<Arc<dyn BlobStore>>,
216}
217
218#[derive(Debug, Clone)]
219enum RemotePrincipal {
220 Device(DevicePrincipal),
221 Gateway(GatewayPrincipal),
222}
223
224#[derive(Debug, Clone)]
225struct RequestLogContext {
226 request_id: String,
227}
228
229#[derive(Debug, Clone)]
230struct ApiErrorLogCode(String);
231
232struct SseLifecycleLog {
233 request_id: String,
234 stream_id: String,
235 stream_kind: &'static str,
236 connector_id: String,
237 session_id: Option<String>,
238 run_id: Option<String>,
239 opened_at: Instant,
240 close_reason: Option<&'static str>,
241}
242
243impl SseLifecycleLog {
244 fn open_agent_session(
245 request: &RequestLogContext,
246 connector_id: &AgentConnectorId,
247 session_id: &AgentSessionId,
248 ) -> Self {
249 Self::open(
250 request,
251 "agent_session",
252 connector_id.as_str(),
253 Some(session_id.as_str()),
254 None,
255 )
256 }
257
258 fn open_run(
259 request: &RequestLogContext,
260 connector_id: Option<&AgentConnectorId>,
261 run_id: &RunId,
262 ) -> Self {
263 Self::open(
264 request,
265 "run",
266 connector_id.map_or("orchestral", AgentConnectorId::as_str),
267 None,
268 Some(run_id.as_str()),
269 )
270 }
271
272 fn open(
273 request: &RequestLogContext,
274 stream_kind: &'static str,
275 connector_id: &str,
276 session_id: Option<&str>,
277 run_id: Option<&str>,
278 ) -> Self {
279 let lifecycle = Self {
280 request_id: request.request_id.clone(),
281 stream_id: uuid::Uuid::new_v4().to_string(),
282 stream_kind,
283 connector_id: connector_id.to_owned(),
284 session_id: session_id.map(str::to_owned),
285 run_id: run_id.map(str::to_owned),
286 opened_at: Instant::now(),
287 close_reason: None,
288 };
289 tracing::info!(
290 request_id = %lifecycle.request_id,
291 stream_id = %lifecycle.stream_id,
292 stream_kind = lifecycle.stream_kind,
293 connector_id = %lifecycle.connector_id,
294 session_id = lifecycle.session_id.as_deref().unwrap_or("-"),
295 run_id = lifecycle.run_id.as_deref().unwrap_or("-"),
296 "SSE stream opened"
297 );
298 lifecycle
299 }
300
301 fn close_as(&mut self, reason: &'static str) {
302 self.close_reason = Some(reason);
303 }
304
305 fn lagged(&self, skipped: u64) {
306 tracing::warn!(
307 request_id = %self.request_id,
308 stream_id = %self.stream_id,
309 stream_kind = self.stream_kind,
310 connector_id = %self.connector_id,
311 session_id = self.session_id.as_deref().unwrap_or("-"),
312 run_id = self.run_id.as_deref().unwrap_or("-"),
313 skipped,
314 "SSE subscriber lagged"
315 );
316 }
317}
318
319impl Drop for SseLifecycleLog {
320 fn drop(&mut self) {
321 let lifetime_ms = u64::try_from(self.opened_at.elapsed().as_millis()).unwrap_or(u64::MAX);
322 tracing::info!(
323 request_id = %self.request_id,
324 stream_id = %self.stream_id,
325 stream_kind = self.stream_kind,
326 connector_id = %self.connector_id,
327 session_id = self.session_id.as_deref().unwrap_or("-"),
328 run_id = self.run_id.as_deref().unwrap_or("-"),
329 close_reason = self.close_reason.unwrap_or("client_disconnected"),
330 lifetime_ms,
331 "SSE stream closed"
332 );
333 }
334}
335
336impl RemotePrincipal {
337 fn current_device_id(&self) -> Option<&str> {
338 match self {
339 Self::Device(principal) => Some(&principal.device_id),
340 Self::Gateway(_) => None,
341 }
342 }
343}
344
345pub fn router(state: RemoteApiState) -> Router {
346 let protected = Router::new()
347 .route("/me", get(me))
348 .route("/devices", get(list_devices))
349 .route("/devices/{device_id}", delete(revoke_device))
350 .route("/sessions", get(list_sessions).post(create_session))
351 .route("/sessions/{session_id}", get(get_session))
352 .route("/sessions/{session_id}/runs", post(start_run))
353 .route("/agent-connectors", get(list_agent_connectors))
354 .route(
355 "/agent-sessions",
356 get(list_agent_sessions).post(create_agent_session),
357 )
358 .route("/agent-session", get(get_agent_session))
359 .route("/agent-session/stream", get(agent_session_stream))
360 .route("/agent-session/actions", post(invoke_agent_session_action))
361 .route(
362 "/agent-session/requests/{request_id}/input",
363 post(resolve_agent_session_input),
364 )
365 .route(
366 "/agent-session/requests/{request_id}/approval",
367 post(resolve_agent_session_approval),
368 )
369 .route("/agent-runs", post(start_agent_run))
370 .route("/runs/{run_id}", get(inspect_run))
371 .route("/runs/{run_id}/events", get(run_events))
372 .route("/runs/{run_id}/stream", get(run_stream))
373 .route("/runs/{run_id}/recover", post(recover_run))
374 .route("/runs/{run_id}/steer", post(steer_run))
375 .route("/runs/{run_id}/cancel", post(cancel_run))
376 .route(
377 "/runs/{run_id}/requests/{request_id}/input",
378 post(resolve_input),
379 )
380 .route(
381 "/runs/{run_id}/requests/{request_id}/approval",
382 post(resolve_approval),
383 )
384 .layer(middleware::from_fn_with_state(state.clone(), authenticate));
385
386 Router::new()
387 .route("/health", get(health))
388 .route("/pairing/claim", post(claim_pairing))
389 .merge(protected)
390 .layer(DefaultBodyLimit::max(256 * 1_024))
391 .layer(middleware::from_fn(no_store))
392 .layer(middleware::from_fn(log_request))
393 .with_state(state)
394}
395
396#[derive(Debug, Serialize)]
397struct HealthResponse {
398 status: &'static str,
399 protocol: &'static str,
400}
401
402async fn health() -> Json<HealthResponse> {
403 Json(HealthResponse {
404 status: "ok",
405 protocol: "orchestral-remote-v1",
406 })
407}
408
409#[derive(Debug, Deserialize)]
410#[serde(deny_unknown_fields)]
411struct PairingClaimRequest {
412 secret: String,
413 device_name: String,
414}
415
416async fn claim_pairing(
417 State(state): State<RemoteApiState>,
418 Json(request): Json<PairingClaimRequest>,
419) -> Result<Json<PairingClaim>, ApiError> {
420 let claim = state
421 .registry
422 .claim_pairing(&request.secret, &request.device_name)
423 .await
424 .map_err(|error| ApiError::unauthorized("pairing_failed", error.to_string()))?;
425 Ok(Json(claim))
426}
427
428#[derive(Debug, Serialize)]
429struct MeResponse {
430 auth_mode: &'static str,
431 #[serde(skip_serializing_if = "Option::is_none")]
432 device_id: Option<String>,
433 #[serde(skip_serializing_if = "Option::is_none")]
434 subject: Option<String>,
435 #[serde(skip_serializing_if = "BTreeMap::is_empty")]
436 attributes: BTreeMap<String, String>,
437}
438
439async fn me(Extension(principal): Extension<RemotePrincipal>) -> Json<MeResponse> {
440 Json(match principal {
441 RemotePrincipal::Device(principal) => MeResponse {
442 auth_mode: "device_token",
443 device_id: Some(principal.device_id),
444 subject: None,
445 attributes: BTreeMap::new(),
446 },
447 RemotePrincipal::Gateway(principal) => MeResponse {
448 auth_mode: "gateway_jwt",
449 device_id: None,
450 subject: principal.subject,
451 attributes: principal.attributes,
452 },
453 })
454}
455
456async fn list_devices(
457 State(state): State<RemoteApiState>,
458 Extension(principal): Extension<RemotePrincipal>,
459) -> Json<Vec<DeviceView>> {
460 Json(
461 state
462 .registry
463 .devices(principal.current_device_id().unwrap_or_default())
464 .await,
465 )
466}
467
468async fn revoke_device(
469 State(state): State<RemoteApiState>,
470 Path(device_id): Path<String>,
471) -> Result<StatusCode, ApiError> {
472 state
473 .registry
474 .revoke_device(&device_id)
475 .await
476 .map_err(|error| ApiError::not_found("device_not_found", error.to_string()))?;
477 Ok(StatusCode::NO_CONTENT)
478}
479
480async fn list_sessions(
481 State(state): State<RemoteApiState>,
482) -> Result<Json<Vec<SessionView>>, ApiError> {
483 Ok(Json(session_views(&state).await?))
484}
485
486#[derive(Debug, Deserialize)]
487#[serde(deny_unknown_fields)]
488struct CreateSessionRequest {
489 #[serde(default)]
490 session_id: Option<String>,
491}
492
493async fn create_session(
494 State(state): State<RemoteApiState>,
495 Json(request): Json<CreateSessionRequest>,
496) -> Result<(StatusCode, Json<SessionView>), ApiError> {
497 let preferred = request.session_id.map(AgentSessionId::new);
498 let session_id = state.agent.create_session(preferred).await?;
499 let timestamp = chrono::Utc::now().timestamp_millis();
500 let session = SessionView {
501 id: session_id.as_str().to_owned(),
502 created_at_unix_ms: timestamp,
503 updated_at_unix_ms: timestamp,
504 run_ids: Vec::new(),
505 cwd: state.native_session_defaults.cwd.clone(),
506 execution_profile: state.native_session_defaults.execution_profile.clone(),
507 };
508 Ok((StatusCode::CREATED, Json(session)))
509}
510
511async fn get_session(
512 State(state): State<RemoteApiState>,
513 Path(session_id): Path<String>,
514) -> Result<Json<SessionView>, ApiError> {
515 session_views(&state)
516 .await?
517 .into_iter()
518 .find(|session| session.id == session_id)
519 .map(Json)
520 .ok_or_else(|| ApiError::not_found("session_not_found", "session was not found"))
521}
522
523async fn list_agent_connectors(
524 State(state): State<RemoteApiState>,
525) -> Json<Vec<orchestral_core::agent_connector::AgentConnectorDescriptor>> {
526 Json(state.agent_directory.connectors().await)
527}
528
529#[derive(Debug, Deserialize)]
530#[serde(deny_unknown_fields)]
531struct AgentSessionsQuery {
532 connector_id: String,
533 #[serde(default)]
534 cursor: Option<String>,
535 #[serde(default = "default_agent_session_limit")]
536 limit: u32,
537 #[serde(default)]
538 cwd: Option<String>,
539 #[serde(default)]
540 search: Option<String>,
541}
542
543const fn default_agent_session_limit() -> u32 {
544 50
545}
546
547async fn list_agent_sessions(
548 State(state): State<RemoteApiState>,
549 Query(query): Query<AgentSessionsQuery>,
550) -> Result<Json<AgentSessionPage>, ApiError> {
551 let connector_id = AgentConnectorId::new(query.connector_id);
552 Ok(Json(
553 state
554 .agent_directory
555 .list_sessions(
556 &connector_id,
557 AgentSessionListQuery {
558 cursor: query.cursor,
559 limit: query.limit,
560 cwd: query.cwd,
561 search: query.search,
562 },
563 )
564 .await?,
565 ))
566}
567
568#[derive(Debug, Deserialize)]
569#[serde(deny_unknown_fields)]
570struct CreateExternalAgentSessionRequest {
571 connector_id: String,
572 #[serde(default)]
573 cwd: Option<String>,
574 #[serde(default)]
575 title: Option<String>,
576 #[serde(default)]
577 options: serde_json::Value,
578 #[serde(default)]
579 extensions: BTreeMap<String, serde_json::Value>,
580}
581
582async fn create_agent_session(
583 State(state): State<RemoteApiState>,
584 Json(request): Json<CreateExternalAgentSessionRequest>,
585) -> Result<(StatusCode, Json<AgentSessionSummary>), ApiError> {
586 let connector_id = AgentConnectorId::new(request.connector_id);
587 let cwd = expand_host_home(request.cwd)?;
588 let summary = state
589 .agent_directory
590 .create_session(
591 &connector_id,
592 CreateAgentSessionRequest {
593 cwd,
594 title: request.title,
595 options: request.options,
596 extensions: request.extensions,
597 },
598 )
599 .await?;
600 Ok((StatusCode::CREATED, Json(summary)))
601}
602
603fn expand_host_home(cwd: Option<String>) -> Result<Option<String>, ApiError> {
604 let Some(cwd) = cwd else {
605 return Ok(None);
606 };
607 let expanded = if cwd == "~" || cwd.starts_with("~/") {
608 #[cfg(target_os = "windows")]
609 let home_variable = "USERPROFILE";
610 #[cfg(not(target_os = "windows"))]
611 let home_variable = "HOME";
612 let home = std::env::var_os(home_variable)
613 .filter(|value| !value.is_empty())
614 .map(PathBuf::from)
615 .ok_or_else(|| {
616 ApiError::new(
617 StatusCode::BAD_REQUEST,
618 "host_home_unavailable",
619 "Host cannot expand ~ because its home directory is unavailable",
620 )
621 })?;
622 if cwd == "~" {
623 home
624 } else {
625 home.join(cwd.trim_start_matches("~/"))
626 }
627 .to_string_lossy()
628 .into_owned()
629 } else {
630 cwd
631 };
632 Ok(Some(expanded))
633}
634
635#[derive(Debug, Deserialize)]
636#[serde(deny_unknown_fields)]
637struct AgentSessionQuery {
638 connector_id: String,
639 session_id: String,
640 #[serde(default)]
641 cursor: Option<String>,
642 #[serde(default)]
643 limit: Option<u32>,
644}
645
646async fn get_agent_session(
647 State(state): State<RemoteApiState>,
648 Query(query): Query<AgentSessionQuery>,
649 headers: HeaderMap,
650) -> Result<Response, ApiError> {
651 let connector_id = AgentConnectorId::new(query.connector_id);
652 let session_id = AgentSessionId::new(query.session_id);
653 let stream_cursor = match state
658 .session_coordinators
659 .get(&connector_id, &session_id)
660 .ensure_hub(state.agent_directory.clone(), &connector_id, &session_id)
661 .await
662 {
663 Ok(hub) => Some(hub.cursor()),
664 Err(error) => {
665 tracing::debug!(
666 connector_id = %connector_id.as_str(),
667 session_id = %session_id.as_str(),
668 %error,
669 "Agent session has no live Hub; bounded polling remains available"
670 );
671 None
672 }
673 };
674 let detail = state
675 .agent_directory
676 .read_session_page(
677 &connector_id,
678 &session_id,
679 AgentSessionReadQuery {
680 cursor: query.cursor,
681 limit: query.limit.unwrap_or(100),
682 },
683 )
684 .await?;
685 let controlled_runs = controlled_session_runs(&state, &connector_id, &session_id).await?;
686 let mut payload = serde_json::to_value(RemoteAgentSessionDetail {
687 detail,
688 controlled_runs,
689 stream_cursor,
690 })
691 .map_err(|error| ApiError::internal("agent_session_encode_failed", error.to_string()))?;
692 enrich_artifact_access(&mut payload, state.artifact_resolver.as_deref()).await;
693 let body = serde_json::to_vec(&payload)
694 .map_err(|error| ApiError::internal("agent_session_encode_failed", error.to_string()))?;
695 let etag = format!(
696 "\"{}\"",
697 orchestral_core::agent_protocol::wire::Digest::sha256(&body)
698 );
699 let mut response = if request_etag_matches(&headers, &etag) {
700 StatusCode::NOT_MODIFIED.into_response()
701 } else {
702 (
703 StatusCode::OK,
704 [(header::CONTENT_TYPE, "application/json")],
705 body,
706 )
707 .into_response()
708 };
709 response.headers_mut().insert(
710 header::ETAG,
711 header::HeaderValue::from_str(&etag).expect("SHA-256 ETag is a valid header value"),
712 );
713 Ok(response)
714}
715
716#[derive(Debug, Serialize)]
717struct RemoteAgentSessionDetail {
718 #[serde(flatten)]
719 detail: orchestral_core::agent_connector::AgentSessionDetail,
720 controlled_runs: Vec<ControlledRemoteRunView>,
725 #[serde(skip_serializing_if = "Option::is_none")]
729 stream_cursor: Option<u64>,
730}
731
732#[derive(Debug, Serialize)]
733struct ControlledRemoteRunView {
734 #[serde(flatten)]
735 run: RemoteRunView,
736 created_at_unix_ms: i64,
740 updated_at_unix_ms: i64,
743 #[serde(skip_serializing_if = "Option::is_none")]
747 after_activity_id: Option<AgentSessionActivityId>,
748}
749
750async fn controlled_session_runs(
751 state: &RemoteApiState,
752 connector_id: &AgentConnectorId,
753 session_id: &AgentSessionId,
754) -> Result<Vec<ControlledRemoteRunView>, ApiError> {
755 let agent = state.agent_directory.agent_api(connector_id).await?;
756 let mut catalog = agent
757 .catalog_runs()
758 .await?
759 .into_iter()
760 .filter(|entry| entry.session_id == *session_id)
761 .collect::<Vec<_>>();
762 catalog.sort_by(|left, right| {
763 left.created_at_unix_ms
764 .cmp(&right.created_at_unix_ms)
765 .then_with(|| left.run_id.cmp(&right.run_id))
766 });
767 let suffix_start = catalog.len().saturating_sub(CONTROLLED_SESSION_RUN_LIMIT);
768 let mut controlled = Vec::with_capacity(catalog.len() - suffix_start);
769 for entry in catalog.into_iter().skip(suffix_start) {
770 if !agent.can_control_run(&entry.run_id).await? {
776 continue;
777 }
778 let view = agent.inspect(&entry.run_id).await?;
779 let history_anchor =
780 AgentSessionHistoryAnchor::from_extensions(&agent.run_extensions(&entry.run_id).await?)
781 .map_err(|error| {
782 ApiError::internal("invalid_session_history_anchor", error.to_string())
783 })?;
784 let remote = RemoteRunView::new(
785 state,
786 Some(connector_id),
787 view,
788 agent.initial_input(&entry.run_id).await?,
789 );
790 if !remote.view.state.is_terminal() {
791 spawn_run_supervisor(
792 state.clone(),
793 agent.clone(),
794 Some(connector_id.clone()),
795 entry.run_id,
796 );
797 }
798 controlled.push(ControlledRemoteRunView {
799 run: remote,
800 created_at_unix_ms: entry.created_at_unix_ms,
801 updated_at_unix_ms: entry.updated_at_unix_ms,
802 after_activity_id: history_anchor.map(|anchor| anchor.after_activity_id),
803 });
804 }
805 Ok(controlled)
806}
807
808#[derive(Debug, Deserialize)]
809#[serde(deny_unknown_fields)]
810struct AgentSessionStreamQuery {
811 connector_id: String,
812 session_id: String,
813 #[serde(default)]
814 after: Option<u64>,
815}
816
817async fn agent_session_stream(
818 State(state): State<RemoteApiState>,
819 Extension(request_context): Extension<RequestLogContext>,
820 Query(query): Query<AgentSessionStreamQuery>,
821 headers: HeaderMap,
822) -> Result<Sse<impl futures_util::Stream<Item = Result<Event, Infallible>>>, ApiError> {
823 let connector_id = AgentConnectorId::new(query.connector_id);
824 let session_id = AgentSessionId::new(query.session_id);
825 let header_cursor = headers
826 .get("last-event-id")
827 .and_then(|value| value.to_str().ok())
828 .and_then(|value| value.parse::<u64>().ok())
829 .unwrap_or_default();
830 let initial_cursor = query.after.unwrap_or_default().max(header_cursor);
831 let hub = state
832 .session_coordinators
833 .get(&connector_id, &session_id)
834 .ensure_hub(state.agent_directory.clone(), &connector_id, &session_id)
835 .await?;
836 let subscription = hub.subscribe(initial_cursor);
837 let replay = subscription.replay;
838 let mut live = subscription.live;
839 let lifecycle =
840 SseLifecycleLog::open_agent_session(&request_context, &connector_id, &session_id);
841 let artifact_resolver = state.artifact_resolver.clone();
842 let stream = async_stream::stream! {
843 let mut lifecycle = lifecycle;
844 let mut cursor = initial_cursor;
845 for change in replay {
846 cursor = change.sequence;
847 match agent_session_change_event(&change, artifact_resolver.as_deref()).await {
848 Ok(event) => yield Ok(event),
849 Err(error) => {
850 lifecycle.close_as("replay_event_encode_failed");
851 tracing::error!(
852 request_id = %lifecycle.request_id,
853 stream_id = %lifecycle.stream_id,
854 %error,
855 "could not encode replayed Agent session SSE event"
856 );
857 yield Ok(api_error_event(&ApiError::internal(
858 "session_stream_encode_failed",
859 error.to_string(),
860 )));
861 return;
862 }
863 }
864 }
865 loop {
866 match live.recv().await {
867 Ok(change) => {
868 if change.sequence != 0 && change.sequence <= cursor {
869 continue;
870 }
871 match agent_session_change_event(&change, artifact_resolver.as_deref()).await {
876 Ok(event) => {
877 cursor = change.sequence;
878 yield Ok(event)
879 },
880 Err(error) => {
881 lifecycle.close_as("event_encode_failed");
882 tracing::error!(
883 request_id = %lifecycle.request_id,
884 stream_id = %lifecycle.stream_id,
885 %error,
886 "could not encode Agent session SSE event"
887 );
888 yield Ok(api_error_event(&ApiError::internal(
889 "session_stream_encode_failed",
890 error.to_string(),
891 )));
892 return;
893 }
894 }
895 }
896 Err(broadcast::error::RecvError::Lagged(skipped)) => {
897 lifecycle.lagged(skipped);
898 let replacement = hub.subscribe(cursor);
901 live = replacement.live;
902 for change in replacement.replay {
903 match agent_session_change_event(&change, artifact_resolver.as_deref()).await {
904 Ok(event) => {
905 cursor = change.sequence;
906 yield Ok(event)
907 },
908 Err(error) => {
909 lifecycle.close_as("gap_replay_encode_failed");
910 tracing::error!(
911 request_id = %lifecycle.request_id,
912 stream_id = %lifecycle.stream_id,
913 %error,
914 "could not encode Agent session gap replay"
915 );
916 yield Ok(api_error_event(&ApiError::internal(
917 "session_stream_encode_failed",
918 error.to_string(),
919 )));
920 return;
921 }
922 }
923 }
924 }
925 Err(broadcast::error::RecvError::Closed) => {
926 lifecycle.close_as("source_closed");
927 return;
928 }
929 }
930 }
931 };
932 Ok(Sse::new(stream).keep_alive(
933 KeepAlive::new()
934 .interval(Duration::from_secs(10))
935 .text("keep-alive"),
936 ))
937}
938
939async fn agent_session_change_event(
940 change: &AgentSessionChange,
941 artifact_resolver: Option<&dyn ArtifactResolver>,
942) -> Result<Event, axum::Error> {
943 let mut payload = serde_json::to_value(change).map_err(axum::Error::new)?;
944 enrich_artifact_access(&mut payload, artifact_resolver).await;
945 Event::default()
946 .event("session_changed")
947 .id(change.sequence.to_string())
948 .json_data(payload)
949}
950
951async fn enrich_artifact_access(
956 payload: &mut serde_json::Value,
957 artifact_resolver: Option<&dyn ArtifactResolver>,
958) {
959 let Some(artifact_resolver) = artifact_resolver else {
960 return;
961 };
962 let mut artifacts = BTreeMap::<String, ArtifactRefWithDigest>::new();
963 collect_artifact_references(payload, &mut artifacts);
964 let mut access = BTreeMap::<String, serde_json::Value>::new();
965 for (reference, artifact) in artifacts {
966 match artifact_resolver.resolve(&artifact).await {
967 Ok(resolved) => {
968 access.insert(
969 reference,
970 serde_json::json!({
971 "uri": resolved.uri,
972 "file_name": resolved.file_name,
973 "media_type": resolved.media_type,
974 "byte_size": resolved.byte_size,
975 "expires_at": resolved.expires_at,
976 }),
977 );
978 }
979 Err(error) => {
980 tracing::warn!(
981 artifact_ref = %artifact.artifact_ref,
982 %error,
983 "could not resolve Artifact access for remote session view"
984 );
985 }
986 }
987 }
988 inject_artifact_access(payload, &access);
989}
990
991fn collect_artifact_references(
992 value: &serde_json::Value,
993 artifacts: &mut BTreeMap<String, ArtifactRefWithDigest>,
994) {
995 match value {
996 serde_json::Value::Array(values) => {
997 for value in values {
998 collect_artifact_references(value, artifacts);
999 }
1000 }
1001 serde_json::Value::Object(object) => {
1002 if object
1003 .get("body")
1004 .and_then(|body| body.get("kind"))
1005 .and_then(serde_json::Value::as_str)
1006 == Some("artifact")
1007 {
1008 if let Some(artifact) = object
1009 .get("body")
1010 .and_then(|body| body.get("value"))
1011 .cloned()
1012 .and_then(|value| serde_json::from_value::<ArtifactRefWithDigest>(value).ok())
1013 .filter(|artifact| artifact.validate_integrity().is_ok())
1014 {
1015 artifacts.insert(artifact.artifact_ref.to_string(), artifact);
1016 }
1017 }
1018 for value in object.values() {
1019 collect_artifact_references(value, artifacts);
1020 }
1021 }
1022 _ => {}
1023 }
1024}
1025
1026fn inject_artifact_access(
1027 value: &mut serde_json::Value,
1028 access: &BTreeMap<String, serde_json::Value>,
1029) {
1030 match value {
1031 serde_json::Value::Array(values) => {
1032 for value in values {
1033 inject_artifact_access(value, access);
1034 }
1035 }
1036 serde_json::Value::Object(object) => {
1037 let reference = object
1038 .get("body")
1039 .and_then(|body| body.get("value"))
1040 .and_then(|value| value.get("artifact_ref"))
1041 .and_then(serde_json::Value::as_str)
1042 .map(str::to_owned);
1043 if let Some(resolved) = reference.and_then(|reference| access.get(&reference)) {
1044 object.insert("access".to_owned(), resolved.clone());
1045 }
1046 for value in object.values_mut() {
1047 inject_artifact_access(value, access);
1048 }
1049 }
1050 _ => {}
1051 }
1052}
1053
1054fn request_etag_matches(headers: &HeaderMap, etag: &str) -> bool {
1055 headers
1056 .get(header::IF_NONE_MATCH)
1057 .and_then(|value| value.to_str().ok())
1058 .is_some_and(|values| {
1059 values
1060 .split(',')
1061 .map(str::trim)
1062 .any(|candidate| candidate == "*" || candidate == etag)
1063 })
1064}
1065
1066#[derive(Debug, Deserialize)]
1067#[serde(deny_unknown_fields)]
1068struct InvokeExternalAgentSessionActionRequest {
1069 connector_id: String,
1070 session_id: String,
1071 action_id: String,
1072 #[serde(default)]
1073 arguments: serde_json::Value,
1074 #[serde(default)]
1075 run_id: Option<String>,
1076}
1077
1078async fn invoke_agent_session_action(
1079 State(state): State<RemoteApiState>,
1080 Json(request): Json<InvokeExternalAgentSessionActionRequest>,
1081) -> Result<Json<AgentSessionActionOutcome>, ApiError> {
1082 Ok(Json(
1083 state
1084 .agent_directory
1085 .invoke_action(
1086 &AgentConnectorId::new(request.connector_id),
1087 InvokeAgentSessionActionRequest {
1088 session_id: AgentSessionId::new(request.session_id),
1089 action_id: AgentSessionActionId::new(request.action_id),
1090 arguments: request.arguments,
1091 run_id: request.run_id.map(RunId::new),
1092 },
1093 )
1094 .await?,
1095 ))
1096}
1097
1098#[derive(Debug, Deserialize)]
1099#[serde(deny_unknown_fields)]
1100struct StartRunRequest {
1101 run_id: String,
1102 input: String,
1103 #[serde(default)]
1104 attachments: Vec<RemoteArtifactInput>,
1105}
1106
1107#[derive(Debug, Serialize)]
1108struct StartRunResponse {
1109 run_id: RunId,
1110 view: RemoteRunView,
1111}
1112
1113#[derive(Debug, Deserialize)]
1114#[serde(deny_unknown_fields)]
1115struct StartAgentRunRequest {
1116 connector_id: String,
1117 session_id: String,
1118 run_id: String,
1119 input: String,
1120 #[serde(default)]
1121 after_activity_id: Option<String>,
1122 #[serde(default)]
1123 attachments: Vec<RemoteArtifactInput>,
1124}
1125
1126#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1127#[serde(deny_unknown_fields)]
1128struct RemoteArtifactInput {
1129 artifact_ref: String,
1130 digest: String,
1131 file_name: String,
1132 media_type: String,
1133 byte_size: u64,
1134}
1135
1136#[derive(Debug, Serialize)]
1137struct StartAgentRunResponse {
1138 connector_id: AgentConnectorId,
1139 run_id: RunId,
1140 operation: &'static str,
1141 #[serde(skip_serializing_if = "Option::is_none")]
1142 command_id: Option<CommandId>,
1143 view: RemoteRunView,
1144}
1145
1146async fn start_agent_run(
1147 State(state): State<RemoteApiState>,
1148 Json(request): Json<StartAgentRunRequest>,
1149) -> Result<(StatusCode, Json<StartAgentRunResponse>), ApiError> {
1150 let input = message_content(&state, &request.input, &request.attachments).await?;
1151 let connector_id = AgentConnectorId::new(request.connector_id);
1152 let session_id = AgentSessionId::new(request.session_id);
1153 let run_id = RunId::new(request.run_id);
1154 let history_anchor =
1155 request
1156 .after_activity_id
1157 .map(|after_activity_id| AgentSessionHistoryAnchor {
1158 after_activity_id: AgentSessionActivityId::new(after_activity_id),
1159 });
1160 let history_extensions = session_history_anchor_extensions(history_anchor.as_ref())?;
1161 let coordinator = state.session_coordinators.get(&connector_id, &session_id);
1162 let _operation_guard = coordinator.operation().lock().await;
1163 let agent = state.agent_directory.agent_api(&connector_id).await?;
1164
1165 if agent.has_run(&run_id).await? {
1168 if agent.inspect(&run_id).await?.execution.session_id != session_id {
1169 return Err(ApiError::conflict(
1170 "run_id_conflict",
1171 "run_id belongs to another session",
1172 ));
1173 }
1174 if agent.initial_input(&run_id).await? != input {
1175 return Err(ApiError::conflict(
1176 "run_id_conflict",
1177 "run_id was already used with different input",
1178 ));
1179 }
1180 let existing_anchor =
1181 AgentSessionHistoryAnchor::from_extensions(&agent.run_extensions(&run_id).await?)
1182 .map_err(|error| {
1183 ApiError::internal("invalid_session_history_anchor", error.to_string())
1184 })?;
1185 if existing_anchor != history_anchor {
1186 return Err(ApiError::conflict(
1187 "run_id_conflict",
1188 "run_id was already used with a different session history anchor",
1189 ));
1190 }
1191 let view = RemoteRunView::new(
1192 &state,
1193 Some(&connector_id),
1194 agent.inspect(&run_id).await?,
1195 agent.initial_input(&run_id).await?,
1196 );
1197 spawn_run_supervisor(
1198 state.clone(),
1199 agent,
1200 Some(connector_id.clone()),
1201 run_id.clone(),
1202 );
1203 log_agent_input_accepted(&connector_id, &session_id, &run_id, "replayed", None);
1204 return Ok((
1205 StatusCode::OK,
1206 Json(StartAgentRunResponse {
1207 connector_id,
1208 run_id,
1209 operation: "replayed",
1210 command_id: None,
1211 view,
1212 }),
1213 ));
1214 }
1215
1216 let submission_command_id = CommandId::new(format!("agent-submit-{}", run_id.as_str()));
1220 for entry in agent
1221 .catalog_runs()
1222 .await?
1223 .into_iter()
1224 .filter(|entry| entry.session_id == session_id)
1225 {
1226 let Some(recorded) = agent
1227 .recorded_command(&entry.run_id, &submission_command_id)
1228 .await?
1229 else {
1230 continue;
1231 };
1232 let expected = AgentCommandEnvelope::new_with_extensions(
1233 submission_command_id.clone(),
1234 entry.run_id.clone(),
1235 None,
1236 AgentCommand::Steer {
1237 content: input.clone(),
1238 },
1239 history_extensions.clone(),
1240 )?;
1241 if recorded != expected {
1242 return Err(ApiError::conflict(
1243 "run_id_conflict",
1244 "session submission identity was already used with different input or metadata",
1245 ));
1246 }
1247 if !agent.can_control_run(&entry.run_id).await? {
1248 return Err(ApiError::conflict(
1249 "submission_contract_changed",
1250 "this submission belongs to an earlier Agent contract and cannot be resubmitted",
1251 ));
1252 }
1253 let ack = command_run(&agent, &entry.run_id, expected).await?;
1254 if !matches!(
1255 ack.state,
1256 CommandAckState::Accepted { .. } | CommandAckState::Applied { .. }
1257 ) {
1258 if rejected_steer_can_start_run(&agent, &entry.run_id, &ack.state).await? {
1259 continue;
1260 }
1261 return Err(ApiError::conflict(
1262 "agent_session_command_rejected",
1263 rejected_session_message(&ack.state),
1264 ));
1265 }
1266 let view = RemoteRunView::new(
1267 &state,
1268 Some(&connector_id),
1269 agent.inspect(&entry.run_id).await?,
1270 agent.initial_input(&entry.run_id).await?,
1271 );
1272 return Ok((
1273 StatusCode::OK,
1274 Json(StartAgentRunResponse {
1275 connector_id,
1276 run_id: entry.run_id,
1277 operation: "steered",
1278 command_id: Some(submission_command_id),
1279 view,
1280 }),
1281 ));
1282 }
1283
1284 'steer: {
1285 if let Some(entry) = latest_session_run(&agent, &session_id).await? {
1286 let mut current = agent.inspect(&entry.run_id).await?;
1287 if current.state.status()
1288 == orchestral_core::agent_protocol::reference::AgentRunStatus::Unknown
1289 {
1290 current = agent.recover(&entry.run_id).await?;
1291 }
1292 if !current.state.is_terminal() {
1293 let command_id = CommandId::new(format!("agent-submit-{}", run_id.as_str()));
1294 let command = AgentCommandEnvelope::new_with_extensions(
1295 command_id.clone(),
1296 entry.run_id.clone(),
1297 None,
1298 AgentCommand::Steer {
1299 content: input.clone(),
1300 },
1301 history_extensions.clone(),
1302 )?;
1303 let ack = command_run(&agent, &entry.run_id, command).await?;
1304 if !matches!(
1305 ack.state,
1306 CommandAckState::Accepted { .. } | CommandAckState::Applied { .. }
1307 ) {
1308 if rejected_steer_can_start_run(&agent, &entry.run_id, &ack.state).await? {
1312 break 'steer;
1313 }
1314 return Err(ApiError::conflict(
1315 "agent_session_command_rejected",
1316 rejected_session_message(&ack.state),
1317 ));
1318 }
1319 let view = RemoteRunView::new(
1320 &state,
1321 Some(&connector_id),
1322 agent.inspect(&entry.run_id).await?,
1323 agent.initial_input(&entry.run_id).await?,
1324 );
1325 spawn_run_supervisor(
1326 state.clone(),
1327 agent,
1328 Some(connector_id.clone()),
1329 entry.run_id.clone(),
1330 );
1331 log_agent_input_accepted(
1332 &connector_id,
1333 &session_id,
1334 &entry.run_id,
1335 "steered",
1336 Some(&command_id),
1337 );
1338 return Ok((
1339 StatusCode::OK,
1340 Json(StartAgentRunResponse {
1341 connector_id,
1342 run_id: entry.run_id,
1343 operation: "steered",
1344 command_id: Some(command_id),
1345 view,
1346 }),
1347 ));
1348 }
1349 }
1350 }
1351
1352 let handle = state
1353 .agent_directory
1354 .start_content_with_extensions(
1355 &connector_id,
1356 &session_id,
1357 Some(run_id.clone()),
1358 input,
1359 history_extensions,
1360 )
1361 .await?;
1362 spawn_run_supervisor(
1363 state.clone(),
1364 agent.clone(),
1365 Some(connector_id.clone()),
1366 run_id.clone(),
1367 );
1368 let view = RemoteRunView::new(
1369 &state,
1370 Some(&connector_id),
1371 handle.inspect().await?,
1372 agent.initial_input(&run_id).await?,
1373 );
1374 log_agent_input_accepted(&connector_id, &session_id, &run_id, "started", None);
1375 Ok((
1376 StatusCode::CREATED,
1377 Json(StartAgentRunResponse {
1378 connector_id,
1379 run_id,
1380 operation: "started",
1381 command_id: None,
1382 view,
1383 }),
1384 ))
1385}
1386
1387async fn rejected_steer_can_start_run(
1388 agent: &AgentApi,
1389 run_id: &RunId,
1390 ack: &CommandAckState,
1391) -> Result<bool, ApiError> {
1392 use orchestral_core::agent_protocol::wire::AgentProtocolErrorCode;
1393 if !matches!(
1394 ack,
1395 CommandAckState::Rejected {
1396 code: AgentProtocolErrorCode::TerminalRun | AgentProtocolErrorCode::InvalidTransition,
1397 ..
1398 }
1399 ) {
1400 return Ok(false);
1401 }
1402 Ok(agent.inspect(run_id).await?.state.is_terminal())
1403}
1404
1405fn rejected_session_message(ack: &CommandAckState) -> String {
1406 match ack {
1407 CommandAckState::Rejected { message, .. } => message.clone(),
1408 _ => "the active Agent Run did not accept this session message".to_owned(),
1409 }
1410}
1411
1412fn log_agent_input_accepted(
1413 connector_id: &AgentConnectorId,
1414 session_id: &AgentSessionId,
1415 run_id: &RunId,
1416 operation: &'static str,
1417 command_id: Option<&CommandId>,
1418) {
1419 tracing::info!(
1420 connector_id = %connector_id.as_str(),
1421 session_id = %session_id.as_str(),
1422 run_id = %run_id.as_str(),
1423 operation,
1424 command_id = command_id.map(CommandId::as_str).unwrap_or("-"),
1425 "Agent session input accepted"
1426 );
1427}
1428
1429async fn latest_session_run(
1430 agent: &AgentApi,
1431 session_id: &AgentSessionId,
1432) -> Result<Option<orchestral_core::agent_protocol::spi::AgentRunCatalogEntry>, ApiError> {
1433 let mut catalog = agent
1434 .catalog_runs()
1435 .await?
1436 .into_iter()
1437 .filter(|entry| entry.session_id == *session_id)
1438 .collect::<Vec<_>>();
1439 catalog.sort_by_key(|entry| {
1440 std::cmp::Reverse((entry.updated_at_unix_ms, entry.created_at_unix_ms))
1441 });
1442 let Some(latest) = catalog.into_iter().next() else {
1443 return Ok(None);
1444 };
1445 if !agent.can_control_run(&latest.run_id).await? {
1451 return Ok(None);
1452 }
1453 Ok(Some(latest))
1454}
1455
1456fn session_history_anchor_extensions(
1457 anchor: Option<&AgentSessionHistoryAnchor>,
1458) -> Result<Extensions, ApiError> {
1459 let mut extensions = Extensions::new();
1460 if let Some(anchor) = anchor {
1461 anchor.insert_into(&mut extensions).map_err(|error| {
1462 ApiError::new(
1463 StatusCode::BAD_REQUEST,
1464 "invalid_session_history_anchor",
1465 error.to_string(),
1466 )
1467 })?;
1468 }
1469 Ok(extensions)
1470}
1471
1472async fn message_content(
1473 state: &RemoteApiState,
1474 text: &str,
1475 attachments: &[RemoteArtifactInput],
1476) -> Result<Vec<Content>, ApiError> {
1477 if text.trim().is_empty() && attachments.is_empty() {
1478 return Err(ApiError::new(
1479 StatusCode::BAD_REQUEST,
1480 "invalid_agent_input",
1481 "message text or at least one attachment is required",
1482 ));
1483 }
1484 if attachments.len() > 10 {
1485 return Err(ApiError::new(
1486 StatusCode::BAD_REQUEST,
1487 "too_many_attachments",
1488 "a message may contain at most 10 attachments",
1489 ));
1490 }
1491 let resolver = if attachments.is_empty() {
1492 None
1493 } else {
1494 Some(state.artifact_resolver.as_ref().ok_or_else(|| {
1495 ApiError::service_unavailable(
1496 "artifact_resolver_unavailable",
1497 "Artifact storage is not configured on this Host",
1498 )
1499 })?)
1500 };
1501
1502 let mut total_bytes = 0_u64;
1503 let mut validated = Vec::with_capacity(attachments.len());
1504 for attachment in attachments {
1505 validate_remote_artifact(attachment)?;
1506 total_bytes = total_bytes
1507 .checked_add(attachment.byte_size)
1508 .ok_or_else(|| {
1509 ApiError::new(
1510 StatusCode::BAD_REQUEST,
1511 "attachments_too_large",
1512 "attachment byte total overflowed",
1513 )
1514 })?;
1515 if total_bytes > 10 * 64 * 1024 * 1024 {
1516 return Err(ApiError::new(
1517 StatusCode::PAYLOAD_TOO_LARGE,
1518 "attachments_too_large",
1519 "attachment total exceeds 640 MiB",
1520 ));
1521 }
1522 let artifact = ArtifactRefWithDigest {
1523 artifact_ref: ArtifactRef::new(&attachment.artifact_ref),
1524 digest: Digest::new(&attachment.digest),
1525 };
1526 let resolved = resolver
1527 .expect("non-empty attachments require a resolver")
1528 .resolve(&artifact)
1529 .await
1530 .map_err(artifact_resolve_error)?;
1531 if resolved.media_type != attachment.media_type
1532 || resolved.byte_size != attachment.byte_size
1533 || resolved.artifact != artifact
1534 {
1535 return Err(ApiError::conflict(
1536 "artifact_metadata_conflict",
1537 "Artifact metadata changed after upload",
1538 ));
1539 }
1540 validated.push((attachment, artifact));
1541 }
1542
1543 let mut description = if text.trim().is_empty() {
1544 "请查看并处理随消息附上的文件。".to_owned()
1545 } else {
1546 text.to_owned()
1547 };
1548 if !validated.is_empty() {
1549 description.push_str("\n\n附件(内容已由 Host 按 SHA-256 校验):");
1550 for (index, (attachment, _)) in validated.iter().enumerate() {
1551 description.push_str(&format!(
1552 "\n{}. {}({},{} bytes,sha256 {})",
1553 index + 1,
1554 attachment.file_name,
1555 attachment.media_type,
1556 attachment.byte_size,
1557 attachment.digest
1558 ));
1559 }
1560 }
1561 let mut content = vec![Content::text(description)];
1562 content.extend(validated.into_iter().map(|(attachment, artifact)| Content {
1563 media_type: attachment.media_type.clone(),
1564 schema_id: None,
1565 body: ContentBody::Artifact(artifact),
1566 }));
1567 Ok(content)
1568}
1569
1570fn validate_remote_artifact(attachment: &RemoteArtifactInput) -> Result<(), ApiError> {
1571 if attachment.artifact_ref != attachment.digest
1572 || attachment.digest.len() != 64
1573 || !attachment
1574 .digest
1575 .bytes()
1576 .all(|byte| byte.is_ascii_hexdigit())
1577 {
1578 return Err(ApiError::new(
1579 StatusCode::BAD_REQUEST,
1580 "invalid_artifact_identity",
1581 "Artifact reference must equal its SHA-256 digest",
1582 ));
1583 }
1584 if attachment.file_name.trim().is_empty()
1585 || attachment.file_name.len() > 255
1586 || attachment.file_name.chars().any(char::is_control)
1587 {
1588 return Err(ApiError::new(
1589 StatusCode::BAD_REQUEST,
1590 "invalid_artifact_file_name",
1591 "Artifact file name is invalid",
1592 ));
1593 }
1594 if attachment.media_type.trim().is_empty()
1595 || attachment.media_type.len() > 160
1596 || attachment.media_type.chars().any(char::is_control)
1597 {
1598 return Err(ApiError::new(
1599 StatusCode::BAD_REQUEST,
1600 "invalid_artifact_media_type",
1601 "Artifact media type is invalid",
1602 ));
1603 }
1604 if attachment.byte_size == 0 || attachment.byte_size > 64 * 1024 * 1024 {
1605 return Err(ApiError::new(
1606 StatusCode::PAYLOAD_TOO_LARGE,
1607 "artifact_too_large",
1608 "Artifact must be between 1 byte and 64 MiB",
1609 ));
1610 }
1611 Ok(())
1612}
1613
1614fn artifact_resolve_error(error: orchestral_core::io::ArtifactResolveError) -> ApiError {
1615 use orchestral_core::io::ArtifactResolveError;
1616 match error {
1617 ArtifactResolveError::Invalid(message) => {
1618 ApiError::new(StatusCode::BAD_REQUEST, "invalid_artifact", message)
1619 }
1620 ArtifactResolveError::NotFound(message) => {
1621 ApiError::not_found("artifact_not_found", message)
1622 }
1623 ArtifactResolveError::Integrity(message) => {
1624 ApiError::conflict("artifact_integrity_failed", message)
1625 }
1626 ArtifactResolveError::Unavailable(message) | ArtifactResolveError::Internal(message) => {
1627 tracing::warn!(%message, "Artifact resolver failed");
1628 ApiError::service_unavailable(
1629 "artifact_resolver_unavailable",
1630 "Artifact storage is temporarily unavailable",
1631 )
1632 }
1633 }
1634}
1635
1636#[derive(Debug, Serialize)]
1637struct RemoteRunView {
1638 #[serde(flatten)]
1639 view: AgentRunView,
1640 input: Vec<Content>,
1643 #[serde(skip_serializing_if = "Option::is_none")]
1647 recovery: Option<RemoteRunRecoveryView>,
1648 #[serde(skip_serializing_if = "Option::is_none")]
1652 supervision: Option<RemoteRunSupervisionView>,
1653}
1654
1655#[derive(Debug, Serialize)]
1656struct RemoteRunRecoveryView {
1657 mode: &'static str,
1658 can_start_new_run: bool,
1659 #[serde(skip_serializing_if = "Option::is_none")]
1660 reason: Option<String>,
1661}
1662
1663impl RemoteRunView {
1664 fn new(
1665 state: &RemoteApiState,
1666 connector_id: Option<&AgentConnectorId>,
1667 view: AgentRunView,
1668 input: Vec<Content>,
1669 ) -> Self {
1670 let recovery = if view.state.status()
1671 == orchestral_core::agent_protocol::reference::AgentRunStatus::Unknown
1672 {
1673 let key = RunSupervisorRegistry::key(connector_id, &view.execution.run_id);
1674 match state.run_supervisors.manual_reason(&key) {
1675 Some(reason) => Some(RemoteRunRecoveryView {
1676 mode: "manual",
1677 can_start_new_run: false,
1681 reason: Some(reason),
1682 }),
1683 None => Some(RemoteRunRecoveryView {
1684 mode: "automatic",
1685 can_start_new_run: false,
1686 reason: None,
1687 }),
1688 }
1689 } else {
1690 None
1691 };
1692 let supervision = state.run_supervisors.issue(&RunSupervisorRegistry::key(
1693 connector_id,
1694 &view.execution.run_id,
1695 ));
1696 Self {
1697 view,
1698 input,
1699 recovery,
1700 supervision,
1701 }
1702 }
1703}
1704
1705async fn start_run(
1706 State(state): State<RemoteApiState>,
1707 Path(session_id): Path<String>,
1708 Json(request): Json<StartRunRequest>,
1709) -> Result<(StatusCode, Json<StartRunResponse>), ApiError> {
1710 if !request.attachments.is_empty() {
1711 return Err(ApiError::new(
1712 StatusCode::NOT_IMPLEMENTED,
1713 "agent_artifacts_unsupported",
1714 "the built-in Agent does not declare Artifact input support",
1715 ));
1716 }
1717 let session_id = AgentSessionId::new(session_id);
1718 state.agent.create_session(Some(session_id.clone())).await?;
1719 let run_id = RunId::new(request.run_id);
1720 let handle = state
1721 .agent
1722 .start_text(&session_id, Some(run_id.clone()), request.input)
1723 .await?;
1724 spawn_remembered_approval_driver(state.clone(), run_id.clone());
1725 let view = RemoteRunView::new(
1726 &state,
1727 None,
1728 handle.inspect().await?,
1729 state.agent.initial_input(&run_id).await?,
1730 );
1731 Ok((StatusCode::CREATED, Json(StartRunResponse { run_id, view })))
1732}
1733
1734pub(super) fn spawn_remembered_approval_driver(state: RemoteApiState, run_id: RunId) {
1735 spawn_run_supervisor(state.clone(), state.agent.clone(), None, run_id);
1736}
1737
1738pub(super) fn spawn_run_supervisor(
1739 state: RemoteApiState,
1740 agent: AgentApi,
1741 connector_id: Option<AgentConnectorId>,
1742 run_id: RunId,
1743) {
1744 let key = RunSupervisorRegistry::key(connector_id.as_ref(), &run_id);
1745 if !state.run_supervisors.begin(&key) {
1746 return;
1747 }
1748 let registry = state.run_supervisors.clone();
1749 tokio::spawn(async move {
1750 supervise_run(state, agent, connector_id, run_id).await;
1751 registry.finish(&key);
1752 });
1753}
1754
1755async fn supervise_run(
1756 state: RemoteApiState,
1757 agent: AgentApi,
1758 connector_id: Option<AgentConnectorId>,
1759 run_id: RunId,
1760) {
1761 let key = RunSupervisorRegistry::key(connector_id.as_ref(), &run_id);
1762 let policy = state.run_supervisors.policy;
1763 let mut failures = 0_u32;
1764 let now = std::time::Instant::now();
1765 let initial_age = agent
1766 .catalog_runs()
1767 .await
1768 .ok()
1769 .and_then(|runs| {
1770 runs.into_iter()
1771 .find(|entry| entry.run_id == run_id)
1772 .map(|entry| {
1773 chrono::Utc::now()
1774 .timestamp_millis()
1775 .saturating_sub(entry.updated_at_unix_ms)
1776 .max(0) as u64
1777 })
1778 })
1779 .map(Duration::from_millis)
1780 .unwrap_or_default();
1781 let mut last_progress = now
1782 .checked_sub(initial_age.min(policy.inactivity_timeout))
1783 .unwrap_or(now);
1784 let mut watchdog_cancel_command = None;
1785 let mut watchdog_stop_requested_at: Option<std::time::Instant> = None;
1786 loop {
1787 let mut live = match agent.subscribe(&run_id).await {
1790 Ok(live) => live,
1791 Err(error) => {
1792 failures = failures.saturating_add(1);
1793 tracing::warn!(
1794 connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1795 run_id = %run_id.as_str(),
1796 %error,
1797 "could not subscribe to supervised Agent Run"
1798 );
1799 tokio::time::sleep(run_supervisor_backoff(failures)).await;
1800 continue;
1801 }
1802 };
1803 let view = match agent.inspect(&run_id).await {
1804 Ok(view) => view,
1805 Err(error) => {
1806 failures = failures.saturating_add(1);
1807 tracing::warn!(
1808 connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1809 run_id = %run_id.as_str(),
1810 %error,
1811 "could not inspect supervised Agent Run"
1812 );
1813 tokio::time::sleep(run_supervisor_backoff(failures)).await;
1814 continue;
1815 }
1816 };
1817 if view.state.is_terminal() {
1818 state.run_supervisors.clear_manual(&key);
1819 state.run_supervisors.clear_issue(&key);
1820 return;
1821 }
1822 if view.state.status()
1823 == orchestral_core::agent_protocol::reference::AgentRunStatus::Unknown
1824 {
1825 state.run_supervisors.clear_issue(&key);
1826 match agent.recover(&run_id).await {
1827 Ok(_) => {
1828 failures = failures.saturating_add(1);
1834 state.run_supervisors.clear_manual(&key);
1835 tracing::info!(
1836 connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1837 run_id = %run_id.as_str(),
1838 "restored Agent Run continuity"
1839 );
1840 tokio::time::sleep(run_supervisor_backoff(failures)).await;
1841 }
1842 Err(error) => {
1843 if !is_retryable_agent_error(&error) {
1844 state
1845 .run_supervisors
1846 .mark_manual(key.clone(), error.to_string());
1847 tracing::info!(
1848 connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1849 run_id = %run_id.as_str(),
1850 %error,
1851 "Agent Run requires manual recovery; stopped automatic retries"
1852 );
1853 return;
1854 }
1855 failures = failures.saturating_add(1);
1856 tracing::warn!(
1857 connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1858 run_id = %run_id.as_str(),
1859 %error,
1860 retry_after_ms = run_supervisor_backoff(failures).as_millis(),
1861 "Agent Run recovery attempt failed"
1862 );
1863 tokio::time::sleep(run_supervisor_backoff(failures)).await;
1864 }
1865 }
1866 continue;
1867 }
1868
1869 failures = 0;
1870 apply_remembered_approvals(&state, &agent, &run_id, &view.pending_requests).await;
1871
1872 if view.state.status()
1873 == orchestral_core::agent_protocol::reference::AgentRunStatus::Waiting
1874 {
1875 last_progress = std::time::Instant::now();
1878 watchdog_cancel_command = None;
1879 watchdog_stop_requested_at = None;
1880 state.run_supervisors.clear_issue(&key);
1881 } else if view.state.status()
1882 == orchestral_core::agent_protocol::reference::AgentRunStatus::Stopping
1883 && watchdog_stop_requested_at.is_none()
1884 {
1885 state.run_supervisors.mark_issue(
1886 key.clone(),
1887 "interrupting",
1888 "Agent stop was accepted; waiting for the Provider to confirm a terminal state"
1889 .to_owned(),
1890 );
1891 watchdog_stop_requested_at = Some(std::time::Instant::now());
1892 } else if let Some(requested_at) = watchdog_stop_requested_at {
1893 if requested_at.elapsed() >= policy.stop_grace {
1894 state.run_supervisors.mark_issue(
1895 key.clone(),
1896 "stalled",
1897 format!(
1898 "Agent execution stopped making progress and did not reach a terminal state within {} seconds after the Host requested cancellation",
1899 policy.stop_grace.as_secs()
1900 ),
1901 );
1902 }
1903 } else if last_progress.elapsed() >= policy.inactivity_timeout {
1904 let reason = format!(
1905 "Agent execution produced no model, Tool, output, or request progress for {} seconds; the Host watchdog requested a safe stop",
1906 policy.inactivity_timeout.as_secs()
1907 );
1908 state
1909 .run_supervisors
1910 .mark_issue(key.clone(), "interrupting", reason.clone());
1911 if watchdog_cancel_command.is_none() {
1912 watchdog_cancel_command = match AgentCommandEnvelope::new(
1913 CommandId::new(format!(
1914 "host-stall-cancel-{}-{}",
1915 run_id.as_str(),
1916 view.last_run_seq.unwrap_or(0)
1917 )),
1918 run_id.clone(),
1919 None,
1920 AgentCommand::Cancel { reason },
1921 ) {
1922 Ok(command) => Some(command),
1923 Err(error) => {
1924 state.run_supervisors.mark_issue(
1925 key.clone(),
1926 "stalled",
1927 format!("could not construct the Agent watchdog cancellation: {error}"),
1928 );
1929 return;
1930 }
1931 };
1932 }
1933 match agent
1934 .command(
1935 watchdog_cancel_command
1936 .clone()
1937 .expect("watchdog cancellation was initialized"),
1938 )
1939 .await
1940 {
1941 Ok(ack)
1942 if matches!(
1943 ack.state,
1944 CommandAckState::Accepted { .. } | CommandAckState::Applied { .. }
1945 ) =>
1946 {
1947 watchdog_stop_requested_at = Some(std::time::Instant::now());
1948 tracing::warn!(
1949 connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1950 run_id = %run_id.as_str(),
1951 inactivity_seconds = policy.inactivity_timeout.as_secs(),
1952 "Agent Run watchdog requested cancellation after execution inactivity"
1953 );
1954 }
1955 Ok(ack) => {
1956 state.run_supervisors.mark_issue(
1957 key.clone(),
1958 "stalled",
1959 format!(
1960 "Agent execution is stalled and its Provider did not accept the Host watchdog cancellation: {:?}",
1961 ack.state
1962 ),
1963 );
1964 return;
1965 }
1966 Err(error) => {
1967 failures = failures.saturating_add(1);
1968 state.run_supervisors.mark_issue(
1969 key.clone(),
1970 "stalled",
1971 format!(
1972 "Agent execution is stalled and the Host watchdog could not request cancellation: {error}"
1973 ),
1974 );
1975 tracing::warn!(
1976 connector_id = connector_id.as_ref().map(AgentConnectorId::as_str),
1977 run_id = %run_id.as_str(),
1978 %error,
1979 "Agent Run watchdog cancellation failed"
1980 );
1981 tokio::time::sleep(run_supervisor_backoff(failures)).await;
1982 continue;
1983 }
1984 }
1985 }
1986
1987 match tokio::time::timeout(RUN_SUPERVISOR_POLL_INTERVAL, live.recv()).await {
1988 Ok(Ok(event)) => {
1989 if agent_control_event_is_execution_progress(&event) {
1990 last_progress = std::time::Instant::now();
1991 if watchdog_stop_requested_at.is_none() {
1992 state.run_supervisors.clear_issue(&key);
1993 }
1994 }
1995 }
1996 Ok(Err(broadcast::error::RecvError::Lagged(_))) | Err(_) => {}
1997 Ok(Err(broadcast::error::RecvError::Closed)) => {
1998 failures = failures.saturating_add(1);
1999 tokio::time::sleep(run_supervisor_backoff(failures)).await;
2000 }
2001 }
2002 }
2003}
2004
2005fn agent_control_event_is_execution_progress(event: &AgentControlEvent) -> bool {
2006 match event {
2007 AgentControlEvent::Telemetry(_) => true,
2008 AgentControlEvent::Durable(record) => {
2009 agent_event_is_execution_progress(&record.event.payload)
2010 }
2011 _ => false,
2012 }
2013}
2014
2015fn agent_event_is_execution_progress(event: &AgentEvent) -> bool {
2016 !matches!(
2017 event,
2018 AgentEvent::RunAccepted { .. }
2019 | AgentEvent::ResourceBindingSkipped { .. }
2020 | AgentEvent::CommandReceived { .. }
2021 | AgentEvent::CommandDispositionRecorded { .. }
2022 | AgentEvent::StopRequested { .. }
2023 | AgentEvent::ContinuityLost { .. }
2024 | AgentEvent::ContinuityRestored { .. }
2025 )
2026}
2027
2028pub(super) fn is_retryable_agent_error(error: &AgentSdkError) -> bool {
2032 match error {
2033 AgentSdkError::InvalidInput(_) => false,
2034 AgentSdkError::Protocol(error)
2035 | AgentSdkError::Control(AgentControlError::Protocol(error)) => error.retryable,
2036 AgentSdkError::Control(AgentControlError::Start(AgentStartError::Rejected(rejection))) => {
2037 rejection.retryable
2038 }
2039 AgentSdkError::Control(AgentControlError::Start(AgentStartError::OutcomeUnknown(_)))
2040 | AgentSdkError::Control(AgentControlError::Journal(_))
2041 | AgentSdkError::ControlStreamClosed(_) => true,
2042 AgentSdkError::Control(
2043 AgentControlError::RunNotFound(_)
2044 | AgentControlError::ContinuityUnknown(_)
2045 | AgentControlError::RecoveryMismatch(_),
2046 ) => false,
2047 _ => true,
2048 }
2049}
2050
2051async fn apply_remembered_approvals(
2052 state: &RemoteApiState,
2053 agent: &AgentApi,
2054 run_id: &RunId,
2055 requests: &[PendingRequest],
2056) {
2057 for request in requests {
2058 if !matches!(request.payload, PendingRequestPayload::Approval { .. }) {
2059 continue;
2060 }
2061 let Ok(Some(grant_ref)) = state
2062 .approvals
2063 .approve_if_remembered(&request.request_id, approval_expiry_ms())
2064 else {
2065 continue;
2066 };
2067 let Ok(command) = AgentCommandEnvelope::new(
2068 CommandId::new(format!(
2069 "host-remembered-approval-{}",
2070 request.request_id.as_str()
2071 )),
2072 run_id.clone(),
2073 Some(request.request_id.clone()),
2074 AgentCommand::ResolveRequest {
2075 response: RequestResolution::Approval {
2076 decision: ApprovalDecision::Allow,
2077 grant_ref: Some(grant_ref),
2078 },
2079 },
2080 ) else {
2081 continue;
2082 };
2083 if let Err(error) = agent.command(command).await {
2084 tracing::warn!(
2085 run_id = %run_id.as_str(),
2086 request_id = %request.request_id.as_str(),
2087 %error,
2088 "could not apply remembered approval"
2089 );
2090 }
2091 }
2092}
2093
2094fn run_supervisor_backoff(failures: u32) -> Duration {
2095 let exponent = failures.saturating_sub(1).min(8);
2096 RUN_SUPERVISOR_INITIAL_BACKOFF
2097 .saturating_mul(2_u32.saturating_pow(exponent))
2098 .min(RUN_SUPERVISOR_MAX_BACKOFF)
2099}
2100
2101async fn inspect_run(
2102 State(state): State<RemoteApiState>,
2103 Path(run_id): Path<String>,
2104 Query(query): Query<RunTargetQuery>,
2105) -> Result<Json<RemoteRunView>, ApiError> {
2106 let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2107 let (agent, run_id) = require_run(
2108 &state,
2109 connector_id.as_ref().map(AgentConnectorId::as_str),
2110 run_id,
2111 )
2112 .await?;
2113 Ok(Json(RemoteRunView::new(
2114 &state,
2115 connector_id.as_ref(),
2116 agent.inspect(&run_id).await?,
2117 agent.initial_input(&run_id).await?,
2118 )))
2119}
2120
2121async fn recover_run(
2122 State(state): State<RemoteApiState>,
2123 Path(run_id): Path<String>,
2124 Query(query): Query<RunTargetQuery>,
2125) -> Result<Json<RemoteRunView>, ApiError> {
2126 let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2127 let (agent, run_id) = require_run(
2128 &state,
2129 connector_id.as_ref().map(AgentConnectorId::as_str),
2130 run_id,
2131 )
2132 .await?;
2133 let current = agent.inspect(&run_id).await?;
2134 let view = if current.state.status()
2135 == orchestral_core::agent_protocol::reference::AgentRunStatus::Unknown
2136 {
2137 let key = RunSupervisorRegistry::key(connector_id.as_ref(), &run_id);
2138 if state.run_supervisors.manual_reason(&key).is_some() {
2139 current
2140 } else {
2141 match agent.recover(&run_id).await {
2142 Ok(view) => {
2143 state.run_supervisors.clear_manual(&key);
2144 view
2145 }
2146 Err(error) if !is_retryable_agent_error(&error) => {
2147 state.run_supervisors.mark_manual(key, error.to_string());
2148 current
2149 }
2150 Err(error) => return Err(error.into()),
2151 }
2152 }
2153 } else {
2154 state
2155 .run_supervisors
2156 .clear_manual(&RunSupervisorRegistry::key(connector_id.as_ref(), &run_id));
2157 current
2158 };
2159 Ok(Json(RemoteRunView::new(
2160 &state,
2161 connector_id.as_ref(),
2162 view,
2163 agent.initial_input(&run_id).await?,
2164 )))
2165}
2166
2167#[derive(Debug, Deserialize, Default)]
2168#[serde(deny_unknown_fields)]
2169struct RunTargetQuery {
2170 #[serde(default)]
2171 connector_id: Option<String>,
2172}
2173
2174#[derive(Debug, Deserialize, Default)]
2175#[serde(deny_unknown_fields)]
2176struct EventsQuery {
2177 #[serde(default)]
2178 after: u64,
2179 #[serde(default)]
2180 connector_id: Option<String>,
2181}
2182
2183#[derive(Debug, Serialize)]
2184struct EventsResponse {
2185 after: u64,
2186 next: u64,
2187 records: Vec<orchestral_core::agent_protocol::wire::AgentJournalRecord>,
2188}
2189
2190async fn run_events(
2191 State(state): State<RemoteApiState>,
2192 Path(run_id): Path<String>,
2193 Query(query): Query<EventsQuery>,
2194) -> Result<Json<EventsResponse>, ApiError> {
2195 let (agent, run_id) = require_run(&state, query.connector_id.as_deref(), run_id).await?;
2196 let records = agent.events(&run_id, query.after).await?;
2197 let next = records
2198 .last()
2199 .map_or(query.after, |record| record.event.run_seq);
2200 Ok(Json(EventsResponse {
2201 after: query.after,
2202 next,
2203 records,
2204 }))
2205}
2206
2207async fn run_stream(
2208 State(state): State<RemoteApiState>,
2209 Extension(request_context): Extension<RequestLogContext>,
2210 Path(run_id): Path<String>,
2211 Query(query): Query<EventsQuery>,
2212 headers: HeaderMap,
2213) -> Result<Sse<impl futures_util::Stream<Item = Result<Event, Infallible>>>, ApiError> {
2214 let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2215 let (agent, run_id) = require_run(
2216 &state,
2217 connector_id.as_ref().map(AgentConnectorId::as_str),
2218 run_id,
2219 )
2220 .await?;
2221 let header_cursor = headers
2222 .get("last-event-id")
2223 .and_then(|value| value.to_str().ok())
2224 .and_then(|value| value.parse::<u64>().ok())
2225 .unwrap_or_default();
2226 let initial_cursor = query.after.max(header_cursor);
2227 let mut live = agent.subscribe(&run_id).await?;
2231 let lifecycle = SseLifecycleLog::open_run(&request_context, connector_id.as_ref(), &run_id);
2232 let stream = async_stream::stream! {
2233 let mut lifecycle = lifecycle;
2234 let mut cursor = initial_cursor;
2235 match replay_events(&agent, &run_id, &mut cursor).await {
2236 Ok(events) => {
2237 for event in events {
2238 yield Ok(event);
2239 }
2240 }
2241 Err(error) => {
2242 lifecycle.close_as("initial_replay_failed");
2243 tracing::error!(
2244 request_id = %lifecycle.request_id,
2245 stream_id = %lifecycle.stream_id,
2246 status = error.status.as_u16(),
2247 error_code = %error.body.code,
2248 "could not replay initial Run SSE events"
2249 );
2250 yield Ok(api_error_event(&error));
2251 return;
2252 }
2253 }
2254
2255 loop {
2256 match live.recv().await {
2257 Ok(AgentControlEvent::Durable(record)) => {
2258 if record.event.run_seq <= cursor {
2259 continue;
2260 }
2261 cursor = record.event.run_seq;
2262 yield Ok(durable_event(record.as_ref()));
2263 }
2264 Ok(AgentControlEvent::Telemetry(telemetry)) => {
2265 match Event::default().event("telemetry").json_data(&telemetry) {
2266 Ok(event) => yield Ok(event),
2267 Err(error) => {
2268 lifecycle.close_as("telemetry_encode_failed");
2269 tracing::error!(
2270 request_id = %lifecycle.request_id,
2271 stream_id = %lifecycle.stream_id,
2272 %error,
2273 "could not encode Run telemetry SSE event"
2274 );
2275 yield Ok(api_error_event(&ApiError::internal(
2276 "stream_encode_failed",
2277 error.to_string(),
2278 )));
2279 return;
2280 }
2281 }
2282 }
2283 Ok(_) => {}
2284 Err(broadcast::error::RecvError::Lagged(skipped)) => {
2285 lifecycle.lagged(skipped);
2286 match replay_events(&agent, &run_id, &mut cursor).await {
2287 Ok(events) => {
2288 for event in events {
2289 yield Ok(event);
2290 }
2291 }
2292 Err(error) => {
2293 lifecycle.close_as("lag_replay_failed");
2294 tracing::error!(
2295 request_id = %lifecycle.request_id,
2296 stream_id = %lifecycle.stream_id,
2297 status = error.status.as_u16(),
2298 error_code = %error.body.code,
2299 "could not replay Run SSE events after subscriber lag"
2300 );
2301 yield Ok(api_error_event(&error));
2302 return;
2303 }
2304 }
2305 }
2306 Err(broadcast::error::RecvError::Closed) => {
2307 match replay_events(&agent, &run_id, &mut cursor).await {
2308 Ok(events) => {
2309 for event in events {
2310 yield Ok(event);
2311 }
2312 lifecycle.close_as("source_closed");
2313 }
2314 Err(error) => {
2315 lifecycle.close_as("source_closed_replay_failed");
2316 tracing::error!(
2317 request_id = %lifecycle.request_id,
2318 stream_id = %lifecycle.stream_id,
2319 status = error.status.as_u16(),
2320 error_code = %error.body.code,
2321 "could not replay final Run SSE events after source closed"
2322 );
2323 }
2324 }
2325 return;
2326 }
2327 }
2328 }
2329 };
2330 Ok(Sse::new(stream).keep_alive(
2331 KeepAlive::new()
2332 .interval(Duration::from_secs(10))
2333 .text("keep-alive"),
2334 ))
2335}
2336
2337async fn replay_events(
2338 agent: &AgentApi,
2339 run_id: &RunId,
2340 cursor: &mut u64,
2341) -> Result<Vec<Event>, ApiError> {
2342 let records = agent.events(run_id, *cursor).await?;
2343 let mut events = Vec::with_capacity(records.len());
2344 for record in records {
2345 if record.event.run_seq <= *cursor {
2346 continue;
2347 }
2348 *cursor = record.event.run_seq;
2349 events.push(durable_event(&record));
2350 }
2351 Ok(events)
2352}
2353
2354fn durable_event(record: &orchestral_core::agent_protocol::wire::AgentJournalRecord) -> Event {
2355 match Event::default()
2356 .event("durable")
2357 .id(record.event.run_seq.to_string())
2358 .json_data(record)
2359 {
2360 Ok(event) => event,
2361 Err(error) => api_error_event(&ApiError::internal(
2362 "stream_encode_failed",
2363 error.to_string(),
2364 )),
2365 }
2366}
2367
2368fn api_error_event(error: &ApiError) -> Event {
2369 Event::default().event("error").data(
2370 serde_json::to_string(&error.body)
2371 .unwrap_or_else(|_| r#"{"code":"stream_failed","message":"stream failed"}"#.to_owned()),
2372 )
2373}
2374
2375#[derive(Debug, Deserialize)]
2376#[serde(deny_unknown_fields)]
2377struct TextCommandRequest {
2378 command_id: String,
2379 text: String,
2380 #[serde(default)]
2381 after_activity_id: Option<String>,
2382 #[serde(default)]
2383 attachments: Vec<RemoteArtifactInput>,
2384}
2385
2386async fn steer_run(
2387 State(state): State<RemoteApiState>,
2388 Path(run_id): Path<String>,
2389 Query(query): Query<RunTargetQuery>,
2390 Json(request): Json<TextCommandRequest>,
2391) -> Result<Json<CommandAck>, ApiError> {
2392 let content = message_content(&state, &request.text, &request.attachments).await?;
2393 let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2394 let (agent, run_id) =
2395 require_commandable_run(&state, query.connector_id.as_deref(), run_id).await?;
2396 let supervision_key = RunSupervisorRegistry::key(connector_id.as_ref(), &run_id);
2397 if let Some(issue) = state.run_supervisors.issue(&supervision_key) {
2398 return Err(ApiError::conflict(
2399 "agent_run_stalled",
2400 format!(
2401 "{}; wait for the watchdog cancellation to finish or stop the Run explicitly",
2402 issue.reason
2403 ),
2404 ));
2405 }
2406 let history_anchor =
2407 request
2408 .after_activity_id
2409 .map(|after_activity_id| AgentSessionHistoryAnchor {
2410 after_activity_id: AgentSessionActivityId::new(after_activity_id),
2411 });
2412 let command = AgentCommandEnvelope::new_with_extensions(
2413 CommandId::new(request.command_id),
2414 run_id.clone(),
2415 None,
2416 AgentCommand::Steer { content },
2417 session_history_anchor_extensions(history_anchor.as_ref())?,
2418 )?;
2419 Ok(Json(
2420 command_run_for_session(&state, connector_id.as_ref(), &agent, &run_id, command).await?,
2421 ))
2422}
2423
2424#[derive(Debug, Deserialize)]
2425#[serde(deny_unknown_fields)]
2426struct CancelRequest {
2427 command_id: String,
2428 reason: String,
2429}
2430
2431async fn cancel_run(
2432 State(state): State<RemoteApiState>,
2433 Path(run_id): Path<String>,
2434 Query(query): Query<RunTargetQuery>,
2435 Json(request): Json<CancelRequest>,
2436) -> Result<Json<CommandAck>, ApiError> {
2437 let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2438 let (agent, run_id) =
2439 require_commandable_run(&state, query.connector_id.as_deref(), run_id).await?;
2440 let command = AgentCommandEnvelope::new(
2441 CommandId::new(request.command_id),
2442 run_id.clone(),
2443 None,
2444 AgentCommand::Cancel {
2445 reason: request.reason,
2446 },
2447 )?;
2448 Ok(Json(
2449 command_run_for_session(&state, connector_id.as_ref(), &agent, &run_id, command).await?,
2450 ))
2451}
2452
2453async fn resolve_input(
2454 State(state): State<RemoteApiState>,
2455 Path((run_id, request_id)): Path<(String, String)>,
2456 Query(query): Query<RunTargetQuery>,
2457 Json(request): Json<TextCommandRequest>,
2458) -> Result<Json<CommandAck>, ApiError> {
2459 let content = message_content(&state, &request.text, &request.attachments).await?;
2460 let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2461 let (agent, run_id) =
2462 require_commandable_run(&state, query.connector_id.as_deref(), run_id).await?;
2463 let request_id = RequestId::new(request_id);
2464 let pending = require_pending_for(&agent, &run_id, &request_id).await?;
2465 if !matches!(pending.payload, PendingRequestPayload::Input { .. }) {
2466 return Err(ApiError::conflict(
2467 "request_kind_mismatch",
2468 "pending request does not accept text input",
2469 ));
2470 }
2471 let command = AgentCommandEnvelope::new(
2472 CommandId::new(request.command_id),
2473 run_id.clone(),
2474 Some(request_id),
2475 AgentCommand::ResolveRequest {
2476 response: RequestResolution::Input { content },
2477 },
2478 )?;
2479 Ok(Json(
2480 command_run_for_session(&state, connector_id.as_ref(), &agent, &run_id, command).await?,
2481 ))
2482}
2483
2484async fn resolve_agent_session_input(
2485 State(state): State<RemoteApiState>,
2486 Path(request_id): Path<String>,
2487 Query(query): Query<AgentSessionQuery>,
2488 Json(request): Json<AgentSessionTextRequest>,
2489) -> Result<Json<Value>, ApiError> {
2490 let content = message_content(&state, &request.text, &request.attachments).await?;
2491 state
2492 .agent_directory
2493 .resolve_request(
2494 &AgentConnectorId::new(query.connector_id),
2495 ResolveAgentSessionRequest {
2496 session_id: AgentSessionId::new(query.session_id),
2497 request_id: RequestId::new(request_id),
2498 response: AgentSessionRequestResolution::Input { content },
2499 },
2500 )
2501 .await?;
2502 Ok(Json(json!({"resolved": true})))
2503}
2504
2505#[derive(Debug, Deserialize)]
2511#[serde(deny_unknown_fields)]
2512struct AgentSessionTextRequest {
2513 text: String,
2514 #[serde(default, rename = "command_id")]
2515 _legacy_command_id: Option<String>,
2516 #[serde(default)]
2517 attachments: Vec<RemoteArtifactInput>,
2518}
2519
2520#[derive(Debug, Clone, Copy, Deserialize)]
2521#[serde(rename_all = "snake_case")]
2522enum ApprovalChoice {
2523 AllowOnce,
2524 AllowSession,
2525 Deny,
2526}
2527
2528#[derive(Debug, Deserialize)]
2529#[serde(deny_unknown_fields)]
2530struct ApprovalRequest {
2531 command_id: String,
2532 decision: ApprovalChoice,
2533}
2534
2535#[derive(Debug, Deserialize)]
2536#[serde(deny_unknown_fields)]
2537struct AgentSessionApprovalRequest {
2538 decision: ApprovalChoice,
2539 #[serde(default, rename = "command_id")]
2540 _legacy_command_id: Option<String>,
2541}
2542
2543async fn resolve_approval(
2544 State(state): State<RemoteApiState>,
2545 Path((run_id, request_id)): Path<(String, String)>,
2546 Query(query): Query<RunTargetQuery>,
2547 Json(request): Json<ApprovalRequest>,
2548) -> Result<Json<CommandAck>, ApiError> {
2549 let connector_id = query.connector_id.as_deref().map(AgentConnectorId::new);
2550 let (agent, run_id) =
2551 require_commandable_run(&state, query.connector_id.as_deref(), run_id).await?;
2552 let request_id = RequestId::new(request_id);
2553 let pending = require_pending_for(&agent, &run_id, &request_id).await?;
2554 let PendingRequestPayload::Approval {
2555 session_approval_scope,
2556 ..
2557 } = pending.payload
2558 else {
2559 return Err(ApiError::conflict(
2560 "request_kind_mismatch",
2561 "pending request is not an approval",
2562 ));
2563 };
2564 let (decision, grant_ref) = match request.decision {
2565 ApprovalChoice::Deny => (ApprovalDecision::Deny, None),
2566 ApprovalChoice::AllowOnce => (
2567 ApprovalDecision::Allow,
2568 Some(state.approvals.approve(&request_id, approval_expiry_ms())?),
2569 ),
2570 ApprovalChoice::AllowSession => {
2571 if session_approval_scope.is_none() {
2572 return Err(ApiError::conflict(
2573 "session_approval_unavailable",
2574 "this operation cannot be approved for the session",
2575 ));
2576 }
2577 (
2578 ApprovalDecision::Allow,
2579 Some(
2580 state
2581 .approvals
2582 .approve_for_session(&request_id, approval_expiry_ms())?,
2583 ),
2584 )
2585 }
2586 };
2587 let command = AgentCommandEnvelope::new(
2588 CommandId::new(request.command_id),
2589 run_id.clone(),
2590 Some(request_id),
2591 AgentCommand::ResolveRequest {
2592 response: RequestResolution::Approval {
2593 decision,
2594 grant_ref,
2595 },
2596 },
2597 )?;
2598 Ok(Json(
2599 command_run_for_session(&state, connector_id.as_ref(), &agent, &run_id, command).await?,
2600 ))
2601}
2602
2603async fn resolve_agent_session_approval(
2604 State(state): State<RemoteApiState>,
2605 Path(request_id): Path<String>,
2606 Query(query): Query<AgentSessionQuery>,
2607 Json(request): Json<AgentSessionApprovalRequest>,
2608) -> Result<Json<Value>, ApiError> {
2609 let decision = match request.decision {
2610 ApprovalChoice::Deny => ApprovalDecision::Deny,
2611 ApprovalChoice::AllowOnce => ApprovalDecision::Allow,
2612 ApprovalChoice::AllowSession => {
2613 return Err(ApiError::conflict(
2614 "session_approval_unavailable",
2615 "provider-native session requests do not declare a remembered approval scope",
2616 ));
2617 }
2618 };
2619 state
2620 .agent_directory
2621 .resolve_request(
2622 &AgentConnectorId::new(query.connector_id),
2623 ResolveAgentSessionRequest {
2624 session_id: AgentSessionId::new(query.session_id),
2625 request_id: RequestId::new(request_id),
2626 response: AgentSessionRequestResolution::Approval { decision },
2627 },
2628 )
2629 .await?;
2630 Ok(Json(json!({"resolved": true})))
2631}
2632
2633async fn require_run(
2634 state: &RemoteApiState,
2635 connector_id: Option<&str>,
2636 run_id: String,
2637) -> Result<(AgentApi, RunId), ApiError> {
2638 if run_id.starts_with("agent-history:") {
2642 return Err(ApiError::conflict(
2643 "client_update_required",
2644 "此页面版本过旧,请刷新页面后重新处理审批或回复;当前操作尚未执行",
2645 ));
2646 }
2647 let agent = match connector_id {
2648 Some(connector_id) => {
2649 state
2650 .agent_directory
2651 .agent_api(&AgentConnectorId::new(connector_id))
2652 .await?
2653 }
2654 None => state.agent.clone(),
2655 };
2656 let run_id = RunId::new(run_id);
2657 if !agent.has_run(&run_id).await? {
2658 tracing::warn!(%run_id, connector_id, "requested Agent Run was not found");
2659 return Err(ApiError::not_found("run_not_found", "run was not found"));
2660 }
2661 spawn_run_supervisor(
2662 state.clone(),
2663 agent.clone(),
2664 connector_id.map(AgentConnectorId::new),
2665 run_id.clone(),
2666 );
2667 Ok((agent, run_id))
2668}
2669
2670async fn require_commandable_run(
2671 state: &RemoteApiState,
2672 connector_id: Option<&str>,
2673 run_id: String,
2674) -> Result<(AgentApi, RunId), ApiError> {
2675 let (agent, run_id) = require_run(state, connector_id, run_id).await?;
2676 if agent.inspect(&run_id).await?.state.status()
2677 == orchestral_core::agent_protocol::reference::AgentRunStatus::Unknown
2678 {
2679 return Err(ApiError::service_unavailable(
2680 "run_recovery_pending",
2681 "Agent Run continuity is being recovered; retry this command shortly",
2682 ));
2683 }
2684 Ok((agent, run_id))
2685}
2686
2687async fn command_run(
2688 agent: &AgentApi,
2689 run_id: &RunId,
2690 command: AgentCommandEnvelope,
2691) -> Result<CommandAck, ApiError> {
2692 let command_id = command.command_id.clone();
2693 let protocol_request_id = command.request_id.clone();
2694 let command_kind = match &command.payload {
2695 AgentCommand::Steer { .. } => "steer",
2696 AgentCommand::ResolveRequest { response } => match response {
2697 RequestResolution::Input { .. } => "resolve_input",
2698 RequestResolution::Approval { .. } => "resolve_approval",
2699 RequestResolution::ExternalResult { .. } => "resolve_external_result",
2700 _ => "resolve_unknown",
2701 },
2702 AgentCommand::Cancel { .. } => "cancel",
2703 _ => "unknown",
2704 };
2705 let result = agent.command(command).await;
2706 match result {
2707 Ok(ack) => {
2708 let ack_state = match &ack.state {
2709 CommandAckState::Accepted { .. } => "accepted",
2710 CommandAckState::Applied { .. } => "applied",
2711 CommandAckState::Rejected { .. } => "rejected",
2712 CommandAckState::Unsupported { .. } => "unsupported",
2713 _ => "unknown",
2714 };
2715 tracing::info!(
2716 run_id = %run_id.as_str(),
2717 command_id = %command_id.as_str(),
2718 protocol_request_id = protocol_request_id
2719 .as_ref()
2720 .map(RequestId::as_str)
2721 .unwrap_or("-"),
2722 command_kind,
2723 ack_state,
2724 duplicate = ack.duplicate,
2725 "Agent command acknowledged"
2726 );
2727 Ok(ack)
2728 }
2729 Err(_error)
2730 if agent.inspect(run_id).await.is_ok_and(|view| {
2731 view.state.status()
2732 == orchestral_core::agent_protocol::reference::AgentRunStatus::Unknown
2733 }) =>
2734 {
2735 tracing::warn!(
2736 run_id = %run_id.as_str(),
2737 command_id = %command_id.as_str(),
2738 protocol_request_id = protocol_request_id
2739 .as_ref()
2740 .map(RequestId::as_str)
2741 .unwrap_or("-"),
2742 command_kind,
2743 "Agent command outcome is pending continuity recovery"
2744 );
2745 Err(ApiError::service_unavailable(
2746 "run_recovery_pending",
2747 "Agent Run continuity is being recovered; retry this command shortly",
2748 ))
2749 }
2750 Err(error) => {
2751 tracing::warn!(
2752 run_id = %run_id.as_str(),
2753 command_id = %command_id.as_str(),
2754 protocol_request_id = protocol_request_id
2755 .as_ref()
2756 .map(RequestId::as_str)
2757 .unwrap_or("-"),
2758 command_kind,
2759 "Agent command failed"
2760 );
2761 Err(error.into())
2762 }
2763 }
2764}
2765
2766async fn command_run_for_session(
2767 state: &RemoteApiState,
2768 connector_id: Option<&AgentConnectorId>,
2769 agent: &AgentApi,
2770 run_id: &RunId,
2771 command: AgentCommandEnvelope,
2772) -> Result<CommandAck, ApiError> {
2773 let Some(connector_id) = connector_id else {
2774 return command_run(agent, run_id, command).await;
2775 };
2776 let session_id = agent
2777 .catalog_runs()
2778 .await?
2779 .into_iter()
2780 .find(|entry| entry.run_id == *run_id)
2781 .map(|entry| entry.session_id)
2782 .ok_or_else(|| ApiError::not_found("run_not_found", "run was not found"))?;
2783 let coordinator = state.session_coordinators.get(connector_id, &session_id);
2784 let _operation_guard = coordinator.operation().lock().await;
2785 command_run(agent, run_id, command).await
2786}
2787
2788async fn session_views(state: &RemoteApiState) -> Result<Vec<SessionView>, AgentSdkError> {
2792 let fallback = chrono::Utc::now().timestamp_millis();
2793 let mut catalog = state.agent.catalog_runs().await?;
2794 catalog.sort_by(|left, right| {
2795 left.created_at_unix_ms
2796 .cmp(&right.created_at_unix_ms)
2797 .then_with(|| left.run_id.cmp(&right.run_id))
2798 });
2799
2800 let mut sessions = BTreeMap::<String, SessionView>::new();
2801 for run in catalog {
2802 let created = if run.created_at_unix_ms > 0 {
2803 run.created_at_unix_ms
2804 } else {
2805 fallback
2806 };
2807 let updated = if run.updated_at_unix_ms > 0 {
2808 run.updated_at_unix_ms
2809 } else {
2810 created
2811 };
2812 let session = sessions
2813 .entry(run.session_id.as_str().to_owned())
2814 .or_insert_with(|| SessionView {
2815 id: run.session_id.as_str().to_owned(),
2816 created_at_unix_ms: created,
2817 updated_at_unix_ms: updated,
2818 run_ids: Vec::new(),
2819 cwd: state.native_session_defaults.cwd.clone(),
2820 execution_profile: state.native_session_defaults.execution_profile.clone(),
2821 });
2822 session.created_at_unix_ms = session.created_at_unix_ms.min(created);
2823 session.updated_at_unix_ms = session.updated_at_unix_ms.max(updated);
2824 session.run_ids.push(run.run_id.as_str().to_owned());
2825 }
2826 let mut sessions = sessions.into_values().collect::<Vec<_>>();
2827 sessions.sort_by_key(|session| std::cmp::Reverse(session.updated_at_unix_ms));
2828 Ok(sessions)
2829}
2830
2831async fn require_pending_for(
2832 agent: &AgentApi,
2833 run_id: &RunId,
2834 request_id: &RequestId,
2835) -> Result<PendingRequest, ApiError> {
2836 let view = agent.inspect(run_id).await?;
2837 view.pending_requests
2838 .into_iter()
2839 .find(|request| request.request_id == *request_id)
2840 .ok_or_else(|| {
2841 ApiError::conflict(
2842 "request_not_pending",
2843 "request is no longer pending for this run",
2844 )
2845 })
2846}
2847
2848async fn authenticate(
2849 State(state): State<RemoteApiState>,
2850 mut request: Request<axum::body::Body>,
2851 next: Next,
2852) -> Result<Response, ApiError> {
2853 if let Some(authenticator) = &state.gateway_authenticator {
2854 let assertion = request
2855 .headers()
2856 .get(authenticator.header_name())
2857 .and_then(|value| value.to_str().ok())
2858 .filter(|value| !value.trim().is_empty())
2859 .ok_or_else(|| {
2860 ApiError::unauthorized(
2861 "gateway_authentication_required",
2862 "a signed gateway identity is required",
2863 )
2864 })?;
2865 let principal = authenticator
2866 .authenticate(assertion)
2867 .await
2868 .map_err(|error| {
2869 ApiError::unauthorized("gateway_authentication_failed", error.to_string())
2870 })?;
2871 request
2872 .extensions_mut()
2873 .insert(RemotePrincipal::Gateway(principal));
2874 return Ok(next.run(request).await);
2875 }
2876
2877 let token = bearer_token(request.headers()).ok_or_else(|| {
2878 ApiError::unauthorized(
2879 "authentication_required",
2880 "device authentication is required",
2881 )
2882 })?;
2883 let principal = state.registry.authenticate(token).await.map_err(|_| {
2884 ApiError::unauthorized(
2885 "authentication_failed",
2886 "device authentication is invalid or revoked",
2887 )
2888 })?;
2889 request
2890 .extensions_mut()
2891 .insert(RemotePrincipal::Device(principal));
2892 Ok(next.run(request).await)
2893}
2894
2895async fn log_request(mut request: Request<axum::body::Body>, next: Next) -> Response {
2896 let context = RequestLogContext {
2897 request_id: uuid::Uuid::new_v4().to_string(),
2898 };
2899 let method = request.method().clone();
2900 let route = request
2901 .extensions()
2902 .get::<MatchedPath>()
2903 .map(MatchedPath::as_str)
2904 .unwrap_or("<unmatched>")
2905 .to_owned();
2906 let cf_ray = request
2907 .headers()
2908 .get("cf-ray")
2909 .and_then(|value| value.to_str().ok())
2910 .unwrap_or("-")
2911 .to_owned();
2912 request.extensions_mut().insert(context.clone());
2913
2914 let started_at = Instant::now();
2915 let span = tracing::info_span!(
2916 "http_request",
2917 request_id = %context.request_id,
2918 method = %method,
2919 route = %route,
2920 cf_ray = %cf_ray,
2921 );
2922 let mut response = next.run(request).instrument(span.clone()).await;
2923 if let Ok(value) = HeaderValue::from_str(&context.request_id) {
2924 response.headers_mut().insert(REQUEST_ID_HEADER, value);
2925 }
2926
2927 let status = response.status();
2928 let response_ready_ms = u64::try_from(started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
2929 let error_code = response
2930 .extensions()
2931 .get::<ApiErrorLogCode>()
2932 .map(|code| code.0.as_str())
2933 .unwrap_or("-");
2934 span.in_scope(|| {
2935 if status.is_server_error() {
2936 tracing::error!(
2937 status = status.as_u16(),
2938 response_ready_ms,
2939 error_code,
2940 "HTTP request completed"
2941 );
2942 } else if status.is_client_error() {
2943 tracing::warn!(
2944 status = status.as_u16(),
2945 response_ready_ms,
2946 error_code,
2947 "HTTP request completed"
2948 );
2949 } else {
2950 tracing::info!(
2951 status = status.as_u16(),
2952 response_ready_ms,
2953 "HTTP request completed"
2954 );
2955 }
2956 });
2957 response
2958}
2959
2960async fn no_store(request: Request<axum::body::Body>, next: Next) -> Response {
2961 let mut response = next.run(request).await;
2962 response.headers_mut().insert(
2963 header::CACHE_CONTROL,
2964 header::HeaderValue::from_static("no-store, private"),
2965 );
2966 response.headers_mut().insert(
2967 header::X_CONTENT_TYPE_OPTIONS,
2968 header::HeaderValue::from_static("nosniff"),
2969 );
2970 response
2971}
2972
2973fn bearer_token(headers: &HeaderMap) -> Option<&str> {
2974 headers
2975 .get(header::AUTHORIZATION)?
2976 .to_str()
2977 .ok()?
2978 .strip_prefix("Bearer ")
2979 .filter(|token| !token.trim().is_empty())
2980}
2981
2982fn approval_expiry_ms() -> i64 {
2983 chrono::Utc::now()
2984 .timestamp_millis()
2985 .saturating_add(APPROVAL_GRANT_TTL_MS)
2986}
2987
2988#[derive(Debug, Clone, Serialize)]
2989struct ApiErrorBody {
2990 code: String,
2991 message: String,
2992}
2993
2994#[derive(Debug)]
2995struct ApiError {
2996 status: StatusCode,
2997 body: ApiErrorBody,
2998}
2999
3000impl ApiError {
3001 fn new(status: StatusCode, code: impl Into<String>, message: impl Into<String>) -> Self {
3002 Self {
3003 status,
3004 body: ApiErrorBody {
3005 code: code.into(),
3006 message: message.into(),
3007 },
3008 }
3009 }
3010
3011 fn unauthorized(code: impl Into<String>, message: impl Into<String>) -> Self {
3012 Self::new(StatusCode::UNAUTHORIZED, code, message)
3013 }
3014
3015 fn not_found(code: impl Into<String>, message: impl Into<String>) -> Self {
3016 Self::new(StatusCode::NOT_FOUND, code, message)
3017 }
3018
3019 fn conflict(code: impl Into<String>, message: impl Into<String>) -> Self {
3020 Self::new(StatusCode::CONFLICT, code, message)
3021 }
3022
3023 fn service_unavailable(code: impl Into<String>, message: impl Into<String>) -> Self {
3024 Self::new(StatusCode::SERVICE_UNAVAILABLE, code, message)
3025 }
3026
3027 fn internal(code: impl Into<String>, message: impl Into<String>) -> Self {
3028 Self::new(StatusCode::INTERNAL_SERVER_ERROR, code, message)
3029 }
3030}
3031
3032impl IntoResponse for ApiError {
3033 fn into_response(self) -> Response {
3034 let code = ApiErrorLogCode(self.body.code.clone());
3035 let mut response = (self.status, Json(self.body)).into_response();
3036 response.extensions_mut().insert(code);
3037 response
3038 }
3039}
3040
3041impl From<anyhow::Error> for ApiError {
3042 fn from(error: anyhow::Error) -> Self {
3043 Self::internal("host_state_failed", error.to_string())
3044 }
3045}
3046
3047impl From<AgentSdkError> for ApiError {
3048 fn from(error: AgentSdkError) -> Self {
3049 match &error {
3050 AgentSdkError::InvalidInput(_) => Self::new(
3051 StatusCode::BAD_REQUEST,
3052 "invalid_agent_input",
3053 error.to_string(),
3054 ),
3055 _ => Self::internal("agent_control_failed", error.to_string()),
3056 }
3057 }
3058}
3059
3060impl From<AgentDirectoryError> for ApiError {
3061 fn from(error: AgentDirectoryError) -> Self {
3062 match &error {
3063 AgentDirectoryError::ConnectorNotFound(_) => {
3064 Self::not_found("agent_connector_not_found", error.to_string())
3065 }
3066 AgentDirectoryError::Connector(connector_error) => {
3067 use orchestral_core::agent_connector::AgentConnectorErrorCode;
3068 match connector_error.code {
3069 AgentConnectorErrorCode::InvalidRequest => Self::new(
3070 StatusCode::BAD_REQUEST,
3071 "invalid_agent_connector_request",
3072 error.to_string(),
3073 ),
3074 AgentConnectorErrorCode::NotFound => {
3075 Self::not_found("agent_session_not_found", error.to_string())
3076 }
3077 AgentConnectorErrorCode::Busy | AgentConnectorErrorCode::LeaseConflict => {
3078 Self::conflict("agent_connector_busy", error.to_string())
3079 }
3080 AgentConnectorErrorCode::Unsupported => Self::new(
3081 StatusCode::NOT_IMPLEMENTED,
3082 "agent_connector_unsupported",
3083 error.to_string(),
3084 ),
3085 AgentConnectorErrorCode::Unavailable => Self::new(
3086 StatusCode::SERVICE_UNAVAILABLE,
3087 "agent_connector_unavailable",
3088 error.to_string(),
3089 ),
3090 AgentConnectorErrorCode::Protocol | AgentConnectorErrorCode::OutcomeUnknown => {
3091 Self::internal("agent_connector_failed", error.to_string())
3092 }
3093 _ => Self::internal("agent_connector_failed", error.to_string()),
3094 }
3095 }
3096 AgentDirectoryError::Agent(AgentSdkError::Control(AgentControlError::Start(
3097 AgentStartError::Rejected(rejection),
3098 ))) if rejection
3099 .details
3100 .get("code")
3101 .and_then(serde_json::Value::as_str)
3102 == Some("live_control_unavailable") =>
3103 {
3104 Self::conflict("live_control_unavailable", rejection.message.clone())
3105 }
3106 AgentDirectoryError::Agent(AgentSdkError::Control(AgentControlError::Start(
3107 AgentStartError::Rejected(rejection),
3108 ))) => match rejection.code {
3109 AgentRejectionCode::SessionConflict | AgentRejectionCode::RunIdConflict => {
3110 Self::conflict("agent_session_conflict", rejection.message.clone())
3111 }
3112 AgentRejectionCode::InvalidSpec => Self::new(
3113 StatusCode::BAD_REQUEST,
3114 "invalid_agent_input",
3115 rejection.message.clone(),
3116 ),
3117 AgentRejectionCode::UnsupportedProtocol
3118 | AgentRejectionCode::UnsupportedCapability
3119 | AgentRejectionCode::UnsupportedResource => Self::new(
3120 StatusCode::NOT_IMPLEMENTED,
3121 "agent_capability_unsupported",
3122 rejection.message.clone(),
3123 ),
3124 AgentRejectionCode::ProviderUnavailable => Self::new(
3125 StatusCode::SERVICE_UNAVAILABLE,
3126 "agent_provider_unavailable",
3127 rejection.message.clone(),
3128 ),
3129 _ => Self::new(
3130 StatusCode::BAD_REQUEST,
3131 "agent_rejected",
3132 rejection.message.clone(),
3133 ),
3134 },
3135 _ => Self::internal("agent_directory_failed", error.to_string()),
3136 }
3137 }
3138}
3139
3140impl From<orchestral_core::agent_protocol::wire::AgentProtocolError> for ApiError {
3141 fn from(error: orchestral_core::agent_protocol::wire::AgentProtocolError) -> Self {
3142 Self::new(
3143 StatusCode::BAD_REQUEST,
3144 "invalid_command",
3145 error.to_string(),
3146 )
3147 }
3148}
3149
3150impl From<ApprovalBridgeError> for ApiError {
3151 fn from(error: ApprovalBridgeError) -> Self {
3152 match error {
3153 ApprovalBridgeError::RequestNotFound(_) => Self::conflict(
3154 "approval_binding_unavailable",
3155 "此入口缺少该审批的授权信息,本次操作未执行。请刷新会话后重试",
3156 ),
3157 ApprovalBridgeError::SessionScopeUnavailable(_) => Self::conflict(
3158 "session_approval_unavailable",
3159 "此审批不支持本会话允许,请选择允许一次或拒绝",
3160 ),
3161 _ => Self::internal("approval_bridge_failed", error.to_string()),
3162 }
3163 }
3164}
3165
3166#[cfg(test)]
3167mod tests {
3168 use super::*;
3169 use orchestral_core::agent_connector::AgentSessionExecutionProfile;
3170 use std::collections::BTreeSet;
3171 use std::sync::atomic::{AtomicUsize, Ordering};
3172 use std::sync::Arc;
3173
3174 use async_trait::async_trait;
3175 use axum::body::Body;
3176 use axum::http::{HeaderName, Request};
3177 use futures_util::{stream, StreamExt};
3178 use http_body_util::BodyExt;
3179 use orchestral_agent_protocol_testkit::{
3180 ProviderFixtureFactory, ProviderScenario, ScriptedStatelessFactory,
3181 SessionfulRecoverFactory, TestProbes,
3182 };
3183 use orchestral_core::agent_connector::{
3184 AgentConnector, AgentConnectorDescriptor, AgentConnectorError, AgentConnectorHealth,
3185 AgentSessionActionDescriptor, AgentSessionActionExecution, AgentSessionActionOutcome,
3186 AgentSessionActionStatus, AgentSessionActivity, AgentSessionActivityId,
3187 AgentSessionActivityKind, AgentSessionActivityStatus, AgentSessionCapabilities,
3188 AgentSessionDetail, AgentSessionState, AgentSessionSummary, AgentSessionTurn,
3189 AgentSessionTurnId, AgentSessionTurnStatus, CreateAgentSessionRequest,
3190 InvokeAgentSessionActionRequest, SESSION_FORK_ACTION, SESSION_RENAME_ACTION,
3191 SESSION_REVIEW_ACTION,
3192 };
3193 use orchestral_core::agent_protocol::{
3194 spi::{
3195 AgentProvider, AgentRecovery, AgentRecoveryRequest, AgentStart, AgentStartError,
3196 InMemoryAgentJournalStore,
3197 },
3198 wire::{
3199 AgentAdmission, AgentCapabilities, AgentDescriptor, AgentDescriptorEnvelope,
3200 AgentEvent, AgentEventDraft, AgentExecutionRef, AgentId, AgentProtocolError,
3201 AgentProtocolErrorCode, AgentProviderId, AgentProviderStreamItem, AgentRunEnvelope,
3202 EffectMediation, PendingRequestKind, ProviderBindingRef, ProviderCommandDisposition,
3203 ProviderCommandOutcome,
3204 },
3205 AGENT_PROTOCOL_V1,
3206 };
3207 use orchestral_core::io::{ArtifactResolveError, ResolvedArtifact};
3208 use orchestral_core::tool_protocol::{
3209 ApprovalBinding, CapabilityRequest, EffectScope, ToolCallId, ToolId,
3210 };
3211 use orchestral_runtime::{AgentApprovalBridge, AgentController};
3212 use tokio::sync::broadcast;
3213 use tower::ServiceExt;
3214
3215 #[test]
3216 fn provider_native_request_bodies_do_not_require_run_command_identity() {
3217 let input: AgentSessionTextRequest = serde_json::from_value(serde_json::json!({
3218 "text": "continue"
3219 }))
3220 .unwrap();
3221 assert_eq!(input.text, "continue");
3222 assert!(input.attachments.is_empty());
3223
3224 let approval: AgentSessionApprovalRequest = serde_json::from_value(serde_json::json!({
3225 "decision": "allow_once"
3226 }))
3227 .unwrap();
3228 assert!(matches!(approval.decision, ApprovalChoice::AllowOnce));
3229 }
3230
3231 #[tokio::test]
3232 async fn native_request_http_accepts_current_and_already_open_pwa_bodies() {
3233 let (app, token) = test_app().await;
3234 for legacy in [false, true] {
3235 for (kind, mut body) in [
3236 ("approval", json!({"decision": "allow_once"})),
3237 ("approval", json!({"decision": "deny"})),
3238 ("input", json!({"text": "continue"})),
3239 ] {
3240 if legacy {
3241 body["command_id"] = json!("old-browser-command");
3242 }
3243 let response = app.clone().oneshot(authorized(
3244 "POST",
3245 &format!("/agent-session/requests/native-{kind}/{kind}?connector_id=fixture%2Flocal&session_id=fixture-session"),
3246 &token,
3247 body,
3248 )).await.unwrap();
3249 assert_eq!(response.status(), StatusCode::OK, "{kind}, legacy={legacy}");
3250 let body = response.into_body().collect().await.unwrap().to_bytes();
3251 assert_eq!(
3252 serde_json::from_slice::<Value>(&body).unwrap(),
3253 json!({"resolved": true})
3254 );
3255 }
3256 }
3257 let response = app.clone().oneshot(authorized(
3260 "POST", "/agent-session/requests/native-approval/approval?connector_id=fixture%2Flocal&session_id=fixture-session",
3261 &token, json!({"decision": "allow_session", "command_id": "legacy"}),
3262 )).await.unwrap();
3263 assert_eq!(response.status(), StatusCode::CONFLICT);
3264 let response = app.oneshot(authorized(
3265 "POST", "/agent-session/requests/native-approval/approval?connector_id=fixture%2Flocal&session_id=fixture-session",
3266 &token, json!({"decision": "allow_once", "grant_ref": "unexpected"}),
3267 )).await.unwrap();
3268 assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
3269 }
3270
3271 #[tokio::test]
3272 async fn history_projection_control_reports_client_update_required() {
3273 let (app, token) = test_app().await;
3274 let response = app.oneshot(authorized(
3275 "POST", "/runs/agent-history%3Afixture%2Flocal%3Afixture-session/requests/native-approval/approval?connector_id=fixture%2Flocal",
3276 &token, json!({"command_id": "old-page", "decision": "allow_once"}),
3277 )).await.unwrap();
3278 assert_eq!(response.status(), StatusCode::CONFLICT);
3279 let body = response.into_body().collect().await.unwrap().to_bytes();
3280 assert_eq!(
3281 serde_json::from_slice::<Value>(&body).unwrap()["code"],
3282 "client_update_required"
3283 );
3284 }
3285
3286 #[test]
3287 fn supervision_retries_only_retryable_agent_failures() {
3288 let permanent =
3289 AgentSdkError::Control(AgentControlError::Protocol(AgentProtocolError::new(
3290 AgentProtocolErrorCode::InvalidTransition,
3291 "manual recovery required",
3292 )));
3293 assert!(!is_retryable_agent_error(&permanent));
3294
3295 let transient = AgentSdkError::Control(AgentControlError::Protocol(
3296 AgentProtocolError::new(
3297 AgentProtocolErrorCode::ProviderUnavailable,
3298 "provider restarting",
3299 )
3300 .with_retryable(true),
3301 ));
3302 assert!(is_retryable_agent_error(&transient));
3303
3304 let missing =
3305 AgentSdkError::Control(AgentControlError::RunNotFound(RunId::new("missing-run")));
3306 assert!(!is_retryable_agent_error(&missing));
3307 }
3308
3309 #[test]
3310 fn session_creation_expands_host_home_shortcuts_without_touching_other_paths() {
3311 #[cfg(target_os = "windows")]
3312 let home_variable = "USERPROFILE";
3313 #[cfg(not(target_os = "windows"))]
3314 let home_variable = "HOME";
3315 let home = std::env::var_os(home_variable)
3316 .filter(|value| !value.is_empty())
3317 .map(PathBuf::from)
3318 .expect("test Host has a home directory");
3319 assert_eq!(
3320 expand_host_home(Some("~".to_owned())).unwrap(),
3321 Some(home.to_string_lossy().into_owned())
3322 );
3323 assert_eq!(
3324 expand_host_home(Some("~/".to_owned())).unwrap(),
3325 Some(home.join("").to_string_lossy().into_owned())
3326 );
3327 assert_eq!(
3328 expand_host_home(Some("~/rust_ws/project".to_owned())).unwrap(),
3329 Some(home.join("rust_ws/project").to_string_lossy().into_owned())
3330 );
3331 assert_eq!(
3332 expand_host_home(Some("/srv/project".to_owned())).unwrap(),
3333 Some("/srv/project".to_owned())
3334 );
3335 }
3336
3337 use super::super::auth::GatewayAuthError;
3338
3339 struct StaticGatewayAuthenticator {
3340 header_name: HeaderName,
3341 }
3342
3343 struct StaticAgentConnector;
3344
3345 struct DirectArtifactResolver;
3346
3347 #[async_trait]
3348 impl ArtifactResolver for DirectArtifactResolver {
3349 async fn resolve(
3350 &self,
3351 artifact: &ArtifactRefWithDigest,
3352 ) -> Result<ResolvedArtifact, ArtifactResolveError> {
3353 Ok(ResolvedArtifact {
3354 artifact: artifact.clone(),
3355 uri: format!(
3356 "https://orchestral-files.example/v1/blobs/{}?capability=signed",
3357 artifact.artifact_ref
3358 ),
3359 file_name: Some("generated.png".to_owned()),
3360 media_type: "image/png".to_owned(),
3361 byte_size: 123,
3362 expires_at: None,
3363 })
3364 }
3365 }
3366
3367 struct ObservableAgentConnector {
3368 subscriptions: Arc<AtomicUsize>,
3369 changes: broadcast::Sender<AgentSessionChange>,
3370 }
3371
3372 struct DisconnectFirstProvider {
3373 inner: Arc<dyn AgentProvider>,
3374 }
3375
3376 struct HoldingProvider {
3377 inner: Arc<dyn AgentProvider>,
3378 finish: Arc<tokio::sync::Notify>,
3379 rejection: Option<AgentProtocolErrorCode>,
3380 }
3381
3382 struct UnrecoverableDisconnectProvider {
3383 inner: Arc<dyn AgentProvider>,
3384 }
3385
3386 #[test]
3387 fn active_external_agent_writer_is_reported_as_a_conflict() {
3388 let rejection = orchestral_core::agent_protocol::wire::AgentRejection::new(
3389 AgentRejectionCode::SessionConflict,
3390 "thread already has an active writer",
3391 );
3392 let error = AgentDirectoryError::Agent(AgentSdkError::Control(AgentControlError::Start(
3393 AgentStartError::Rejected(rejection),
3394 )));
3395
3396 let response = ApiError::from(error);
3397 assert_eq!(response.status, StatusCode::CONFLICT);
3398 assert_eq!(response.body.code, "agent_session_conflict");
3399 }
3400
3401 #[test]
3402 fn realtime_only_agent_writer_conflict_has_a_stable_api_code() {
3403 let rejection = orchestral_core::agent_protocol::wire::AgentRejection::new(
3404 AgentRejectionCode::UnsupportedCapability,
3405 "live control unavailable",
3406 )
3407 .with_details(serde_json::json!({"code": "live_control_unavailable"}));
3408 let error = AgentDirectoryError::Agent(AgentSdkError::Control(AgentControlError::Start(
3409 AgentStartError::Rejected(rejection),
3410 )));
3411
3412 let response = ApiError::from(error);
3413 assert_eq!(response.status, StatusCode::CONFLICT);
3414 assert_eq!(response.body.code, "live_control_unavailable");
3415 }
3416
3417 #[tokio::test]
3418 async fn session_views_add_direct_storage_access_without_changing_artifact_identity() {
3419 let digest = "a".repeat(64);
3420 let mut payload = serde_json::json!({
3421 "turns": [{
3422 "activities": [{
3423 "content": [{
3424 "media_type": "image/png",
3425 "schema_id": null,
3426 "body": {
3427 "kind": "artifact",
3428 "value": {
3429 "artifact_ref": digest,
3430 "digest": "a".repeat(64)
3431 }
3432 }
3433 }]
3434 }]
3435 }]
3436 });
3437
3438 enrich_artifact_access(&mut payload, Some(&DirectArtifactResolver)).await;
3439
3440 let content = payload.pointer("/turns/0/activities/0/content/0").unwrap();
3441 assert_eq!(
3442 content.pointer("/body/value/artifact_ref"),
3443 Some(&serde_json::Value::String("a".repeat(64)))
3444 );
3445 assert_eq!(
3446 content.pointer("/access/uri").and_then(serde_json::Value::as_str),
3447 Some(
3448 "https://orchestral-files.example/v1/blobs/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?capability=signed"
3449 )
3450 );
3451 assert_eq!(
3452 content.pointer("/access/byte_size"),
3453 Some(&serde_json::json!(123))
3454 );
3455 }
3456
3457 #[tokio::test]
3458 async fn latest_session_run_ignores_a_journal_from_an_old_controller_contract() {
3459 let factory = ScriptedStatelessFactory::conformant().expect("fixture descriptor");
3460 let scenario = ProviderScenario::standard(&factory.descriptor()).expect("fixture scenario");
3461 let journal = Arc::new(InMemoryAgentJournalStore::default());
3462 let previous = Arc::new(
3463 AgentController::with_journal_store(
3464 factory.create(scenario.clone(), TestProbes::default()),
3465 ProviderBindingRef::new("previous-binding"),
3466 journal.clone(),
3467 )
3468 .expect("previous controller binds"),
3469 );
3470 let execution = previous
3471 .start(scenario.start_request.run.clone())
3472 .await
3473 .expect("previous Run starts");
3474 previous
3475 .wait_for_terminal(&execution.run_id)
3476 .await
3477 .expect("previous Run completes");
3478 drop(previous);
3479
3480 let upgraded = Arc::new(
3481 AgentController::with_journal_store(
3482 factory.create(scenario.clone(), TestProbes::default()),
3483 ProviderBindingRef::new("upgraded-binding"),
3484 journal,
3485 )
3486 .expect("upgraded controller binds"),
3487 );
3488 let agent = AgentApi::new(upgraded);
3489
3490 assert!(
3491 latest_session_run(&agent, &scenario.start_request.run.spec.session_id)
3492 .await
3493 .expect("catalog remains readable")
3494 .is_none()
3495 );
3496 }
3497
3498 #[async_trait]
3499 impl AgentProvider for DisconnectFirstProvider {
3500 fn describe(&self) -> AgentDescriptorEnvelope {
3501 self.inner.describe()
3502 }
3503
3504 async fn start(
3505 &self,
3506 request: orchestral_core::agent_protocol::wire::AgentStartRequest,
3507 ) -> Result<AgentStart, AgentStartError> {
3508 let started = self.inner.start(request).await?;
3509 Ok(AgentStart {
3510 execution: started.execution,
3511 admission: started.admission,
3512 stream: started.stream.take(1).boxed(),
3513 })
3514 }
3515
3516 async fn command(
3517 &self,
3518 execution: &AgentExecutionRef,
3519 command: AgentCommandEnvelope,
3520 ) -> Result<ProviderCommandDisposition, AgentProtocolError> {
3521 self.inner.command(execution, command).await
3522 }
3523
3524 async fn recover(
3525 &self,
3526 request: AgentRecoveryRequest,
3527 ) -> Result<AgentRecovery, AgentProtocolError> {
3528 self.inner.recover(request).await
3529 }
3530 }
3531
3532 #[async_trait]
3533 impl AgentProvider for HoldingProvider {
3534 fn describe(&self) -> AgentDescriptorEnvelope {
3535 self.inner.describe()
3536 }
3537
3538 async fn start(
3539 &self,
3540 request: orchestral_core::agent_protocol::wire::AgentStartRequest,
3541 ) -> Result<AgentStart, AgentStartError> {
3542 let run_id = request.run.spec.run_id.clone();
3543 let finish = self.finish.clone();
3544 let started = self.inner.start(request).await?;
3545 let terminal = stream::once(async move {
3546 finish.notified().await;
3547 Ok(AgentProviderStreamItem::Event(Box::new(AgentEventDraft {
3548 event_id: orchestral_core::agent_protocol::wire::AgentEventId::new(format!(
3549 "fixture-finished-{run_id}"
3550 )),
3551 run_id,
3552 causation_id: None,
3553 source_fingerprint: None,
3554 payload: AgentEvent::RunFailed {
3555 failure: orchestral_core::agent_protocol::wire::AgentFailure {
3556 code: "fixture_finished".to_owned(),
3557 message: "controlled fixture completed".to_owned(),
3558 retryable: false,
3559 details: Value::Null,
3560 },
3561 },
3562 })))
3563 });
3564 Ok(AgentStart {
3565 execution: started.execution,
3566 admission: started.admission,
3567 stream: started.stream.take(1).chain(terminal).boxed(),
3568 })
3569 }
3570
3571 async fn command(
3572 &self,
3573 _execution: &AgentExecutionRef,
3574 command: AgentCommandEnvelope,
3575 ) -> Result<ProviderCommandDisposition, AgentProtocolError> {
3576 Ok(ProviderCommandDisposition {
3577 command_id: command.command_id,
3578 run_id: command.run_id,
3579 outcome: self
3580 .rejection
3581 .clone()
3582 .map_or(ProviderCommandOutcome::Accepted, |code| {
3583 ProviderCommandOutcome::Rejected {
3584 code,
3585 message: "fixture command was not dispatched".to_owned(),
3586 }
3587 }),
3588 duplicate: false,
3589 })
3590 }
3591
3592 async fn recover(
3593 &self,
3594 request: AgentRecoveryRequest,
3595 ) -> Result<AgentRecovery, AgentProtocolError> {
3596 self.inner.recover(request).await
3597 }
3598 }
3599
3600 #[async_trait]
3601 impl AgentProvider for UnrecoverableDisconnectProvider {
3602 fn describe(&self) -> AgentDescriptorEnvelope {
3603 self.inner.describe()
3604 }
3605
3606 async fn start(
3607 &self,
3608 request: orchestral_core::agent_protocol::wire::AgentStartRequest,
3609 ) -> Result<AgentStart, AgentStartError> {
3610 let started = self.inner.start(request).await?;
3611 Ok(AgentStart {
3612 execution: started.execution,
3613 admission: started.admission,
3614 stream: started.stream.take(1).boxed(),
3615 })
3616 }
3617
3618 async fn command(
3619 &self,
3620 execution: &AgentExecutionRef,
3621 command: AgentCommandEnvelope,
3622 ) -> Result<ProviderCommandDisposition, AgentProtocolError> {
3623 self.inner.command(execution, command).await
3624 }
3625
3626 async fn recover(
3627 &self,
3628 _request: AgentRecoveryRequest,
3629 ) -> Result<AgentRecovery, AgentProtocolError> {
3630 Err(AgentProtocolError::new(
3631 AgentProtocolErrorCode::ProviderUnavailable,
3632 "fixture recovery remains unavailable",
3633 ))
3634 }
3635 }
3636
3637 impl StaticAgentConnector {
3638 fn summary() -> AgentSessionSummary {
3639 AgentSessionSummary {
3640 connector_id: AgentConnectorId::new("fixture/local"),
3641 session_id: AgentSessionId::new("fixture-session"),
3642 title: Some("Existing fixture session".to_owned()),
3643 preview: Some("resume me".to_owned()),
3644 cwd: Some("/fixture/workspace".to_owned()),
3645 created_at_unix_ms: Some(1_000),
3646 updated_at_unix_ms: Some(2_000),
3647 state: AgentSessionState::Idle,
3648 execution_profile: Default::default(),
3649 extensions: BTreeMap::new(),
3650 }
3651 }
3652
3653 fn created_summary() -> AgentSessionSummary {
3654 AgentSessionSummary {
3655 session_id: AgentSessionId::new("fixture-created"),
3656 title: Some("Created from HTTP".to_owned()),
3657 ..Self::summary()
3658 }
3659 }
3660 }
3661
3662 #[async_trait]
3663 impl AgentConnector for StaticAgentConnector {
3664 fn describe(&self) -> AgentConnectorDescriptor {
3665 AgentConnectorDescriptor {
3666 connector_id: AgentConnectorId::new("fixture/local"),
3667 provider_binding: ProviderBindingRef::new("fixture/external"),
3668 agent_family: "test-agent".to_owned(),
3669 display_name: "Fixture Agent".to_owned(),
3670 capabilities: AgentSessionCapabilities {
3671 create: true,
3672 resolve_requests: true,
3673 ..AgentSessionCapabilities::discoverable()
3674 },
3675 creation: None,
3676 actions: vec![
3677 AgentSessionActionDescriptor {
3678 action_id: AgentSessionActionId::new(SESSION_FORK_ACTION),
3679 title: "Fork".to_owned(),
3680 description: "Fork a fixture session".to_owned(),
3681 input_schema: None,
3682 execution: AgentSessionActionExecution::Immediate,
3683 },
3684 AgentSessionActionDescriptor {
3685 action_id: AgentSessionActionId::new(SESSION_RENAME_ACTION),
3686 title: "Rename".to_owned(),
3687 description: "Rename a fixture session".to_owned(),
3688 input_schema: Some(serde_json::json!({"type": "object"})),
3689 execution: AgentSessionActionExecution::Immediate,
3690 },
3691 AgentSessionActionDescriptor {
3692 action_id: AgentSessionActionId::new(SESSION_REVIEW_ACTION),
3693 title: "Review".to_owned(),
3694 description: "Review fixture changes".to_owned(),
3695 input_schema: Some(serde_json::json!({"type": "object"})),
3696 execution: AgentSessionActionExecution::Run,
3697 },
3698 ],
3699 }
3700 }
3701
3702 async fn health(&self) -> Result<AgentConnectorHealth, AgentConnectorError> {
3703 Ok(AgentConnectorHealth::ready(Some("test".to_owned())))
3704 }
3705
3706 async fn list_sessions(
3707 &self,
3708 _query: AgentSessionListQuery,
3709 ) -> Result<AgentSessionPage, AgentConnectorError> {
3710 Ok(AgentSessionPage {
3711 sessions: vec![Self::summary()],
3712 next_cursor: None,
3713 })
3714 }
3715
3716 async fn read_session(
3717 &self,
3718 session_id: &AgentSessionId,
3719 ) -> Result<AgentSessionDetail, AgentConnectorError> {
3720 if session_id.as_str() == "fixture-created" {
3721 return Ok(AgentSessionDetail {
3722 summary: Self::created_summary(),
3723 turns: Vec::new(),
3724 pending_requests: Vec::new(),
3725 next_cursor: None,
3726 });
3727 }
3728 if session_id.as_str() != "fixture-session" {
3729 return Err(AgentConnectorError::new(
3730 orchestral_core::agent_connector::AgentConnectorErrorCode::NotFound,
3731 "fixture session not found",
3732 false,
3733 ));
3734 }
3735 Ok(AgentSessionDetail {
3736 summary: Self::summary(),
3737 turns: vec![AgentSessionTurn {
3738 turn_id: AgentSessionTurnId::new("turn-large"),
3739 status: AgentSessionTurnStatus::Completed,
3740 failure: None,
3741 activities: (0..60)
3742 .map(|index| AgentSessionActivity {
3743 activity_id: AgentSessionActivityId::new(format!("activity-{index}")),
3744 kind: AgentSessionActivityKind::Command,
3745 status: AgentSessionActivityStatus::Completed,
3746 title: Some(format!("command-{index}")),
3747 content: vec![Content::text("x".repeat(10_000))],
3748 details: serde_json::Value::Null,
3749 })
3750 .collect(),
3751 }],
3752 pending_requests: Vec::new(),
3753 next_cursor: None,
3754 })
3755 }
3756
3757 async fn create_session(
3758 &self,
3759 request: CreateAgentSessionRequest,
3760 ) -> Result<AgentSessionSummary, AgentConnectorError> {
3761 assert_eq!(request.cwd.as_deref(), Some("/fixture/new"));
3762 assert_eq!(request.title.as_deref(), Some("Created from HTTP"));
3763 assert!(request.extensions.is_empty());
3764 Ok(Self::created_summary())
3765 }
3766
3767 async fn invoke_action(
3768 &self,
3769 request: InvokeAgentSessionActionRequest,
3770 ) -> Result<AgentSessionActionOutcome, AgentConnectorError> {
3771 assert_eq!(request.session_id.as_str(), "fixture-session");
3772 assert_eq!(request.action_id.as_str(), SESSION_RENAME_ACTION);
3773 assert_eq!(request.arguments["name"], "Renamed over HTTP");
3774 let mut summary = Self::summary();
3775 summary.title = Some("Renamed over HTTP".to_owned());
3776 Ok(AgentSessionActionOutcome {
3777 status: AgentSessionActionStatus::Completed,
3778 session: Some(summary),
3779 content: Vec::new(),
3780 details: serde_json::Value::Null,
3781 })
3782 }
3783
3784 async fn resolve_request(
3785 &self,
3786 request: ResolveAgentSessionRequest,
3787 ) -> Result<(), AgentConnectorError> {
3788 self.read_session(&request.session_id).await?;
3789 request.response.validate()?;
3790 Ok(())
3791 }
3792 }
3793
3794 #[async_trait]
3795 impl AgentConnector for ObservableAgentConnector {
3796 fn describe(&self) -> AgentConnectorDescriptor {
3797 StaticAgentConnector.describe()
3798 }
3799
3800 async fn health(&self) -> Result<AgentConnectorHealth, AgentConnectorError> {
3801 StaticAgentConnector.health().await
3802 }
3803
3804 async fn list_sessions(
3805 &self,
3806 query: AgentSessionListQuery,
3807 ) -> Result<AgentSessionPage, AgentConnectorError> {
3808 StaticAgentConnector.list_sessions(query).await
3809 }
3810
3811 async fn read_session(
3812 &self,
3813 session_id: &AgentSessionId,
3814 ) -> Result<AgentSessionDetail, AgentConnectorError> {
3815 StaticAgentConnector.read_session(session_id).await
3816 }
3817
3818 async fn subscribe_session_changes(
3819 &self,
3820 session_id: &AgentSessionId,
3821 ) -> Result<broadcast::Receiver<AgentSessionChange>, AgentConnectorError> {
3822 if session_id.as_str() != "fixture-session" {
3823 return Err(AgentConnectorError::invalid("unexpected fixture session"));
3824 }
3825 self.subscriptions.fetch_add(1, Ordering::SeqCst);
3826 Ok(self.changes.subscribe())
3827 }
3828 }
3829
3830 #[async_trait]
3831 impl GatewayAuthenticator for StaticGatewayAuthenticator {
3832 fn header_name(&self) -> &HeaderName {
3833 &self.header_name
3834 }
3835
3836 async fn authenticate(&self, token: &str) -> Result<GatewayPrincipal, GatewayAuthError> {
3837 if token != "valid-gateway-assertion" {
3838 return Err(GatewayAuthError::Invalid(
3839 "test assertion was rejected".to_owned(),
3840 ));
3841 }
3842 Ok(GatewayPrincipal {
3843 subject: Some("gateway-user".to_owned()),
3844 attributes: BTreeMap::from([("email".to_owned(), "person@example.com".to_owned())]),
3845 })
3846 }
3847 }
3848
3849 struct ApprovalProvider {
3850 descriptor: AgentDescriptorEnvelope,
3851 events: broadcast::Sender<AgentEventDraft>,
3852 approvals: Arc<InMemoryHostApprovalBroker>,
3853 }
3854
3855 impl ApprovalProvider {
3856 fn new(approvals: Arc<InMemoryHostApprovalBroker>) -> Self {
3857 let descriptor = AgentDescriptorEnvelope::seal(AgentDescriptor {
3858 provider_id: AgentProviderId::new("test.remote-approval"),
3859 agent_id: AgentId::new("approval-v1"),
3860 supported_protocol_versions: vec![AGENT_PROTOCOL_V1],
3861 accepted_content_types: BTreeSet::from(["text/plain".to_owned()]),
3862 capabilities: AgentCapabilities {
3863 session_reuse: true,
3864 pending_request_kinds: BTreeSet::from([PendingRequestKind::Approval]),
3865 effect_mediation: EffectMediation::HostMediated,
3866 ..AgentCapabilities::default()
3867 },
3868 extensions: Default::default(),
3869 })
3870 .unwrap();
3871 let (events, _) = broadcast::channel(32);
3872 Self {
3873 descriptor,
3874 events,
3875 approvals,
3876 }
3877 }
3878
3879 fn draft(
3880 run_id: &RunId,
3881 event_id: impl Into<String>,
3882 causation_id: Option<CommandId>,
3883 payload: AgentEvent,
3884 ) -> AgentEventDraft {
3885 AgentEventDraft {
3886 event_id: orchestral_core::agent_protocol::wire::AgentEventId::new(event_id),
3887 run_id: run_id.clone(),
3888 causation_id,
3889 source_fingerprint: None,
3890 payload,
3891 }
3892 }
3893 }
3894
3895 #[async_trait]
3896 impl AgentProvider for ApprovalProvider {
3897 fn describe(&self) -> AgentDescriptorEnvelope {
3898 self.descriptor.clone()
3899 }
3900
3901 async fn start(
3902 &self,
3903 request: orchestral_core::agent_protocol::wire::AgentStartRequest,
3904 ) -> Result<AgentStart, AgentStartError> {
3905 request
3906 .validate_for_descriptor(&self.descriptor)
3907 .map_err(AgentStartError::OutcomeUnknown)?;
3908 let execution = AgentExecutionRef::for_start(&request, &self.descriptor)
3909 .map_err(AgentStartError::OutcomeUnknown)?;
3910 let run_id = execution.run_id.clone();
3911 self.approvals
3912 .stage(
3913 &RequestId::new("approval-request"),
3914 ApprovalBinding {
3915 run_id: run_id.clone(),
3916 call_id: ToolCallId::new("approval-call"),
3917 tool_id: ToolId::new("test/write"),
3918 args_digest: orchestral_core::agent_protocol::wire::Digest::sha256(
3919 "write args",
3920 ),
3921 operation_digest: orchestral_core::agent_protocol::wire::Digest::sha256(
3922 "write operation",
3923 ),
3924 permission_digest: orchestral_core::agent_protocol::wire::Digest::sha256(
3925 "write permission",
3926 ),
3927 requested_capabilities: CapabilityRequest::from_effects(BTreeSet::from([
3928 EffectScope::FilesystemWrite,
3929 ])),
3930 session_approval_scope: None,
3931 policy_digest: orchestral_core::agent_protocol::wire::Digest::sha256(
3932 "test policy",
3933 ),
3934 },
3935 )
3936 .await
3937 .map_err(|error| {
3938 AgentStartError::OutcomeUnknown(AgentProtocolError::new(
3939 AgentProtocolErrorCode::Internal,
3940 error.to_string(),
3941 ))
3942 })?;
3943 let mut receiver = self.events.subscribe();
3944 let _ = self.events.send(Self::draft(
3945 &run_id,
3946 "approval-run-started",
3947 None,
3948 AgentEvent::RunStarted,
3949 ));
3950 let _ = self.events.send(Self::draft(
3951 &run_id,
3952 "approval-request-opened",
3953 None,
3954 AgentEvent::RequestOpened {
3955 request: PendingRequest {
3956 request_id: RequestId::new("approval-request"),
3957 blocking: true,
3958 payload: PendingRequestPayload::Approval {
3959 operation_digest: orchestral_core::agent_protocol::wire::Digest::sha256(
3960 "write operation",
3961 ),
3962 requested_scope: vec!["filesystem_write".to_owned()],
3963 session_approval_scope: None,
3964 reason: "write the requested file".to_owned(),
3965 },
3966 },
3967 },
3968 ));
3969 let stream = async_stream::stream! {
3970 loop {
3971 match receiver.recv().await {
3972 Ok(event) => yield Ok(AgentProviderStreamItem::Event(Box::new(event))),
3973 Err(broadcast::error::RecvError::Lagged(_)) => continue,
3974 Err(broadcast::error::RecvError::Closed) => break,
3975 }
3976 }
3977 }
3978 .boxed();
3979 Ok(AgentStart {
3980 execution,
3981 admission: AgentAdmission {
3982 skipped_optional_bindings: Vec::new(),
3983 },
3984 stream,
3985 })
3986 }
3987
3988 async fn command(
3989 &self,
3990 execution: &AgentExecutionRef,
3991 command: AgentCommandEnvelope,
3992 ) -> Result<ProviderCommandDisposition, AgentProtocolError> {
3993 command.verify_digest()?;
3994 let outcome = match (&command.request_id, &command.payload) {
3995 (Some(request_id), AgentCommand::ResolveRequest { response }) => {
3996 let resolution = response.clone();
3997 self.events
3998 .send(Self::draft(
3999 &execution.run_id,
4000 format!("approval-resolved-{}", command.command_id.as_str()),
4001 Some(command.command_id.clone()),
4002 AgentEvent::RequestResolved {
4003 request_id: request_id.clone(),
4004 resolution_digest: resolution.digest()?,
4005 resolution,
4006 },
4007 ))
4008 .map_err(|_| {
4009 AgentProtocolError::new(
4010 AgentProtocolErrorCode::Internal,
4011 "approval test stream is closed",
4012 )
4013 })?;
4014 ProviderCommandOutcome::Accepted
4015 }
4016 _ => ProviderCommandOutcome::Unsupported {
4017 feature: "command".to_owned(),
4018 },
4019 };
4020 Ok(ProviderCommandDisposition {
4021 command_id: command.command_id,
4022 run_id: command.run_id,
4023 outcome,
4024 duplicate: false,
4025 })
4026 }
4027
4028 async fn recover(
4029 &self,
4030 _request: AgentRecoveryRequest,
4031 ) -> Result<AgentRecovery, AgentProtocolError> {
4032 Err(AgentProtocolError::new(
4033 AgentProtocolErrorCode::Unsupported,
4034 "approval fixture does not recover",
4035 ))
4036 }
4037 }
4038
4039 async fn test_app() -> (Router, String) {
4040 test_app_with_gateway(None).await
4041 }
4042
4043 async fn holding_agent_app() -> (Router, String) {
4044 let (app, token, _) = completable_agent_app().await;
4045 (app, token)
4046 }
4047
4048 async fn completable_agent_app() -> (Router, String, Arc<tokio::sync::Notify>) {
4049 completable_agent_app_with_rejection(None).await
4050 }
4051
4052 async fn completable_agent_app_with_rejection(
4053 rejection: Option<AgentProtocolErrorCode>,
4054 ) -> (Router, String, Arc<tokio::sync::Notify>) {
4055 let finish = Arc::new(tokio::sync::Notify::new());
4056 let factory = ScriptedStatelessFactory::conformant().unwrap();
4057 let descriptor = factory.descriptor();
4058 let scenario = ProviderScenario::standard(&descriptor).unwrap();
4059 let controller = Arc::new(
4060 AgentController::new(
4061 factory.create(scenario, TestProbes::default()),
4062 ProviderBindingRef::new("remote-test"),
4063 )
4064 .unwrap(),
4065 );
4066 let ticket = super::super::state::PairingTicket::issue(60_000).unwrap();
4067 let secret = ticket.secret().to_owned();
4068 let registry = RemoteRegistry::in_memory(Some(ticket));
4069 let claim = registry.claim_pairing(&secret, "Two tabs").await.unwrap();
4070 let approvals =
4071 Arc::new(InMemoryHostApprovalBroker::new(b"0123456789abcdef0123456789abcdef").unwrap());
4072 let external_factory = SessionfulRecoverFactory::new().unwrap();
4073 let external_scenario = ProviderScenario::standard(&external_factory.descriptor()).unwrap();
4074 let external_provider = Arc::new(HoldingProvider {
4075 finish: finish.clone(),
4076 rejection,
4077 inner: external_factory.create(external_scenario, TestProbes::default()),
4078 });
4079 let agent_directory = Arc::new(AgentDirectory::new());
4080 agent_directory
4081 .register(Arc::new(StaticAgentConnector), external_provider)
4082 .await
4083 .unwrap();
4084 (
4085 router(RemoteApiState {
4086 agent: AgentApi::new(controller),
4087 agent_directory,
4088 native_session_defaults: NativeSessionDefaults::default(),
4089 approvals,
4090 registry,
4091 gateway_authenticator: None,
4092 run_supervisors: Arc::default(),
4093 session_coordinators: Arc::default(),
4094 artifact_resolver: None,
4095 artifact_blob_store: None,
4096 }),
4097 claim.token,
4098 finish,
4099 )
4100 }
4101
4102 async fn test_app_with_gateway(
4103 gateway_authenticator: Option<Arc<dyn GatewayAuthenticator>>,
4104 ) -> (Router, String) {
4105 let factory = ScriptedStatelessFactory::conformant().unwrap();
4106 let descriptor = factory.descriptor();
4107 let scenario = ProviderScenario::standard(&descriptor).unwrap();
4108 let provider = factory.create(scenario, TestProbes::default());
4109 let controller = Arc::new(
4110 AgentController::new(provider, ProviderBindingRef::new("remote-test")).unwrap(),
4111 );
4112 let ticket = super::super::state::PairingTicket::issue(60_000).unwrap();
4113 let secret = ticket.secret().to_owned();
4114 let registry = RemoteRegistry::in_memory(Some(ticket));
4115 let claim = registry.claim_pairing(&secret, "Test phone").await.unwrap();
4116 let approvals =
4117 Arc::new(InMemoryHostApprovalBroker::new(b"0123456789abcdef0123456789abcdef").unwrap());
4118 let external_factory = SessionfulRecoverFactory::new().unwrap();
4119 let external_scenario = ProviderScenario::standard(&external_factory.descriptor()).unwrap();
4120 let external_provider = Arc::new(DisconnectFirstProvider {
4121 inner: external_factory.create(external_scenario, TestProbes::default()),
4122 });
4123 let agent_directory = Arc::new(AgentDirectory::new());
4124 agent_directory
4125 .register(Arc::new(StaticAgentConnector), external_provider)
4126 .await
4127 .unwrap();
4128 (
4129 router(RemoteApiState {
4130 agent: AgentApi::new(controller),
4131 agent_directory,
4132 native_session_defaults: NativeSessionDefaults {
4133 cwd: Some("/fixture/workspace".to_owned()),
4134 execution_profile: AgentSessionExecutionProfile {
4135 model: Some("fixture-model".to_owned()),
4136 reasoning_effort: Some("default".to_owned()),
4137 permissions: Default::default(),
4138 },
4139 },
4140 approvals,
4141 registry,
4142 gateway_authenticator,
4143 run_supervisors: Arc::default(),
4144 session_coordinators: Arc::default(),
4145 artifact_resolver: None,
4146 artifact_blob_store: None,
4147 }),
4148 claim.token,
4149 )
4150 }
4151
4152 async fn stale_external_agent_app() -> (Router, String) {
4153 let journal = Arc::new(InMemoryAgentJournalStore::default());
4154 let previous_factory = ScriptedStatelessFactory::conformant().unwrap();
4155 let previous_scenario = ProviderScenario::standard(&previous_factory.descriptor()).unwrap();
4156 let previous = Arc::new(
4157 AgentController::with_journal_store(
4158 previous_factory.create(previous_scenario.clone(), TestProbes::default()),
4159 ProviderBindingRef::new("fixture/previous"),
4160 journal.clone(),
4161 )
4162 .unwrap(),
4163 );
4164 let mut previous_spec = previous_scenario.start_request.run.spec;
4165 previous_spec.session_id = AgentSessionId::new("fixture-session");
4166 previous_spec.run_id = RunId::new("fixture-stale-run");
4167 let previous_execution = previous
4168 .start(AgentRunEnvelope::seal(previous_spec).unwrap())
4169 .await
4170 .unwrap();
4171 previous
4172 .wait_for_terminal(&previous_execution.run_id)
4173 .await
4174 .unwrap();
4175 drop(previous);
4176
4177 let external_factory = SessionfulRecoverFactory::new().unwrap();
4178 let external_scenario = ProviderScenario::standard(&external_factory.descriptor()).unwrap();
4179 let external_provider = Arc::new(DisconnectFirstProvider {
4180 inner: external_factory.create(external_scenario, TestProbes::default()),
4181 });
4182 let agent_directory = Arc::new(AgentDirectory::new());
4183 agent_directory
4184 .register_with_journal(Arc::new(StaticAgentConnector), external_provider, journal)
4185 .await
4186 .unwrap();
4187
4188 let generic_factory = ScriptedStatelessFactory::conformant().unwrap();
4189 let generic_scenario = ProviderScenario::standard(&generic_factory.descriptor()).unwrap();
4190 let generic_controller = Arc::new(
4191 AgentController::new(
4192 generic_factory.create(generic_scenario, TestProbes::default()),
4193 ProviderBindingRef::new("remote-test"),
4194 )
4195 .unwrap(),
4196 );
4197 let ticket = super::super::state::PairingTicket::issue(60_000).unwrap();
4198 let secret = ticket.secret().to_owned();
4199 let registry = RemoteRegistry::in_memory(Some(ticket));
4200 let claim = registry
4201 .claim_pairing(&secret, "Descriptor upgrade phone")
4202 .await
4203 .unwrap();
4204 let approvals =
4205 Arc::new(InMemoryHostApprovalBroker::new(b"0123456789abcdef0123456789abcdef").unwrap());
4206 (
4207 router(RemoteApiState {
4208 agent: AgentApi::new(generic_controller),
4209 agent_directory,
4210 native_session_defaults: NativeSessionDefaults::default(),
4211 approvals,
4212 registry,
4213 gateway_authenticator: None,
4214 run_supervisors: Arc::default(),
4215 session_coordinators: Arc::default(),
4216 artifact_resolver: None,
4217 artifact_blob_store: None,
4218 }),
4219 claim.token,
4220 )
4221 }
4222
4223 async fn approval_app() -> (Router, String) {
4224 let approvals =
4225 Arc::new(InMemoryHostApprovalBroker::new(b"0123456789abcdef0123456789abcdef").unwrap());
4226 let provider = Arc::new(ApprovalProvider::new(approvals.clone()));
4227 let controller = Arc::new(
4228 AgentController::new(provider, ProviderBindingRef::new("remote-test")).unwrap(),
4229 );
4230 let ticket = super::super::state::PairingTicket::issue(60_000).unwrap();
4231 let secret = ticket.secret().to_owned();
4232 let registry = RemoteRegistry::in_memory(Some(ticket));
4233 let claim = registry
4234 .claim_pairing(&secret, "Approval phone")
4235 .await
4236 .unwrap();
4237 (
4238 router(RemoteApiState {
4239 agent: AgentApi::new(controller),
4240 agent_directory: Arc::new(AgentDirectory::new()),
4241 native_session_defaults: NativeSessionDefaults::default(),
4242 approvals,
4243 registry,
4244 gateway_authenticator: None,
4245 run_supervisors: Arc::default(),
4246 session_coordinators: Arc::default(),
4247 artifact_resolver: None,
4248 artifact_blob_store: None,
4249 }),
4250 claim.token,
4251 )
4252 }
4253
4254 async fn unrecoverable_app() -> (Router, String) {
4255 let approvals =
4256 Arc::new(InMemoryHostApprovalBroker::new(b"0123456789abcdef0123456789abcdef").unwrap());
4257 let provider = Arc::new(UnrecoverableDisconnectProvider {
4258 inner: Arc::new(ApprovalProvider::new(approvals.clone())),
4259 });
4260 let controller = Arc::new(
4261 AgentController::new(provider, ProviderBindingRef::new("remote-test")).unwrap(),
4262 );
4263 let ticket = super::super::state::PairingTicket::issue(60_000).unwrap();
4264 let secret = ticket.secret().to_owned();
4265 let registry = RemoteRegistry::in_memory(Some(ticket));
4266 let claim = registry
4267 .claim_pairing(&secret, "Recovery phone")
4268 .await
4269 .unwrap();
4270 (
4271 router(RemoteApiState {
4272 agent: AgentApi::new(controller),
4273 agent_directory: Arc::new(AgentDirectory::new()),
4274 native_session_defaults: NativeSessionDefaults::default(),
4275 approvals,
4276 registry,
4277 gateway_authenticator: None,
4278 run_supervisors: Arc::default(),
4279 session_coordinators: Arc::default(),
4280 artifact_resolver: None,
4281 artifact_blob_store: None,
4282 }),
4283 claim.token,
4284 )
4285 }
4286
4287 fn authorized(method: &str, uri: &str, token: &str, body: serde_json::Value) -> Request<Body> {
4288 Request::builder()
4289 .method(method)
4290 .uri(uri)
4291 .header(header::AUTHORIZATION, format!("Bearer {token}"))
4292 .header(header::CONTENT_TYPE, "application/json")
4293 .body(Body::from(body.to_string()))
4294 .unwrap()
4295 }
4296
4297 #[tokio::test]
4298 async fn multiple_clients_share_one_connector_subscription_and_sequence() {
4299 let subscriptions = Arc::new(AtomicUsize::new(0));
4300 let (changes, _) = broadcast::channel(16);
4301 let connector = Arc::new(ObservableAgentConnector {
4302 subscriptions: subscriptions.clone(),
4303 changes: changes.clone(),
4304 });
4305 let factory = SessionfulRecoverFactory::new().unwrap();
4306 let scenario = ProviderScenario::standard(&factory.descriptor()).unwrap();
4307 let directory = Arc::new(AgentDirectory::new());
4308 directory
4309 .register(connector, factory.create(scenario, TestProbes::default()))
4310 .await
4311 .unwrap();
4312
4313 let connector_id = AgentConnectorId::new("fixture/local");
4314 let session_id = AgentSessionId::new("fixture-session");
4315 let registry = AgentSessionCoordinatorRegistry::default();
4316 let coordinator = registry.get(&connector_id, &session_id);
4317 let first_hub = coordinator
4318 .ensure_hub(directory.clone(), &connector_id, &session_id)
4319 .await
4320 .unwrap();
4321 let second_hub = registry
4322 .get(&connector_id, &session_id)
4323 .ensure_hub(directory, &connector_id, &session_id)
4324 .await
4325 .unwrap();
4326 assert!(Arc::ptr_eq(&first_hub, &second_hub));
4327 assert_eq!(subscriptions.load(Ordering::SeqCst), 1);
4328
4329 let mut phone = first_hub.subscribe(0).live;
4330 let mut desktop = second_hub.subscribe(0).live;
4331 changes
4332 .send(AgentSessionChange {
4333 connector_id: connector_id.clone(),
4334 session_id: session_id.clone(),
4335 sequence: 900,
4336 change: orchestral_core::agent_connector::AgentSessionChangeKind::RefreshRequired {
4337 reason: "fixture-change".to_owned(),
4338 },
4339 })
4340 .unwrap();
4341 let phone_change = tokio::time::timeout(Duration::from_secs(1), phone.recv())
4342 .await
4343 .unwrap()
4344 .unwrap();
4345 let desktop_change = tokio::time::timeout(Duration::from_secs(1), desktop.recv())
4346 .await
4347 .unwrap()
4348 .unwrap();
4349 assert_eq!(phone_change, desktop_change);
4350 assert_eq!(phone_change.sequence, 1);
4351 }
4352
4353 #[tokio::test]
4354 async fn protected_routes_require_a_paired_device() {
4355 let (app, _) = test_app().await;
4356 let response = app
4357 .oneshot(
4358 Request::builder()
4359 .uri("/sessions")
4360 .body(Body::empty())
4361 .unwrap(),
4362 )
4363 .await
4364 .unwrap();
4365 assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
4366 let request_id = response
4367 .headers()
4368 .get(REQUEST_ID_HEADER)
4369 .and_then(|value| value.to_str().ok())
4370 .expect("every API response carries its log correlation id");
4371 uuid::Uuid::parse_str(request_id).expect("request id is a UUID");
4372 assert!(response.extensions().get::<ApiErrorLogCode>().is_some());
4373 }
4374
4375 #[tokio::test]
4376 async fn native_sessions_expose_the_composed_host_execution_profile() {
4377 let (app, token) = test_app().await;
4378 let response = app
4379 .clone()
4380 .oneshot(authorized(
4381 "POST",
4382 "/sessions",
4383 &token,
4384 serde_json::json!({"session_id": "profile-session"}),
4385 ))
4386 .await
4387 .unwrap();
4388 assert_eq!(response.status(), StatusCode::CREATED);
4389 let body = response.into_body().collect().await.unwrap().to_bytes();
4390 let created: SessionView = serde_json::from_slice(&body).unwrap();
4391 assert_eq!(created.cwd.as_deref(), Some("/fixture/workspace"));
4392 assert_eq!(
4393 created.execution_profile,
4394 AgentSessionExecutionProfile {
4395 model: Some("fixture-model".to_owned()),
4396 reasoning_effort: Some("default".to_owned()),
4397 permissions: Default::default(),
4398 }
4399 );
4400
4401 let response = app
4402 .clone()
4403 .oneshot(authorized(
4404 "POST",
4405 "/sessions/profile-session/runs",
4406 &token,
4407 serde_json::json!({
4408 "run_id": "profile-run",
4409 "input": "complete the deterministic fixture"
4410 }),
4411 ))
4412 .await
4413 .unwrap();
4414 assert_eq!(response.status(), StatusCode::CREATED);
4415
4416 let response = app
4417 .oneshot(authorized(
4418 "GET",
4419 "/sessions",
4420 &token,
4421 serde_json::Value::Null,
4422 ))
4423 .await
4424 .unwrap();
4425 assert_eq!(response.status(), StatusCode::OK);
4426 let body = response.into_body().collect().await.unwrap().to_bytes();
4427 let sessions: Vec<SessionView> = serde_json::from_slice(&body).unwrap();
4428 let listed = sessions
4429 .iter()
4430 .find(|session| session.id == "profile-session")
4431 .unwrap();
4432 assert_eq!(listed.cwd.as_deref(), Some("/fixture/workspace"));
4433 assert_eq!(listed.execution_profile, created.execution_profile);
4434 }
4435
4436 #[tokio::test]
4437 async fn rejected_session_submission_restarts_only_after_confirmed_completion() {
4438 for code in [
4439 AgentProtocolErrorCode::InvalidTransition,
4440 AgentProtocolErrorCode::TerminalRun,
4441 AgentProtocolErrorCode::Unsupported,
4442 ] {
4443 let (app, token, finish) =
4444 completable_agent_app_with_rejection(Some(code.clone())).await;
4445 let request = |id: &str| {
4446 authorized(
4447 "POST",
4448 "/agent-runs",
4449 &token,
4450 serde_json::json!({
4451 "connector_id": "fixture/local",
4452 "session_id": "fixture-session",
4453 "run_id": id,
4454 "input": format!("input for {id}")
4455 }),
4456 )
4457 };
4458 let first = app.clone().oneshot(request("first")).await.unwrap();
4459 assert_eq!(first.status(), StatusCode::CREATED);
4460 let rejected = app.clone().oneshot(request("next")).await.unwrap();
4461 assert_eq!(rejected.status(), StatusCode::CONFLICT);
4462 let body = rejected.into_body().collect().await.unwrap().to_bytes();
4463 let error: Value = serde_json::from_slice(&body).unwrap();
4464 assert_eq!(error["message"], "fixture command was not dispatched");
4465
4466 let retry = app.clone().oneshot(request("next")).await.unwrap();
4469 assert_eq!(retry.status(), StatusCode::CONFLICT);
4470 finish.notify_one();
4471 tokio::time::timeout(Duration::from_secs(2), async {
4472 loop {
4473 let response = app
4474 .clone()
4475 .oneshot(authorized(
4476 "GET",
4477 "/runs/first?connector_id=fixture%2Flocal",
4478 &token,
4479 Value::Null,
4480 ))
4481 .await
4482 .unwrap();
4483 let body = response.into_body().collect().await.unwrap().to_bytes();
4484 let view: Value = serde_json::from_slice(&body).unwrap();
4485 if view["state"]["state"] == "terminal" {
4486 break;
4487 }
4488 tokio::task::yield_now().await;
4489 }
4490 })
4491 .await
4492 .expect("fixture must commit its terminal state");
4493
4494 let retry = app.clone().oneshot(request("next")).await.unwrap();
4495 if code == AgentProtocolErrorCode::Unsupported {
4496 assert_eq!(retry.status(), StatusCode::CONFLICT);
4497 continue;
4498 }
4499 assert_eq!(retry.status(), StatusCode::CREATED);
4500 let body = retry.into_body().collect().await.unwrap().to_bytes();
4501 let result: Value = serde_json::from_slice(&body).unwrap();
4502 assert_eq!(result["run_id"], "next");
4503 assert_eq!(result["operation"], "started");
4504 let duplicate = app.clone().oneshot(request("next")).await.unwrap();
4505 assert_eq!(duplicate.status(), StatusCode::OK);
4506 let body = duplicate.into_body().collect().await.unwrap().to_bytes();
4507 let result: Value = serde_json::from_slice(&body).unwrap();
4508 assert_eq!(result["operation"], "replayed");
4509 assert_eq!(result["run_id"], "next");
4510 }
4511 }
4512
4513 #[tokio::test]
4514 async fn concurrent_agent_session_posts_share_one_run_and_retry_one_command() {
4515 let (app, token) = holding_agent_app().await;
4516 let request = |run_id: &str, input: &str| {
4517 authorized(
4518 "POST",
4519 "/agent-runs",
4520 &token,
4521 serde_json::json!({
4522 "connector_id": "fixture/local",
4523 "session_id": "fixture-session",
4524 "run_id": run_id,
4525 "input": input
4526 }),
4527 )
4528 };
4529
4530 let (left, right) = tokio::join!(
4531 app.clone().oneshot(request("tab-a", "message from tab A")),
4532 app.clone().oneshot(request("tab-b", "message from tab B")),
4533 );
4534 let mut responses = Vec::new();
4535 for response in [left.unwrap(), right.unwrap()] {
4536 assert!(matches!(
4537 response.status(),
4538 StatusCode::CREATED | StatusCode::OK
4539 ));
4540 let body = response.into_body().collect().await.unwrap().to_bytes();
4541 responses.push(serde_json::from_slice::<serde_json::Value>(&body).unwrap());
4542 }
4543 let started = responses
4544 .iter()
4545 .find(|response| response["operation"] == "started")
4546 .expect("one tab starts the session Run");
4547 let steered = responses
4548 .iter()
4549 .find(|response| response["operation"] == "steered")
4550 .expect("the other tab steers that same Run");
4551 assert_eq!(started["run_id"], steered["run_id"]);
4552 let active_run_id = started["run_id"].as_str().unwrap();
4553 let steered_operation_id = if active_run_id == "tab-a" {
4554 "tab-b"
4555 } else {
4556 "tab-a"
4557 };
4558
4559 let retry = app
4560 .clone()
4561 .oneshot(request(
4562 steered_operation_id,
4563 if steered_operation_id == "tab-a" {
4564 "message from tab A"
4565 } else {
4566 "message from tab B"
4567 },
4568 ))
4569 .await
4570 .unwrap();
4571 assert_eq!(retry.status(), StatusCode::OK);
4572 let body = retry.into_body().collect().await.unwrap().to_bytes();
4573 let retry: serde_json::Value = serde_json::from_slice(&body).unwrap();
4574 assert_eq!(retry["operation"], "steered");
4575 assert_eq!(retry["run_id"], active_run_id);
4576
4577 let response = app
4578 .oneshot(authorized(
4579 "GET",
4580 &format!("/runs/{active_run_id}/events?connector_id=fixture%2Flocal&after=0"),
4581 &token,
4582 serde_json::Value::Null,
4583 ))
4584 .await
4585 .unwrap();
4586 let body = response.into_body().collect().await.unwrap().to_bytes();
4587 let events: serde_json::Value = serde_json::from_slice(&body).unwrap();
4588 let command_id = format!("agent-submit-{steered_operation_id}");
4589 let command_count = events["records"]
4590 .as_array()
4591 .unwrap()
4592 .iter()
4593 .filter(|record| {
4594 record["event"]["payload"]["type"] == "command_received"
4595 && record["event"]["payload"]["command"]["command_id"] == command_id
4596 })
4597 .count();
4598 assert_eq!(command_count, 1, "network retry must not steer twice");
4599 }
4600
4601 #[tokio::test]
4602 async fn external_agent_session_can_be_discovered_read_and_started_through_http() {
4603 let (app, token) = test_app().await;
4604
4605 let response = app
4606 .clone()
4607 .oneshot(authorized(
4608 "GET",
4609 "/agent-connectors",
4610 &token,
4611 serde_json::Value::Null,
4612 ))
4613 .await
4614 .unwrap();
4615 assert_eq!(response.status(), StatusCode::OK);
4616 let body = response.into_body().collect().await.unwrap().to_bytes();
4617 let connectors: serde_json::Value = serde_json::from_slice(&body).unwrap();
4618 assert_eq!(connectors[0]["connector_id"], "fixture/local");
4619
4620 let response = app
4621 .clone()
4622 .oneshot(authorized(
4623 "GET",
4624 "/agent-sessions?connector_id=fixture%2Flocal&limit=25",
4625 &token,
4626 serde_json::Value::Null,
4627 ))
4628 .await
4629 .unwrap();
4630 assert_eq!(response.status(), StatusCode::OK);
4631 let body = response.into_body().collect().await.unwrap().to_bytes();
4632 let sessions: serde_json::Value = serde_json::from_slice(&body).unwrap();
4633 assert_eq!(sessions["sessions"][0]["session_id"], "fixture-session");
4634
4635 let response = app
4636 .clone()
4637 .oneshot(authorized(
4638 "GET",
4639 "/agent-session?connector_id=fixture%2Flocal&session_id=fixture-session&limit=100",
4640 &token,
4641 serde_json::Value::Null,
4642 ))
4643 .await
4644 .unwrap();
4645 assert_eq!(response.status(), StatusCode::OK);
4646 let session_etag = response
4647 .headers()
4648 .get(header::ETAG)
4649 .unwrap()
4650 .to_str()
4651 .unwrap()
4652 .to_owned();
4653 let body = response.into_body().collect().await.unwrap().to_bytes();
4654 assert!(body.len() < 512 * 1_024);
4655 let session: serde_json::Value = serde_json::from_slice(&body).unwrap();
4656 assert_eq!(session["summary"]["title"], "Existing fixture session");
4657 assert!(session["next_cursor"].is_string());
4658 assert_eq!(
4659 session["turns"][0]["activities"]
4660 .as_array()
4661 .unwrap()
4662 .last()
4663 .unwrap()["activity_id"],
4664 "activity-59"
4665 );
4666
4667 let mut conditional = authorized(
4668 "GET",
4669 "/agent-session?connector_id=fixture%2Flocal&session_id=fixture-session&limit=100",
4670 &token,
4671 serde_json::Value::Null,
4672 );
4673 conditional.headers_mut().insert(
4674 header::IF_NONE_MATCH,
4675 header::HeaderValue::from_str(&session_etag).unwrap(),
4676 );
4677 let response = app.clone().oneshot(conditional).await.unwrap();
4678 assert_eq!(response.status(), StatusCode::NOT_MODIFIED);
4679 assert_eq!(response.headers()[header::ETAG], session_etag);
4680 assert!(response
4681 .into_body()
4682 .collect()
4683 .await
4684 .unwrap()
4685 .to_bytes()
4686 .is_empty());
4687
4688 let response = app
4689 .clone()
4690 .oneshot(authorized(
4691 "POST",
4692 "/agent-sessions",
4693 &token,
4694 serde_json::json!({
4695 "connector_id": "fixture/local",
4696 "cwd": "/fixture/new",
4697 "title": "Created from HTTP"
4698 }),
4699 ))
4700 .await
4701 .unwrap();
4702 assert_eq!(response.status(), StatusCode::CREATED);
4703 let body = response.into_body().collect().await.unwrap().to_bytes();
4704 let created: serde_json::Value = serde_json::from_slice(&body).unwrap();
4705 assert_eq!(created["session_id"], "fixture-created");
4706
4707 let response = app
4708 .clone()
4709 .oneshot(authorized(
4710 "POST",
4711 "/agent-session/actions",
4712 &token,
4713 serde_json::json!({
4714 "connector_id": "fixture/local",
4715 "session_id": "fixture-session",
4716 "action_id": "session.rename",
4717 "arguments": {"name": "Renamed over HTTP"}
4718 }),
4719 ))
4720 .await
4721 .unwrap();
4722 assert_eq!(response.status(), StatusCode::OK);
4723 let body = response.into_body().collect().await.unwrap().to_bytes();
4724 let renamed: serde_json::Value = serde_json::from_slice(&body).unwrap();
4725 assert_eq!(renamed["session"]["title"], "Renamed over HTTP");
4726
4727 let response = app
4728 .clone()
4729 .oneshot(authorized(
4730 "POST",
4731 "/agent-session/actions",
4732 &token,
4733 serde_json::json!({
4734 "connector_id": "fixture/local",
4735 "session_id": "fixture-session",
4736 "action_id": "session.review",
4737 "arguments": {"target": "uncommitted_changes"},
4738 "run_id": "fixture-review-run"
4739 }),
4740 ))
4741 .await
4742 .unwrap();
4743 assert_eq!(response.status(), StatusCode::OK);
4744 let body = response.into_body().collect().await.unwrap().to_bytes();
4745 let review: serde_json::Value = serde_json::from_slice(&body).unwrap();
4746 assert_eq!(review["status"]["state"], "running");
4747 assert_eq!(review["status"]["run_id"], "fixture-review-run");
4748
4749 let response = app
4750 .clone()
4751 .oneshot(authorized(
4752 "POST",
4753 "/agent-session/actions",
4754 &token,
4755 serde_json::json!({
4756 "connector_id": "fixture/local",
4757 "session_id": "fixture-session",
4758 "action_id": "session.undeclared"
4759 }),
4760 ))
4761 .await
4762 .unwrap();
4763 assert_eq!(response.status(), StatusCode::NOT_IMPLEMENTED);
4764
4765 let response = app
4766 .clone()
4767 .oneshot(authorized(
4768 "POST",
4769 "/agent-runs",
4770 &token,
4771 serde_json::json!({
4772 "connector_id": "fixture/local",
4773 "session_id": "fixture-created",
4774 "run_id": "fixture-external-run",
4775 "input": "continue the existing session",
4776 "after_activity_id": "fixture-visible-tail"
4777 }),
4778 ))
4779 .await
4780 .unwrap();
4781 assert_eq!(response.status(), StatusCode::CREATED);
4782 let body = response.into_body().collect().await.unwrap().to_bytes();
4783 let started: serde_json::Value = serde_json::from_slice(&body).unwrap();
4784 assert_eq!(started["connector_id"], "fixture/local");
4785 assert_eq!(started["run_id"], "fixture-external-run");
4786
4787 let response = app
4788 .clone()
4789 .oneshot(authorized(
4790 "GET",
4791 "/agent-session?connector_id=fixture%2Flocal&session_id=fixture-created&limit=100",
4792 &token,
4793 serde_json::Value::Null,
4794 ))
4795 .await
4796 .unwrap();
4797 assert_eq!(response.status(), StatusCode::OK);
4798 let body = response.into_body().collect().await.unwrap().to_bytes();
4799 let refreshed: serde_json::Value = serde_json::from_slice(&body).unwrap();
4800 assert_eq!(
4801 refreshed["controlled_runs"][0]["execution"]["run_id"],
4802 "fixture-external-run"
4803 );
4804 assert!(refreshed["controlled_runs"][0]["created_at_unix_ms"].is_i64());
4805 assert!(refreshed["controlled_runs"][0]["updated_at_unix_ms"].is_i64());
4806 assert_eq!(
4807 refreshed["controlled_runs"][0]["after_activity_id"],
4808 "fixture-visible-tail"
4809 );
4810
4811 let response = app
4812 .clone()
4813 .oneshot(authorized(
4814 "GET",
4815 "/runs/fixture-external-run?connector_id=fixture%2Flocal",
4816 &token,
4817 serde_json::Value::Null,
4818 ))
4819 .await
4820 .unwrap();
4821 assert_eq!(response.status(), StatusCode::OK);
4822 let body = response.into_body().collect().await.unwrap().to_bytes();
4823 let run: serde_json::Value = serde_json::from_slice(&body).unwrap();
4824 assert_eq!(
4825 run["input"][0]["body"]["value"],
4826 "continue the existing session"
4827 );
4828
4829 let response = app
4830 .clone()
4831 .oneshot(authorized(
4832 "GET",
4833 "/runs/fixture-external-run/events?connector_id=fixture%2Flocal&after=0",
4834 &token,
4835 serde_json::Value::Null,
4836 ))
4837 .await
4838 .unwrap();
4839 assert_eq!(response.status(), StatusCode::OK);
4840 let body = response.into_body().collect().await.unwrap().to_bytes();
4841 let events: serde_json::Value = serde_json::from_slice(&body).unwrap();
4842 assert!(events["records"]
4843 .as_array()
4844 .is_some_and(|items| !items.is_empty()));
4845 assert!(events["next"].as_u64().is_some_and(|next| next > 0));
4846
4847 let mut recovered_terminal = false;
4848 for _ in 0..100 {
4849 let response = app
4850 .clone()
4851 .oneshot(authorized(
4852 "GET",
4853 "/runs/fixture-external-run?connector_id=fixture%2Flocal",
4854 &token,
4855 serde_json::Value::Null,
4856 ))
4857 .await
4858 .unwrap();
4859 let body = response.into_body().collect().await.unwrap().to_bytes();
4860 let view: serde_json::Value = serde_json::from_slice(&body).unwrap();
4861 if view["state"]["state"] == "terminal" {
4862 recovered_terminal = true;
4863 break;
4864 }
4865 tokio::task::yield_now().await;
4866 }
4867 assert!(
4868 recovered_terminal,
4869 "the Host supervisor automatically recovers the disconnected stream"
4870 );
4871
4872 let response = app
4873 .clone()
4874 .oneshot(authorized(
4875 "GET",
4876 "/runs/fixture-external-run/events?connector_id=fixture%2Flocal&after=0",
4877 &token,
4878 serde_json::Value::Null,
4879 ))
4880 .await
4881 .unwrap();
4882 let body = response.into_body().collect().await.unwrap().to_bytes();
4883 let events: serde_json::Value = serde_json::from_slice(&body).unwrap();
4884 let event_types = events["records"]
4885 .as_array()
4886 .unwrap()
4887 .iter()
4888 .filter_map(|record| record["event"]["payload"]["type"].as_str())
4889 .collect::<Vec<_>>();
4890 assert!(event_types.contains(&"continuity_lost"));
4891 assert!(event_types.contains(&"continuity_restored"));
4892
4893 let response = app
4894 .clone()
4895 .oneshot(authorized(
4896 "POST",
4897 "/agent-runs",
4898 &token,
4899 serde_json::json!({
4900 "connector_id": "fixture/local",
4901 "session_id": "fixture-created",
4902 "run_id": "fixture-external-run-2",
4903 "input": "continue once more",
4904 "after_activity_id": "fixture-newer-tail"
4905 }),
4906 ))
4907 .await
4908 .unwrap();
4909 assert_eq!(response.status(), StatusCode::CREATED);
4910
4911 let response = app
4912 .oneshot(authorized(
4913 "GET",
4914 "/agent-session?connector_id=fixture%2Flocal&session_id=fixture-created&limit=100",
4915 &token,
4916 serde_json::Value::Null,
4917 ))
4918 .await
4919 .unwrap();
4920 assert_eq!(response.status(), StatusCode::OK);
4921 let body = response.into_body().collect().await.unwrap().to_bytes();
4922 let refreshed: serde_json::Value = serde_json::from_slice(&body).unwrap();
4923 let controlled_run_ids = refreshed["controlled_runs"]
4924 .as_array()
4925 .unwrap()
4926 .iter()
4927 .map(|run| run["execution"]["run_id"].as_str().unwrap())
4928 .collect::<Vec<_>>();
4929 assert_eq!(
4930 controlled_run_ids,
4931 ["fixture-external-run", "fixture-external-run-2"],
4932 "a stale native transcript must not collapse the durable Host suffix to one Run"
4933 );
4934 }
4935
4936 #[tokio::test]
4937 async fn old_connector_contract_does_not_block_session_read_or_new_http_run() {
4938 let (app, token) = stale_external_agent_app().await;
4939
4940 let response = app
4941 .clone()
4942 .oneshot(authorized(
4943 "GET",
4944 "/agent-session?connector_id=fixture%2Flocal&session_id=fixture-session&limit=100",
4945 &token,
4946 serde_json::Value::Null,
4947 ))
4948 .await
4949 .unwrap();
4950 assert_eq!(response.status(), StatusCode::OK);
4951 let body = response.into_body().collect().await.unwrap().to_bytes();
4952 let session: serde_json::Value = serde_json::from_slice(&body).unwrap();
4953 assert_eq!(session["summary"]["session_id"], "fixture-session");
4954 assert!(session["controlled_runs"].as_array().unwrap().is_empty());
4955
4956 let response = app
4957 .clone()
4958 .oneshot(authorized(
4959 "POST",
4960 "/agent-runs",
4961 &token,
4962 serde_json::json!({
4963 "connector_id": "fixture/local",
4964 "session_id": "fixture-session",
4965 "run_id": "fixture-current-run",
4966 "input": "continue after the connector upgrade"
4967 }),
4968 ))
4969 .await
4970 .unwrap();
4971 assert_eq!(response.status(), StatusCode::CREATED);
4972 let body = response.into_body().collect().await.unwrap().to_bytes();
4973 let started: serde_json::Value = serde_json::from_slice(&body).unwrap();
4974 assert_eq!(started["operation"], "started");
4975 assert_eq!(started["run_id"], "fixture-current-run");
4976
4977 let retry = app
4978 .oneshot(authorized(
4979 "POST",
4980 "/agent-runs",
4981 &token,
4982 serde_json::json!({
4983 "connector_id": "fixture/local",
4984 "session_id": "fixture-session",
4985 "run_id": "fixture-current-run",
4986 "input": "continue after the connector upgrade"
4987 }),
4988 ))
4989 .await
4990 .unwrap();
4991 assert_eq!(retry.status(), StatusCode::OK);
4992 let body = retry.into_body().collect().await.unwrap().to_bytes();
4993 let replayed: serde_json::Value = serde_json::from_slice(&body).unwrap();
4994 assert_eq!(replayed["operation"], "replayed");
4995 assert_eq!(replayed["run_id"], "fixture-current-run");
4996 }
4997
4998 #[tokio::test]
4999 async fn gateway_mode_accepts_a_verified_identity_without_a_bearer_token() {
5000 let authenticator = Arc::new(StaticGatewayAuthenticator {
5001 header_name: HeaderName::from_static("x-gateway-jwt"),
5002 });
5003 let (app, _) = test_app_with_gateway(Some(authenticator)).await;
5004
5005 let response = app
5006 .clone()
5007 .oneshot(
5008 Request::builder()
5009 .uri("/me")
5010 .header("x-gateway-jwt", "valid-gateway-assertion")
5011 .body(Body::empty())
5012 .unwrap(),
5013 )
5014 .await
5015 .unwrap();
5016 assert_eq!(response.status(), StatusCode::OK);
5017 let body = response.into_body().collect().await.unwrap().to_bytes();
5018 let me: serde_json::Value = serde_json::from_slice(&body).unwrap();
5019 assert_eq!(me["auth_mode"], "gateway_jwt");
5020 assert_eq!(me["attributes"]["email"], "person@example.com");
5021
5022 for assertion in [None, Some("forged-assertion")] {
5023 let mut request = Request::builder().uri("/sessions");
5024 if let Some(assertion) = assertion {
5025 request = request.header("x-gateway-jwt", assertion);
5026 }
5027 let response = app
5028 .clone()
5029 .oneshot(request.body(Body::empty()).unwrap())
5030 .await
5031 .unwrap();
5032 assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
5033 }
5034 }
5035
5036 #[tokio::test]
5037 async fn commands_return_recovery_pending_while_continuity_is_unknown() {
5038 let (app, token) = unrecoverable_app().await;
5039 let response = app
5040 .clone()
5041 .oneshot(authorized(
5042 "POST",
5043 "/sessions",
5044 &token,
5045 serde_json::json!({"session_id": "recovery-session"}),
5046 ))
5047 .await
5048 .unwrap();
5049 assert_eq!(response.status(), StatusCode::CREATED);
5050
5051 let response = app
5052 .clone()
5053 .oneshot(authorized(
5054 "POST",
5055 "/sessions/recovery-session/runs",
5056 &token,
5057 serde_json::json!({
5058 "run_id": "recovery-run",
5059 "input": "wait for recovery"
5060 }),
5061 ))
5062 .await
5063 .unwrap();
5064 assert_eq!(response.status(), StatusCode::CREATED);
5065
5066 let mut unknown = false;
5067 for _ in 0..50 {
5068 let response = app
5069 .clone()
5070 .oneshot(authorized(
5071 "GET",
5072 "/runs/recovery-run",
5073 &token,
5074 serde_json::Value::Null,
5075 ))
5076 .await
5077 .unwrap();
5078 let body = response.into_body().collect().await.unwrap().to_bytes();
5079 let view: serde_json::Value = serde_json::from_slice(&body).unwrap();
5080 if view["state"]["state"] == "unknown" {
5081 assert_eq!(view["recovery"]["mode"], "manual");
5082 assert_eq!(view["recovery"]["can_start_new_run"], false);
5083 unknown = true;
5084 break;
5085 }
5086 tokio::time::sleep(Duration::from_millis(10)).await;
5087 }
5088 assert!(unknown, "fixture Run did not enter Unknown continuity");
5089
5090 let response = app
5091 .clone()
5092 .oneshot(authorized(
5093 "POST",
5094 "/runs/recovery-run/steer",
5095 &token,
5096 serde_json::json!({
5097 "command_id": "steer-during-recovery",
5098 "text": "do not duplicate this"
5099 }),
5100 ))
5101 .await
5102 .unwrap();
5103 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
5104 let body = response.into_body().collect().await.unwrap().to_bytes();
5105 let error: serde_json::Value = serde_json::from_slice(&body).unwrap();
5106 assert_eq!(error["code"], "run_recovery_pending");
5107 }
5108
5109 #[test]
5110 fn manual_recovery_registry_stops_repeat_supervision() {
5111 let registry = RunSupervisorRegistry::default();
5112 let run_id = RunId::new("manual-run");
5113 let key = RunSupervisorRegistry::key(None, &run_id);
5114
5115 assert!(registry.begin(&key));
5116 registry.finish(&key);
5117 registry.mark_manual(key.clone(), "unsafe model boundary".to_owned());
5118
5119 assert!(!registry.begin(&key));
5120 assert_eq!(
5121 registry.manual_reason(&key).as_deref(),
5122 Some("unsafe model boundary")
5123 );
5124
5125 registry.clear_manual(&key);
5126 assert!(registry.begin(&key));
5127 registry.finish(&key);
5128 }
5129
5130 #[test]
5131 fn supervision_registry_exposes_and_clears_stall_state() {
5132 let registry = RunSupervisorRegistry::default();
5133 let key = RunSupervisorRegistry::key(None, &RunId::new("stalled-run"));
5134
5135 registry.mark_issue(
5136 key.clone(),
5137 "interrupting",
5138 "execution lease expired".to_owned(),
5139 );
5140 let interrupting = registry.issue(&key).expect("issue is visible");
5141 assert_eq!(interrupting.state, "interrupting");
5142 assert_eq!(interrupting.reason, "execution lease expired");
5143
5144 registry.mark_issue(key.clone(), "stalled", "stop did not converge".to_owned());
5145 let stalled = registry.issue(&key).expect("updated issue is visible");
5146 assert_eq!(stalled.state, "stalled");
5147 assert_eq!(
5148 stalled.detected_at_unix_ms,
5149 interrupting.detected_at_unix_ms
5150 );
5151
5152 registry.clear_issue(&key);
5153 assert!(registry.issue(&key).is_none());
5154 }
5155
5156 #[test]
5157 fn control_plane_chatter_does_not_renew_the_execution_lease() {
5158 assert!(!agent_event_is_execution_progress(
5159 &AgentEvent::CommandDispositionRecorded {
5160 command_id: CommandId::new("accepted-but-not-applied"),
5161 outcome: ProviderCommandOutcome::Accepted,
5162 }
5163 ));
5164 assert!(!agent_event_is_execution_progress(
5165 &AgentEvent::StopRequested {
5166 reason: "watchdog".to_owned(),
5167 }
5168 ));
5169 assert!(agent_event_is_execution_progress(&AgentEvent::RunStarted));
5170 assert!(agent_event_is_execution_progress(
5171 &AgentEvent::RequestClosed {
5172 request_id: RequestId::new("native-request"),
5173 reason: "Provider completed the request".to_owned(),
5174 }
5175 ));
5176 }
5177
5178 #[tokio::test]
5179 async fn approval_is_host_signed_and_resolves_the_pending_request() {
5180 let (app, token) = approval_app().await;
5181 let response = app
5182 .clone()
5183 .oneshot(authorized(
5184 "POST",
5185 "/sessions",
5186 &token,
5187 serde_json::json!({"session_id": "approval-session"}),
5188 ))
5189 .await
5190 .unwrap();
5191 assert_eq!(response.status(), StatusCode::CREATED);
5192
5193 let response = app
5194 .clone()
5195 .oneshot(authorized(
5196 "POST",
5197 "/sessions/approval-session/runs",
5198 &token,
5199 serde_json::json!({
5200 "run_id": "approval-run",
5201 "input": "perform the write"
5202 }),
5203 ))
5204 .await
5205 .unwrap();
5206 assert_eq!(response.status(), StatusCode::CREATED);
5207
5208 let mut pending_seen = false;
5209 for _ in 0..50 {
5210 let response = app
5211 .clone()
5212 .oneshot(authorized(
5213 "GET",
5214 "/runs/approval-run",
5215 &token,
5216 serde_json::Value::Null,
5217 ))
5218 .await
5219 .unwrap();
5220 let body = response.into_body().collect().await.unwrap().to_bytes();
5221 let view: serde_json::Value = serde_json::from_slice(&body).unwrap();
5222 if view["pending_requests"]
5223 .as_array()
5224 .is_some_and(|items| items.len() == 1)
5225 {
5226 pending_seen = true;
5227 break;
5228 }
5229 tokio::time::sleep(Duration::from_millis(10)).await;
5230 }
5231 assert!(pending_seen, "approval request did not become visible");
5232
5233 let response = app
5234 .clone()
5235 .oneshot(authorized(
5236 "POST",
5237 "/runs/approval-run/requests/approval-request/approval",
5238 &token,
5239 serde_json::json!({
5240 "command_id": "approve-from-phone",
5241 "decision": "allow_once"
5242 }),
5243 ))
5244 .await
5245 .unwrap();
5246 assert_eq!(response.status(), StatusCode::OK);
5247 let body = response.into_body().collect().await.unwrap().to_bytes();
5248 let ack: serde_json::Value = serde_json::from_slice(&body).unwrap();
5249 assert!(matches!(
5250 ack["state"]["state"].as_str(),
5251 Some("accepted" | "applied")
5252 ));
5253
5254 let mut resolved = None;
5255 for _ in 0..50 {
5256 let response = app
5257 .clone()
5258 .oneshot(authorized(
5259 "GET",
5260 "/runs/approval-run/events?after=0",
5261 &token,
5262 serde_json::Value::Null,
5263 ))
5264 .await
5265 .unwrap();
5266 let body = response.into_body().collect().await.unwrap().to_bytes();
5267 let page: serde_json::Value = serde_json::from_slice(&body).unwrap();
5268 resolved = page["records"]
5269 .as_array()
5270 .and_then(|records| {
5271 records
5272 .iter()
5273 .find(|record| record["event"]["payload"]["type"] == "request_resolved")
5274 })
5275 .cloned();
5276 if resolved.is_some() {
5277 break;
5278 }
5279 tokio::time::sleep(Duration::from_millis(10)).await;
5280 }
5281 let resolved = resolved.expect("approval resolution is durable");
5282 let resolution = &resolved["event"]["payload"]["resolution"];
5283 assert_eq!(resolution["type"], "approval");
5284 assert_eq!(resolution["decision"], "allow");
5285 assert!(resolution["grant_ref"]
5286 .as_str()
5287 .is_some_and(|grant| !grant.is_empty()));
5288 }
5289
5290 #[tokio::test]
5291 async fn revoking_the_current_device_invalidates_its_next_request() {
5292 let (app, token) = test_app().await;
5293 let response = app
5294 .clone()
5295 .oneshot(authorized("GET", "/me", &token, serde_json::Value::Null))
5296 .await
5297 .unwrap();
5298 let body = response.into_body().collect().await.unwrap().to_bytes();
5299 let me: serde_json::Value = serde_json::from_slice(&body).unwrap();
5300 let device_id = me["device_id"].as_str().unwrap();
5301
5302 let response = app
5303 .clone()
5304 .oneshot(authorized(
5305 "DELETE",
5306 &format!("/devices/{device_id}"),
5307 &token,
5308 serde_json::Value::Null,
5309 ))
5310 .await
5311 .unwrap();
5312 assert_eq!(response.status(), StatusCode::NO_CONTENT);
5313
5314 let response = app
5315 .oneshot(authorized(
5316 "GET",
5317 "/sessions",
5318 &token,
5319 serde_json::Value::Null,
5320 ))
5321 .await
5322 .unwrap();
5323 assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
5324 }
5325
5326 #[tokio::test]
5327 async fn session_start_inspect_and_cursor_replay_use_real_agent_api() {
5328 let (app, token) = test_app().await;
5329 let response = app
5330 .clone()
5331 .oneshot(authorized(
5332 "POST",
5333 "/sessions",
5334 &token,
5335 serde_json::json!({"session_id": "mobile-session"}),
5336 ))
5337 .await
5338 .unwrap();
5339 assert_eq!(response.status(), StatusCode::CREATED);
5340
5341 let response = app
5342 .clone()
5343 .oneshot(authorized(
5344 "POST",
5345 "/sessions/mobile-session/runs",
5346 &token,
5347 serde_json::json!({
5348 "run_id": "mobile-run",
5349 "input": "complete the deterministic fixture"
5350 }),
5351 ))
5352 .await
5353 .unwrap();
5354 assert_eq!(response.status(), StatusCode::CREATED);
5355 let bytes = response.into_body().collect().await.unwrap().to_bytes();
5356 let started: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
5357 assert_eq!(
5358 started["view"]["input"][0]["body"]["value"],
5359 "complete the deterministic fixture"
5360 );
5361
5362 tokio::time::sleep(Duration::from_millis(20)).await;
5363 let response = app
5364 .clone()
5365 .oneshot(authorized(
5366 "GET",
5367 "/runs/mobile-run",
5368 &token,
5369 serde_json::Value::Null,
5370 ))
5371 .await
5372 .unwrap();
5373 assert_eq!(response.status(), StatusCode::OK);
5374 let bytes = response.into_body().collect().await.unwrap().to_bytes();
5375 let inspected: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
5376 assert_eq!(
5377 inspected["input"][0]["body"]["value"],
5378 "complete the deterministic fixture"
5379 );
5380
5381 let response = app
5382 .clone()
5383 .oneshot(authorized(
5384 "GET",
5385 "/sessions",
5386 &token,
5387 serde_json::Value::Null,
5388 ))
5389 .await
5390 .unwrap();
5391 assert_eq!(response.status(), StatusCode::OK);
5392 let bytes = response.into_body().collect().await.unwrap().to_bytes();
5393 let sessions: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
5394 assert_eq!(sessions[0]["id"], "mobile-session");
5395 assert_eq!(sessions[0]["run_ids"], serde_json::json!(["mobile-run"]));
5396
5397 let response = app
5398 .clone()
5399 .oneshot(authorized(
5400 "GET",
5401 "/runs/mobile-run/events?after=0",
5402 &token,
5403 serde_json::Value::Null,
5404 ))
5405 .await
5406 .unwrap();
5407 assert_eq!(response.status(), StatusCode::OK);
5408 let bytes = response.into_body().collect().await.unwrap().to_bytes();
5409 let payload: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
5410 let records = payload["records"].as_array().unwrap();
5411 assert!(!records.is_empty());
5412 let next = payload["next"].as_u64().unwrap();
5413
5414 let response = app
5415 .oneshot(authorized(
5416 "GET",
5417 &format!("/runs/mobile-run/events?after={next}"),
5418 &token,
5419 serde_json::Value::Null,
5420 ))
5421 .await
5422 .unwrap();
5423 let bytes = response.into_body().collect().await.unwrap().to_bytes();
5424 let payload: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
5425 assert_eq!(payload["records"], serde_json::json!([]));
5426 }
5427
5428 #[tokio::test]
5429 async fn command_identity_is_forwarded_for_retry_deduplication() {
5430 let (app, token) = test_app().await;
5431 app.clone()
5432 .oneshot(authorized(
5433 "POST",
5434 "/sessions",
5435 &token,
5436 serde_json::json!({"session_id": "mobile-session"}),
5437 ))
5438 .await
5439 .unwrap();
5440 app.clone()
5441 .oneshot(authorized(
5442 "POST",
5443 "/sessions/mobile-session/runs",
5444 &token,
5445 serde_json::json!({"run_id": "mobile-run", "input": "do it"}),
5446 ))
5447 .await
5448 .unwrap();
5449
5450 let command = || {
5451 authorized(
5452 "POST",
5453 "/runs/mobile-run/cancel",
5454 &token,
5455 serde_json::json!({
5456 "command_id": "mobile-command-1",
5457 "reason": "stop from phone"
5458 }),
5459 )
5460 };
5461 let first = app.clone().oneshot(command()).await.unwrap();
5462 let second = app.oneshot(command()).await.unwrap();
5463 assert_eq!(first.status(), StatusCode::OK);
5464 assert_eq!(second.status(), StatusCode::OK);
5465 let body = second.into_body().collect().await.unwrap().to_bytes();
5466 let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
5467 assert_eq!(payload["duplicate"], true);
5468 }
5469
5470 #[tokio::test]
5471 async fn sse_replay_uses_durable_sequence_ids_after_the_requested_cursor() {
5472 let (app, token) = test_app().await;
5473 app.clone()
5474 .oneshot(authorized(
5475 "POST",
5476 "/sessions",
5477 &token,
5478 serde_json::json!({"session_id": "stream-session"}),
5479 ))
5480 .await
5481 .unwrap();
5482 app.clone()
5483 .oneshot(authorized(
5484 "POST",
5485 "/sessions/stream-session/runs",
5486 &token,
5487 serde_json::json!({"run_id": "stream-run", "input": "do it"}),
5488 ))
5489 .await
5490 .unwrap();
5491 tokio::time::sleep(Duration::from_millis(20)).await;
5492
5493 let request = Request::builder()
5494 .uri("/runs/stream-run/stream?after=0")
5495 .header(header::AUTHORIZATION, format!("Bearer {token}"))
5496 .header("last-event-id", "2")
5497 .body(Body::empty())
5498 .unwrap();
5499 let response = app.oneshot(request).await.unwrap();
5500 assert_eq!(response.status(), StatusCode::OK);
5501 assert_eq!(
5502 response.headers()[header::CACHE_CONTROL],
5503 "no-store, private"
5504 );
5505 let request_id = response
5506 .headers()
5507 .get(REQUEST_ID_HEADER)
5508 .and_then(|value| value.to_str().ok())
5509 .expect("SSE response carries its lifecycle correlation id");
5510 uuid::Uuid::parse_str(request_id).expect("request id is a UUID");
5511 let mut stream = response.into_body().into_data_stream();
5512 let frame = tokio::time::timeout(Duration::from_secs(1), stream.next())
5513 .await
5514 .unwrap()
5515 .unwrap()
5516 .unwrap();
5517 let text = String::from_utf8(frame.to_vec()).unwrap();
5518 assert!(text.contains("event: durable"));
5519 assert!(!text.contains("id: 1\n"));
5520 assert!(!text.contains("id: 2\n"));
5521 }
5522
5523 #[tokio::test]
5524 async fn redirected_submission_retry_keeps_its_original_run_after_completion() {
5525 let (app, token, finish) = completable_agent_app().await;
5526 let submit = |id: &str, input: &str, session: &str| {
5527 authorized(
5528 "POST",
5529 "/agent-runs",
5530 &token,
5531 json!({"connector_id": "fixture/local", "session_id": session, "run_id": id, "input": input}),
5532 )
5533 };
5534 assert_eq!(
5535 app.clone()
5536 .oneshot(submit("first", "start", "fixture-session"))
5537 .await
5538 .unwrap()
5539 .status(),
5540 StatusCode::CREATED
5541 );
5542 assert_eq!(
5543 app.clone()
5544 .oneshot(submit("second", "follow up", "fixture-session"))
5545 .await
5546 .unwrap()
5547 .status(),
5548 StatusCode::OK
5549 );
5550 assert_eq!(
5551 app.clone()
5552 .oneshot(submit("second", "changed", "fixture-session"))
5553 .await
5554 .unwrap()
5555 .status(),
5556 StatusCode::CONFLICT
5557 );
5558 assert_eq!(
5559 app.clone()
5560 .oneshot(submit("first", "start", "other-session"))
5561 .await
5562 .unwrap()
5563 .status(),
5564 StatusCode::CONFLICT
5565 );
5566 finish.notify_one();
5567 tokio::time::timeout(Duration::from_secs(2), async {
5568 loop {
5569 let response = app
5570 .clone()
5571 .oneshot(authorized(
5572 "GET",
5573 "/runs/first?connector_id=fixture%2Flocal",
5574 &token,
5575 Value::Null,
5576 ))
5577 .await
5578 .unwrap();
5579 let body = response.into_body().collect().await.unwrap().to_bytes();
5580 let view: Value = serde_json::from_slice(&body).unwrap();
5581 if view["state"]["state"] == "terminal" {
5582 break;
5583 }
5584 tokio::task::yield_now().await;
5585 }
5586 })
5587 .await
5588 .expect("first Run should reach a terminal state");
5589 assert_eq!(
5591 app.clone()
5592 .oneshot(submit("third", "new turn", "fixture-session"))
5593 .await
5594 .unwrap()
5595 .status(),
5596 StatusCode::CREATED
5597 );
5598 let retry = app
5599 .clone()
5600 .oneshot(submit("second", "follow up", "fixture-session"))
5601 .await
5602 .unwrap();
5603 assert_eq!(retry.status(), StatusCode::OK);
5604 let body = retry.into_body().collect().await.unwrap().to_bytes();
5605 let response: Value = serde_json::from_slice(&body).unwrap();
5606 assert_eq!(response["run_id"], "first");
5607 assert_eq!(response["command_id"], "agent-submit-second");
5608 let events = app
5609 .oneshot(authorized(
5610 "GET",
5611 "/runs/third/events?connector_id=fixture%2Flocal&after=0",
5612 &token,
5613 Value::Null,
5614 ))
5615 .await
5616 .unwrap();
5617 let body = events.into_body().collect().await.unwrap().to_bytes();
5618 let events: Value = serde_json::from_slice(&body).unwrap();
5619 assert!(!events["records"]
5620 .as_array()
5621 .unwrap()
5622 .iter()
5623 .any(|record| record["event"]["payload"]["type"] == "command_received"));
5624 }
5625}