1use async_stream::stream;
4use axum::extract::{DefaultBodyLimit, Multipart, Path as AxumPath, Query, State};
5use axum::http::{HeaderMap, HeaderValue, StatusCode, Uri, header};
6use axum::response::sse::{Event, KeepAlive, Sse};
7use axum::response::{IntoResponse, Redirect};
8use axum::routing::{get, post};
9use axum::{Json, Router};
10use base64::Engine;
11use futures::StreamExt;
12use futures::future::join_all;
13use meerkat_contracts::WireRuntimeBinding;
14use meerkat_core::ContentInput;
15use meerkat_core::comms::TrustedPeerDescriptor;
16use meerkat_core::event::agent_event_type;
17use meerkat_mob::MobState;
18use meerkat_mob::ids::{AgentIdentity, MobId};
19use meerkat_mob::launch::MemberLaunchMode;
20use meerkat_mob::runtime::reconcile::MemberFilter;
21use meerkat_mob::{
22 MobBackendKind, MobHandle, MobRuntimeMode, PeerTarget, ProfileName, SpawnMemberSpec,
23};
24
25use crate::mob_handle_runtime::{
26 is_recoverable_lifecycle_cleanup_error, member_entry_to_json,
27 model_capabilities_for_member_entry, model_capabilities_for_role,
28 topology_restore_failed_peer_ids, topology_restore_warning_json,
29};
30use serde_json::{Value, json};
31use std::collections::{BTreeMap, BTreeSet};
32use std::convert::Infallible;
33use std::sync::Arc;
34use std::time::{Duration, Instant};
35
36use crate::access::{
37 ACCESS_ACTIONS, ACTION_AGENT_RESET, ACTION_AGENT_RESPAWN, ACTION_AGENT_RETIRE,
38 ACTION_AGENT_SEND, ACTION_AGENT_SPAWN, ACTION_AGENT_VIEW, ACTION_GATING_DECIDE,
39 ACTION_GATING_VIEW, ACTION_MOB_OBSERVE, ACTION_RUNTIME_ADMIN, AccessController, AccessGroup,
40 AccessResource, AccessRule, AccessView, AgentResourceAttributes,
41};
42use crate::blob_store::{BinaryBlobPayload, BinaryBlobStore, is_valid_blob_id_value};
43use crate::console_aggregator::{
44 ConsoleCursor, ConsoleFrame, ConsoleIdentityRecord, ConsoleLogError, ConsoleLogResult,
45 ConsoleLogStore, ConsoleReplayUnavailable, ConsoleSendError, ConsoleSendRequest,
46 ConsoleTimelineEvent, ConsoleTimelineMode, ConsoleTimelineQuery, ConsoleTimelineWindowPage,
47 ConsoleTimelineWindowQuery, ConsoleVisibility, ConsoleVisibilityPolicy,
48 HideImplicitDelegateMembersConsoleVisibilityPolicy, MobKitConsoleAggregator,
49};
50use crate::contact_directory::ContactDirectory;
51use crate::http_sse::{DEFAULT_KEEP_ALIVE_INTERVAL, KEEP_ALIVE_TEXT};
52use crate::mob_handle_runtime::{
53 MEMBER_STATE_ACTIVE, MEMBER_STATE_RETIRING, MobRuntime, member_status_state_string,
54};
55use crate::rpc::{JSONRPC_VERSION, JsonRpcError, JsonRpcRequest, JsonRpcResponse};
56use crate::runtime::MobkitRuntimeHandle;
57use crate::runtime::{
58 ConsoleAgentLiveSnapshot, ConsoleLiveSnapshot, ConsoleMember, ConsoleModelCapabilities,
59 ConsoleRestJsonRequest, DeliveryHistoryRequest, GatingDecideRequest, GatingDecision,
60 RuntimeDecisionState, extract_bearer_token_from_header,
61 handle_console_rest_json_route_with_snapshot_and_access,
62 resolve_authorized_console_auth_from_token,
63};
64use crate::runtime::{MetadataScope, RuntimeMetadataTable, labels_to_json_value};
65use crate::types::{EventEnvelope, UnifiedEvent};
66use crate::unified_runtime::console_events::ConsoleEventStore;
67use crate::unified_runtime::mob_events::MobEventsStore;
68use crate::unified_runtime::{EventLogStore, EventQuery};
69
70#[derive(Clone)]
71pub struct ConsoleJsonState {
72 pub decisions: RuntimeDecisionState,
73 pub runtime: Option<MobRuntime>,
74 pub module_runtime: Option<std::sync::Arc<tokio::sync::Mutex<MobkitRuntimeHandle>>>,
75 pub contact_directory: Option<ContactDirectory>,
76 pub event_log: Option<std::sync::Arc<dyn EventLogStore>>,
77 pub gateway_peer_keys: Option<crate::auth::peer_keys::GatewayPeerKeys>,
81 pub(crate) identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
82 pub(crate) console_events: Option<ConsoleEventStore>,
83 pub(crate) console_aggregator: Option<MobKitConsoleAggregator>,
84 pub(crate) mob_events: Option<MobEventsStore>,
85 pub(crate) metadata_table: Option<std::sync::Arc<RuntimeMetadataTable>>,
86 pub(crate) visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
87 pub(crate) snapshot_read_model: ConsoleSnapshotReadModel,
88 pub(crate) access: Option<AccessController>,
91}
92
93#[derive(Debug, Clone)]
94struct ConsoleHttpAuthContext {
95 principal: Option<String>,
96 access_view: Option<AccessView>,
98}
99
100#[derive(Clone, Default)]
101pub(crate) struct ConsoleSnapshotReadModel {
102 inner: Arc<tokio::sync::RwLock<ConsoleSnapshotReadModelState>>,
103 refresh_lock: Arc<tokio::sync::Mutex<()>>,
109 primed: Arc<std::sync::atomic::AtomicBool>,
113}
114
115#[derive(Clone, Default)]
116struct ConsoleSnapshotReadModelState {
117 running: Option<bool>,
118 session_id_by_identity: BTreeMap<String, String>,
119 session_owner_by_id: BTreeMap<String, String>,
120 primary_members: Vec<ConsoleMember>,
125 delegate_member_groups: Vec<Vec<ConsoleMember>>,
130}
131
132impl ConsoleSnapshotReadModel {
133 async fn snapshot(&self, runtime: &MobRuntime) -> ConsoleSnapshotReadModelState {
140 if !self.primed.load(std::sync::atomic::Ordering::Acquire) {
141 Box::pin(self.prime_now(runtime)).await;
142 }
143 self.inner.read().await.clone()
144 }
145
146 async fn prime_now(&self, runtime: &MobRuntime) {
166 if self.primed.load(std::sync::atomic::Ordering::Acquire) {
167 return;
168 }
169 let _guard = self.refresh_lock.clone().lock_owned().await;
170 if self.primed.load(std::sync::atomic::Ordering::Acquire) {
171 return;
172 }
173 let refreshed = Box::pin(collect_console_snapshot_read_model(runtime)).await;
174 *self.inner.write().await = refreshed;
175 self.primed
176 .store(true, std::sync::atomic::Ordering::Release);
177 }
181
182 fn refresh_soon(&self, runtime: MobRuntime) {
189 let Ok(runtime_handle) = tokio::runtime::Handle::try_current() else {
190 return;
191 };
192 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
193 return;
194 };
195 let inner = Arc::clone(&self.inner);
196 let primed = Arc::clone(&self.primed);
197 runtime_handle.spawn(async move {
198 let _guard = guard;
199 let refreshed = Box::pin(collect_console_snapshot_read_model(&runtime)).await;
200 *inner.write().await = refreshed;
201 primed.store(true, std::sync::atomic::Ordering::Release);
202 });
206 }
207}
208
209const CONSOLE_FRONTEND_INDEX_HTML: &str = include_str!("../console-dist/index.html");
210const CONSOLE_FRONTEND_APP_JS: &str = include_str!("../console-dist/console-app.js");
211const CONSOLE_FRONTEND_APP_CSS: &str = include_str!("../console-dist/console-app.css");
212const MAX_MULTIPART_IMAGE_BYTES: usize = 25 * 1024 * 1024;
213const MAX_MULTIPART_IMAGES: usize = 4;
214const MAX_MULTIPART_BODY_BYTES: usize =
215 (MAX_MULTIPART_IMAGE_BYTES * MAX_MULTIPART_IMAGES) + 1024 * 1024;
216
217pub fn console_json_router(decisions: RuntimeDecisionState) -> Router {
218 console_json_router_with_state(ConsoleJsonState {
219 decisions,
220 runtime: None,
221 module_runtime: None,
222 contact_directory: None,
223 event_log: None,
224 gateway_peer_keys: None,
225 identity_runtime: None,
226 console_events: None,
227 console_aggregator: None,
228 mob_events: None,
229 metadata_table: None,
230 visibility_policy: Arc::new(HideImplicitDelegateMembersConsoleVisibilityPolicy),
231 snapshot_read_model: ConsoleSnapshotReadModel::default(),
232 access: None,
233 })
234}
235
236pub fn console_json_router_with_aggregator(
237 decisions: RuntimeDecisionState,
238 console_aggregator: MobKitConsoleAggregator,
239) -> Router {
240 console_json_router_with_aggregator_and_access(decisions, console_aggregator, None)
241}
242
243pub fn console_json_router_with_aggregator_and_access(
244 decisions: RuntimeDecisionState,
245 console_aggregator: MobKitConsoleAggregator,
246 access: Option<AccessController>,
247) -> Router {
248 console_json_router_with_state(ConsoleJsonState {
249 decisions,
250 runtime: None,
251 module_runtime: None,
252 contact_directory: None,
253 event_log: None,
254 gateway_peer_keys: None,
255 identity_runtime: None,
256 console_events: None,
257 console_aggregator: Some(console_aggregator),
258 mob_events: None,
259 metadata_table: None,
260 visibility_policy: Arc::new(HideImplicitDelegateMembersConsoleVisibilityPolicy),
261 snapshot_read_model: ConsoleSnapshotReadModel::default(),
262 access,
263 })
264}
265
266pub fn console_json_router_with_runtime(
267 decisions: RuntimeDecisionState,
268 runtime: MobRuntime,
269 contact_directory: Option<ContactDirectory>,
270 event_log: Option<std::sync::Arc<dyn EventLogStore>>,
271) -> Router {
272 console_json_router_with_runtime_and_events(
273 decisions,
274 runtime,
275 None,
276 contact_directory,
277 event_log,
278 None,
279 None,
280 None,
281 None,
282 None,
283 None,
284 )
285}
286
287#[allow(clippy::too_many_arguments)]
288pub(crate) fn console_json_router_with_runtime_and_events(
289 decisions: RuntimeDecisionState,
290 runtime: MobRuntime,
291 module_runtime: Option<std::sync::Arc<tokio::sync::Mutex<MobkitRuntimeHandle>>>,
292 contact_directory: Option<ContactDirectory>,
293 event_log: Option<std::sync::Arc<dyn EventLogStore>>,
294 gateway_peer_keys: Option<crate::auth::peer_keys::GatewayPeerKeys>,
295 console_events: Option<ConsoleEventStore>,
296 console_log_store: Option<std::sync::Arc<dyn ConsoleLogStore>>,
297 mob_events: Option<MobEventsStore>,
298 metadata_table: Option<std::sync::Arc<RuntimeMetadataTable>>,
299 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
300) -> Router {
301 console_json_router_with_runtime_events_and_policy(
302 decisions,
303 runtime,
304 module_runtime,
305 contact_directory,
306 event_log,
307 gateway_peer_keys,
308 console_events,
309 console_log_store,
310 mob_events,
311 metadata_table,
312 identity_runtime,
313 Arc::new(HideImplicitDelegateMembersConsoleVisibilityPolicy),
314 None,
315 )
316}
317
318#[allow(clippy::too_many_arguments)]
319pub(crate) fn console_json_router_with_runtime_events_and_policy(
320 decisions: RuntimeDecisionState,
321 runtime: MobRuntime,
322 module_runtime: Option<std::sync::Arc<tokio::sync::Mutex<MobkitRuntimeHandle>>>,
323 contact_directory: Option<ContactDirectory>,
324 event_log: Option<std::sync::Arc<dyn EventLogStore>>,
325 gateway_peer_keys: Option<crate::auth::peer_keys::GatewayPeerKeys>,
326 console_events: Option<ConsoleEventStore>,
327 console_log_store: Option<std::sync::Arc<dyn ConsoleLogStore>>,
328 mob_events: Option<MobEventsStore>,
329 metadata_table: Option<std::sync::Arc<RuntimeMetadataTable>>,
330 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
331 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
332 access: Option<AccessController>,
333) -> Router {
334 let console_aggregator = console_events.clone().map(|events| {
335 if let Some(store) = console_log_store {
336 let aggregator = MobKitConsoleAggregator::new(store);
337 aggregator.register_runtime_handles_with_policy(
338 "default",
339 "",
340 runtime.clone(),
341 identity_runtime.clone(),
342 events,
343 visibility_policy.clone(),
344 );
345 aggregator
346 } else {
347 let aggregator = MobKitConsoleAggregator::in_memory();
348 aggregator.register_runtime_handles_with_policy(
349 "default",
350 "",
351 runtime.clone(),
352 identity_runtime.clone(),
353 events,
354 visibility_policy.clone(),
355 );
356 aggregator
357 }
358 });
359 let snapshot_read_model = ConsoleSnapshotReadModel::default();
360 snapshot_read_model.refresh_soon(runtime.clone());
361 console_json_router_with_state(ConsoleJsonState {
362 decisions,
363 runtime: Some(runtime),
364 module_runtime,
365 contact_directory,
366 event_log,
367 gateway_peer_keys,
368 identity_runtime,
369 console_events,
370 console_aggregator,
371 mob_events,
372 metadata_table,
373 visibility_policy,
374 snapshot_read_model,
375 access,
376 })
377}
378
379pub fn console_frontend_router() -> Router {
380 Router::new()
381 .route("/", get(|| async { Redirect::temporary("/console") }))
382 .route("/favicon.ico", get(|| async { StatusCode::NO_CONTENT }))
383 .route("/console", get(console_frontend_index_handler))
384 .route("/console/", get(console_frontend_index_handler))
385 .route(
386 "/console/assets/console-app.js",
387 get(console_frontend_app_js_handler),
388 )
389 .route(
390 "/console/assets/console-app.css",
391 get(console_frontend_app_css_handler),
392 )
393}
394
395fn console_json_router_with_state(state: ConsoleJsonState) -> Router {
396 let router = Router::new()
397 .route("/console/experience", get(console_json_handler))
398 .route("/console/modules", get(console_json_handler))
399 .route("/console/identities", get(console_identities_handler))
400 .route("/console/timeline", get(console_timeline_handler))
401 .route(
402 "/console/timeline/stream",
403 get(console_timeline_stream_handler),
404 )
405 .route(
406 "/console/identity/{identity}/stream",
407 get(console_identity_timeline_stream_handler),
408 )
409 .route("/console/send", post(console_send_handler))
410 .route("/console/rpc", post(console_rpc_handler))
411 .route(
412 "/console/rpc/multipart",
413 post(console_rpc_multipart_handler)
414 .layer(DefaultBodyLimit::max(MAX_MULTIPART_BODY_BYTES)),
415 )
416 .route("/blobs/{blob_id}", get(blob_get_handler));
417 router.with_state(state)
418}
419
420pub async fn console_json_handler(
421 State(state): State<ConsoleJsonState>,
422 headers: HeaderMap,
423 uri: Uri,
424) -> impl IntoResponse {
425 let mut path = uri
426 .path_and_query()
427 .map(|path_and_query| path_and_query.as_str().to_string())
428 .unwrap_or_else(|| uri.path().to_string());
429
430 let already_has_token = path
442 .split_once('?')
443 .map(|(_, q)| form_urlencoded::parse(q.as_bytes()).any(|(key, _)| key == "auth_token"))
444 .unwrap_or(false);
445 if !already_has_token
446 && let Some(bearer) = headers
447 .get(header::AUTHORIZATION)
448 .and_then(|v| v.to_str().ok())
449 .and_then(extract_bearer_token_from_header)
450 {
451 let encoded: String = form_urlencoded::byte_serialize(bearer.as_bytes()).collect();
452 let sep = if path.contains('?') { '&' } else { '?' };
453 path = format!("{path}{sep}auth_token={encoded}");
454 }
455
456 let config_module_ids: Vec<String> = state
457 .decisions
458 .modules
459 .iter()
460 .map(|m| m.id.clone())
461 .collect();
462 let live_snapshot = match &state.runtime {
463 Some(runtime) => {
464 state.snapshot_read_model.refresh_soon(runtime.clone());
465 Some(
466 Box::pin(build_live_snapshot(
467 runtime,
468 &config_module_ids,
469 state.console_events.as_ref(),
470 state.visibility_policy.as_ref(),
471 &state.snapshot_read_model,
472 ))
473 .await,
474 )
475 }
476 None => match &state.console_aggregator {
477 Some(aggregator) => Box::pin(build_aggregator_live_snapshot(
478 aggregator,
479 &config_module_ids,
480 ))
481 .await
482 .ok(),
483 None => None,
484 },
485 }
486 .map(|mut snapshot| {
487 apply_console_visibility_policy(&mut snapshot, state.visibility_policy.as_ref());
488 snapshot
489 });
490
491 let response = handle_console_rest_json_route_with_snapshot_and_access(
492 &state.decisions,
493 &ConsoleRestJsonRequest {
494 method: "GET".to_string(),
495 path,
496 auth: None,
497 },
498 live_snapshot.as_ref(),
499 state.access.as_ref(),
500 );
501 let status = StatusCode::from_u16(response.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
502 (status, Json::<Value>(response.body))
503}
504
505pub async fn console_rpc_handler(
506 State(state): State<ConsoleJsonState>,
507 headers: HeaderMap,
508 uri: Uri,
509 Json(request): Json<Value>,
510) -> impl IntoResponse {
511 let parsed_request = match serde_json::from_value::<JsonRpcRequest>(request) {
513 Ok(req) => req,
514 Err(_) => {
515 return (
516 StatusCode::OK,
517 Json::<Value>(serde_json::json!({
518 "jsonrpc": JSONRPC_VERSION,
519 "id": Value::Null,
520 "error": { "code": -32600, "message": "Invalid Request" }
521 })),
522 );
523 }
524 };
525
526 let auth_context = match console_request_auth_context(&state, &headers, &uri) {
532 Some(context) => context,
533 None => {
534 return (
535 StatusCode::UNAUTHORIZED,
536 Json::<Value>(serde_json::json!({
537 "jsonrpc": JSONRPC_VERSION,
538 "id": parsed_request.id.unwrap_or(Value::Null),
539 "error": {
540 "code": -32600,
541 "message": "unauthorized: console rpc requires a valid auth token",
542 }
543 })),
544 );
545 }
546 };
547 let is_authenticated = true;
552 let read_only = state.decisions.console.read_only;
553 let Some(runtime) = &state.runtime else {
554 let response_value = Box::pin(handle_console_aggregator_rpc(
555 state.console_aggregator.clone(),
556 parsed_request,
557 is_authenticated,
558 read_only,
559 state.access.as_ref(),
560 auth_context.access_view.as_ref(),
561 ))
562 .await;
563 return (StatusCode::OK, Json::<Value>(response_value));
564 };
565
566 let response_value = Box::pin(handle_console_runtime_rpc_with_visibility(
567 runtime,
568 state.module_runtime.clone(),
569 state.contact_directory.as_ref(),
570 state.gateway_peer_keys.as_ref(),
571 state.console_events.clone(),
572 state.console_aggregator.clone(),
573 state.identity_runtime.clone(),
574 state.metadata_table.clone(),
575 state.mob_events.clone(),
576 state.visibility_policy.as_ref(),
577 parsed_request,
578 is_authenticated,
579 read_only,
580 auth_context.principal.as_deref(),
581 state.access.as_ref(),
582 auth_context.access_view.as_ref(),
583 ))
584 .await;
585 (StatusCode::OK, Json::<Value>(response_value))
586}
587
588#[derive(Debug, serde::Deserialize)]
589struct ConsoleTimelineHttpQuery {
590 #[serde(default)]
591 identity: Option<String>,
592 #[serde(default)]
593 conversation_id: Option<String>,
594 #[serde(default)]
595 after: Option<String>,
596 #[serde(default)]
597 before: Option<String>,
598 #[serde(default)]
599 mode: Option<ConsoleTimelineMode>,
600 #[serde(default)]
601 limit: Option<usize>,
602}
603
604async fn console_identities_handler(
605 State(state): State<ConsoleJsonState>,
606 headers: HeaderMap,
607 uri: Uri,
608) -> impl IntoResponse {
609 let Some(auth_context) = console_request_auth_context(&state, &headers, &uri) else {
610 return console_json_error(
611 StatusCode::UNAUTHORIZED,
612 "unauthorized",
613 "console identities require a valid auth token",
614 );
615 };
616 let Some(aggregator) = &state.console_aggregator else {
617 return console_json_error(
618 StatusCode::NOT_FOUND,
619 "unavailable",
620 "console aggregator unavailable",
621 );
622 };
623 let aggregator = aggregator.clone();
624 match Box::pin(aggregator.list_identities()).await {
625 Ok(mut identities) => {
626 retain_visible_identity_records(&mut identities, auth_context.access_view.as_ref());
627 (
628 StatusCode::OK,
629 Json::<Value>(json!({ "identities": identities })),
630 )
631 .into_response()
632 }
633 Err(err) => {
634 tracing::warn!(target: "mobkit::console", error = %err, "console identities request failed");
635 console_json_error(
636 StatusCode::INTERNAL_SERVER_ERROR,
637 "internal_error",
638 "console identities unavailable",
639 )
640 }
641 }
642}
643
644async fn console_timeline_handler(
645 State(state): State<ConsoleJsonState>,
646 headers: HeaderMap,
647 uri: Uri,
648 Query(query): Query<ConsoleTimelineHttpQuery>,
649) -> impl IntoResponse {
650 let Some(auth_context) = console_request_auth_context(&state, &headers, &uri) else {
651 return console_json_error(
652 StatusCode::UNAUTHORIZED,
653 "unauthorized",
654 "console timeline requires a valid auth token",
655 );
656 };
657 let Some(aggregator) = &state.console_aggregator else {
658 return console_json_error(
659 StatusCode::NOT_FOUND,
660 "unavailable",
661 "console aggregator unavailable",
662 );
663 };
664 if let Some(runtime) = &state.runtime
668 && let Some(controller) = state.access.as_ref().filter(|c| c.enabled())
669 {
670 prime_access_cache_from_runtime(runtime, controller).await;
671 }
672 let timeline_query = timeline_query_from_http(query, None);
673 match Box::pin(aggregator.query_timeline_windowed(timeline_query)).await {
674 Ok(mut page) => {
675 if let Some(view) = auth_context.access_view.as_ref() {
676 page.frames
677 .retain(|frame| view.can_view_agent(frame.identity.as_str()));
678 }
679 (
680 StatusCode::OK,
681 Json::<Value>(
682 serde_json::to_value(page).unwrap_or_else(|_| json!({ "frames": [] })),
683 ),
684 )
685 .into_response()
686 }
687 Err(err) => {
688 console_json_error(StatusCode::CONFLICT, "replay_unavailable", &err.to_string())
689 }
690 }
691}
692
693async fn console_send_handler(
694 State(state): State<ConsoleJsonState>,
695 headers: HeaderMap,
696 uri: Uri,
697 Json(request): Json<ConsoleSendRequest>,
698) -> impl IntoResponse {
699 let Some(auth_context) = console_request_auth_context(&state, &headers, &uri) else {
700 return console_json_error(
701 StatusCode::UNAUTHORIZED,
702 "unauthorized",
703 "console send requires a valid auth token",
704 );
705 };
706 if state.decisions.console.read_only {
707 return console_json_error(StatusCode::FORBIDDEN, "read_only", "console is read-only");
708 }
709 if let Some(runtime) = &state.runtime
717 && let Some(controller) = state.access.as_ref().filter(|c| c.enabled())
718 {
719 prime_access_cache_from_runtime(runtime, controller).await;
720 }
721 if let Some(view) = auth_context.access_view.as_ref()
722 && view.enforced()
723 && !view.allows_agent(ACTION_AGENT_SEND, request.identity.as_str())
724 {
725 return console_json_error(
726 StatusCode::FORBIDDEN,
727 "access_denied",
728 "you are not allowed to send to this agent",
729 );
730 }
731 let Some(aggregator) = &state.console_aggregator else {
732 return console_json_error(
733 StatusCode::NOT_FOUND,
734 "unavailable",
735 "console aggregator unavailable",
736 );
737 };
738 if let Some(identity_runtime) = &state.identity_runtime {
739 return match Box::pin(console_send_with_identity_first_fallback(
740 aggregator,
741 identity_runtime.clone(),
742 state.runtime.as_ref(),
743 state.console_events.as_ref(),
744 request,
745 ))
746 .await
747 {
748 Ok(accepted) => (
749 StatusCode::OK,
750 Json::<Value>(
751 serde_json::to_value(accepted).unwrap_or_else(|_| json!({ "accepted": true })),
752 ),
753 )
754 .into_response(),
755 Err(err) => console_send_error_response(err),
756 };
757 }
758 match Box::pin(aggregator.send(request)).await {
759 Ok(accepted) => (
760 StatusCode::OK,
761 Json::<Value>(
762 serde_json::to_value(accepted).unwrap_or_else(|_| json!({ "accepted": true })),
763 ),
764 )
765 .into_response(),
766 Err(err) => console_send_error_response(err),
767 }
768}
769
770async fn console_send_with_identity_first_fallback(
771 aggregator: &MobKitConsoleAggregator,
772 identity_runtime: Arc<crate::identity_first::IdentityRuntime>,
773 runtime: Option<&MobRuntime>,
774 console_events: Option<&ConsoleEventStore>,
775 request: ConsoleSendRequest,
776) -> Result<crate::console_aggregator::ConsoleInteractionAccepted, ConsoleSendError> {
777 let member_send_request = request.clone();
778 match Box::pin(console_send_identity_first(
779 aggregator,
780 identity_runtime,
781 runtime,
782 console_events,
783 request,
784 ))
785 .await
786 {
787 Err(ConsoleSendError::UnknownIdentity(_)) => {
788 Box::pin(aggregator.send(member_send_request)).await
789 }
790 result => result,
791 }
792}
793
794async fn console_send_identity_first(
795 aggregator: &MobKitConsoleAggregator,
796 identity_runtime: Arc<crate::identity_first::IdentityRuntime>,
797 runtime: Option<&MobRuntime>,
798 console_events: Option<&ConsoleEventStore>,
799 mut request: ConsoleSendRequest,
800) -> Result<crate::console_aggregator::ConsoleInteractionAccepted, ConsoleSendError> {
801 let requested_identity = request.identity.clone();
802 let parsed_identity = crate::identity_first::AgentIdentity::parse(request.identity.as_str())
803 .map_err(|err| ConsoleSendError::InvalidRequest(format!("invalid identity: {err}")))?;
804 let content: ContentInput = serde_json::from_value(request.content.clone())
805 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
806 if let ContentInput::Text(text) = &content
807 && text.trim().is_empty()
808 {
809 return Err(ConsoleSendError::InvalidContent(
810 "content must be non-empty".to_string(),
811 ));
812 }
813 if let ContentInput::Blocks(blocks) = &content
814 && blocks.is_empty()
815 {
816 return Err(ConsoleSendError::InvalidContent(
817 "content blocks must be non-empty".to_string(),
818 ));
819 }
820 let handling_mode = parse_identity_first_handling_mode(request.handling_mode.as_deref())?;
821
822 let (identity, status) = match identity_runtime.status(&parsed_identity).await {
823 Ok(status) => (parsed_identity, status),
824 Err(original_err) => {
825 let Some(canonical_identity) = Box::pin(resolve_console_send_identity_alias(
826 aggregator,
827 &requested_identity,
828 ))
829 .await
830 else {
831 return Err(identity_runtime_error_to_console_send_error(
832 requested_identity.as_str(),
833 original_err,
834 ));
835 };
836 let identity = crate::identity_first::AgentIdentity::parse(canonical_identity.as_str())
837 .map_err(|err| {
838 ConsoleSendError::InvalidRequest(format!("invalid aliased identity: {err}"))
839 })?;
840 let status = identity_runtime.status(&identity).await.map_err(|_| {
841 identity_runtime_error_to_console_send_error(
842 requested_identity.as_str(),
843 original_err,
844 )
845 })?;
846 request.identity = canonical_identity;
847 (identity, status)
848 }
849 };
850 let session_id = status
851 .session_id
852 .as_ref()
853 .map(std::string::ToString::to_string);
854 let runtime_member_id = status
855 .agent_runtime_id
856 .as_ref()
857 .map(|id| id.as_str().to_string());
858 let accepted = Box::pin(
859 aggregator.reserve_identity_first_interaction(request.clone(), session_id.as_deref()),
860 )
861 .await?;
862
863 if let Some(events) = console_events {
864 events
865 .reserve_interaction_value(
866 identity.as_str(),
867 runtime_member_id.as_deref(),
868 &accepted.interaction_id,
869 &request.origin,
870 request.content.clone(),
871 )
872 .await
873 .map_err(ConsoleSendError::State)?;
874 }
875
876 if let (Some(runtime), Some(events), Some(runtime_member_id)) = (
877 runtime,
878 console_events.cloned(),
879 runtime_member_id.as_deref(),
880 ) {
881 start_identity_first_console_live_projection(
882 runtime,
883 events,
884 identity.as_str(),
885 runtime_member_id,
886 )
887 .await;
888 }
889
890 if handling_mode == meerkat_core::types::HandlingMode::Steer {
891 match identity_runtime
892 .send_with_mode(&identity, &content, handling_mode)
893 .await
894 {
895 Ok(_) => {
896 if let Err(err) = aggregator
897 .mark_steer_interaction_delivered(
898 &accepted.input_frame_id,
899 &accepted.interaction_id,
900 )
901 .await
902 {
903 tracing::warn!(
904 identity = %identity,
905 error = %err,
906 "console identity-first steer was admitted but delivery status projection failed"
907 );
908 }
909 }
910 Err(err) => {
911 let _ = aggregator
912 .mark_interaction_delivery_failed(&accepted.input_frame_id)
913 .await;
914 if let Some(events) = console_events {
915 events
916 .record_lifecycle(
917 identity.as_str(),
918 "interaction_failed",
919 json!({
920 "interaction_id": accepted.interaction_id,
921 "origin": request.origin,
922 "error": err.to_string(),
923 }),
924 )
925 .await;
926 }
927 tracing::warn!(
928 identity = %identity,
929 error = %err,
930 "console identity-first steer was accepted but delivery failed"
931 );
932 return Err(identity_runtime_error_to_console_send_error(
933 identity.as_str(),
934 err,
935 ));
936 }
937 }
938 return Ok(accepted);
939 }
940
941 let dispatch_aggregator = aggregator.clone();
942 let dispatch_events = console_events.cloned();
943 let dispatch_identity = identity.clone();
944 let dispatch_content = content.clone();
945 let dispatch_origin = request.origin.clone();
946 let dispatch_accepted = accepted.clone();
947 tokio::spawn(async move {
948 match identity_runtime
949 .send_with_mode(&dispatch_identity, &dispatch_content, handling_mode)
950 .await
951 {
952 Ok(_) => {
953 if let Err(err) = dispatch_aggregator
954 .mark_interaction_delivered(&dispatch_accepted.input_frame_id)
955 .await
956 {
957 tracing::warn!(
958 identity = %dispatch_identity,
959 error = %err,
960 "console identity-first send was accepted but delivery status projection failed"
961 );
962 }
963 }
964 Err(err) => {
965 let _ = dispatch_aggregator
966 .mark_interaction_delivery_failed(&dispatch_accepted.input_frame_id)
967 .await;
968 if let Some(events) = dispatch_events {
969 events
970 .record_lifecycle(
971 dispatch_identity.as_str(),
972 "interaction_failed",
973 json!({
974 "interaction_id": dispatch_accepted.interaction_id,
975 "origin": dispatch_origin,
976 "error": err.to_string(),
977 }),
978 )
979 .await;
980 }
981 tracing::warn!(
982 identity = %dispatch_identity,
983 error = %err,
984 "console identity-first send was accepted but delivery failed"
985 );
986 }
987 }
988 });
989 Ok(accepted)
990}
991
992async fn start_identity_first_console_live_projection(
993 runtime: &MobRuntime,
994 console_events: ConsoleEventStore,
995 identity: &str,
996 runtime_member_id: &str,
997) {
998 let identity = identity.to_string();
999 let runtime_member_id = runtime_member_id.to_string();
1000 let mut stream = match runtime
1001 .handle()
1002 .subscribe_agent_events(&crate::member_comms_id::mob_member_id(
1003 runtime_member_id.as_str(),
1004 ))
1005 .await
1006 {
1007 Ok(stream) => stream,
1008 Err(err) => {
1009 tracing::warn!(
1010 identity = %identity,
1011 runtime_member_id = %runtime_member_id,
1012 error = %err,
1013 "console identity-first live projection could not subscribe to agent events"
1014 );
1015 return;
1016 }
1017 };
1018
1019 tokio::spawn(async move {
1020 while let Some(envelope) = stream.next().await {
1021 let event_type = agent_event_type(&envelope.payload).to_string();
1022 let unified = EventEnvelope {
1023 event_id: format!("evt-agent-{}", envelope.event_id),
1024 source: "agent".to_string(),
1025 timestamp_ms: envelope.timestamp_ms,
1026 event: UnifiedEvent::Agent {
1027 agent_id: runtime_member_id.clone(),
1028 event_type: event_type.clone(),
1029 payload: Some(crate::mob_handle_runtime::console_agent_event_payload(
1034 &envelope.payload,
1035 )),
1036 },
1037 };
1038 console_events.project_unified_event(&unified).await;
1039 if matches!(event_type.as_str(), "run_completed" | "run_failed") {
1040 break;
1041 }
1042 }
1043 });
1044}
1045
1046fn parse_identity_first_handling_mode(
1047 value: Option<&str>,
1048) -> Result<meerkat_core::types::HandlingMode, ConsoleSendError> {
1049 match value.unwrap_or("queue") {
1050 "queue" => Ok(meerkat_core::types::HandlingMode::Queue),
1051 "steer" => Ok(meerkat_core::types::HandlingMode::Steer),
1052 other => Err(ConsoleSendError::InvalidHandlingMode(other.to_string())),
1053 }
1054}
1055
1056async fn resolve_console_send_identity_alias(
1057 aggregator: &MobKitConsoleAggregator,
1058 requested_identity: &str,
1059) -> Option<String> {
1060 let identities = Box::pin(aggregator.list_identities()).await.ok()?;
1061 identities
1062 .into_iter()
1063 .find(|record| {
1064 record.identity == requested_identity || record.runtime_member_id == requested_identity
1065 })
1066 .map(|record| record.identity)
1067}
1068
1069fn identity_runtime_error_to_console_send_error(
1070 identity: &str,
1071 err: crate::identity_first::IdentityRuntimeError,
1072) -> ConsoleSendError {
1073 match err {
1074 crate::identity_first::IdentityRuntimeError::UnknownIdentity(_) => {
1075 ConsoleSendError::UnknownIdentity(identity.to_string())
1076 }
1077 crate::identity_first::IdentityRuntimeError::NotAddressable(_) => {
1078 ConsoleSendError::NotAddressable(identity.to_string())
1079 }
1080 crate::identity_first::IdentityRuntimeError::InvalidState { .. } => {
1081 ConsoleSendError::Retired(identity.to_string())
1082 }
1083 other => ConsoleSendError::Dispatch(other.to_string()),
1084 }
1085}
1086
1087async fn console_timeline_stream_handler(
1088 State(state): State<ConsoleJsonState>,
1089 headers: HeaderMap,
1090 uri: Uri,
1091 Query(query): Query<ConsoleTimelineHttpQuery>,
1092) -> impl IntoResponse {
1093 let Some(auth_context) = console_request_auth_context(&state, &headers, &uri) else {
1094 return console_json_error(
1095 StatusCode::UNAUTHORIZED,
1096 "unauthorized",
1097 "console timeline stream requires a valid auth token",
1098 );
1099 };
1100 let access_view = auth_context.access_view.filter(AccessView::enforced);
1101 if access_view.is_some()
1108 && let Some(runtime) = &state.runtime
1109 && let Some(controller) = state
1110 .access
1111 .as_ref()
1112 .filter(|controller| controller.enabled())
1113 {
1114 prime_access_cache_from_runtime(runtime, controller).await;
1115 }
1116 let Some(aggregator) = &state.console_aggregator else {
1117 return console_json_error(
1118 StatusCode::NOT_FOUND,
1119 "unavailable",
1120 "console aggregator unavailable",
1121 );
1122 };
1123 let aggregator = aggregator.clone();
1124 let last_event_id = headers
1125 .get("last-event-id")
1126 .and_then(|value| value.to_str().ok())
1127 .map(str::trim)
1128 .filter(|value| !value.is_empty())
1129 .map(ToString::to_string);
1130 let timeline_query = timeline_query_from_http(query, last_event_id);
1131 let mut rx = aggregator.subscribe();
1132 let (snapshot_frames, snapshot_cursor) =
1133 match Box::pin(query_timeline_snapshot(&aggregator, timeline_query.clone())).await {
1134 Ok(snapshot) => snapshot,
1135 Err(_) => {
1136 let latest_cursor = aggregator.latest_cursor().await.ok().flatten();
1137 let requested_cursor = timeline_query
1138 .after
1139 .as_ref()
1140 .map(ToString::to_string)
1141 .unwrap_or_default();
1142 return (
1143 StatusCode::CONFLICT,
1144 Json::<Value>(
1145 serde_json::to_value(ConsoleReplayUnavailable {
1146 error: "replay_unavailable".to_string(),
1147 requested_cursor,
1148 latest_cursor,
1149 })
1150 .unwrap_or_else(|_| json!({ "error": "replay_unavailable" })),
1151 ),
1152 )
1153 .into_response();
1154 }
1155 };
1156 let identity = timeline_query.identity.clone();
1157 let conversation_id = timeline_query.conversation_id.clone();
1158 let snapshot_after = timeline_query.after.clone();
1159 let stream = stream! {
1160 if let Some(event) = sse_event_from_timeline_event(&ConsoleTimelineEvent::SnapshotStarted { after: snapshot_after }) {
1161 yield Ok::<Event, Infallible>(event);
1162 }
1163 let mut latest_cursor = snapshot_cursor;
1164 for frame in snapshot_frames {
1165 latest_cursor = Some(frame.cursor.clone());
1166 if access_view.as_ref().is_some_and(|view| !view.can_view_agent(frame.identity.as_str())) {
1167 continue;
1168 }
1169 if let Some(event) = sse_event_from_timeline_event(&ConsoleTimelineEvent::ConsoleFrame { frame }) {
1170 yield Ok::<Event, Infallible>(event);
1171 }
1172 }
1173 if let Some(event) = sse_event_from_timeline_event(&ConsoleTimelineEvent::SnapshotComplete { cursor: latest_cursor.clone() }) {
1174 yield Ok::<Event, Infallible>(event);
1175 }
1176 loop {
1177 match rx.recv().await {
1178 Ok(event) if timeline_event_matches(&event, identity.as_deref(), conversation_id.as_deref()) => {
1179 if !Box::pin(aggregator.timeline_event_visible(&event)).await {
1180 continue;
1181 }
1182 if let Some(view) = access_view.as_ref()
1183 && let Some(frame_identity) = timeline_event_identity(&event)
1184 && !view.can_view_agent(frame_identity)
1185 {
1186 continue;
1187 }
1188 if let Some(event_cursor) = timeline_event_cursor(&event)
1189 && let Some(current_cursor) = latest_cursor.as_ref()
1190 && !cursor_is_after(event_cursor, current_cursor)
1191 {
1192 continue;
1193 }
1194 if let Some(sse) = sse_event_from_timeline_event(&event) {
1195 if let Some(event_cursor) = timeline_event_cursor(&event) {
1196 latest_cursor = Some(event_cursor.clone());
1197 }
1198 yield Ok::<Event, Infallible>(sse);
1199 }
1200 }
1201 Ok(_) => {}
1202 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => break,
1203 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
1204 }
1205 }
1206 };
1207 Sse::new(stream)
1208 .keep_alive(
1209 KeepAlive::new()
1210 .interval(DEFAULT_KEEP_ALIVE_INTERVAL)
1211 .text(KEEP_ALIVE_TEXT),
1212 )
1213 .into_response()
1214}
1215
1216async fn console_identity_timeline_stream_handler(
1217 State(state): State<ConsoleJsonState>,
1218 headers: HeaderMap,
1219 uri: Uri,
1220 AxumPath(identity): AxumPath<String>,
1221 Query(mut query): Query<ConsoleTimelineHttpQuery>,
1222) -> impl IntoResponse {
1223 query.identity = Some(identity);
1224 Box::pin(console_timeline_stream_handler(
1225 State(state),
1226 headers,
1227 uri,
1228 Query(query),
1229 ))
1230 .await
1231 .into_response()
1232}
1233
1234fn timeline_query_from_http(
1235 query: ConsoleTimelineHttpQuery,
1236 fallback_after: Option<String>,
1237) -> ConsoleTimelineWindowQuery {
1238 let after = fallback_after.or(query.after).map(ConsoleCursor::from);
1239 let before = query.before.map(ConsoleCursor::from);
1240 ConsoleTimelineWindowQuery {
1241 identity: query
1242 .identity
1243 .map(|value| value.trim().to_string())
1244 .filter(|value| !value.is_empty()),
1245 conversation_id: query
1246 .conversation_id
1247 .map(|value| value.trim().to_string())
1248 .filter(|value| !value.is_empty()),
1249 after,
1250 before,
1251 mode: query.mode.unwrap_or_default(),
1252 limit: query.limit.unwrap_or(200),
1253 }
1254}
1255
1256async fn query_timeline_snapshot(
1257 aggregator: &MobKitConsoleAggregator,
1258 mut query: ConsoleTimelineWindowQuery,
1259) -> ConsoleLogResult<(Vec<ConsoleFrame>, Option<ConsoleCursor>)> {
1260 const DEFAULT_SNAPSHOT_LIMIT: usize = 200;
1261 query.limit = if query.limit == 0 {
1262 DEFAULT_SNAPSHOT_LIMIT
1263 } else {
1264 query.limit
1265 };
1266 if query.after.is_none() && query.mode == ConsoleTimelineMode::Since {
1267 query.mode = ConsoleTimelineMode::Recent;
1268 }
1269 let mode = query.mode;
1270 match mode {
1271 ConsoleTimelineMode::Recent => {
1272 let page = Box::pin(aggregator.query_timeline_windowed(query)).await?;
1273 Ok((page.frames, page.latest_cursor.or(page.next_cursor)))
1274 }
1275 ConsoleTimelineMode::Since => {
1276 if let (Some(after), Some(latest)) =
1277 (query.after.as_ref(), aggregator.latest_cursor().await?)
1278 && let (Some(after_seq), Some(latest_seq)) = (after.seq(), latest.seq())
1279 && after_seq > latest_seq
1280 {
1281 return Err(std::io::Error::other(
1282 "timeline replay cursor is beyond the current store frontier",
1283 )
1284 .into());
1285 }
1286 let mut frames = Vec::new();
1287 let mut cursor = query.after.clone();
1288 let mut latest_cursor = None;
1289 loop {
1290 let page = Box::pin(aggregator.query_timeline_windowed(query.clone())).await?;
1291 latest_cursor = page.latest_cursor.clone().or(latest_cursor);
1292 if !page.frames.is_empty() {
1293 cursor = page
1294 .next_cursor
1295 .clone()
1296 .or_else(|| page.frames.last().map(|frame| frame.cursor.clone()));
1297 frames.extend(page.frames);
1298 } else if page.next_cursor.is_some() {
1299 cursor = page.next_cursor.clone();
1300 }
1301 if page.exhausted || page.next_cursor.is_none() {
1302 return Ok((frames, cursor.or(latest_cursor)));
1303 }
1304 if page.next_cursor == query.after {
1305 return Err(
1306 std::io::Error::other("timeline replay made no cursor progress").into(),
1307 );
1308 }
1309 query.after = page.next_cursor;
1310 }
1311 }
1312 }
1313}
1314
1315fn console_json_error(status: StatusCode, error: &str, message: &str) -> axum::response::Response {
1316 (
1317 status,
1318 Json::<Value>(json!({
1319 "error": error,
1320 "message": message,
1321 })),
1322 )
1323 .into_response()
1324}
1325
1326fn is_console_mutating_rpc_method(method: &str) -> bool {
1327 matches!(
1328 method,
1329 "mobkit/retire"
1330 | "mobkit/reset_all"
1331 | "mobkit/console/send"
1332 | "mobkit/blob/upload"
1333 | "mobkit/ensure_member"
1334 | "mobkit/retire_member"
1335 | "mobkit/respawn_member"
1336 | "mobkit/force_cancel_member"
1337 | "mobkit/cancel_flow"
1338 | "mobkit/collect_completed"
1339 | "mobkit/run_flow"
1340 | "mobkit/spawn_helper"
1341 | "mobkit/fork_helper"
1342 | "mobkit/attach_existing_session"
1343 | "mobkit/reconcile_edges"
1344 | "mobkit/cross_mob/wire_local"
1345 | "mobkit/cross_mob/unwire_local"
1346 | "mobkit/respawn"
1347 | "mobkit/reset"
1348 | "mobkit/delete_identity"
1349 | "mobkit/gating/decide"
1350 | "mobkit/mob_labels/set"
1351 | "mobkit/mob_labels/delete"
1352 | "mobkit/run_labels/set"
1353 | "mobkit/run_labels/delete"
1354 | "mobkit/access/set"
1355 | "mobkit/access/enable"
1356 | "mobkit/access/rules/upsert"
1357 | "mobkit/access/rules/delete"
1358 | "mobkit/access/groups/set"
1359 | "mobkit/access/groups/delete"
1360 )
1361}
1362
1363fn console_read_only_rpc_error(response_id: Value) -> Value {
1364 response_value(
1365 response_id,
1366 None,
1367 Some(JsonRpcError {
1368 code: -32010,
1369 message: "console is read-only".to_string(),
1370 data: Some(json!({ "kind": "read_only" })),
1371 }),
1372 )
1373}
1374
1375fn console_send_error_response(err: ConsoleSendError) -> axum::response::Response {
1376 let (status, code) = match &err {
1377 ConsoleSendError::UnknownIdentity(_) => (StatusCode::NOT_FOUND, "unknown_identity"),
1378 ConsoleSendError::AmbiguousIdentity { .. } => {
1379 (StatusCode::CONFLICT, "ambiguous_live_identity_alias")
1380 }
1381 ConsoleSendError::NotAddressable(_) => (StatusCode::CONFLICT, "not_addressable"),
1382 ConsoleSendError::Retired(_) => (StatusCode::CONFLICT, "retired"),
1383 ConsoleSendError::InvalidContent(_)
1384 | ConsoleSendError::InvalidHandlingMode(_)
1385 | ConsoleSendError::InvalidRequest(_) => (StatusCode::BAD_REQUEST, "invalid_request"),
1386 ConsoleSendError::IdempotencyConflict(_) => (StatusCode::CONFLICT, "idempotency_conflict"),
1387 ConsoleSendError::State(_) | ConsoleSendError::Dispatch(_) | ConsoleSendError::Log(_) => {
1388 (StatusCode::INTERNAL_SERVER_ERROR, "internal_error")
1389 }
1390 };
1391 console_json_error(status, code, &console_send_public_message(&err))
1392}
1393
1394fn console_send_rpc_code(err: &ConsoleSendError) -> i64 {
1395 match err {
1396 ConsoleSendError::UnknownIdentity(_) => -32001,
1397 ConsoleSendError::AmbiguousIdentity { .. } => -32602,
1398 ConsoleSendError::NotAddressable(_) => -32002,
1399 ConsoleSendError::InvalidContent(_)
1400 | ConsoleSendError::InvalidHandlingMode(_)
1401 | ConsoleSendError::InvalidRequest(_) => -32602,
1402 ConsoleSendError::IdempotencyConflict(_) => -32009,
1403 ConsoleSendError::Retired(_) => -32004,
1404 ConsoleSendError::State(_) | ConsoleSendError::Dispatch(_) | ConsoleSendError::Log(_) => {
1405 -32000
1406 }
1407 }
1408}
1409
1410fn console_send_rpc_error(response_id: Value, err: ConsoleSendError) -> Value {
1411 response_value(response_id, None, Some(console_send_json_rpc_error(err)))
1412}
1413
1414fn console_send_json_rpc_error(err: ConsoleSendError) -> JsonRpcError {
1415 let code = console_send_rpc_code(&err);
1416 JsonRpcError {
1417 code,
1418 message: console_send_public_message(&err),
1419 data: None,
1420 }
1421}
1422
1423fn console_send_public_message(err: &ConsoleSendError) -> String {
1424 match err {
1425 ConsoleSendError::State(_) | ConsoleSendError::Dispatch(_) | ConsoleSendError::Log(_) => {
1426 tracing::warn!(target: "mobkit::console", error = %err, "console send internal error");
1427 "console send failed".to_string()
1428 }
1429 _ => err.to_string(),
1430 }
1431}
1432
1433fn timeline_event_matches(
1434 event: &ConsoleTimelineEvent,
1435 identity: Option<&str>,
1436 conversation_id: Option<&str>,
1437) -> bool {
1438 let frame = match event {
1439 ConsoleTimelineEvent::ConsoleFrame { frame }
1440 | ConsoleTimelineEvent::FrameUpdated { frame } => frame,
1441 ConsoleTimelineEvent::SnapshotStarted { .. }
1442 | ConsoleTimelineEvent::SnapshotComplete { .. }
1443 | ConsoleTimelineEvent::ReplayUnavailable { .. } => return true,
1444 };
1445 if identity.is_some_and(|value| frame.identity != value) {
1446 return false;
1447 }
1448 if conversation_id.is_some_and(|value| frame.conversation_id.as_deref() != Some(value)) {
1449 return false;
1450 }
1451 true
1452}
1453
1454fn timeline_event_cursor(event: &ConsoleTimelineEvent) -> Option<&ConsoleCursor> {
1455 match event {
1456 ConsoleTimelineEvent::ConsoleFrame { frame }
1457 | ConsoleTimelineEvent::FrameUpdated { frame } => Some(&frame.cursor),
1458 ConsoleTimelineEvent::SnapshotStarted { .. }
1459 | ConsoleTimelineEvent::SnapshotComplete { .. }
1460 | ConsoleTimelineEvent::ReplayUnavailable { .. } => None,
1461 }
1462}
1463
1464fn timeline_event_identity(event: &ConsoleTimelineEvent) -> Option<&str> {
1465 match event {
1466 ConsoleTimelineEvent::ConsoleFrame { frame }
1467 | ConsoleTimelineEvent::FrameUpdated { frame } => Some(frame.identity.as_str()),
1468 ConsoleTimelineEvent::SnapshotStarted { .. }
1469 | ConsoleTimelineEvent::SnapshotComplete { .. }
1470 | ConsoleTimelineEvent::ReplayUnavailable { .. } => None,
1471 }
1472}
1473
1474pub(crate) const ACCESS_DENIED_RPC_CODE: i64 = -32030;
1479
1480fn retain_visible_timeline_frames(page: &mut ConsoleTimelineWindowPage, view: Option<&AccessView>) {
1481 let Some(view) = view.filter(|view| view.enforced()) else {
1482 return;
1483 };
1484 page.frames
1485 .retain(|frame| view.can_view_agent(frame.identity.as_str()));
1486}
1487
1488fn retain_visible_member_rows(rows: &mut Vec<Value>, view: Option<&AccessView>) {
1492 let Some(view) = view.filter(|view| view.enforced()) else {
1493 return;
1494 };
1495 rows.retain(|row| {
1496 let labels: Option<std::collections::BTreeMap<String, String>> = row
1497 .get("labels")
1498 .and_then(|value| serde_json::from_value(value.clone()).ok());
1499 let agent_id = row
1500 .get("agent_identity")
1501 .or_else(|| row.get("member_id"))
1502 .or_else(|| row.get("agent_id"))
1503 .and_then(Value::as_str);
1504 let identity = labels
1505 .as_ref()
1506 .and_then(|labels| labels.get("agent_identity"))
1507 .map(String::as_str)
1508 .or_else(|| row.get("identity").and_then(Value::as_str))
1509 .or(agent_id);
1510 let role = row
1511 .get("role")
1512 .or_else(|| row.get("profile"))
1513 .and_then(Value::as_str);
1514 view.decide(
1515 ACTION_AGENT_VIEW,
1516 &AccessResource {
1517 identity,
1518 agent_id,
1519 role,
1520 labels: labels.as_ref(),
1521 },
1522 )
1523 .is_allow()
1524 });
1525}
1526
1527fn retain_visible_identity_records(
1528 identities: &mut Vec<ConsoleIdentityRecord>,
1529 view: Option<&AccessView>,
1530) {
1531 let Some(view) = view.filter(|view| view.enforced()) else {
1532 return;
1533 };
1534 identities.retain(|record| {
1535 view.decide(
1536 ACTION_AGENT_VIEW,
1537 &AccessResource {
1538 identity: Some(record.identity.as_str()),
1539 agent_id: Some(record.runtime_member_id.as_str()),
1540 role: None,
1541 labels: Some(&record.labels),
1542 },
1543 )
1544 .is_allow()
1545 });
1546}
1547
1548fn console_rpc_access_requirement<'a>(
1553 method: &str,
1554 params: &'a Value,
1555) -> Option<(&'static str, Option<&'a str>)> {
1556 let identity = params.get("identity").and_then(Value::as_str);
1557 let target = identity
1558 .or_else(|| params.get("member_id").and_then(Value::as_str))
1559 .or_else(|| params.get("agent_id").and_then(Value::as_str));
1560 match method {
1561 "mobkit/console/send" => Some((ACTION_AGENT_SEND, identity)),
1562 "mobkit/retire"
1563 | "mobkit/retire_member"
1564 | "mobkit/force_cancel_member"
1565 | "mobkit/delete_identity" => Some((ACTION_AGENT_RETIRE, target)),
1566 "mobkit/respawn" | "mobkit/respawn_member" => Some((ACTION_AGENT_RESPAWN, target)),
1567 "mobkit/reset" | "mobkit/reset_all" => Some((ACTION_AGENT_RESET, target)),
1568 "mobkit/ensure_member"
1569 | "mobkit/spawn_helper"
1570 | "mobkit/fork_helper"
1571 | "mobkit/attach_existing_session"
1572 | "mobkit/run_flow"
1573 | "mobkit/cancel_flow"
1574 | "mobkit/collect_completed" => Some((ACTION_AGENT_SPAWN, target)),
1575 "mobkit/flow_status" | "mobkit/list_flows" | "mobkit/list_runs" => {
1578 Some((ACTION_AGENT_SPAWN, None))
1579 }
1580 "mobkit/gating/decide" => Some((ACTION_GATING_DECIDE, None)),
1581 "mobkit/gating/pending" | "mobkit/gating/audit" => Some((ACTION_GATING_VIEW, None)),
1582 "mobkit/mob_events/query" | "mobkit/mob_events/subscribe" => {
1583 Some((ACTION_MOB_OBSERVE, None))
1584 }
1585 "mobkit/reconcile_edges"
1586 | "mobkit/cross_mob/wire_local"
1587 | "mobkit/cross_mob/unwire_local"
1588 | "mobkit/mob_labels/set"
1589 | "mobkit/mob_labels/delete"
1590 | "mobkit/run_labels/set"
1591 | "mobkit/run_labels/delete"
1592 | "mobkit/routing/routes/list"
1597 | "mobkit/delivery/history"
1598 | "mobkit/cross_mob/directory"
1599 | "mobkit/cross_mob/peer_info"
1600 | "mobkit/mob_labels/get"
1601 | "mobkit/run_labels/get" => Some((ACTION_RUNTIME_ADMIN, None)),
1602 "mobkit/get_member"
1603 | "mobkit/member_status"
1604 | "mobkit/inspect_identity"
1605 | "mobkit/status_identity"
1606 | "mobkit/console/inspect_identity" => Some((ACTION_AGENT_VIEW, target)),
1607 _ => None,
1608 }
1609}
1610
1611fn console_rpc_access_violation(
1613 view: Option<&AccessView>,
1614 method: &str,
1615 params: &Value,
1616) -> Option<JsonRpcError> {
1617 let view = view.filter(|view| view.enforced())?;
1618 let (action, target) = console_rpc_access_requirement(method, params)?;
1619 let allowed = match target {
1620 Some(identity) => view.allows_agent(action, identity),
1621 None => view.allows(action),
1622 };
1623 if allowed {
1624 return None;
1625 }
1626 Some(JsonRpcError {
1627 code: ACCESS_DENIED_RPC_CODE,
1628 message: format!("access denied: {action}"),
1629 data: Some(json!({
1630 "kind": "access_denied",
1631 "action": action,
1632 "resource": target,
1633 })),
1634 })
1635}
1636
1637fn access_denied_rpc_error(response_id: Value, message: impl Into<String>) -> Value {
1638 response_value(
1639 response_id,
1640 None,
1641 Some(JsonRpcError {
1642 code: ACCESS_DENIED_RPC_CODE,
1643 message: message.into(),
1644 data: Some(json!({ "kind": "access_denied" })),
1645 }),
1646 )
1647}
1648
1649fn access_unavailable_rpc_error(response_id: Value) -> Value {
1650 response_value(
1651 response_id,
1652 None,
1653 Some(JsonRpcError {
1654 code: -32004,
1655 message: "access control is not configured on this runtime".to_string(),
1656 data: Some(json!({ "kind": "access_unavailable" })),
1657 }),
1658 )
1659}
1660
1661fn access_config_rpc_error(response_id: Value, err: crate::access::AccessConfigError) -> Value {
1662 response_value(
1663 response_id,
1664 None,
1665 Some(JsonRpcError {
1666 code: -32602,
1667 message: err.to_string(),
1668 data: Some(json!({ "kind": "invalid_access_config" })),
1669 }),
1670 )
1671}
1672
1673fn access_status_result(access: Option<&AccessController>, view: Option<&AccessView>) -> Value {
1674 match (access, view) {
1675 (Some(controller), Some(view)) => {
1676 let (_, revision) = controller.snapshot();
1677 json!({
1678 "available": true,
1679 "enabled": view.enforced(),
1680 "revision": revision,
1681 "subject": view.subject(),
1682 "groups": view.groups().iter().collect::<Vec<_>>(),
1683 "is_admin": view.is_admin(),
1684 "can_administer": view.can_administer(),
1685 "actions": ACCESS_ACTIONS,
1686 })
1687 }
1688 _ => json!({
1689 "available": false,
1690 "enabled": false,
1691 "actions": ACCESS_ACTIONS,
1692 }),
1693 }
1694}
1695
1696fn handle_access_admin_rpc(
1699 access: Option<&AccessController>,
1700 view: Option<&AccessView>,
1701 request: &JsonRpcRequest,
1702) -> Option<Value> {
1703 if !request.method.starts_with("mobkit/access/") {
1704 return None;
1705 }
1706 let response_id = request.id.clone().unwrap_or(Value::Null);
1707 let controller = match request.method.as_str() {
1708 "mobkit/access/status" => {
1709 return Some(response_value(
1710 response_id,
1711 Some(access_status_result(access, view)),
1712 None,
1713 ));
1714 }
1715 _ => {
1716 let Some(controller) = access else {
1717 return Some(access_unavailable_rpc_error(response_id));
1718 };
1719 if !view.is_some_and(AccessView::can_administer) {
1720 return Some(access_denied_rpc_error(
1721 response_id,
1722 "access denied: access.admin",
1723 ));
1724 }
1725 controller
1726 }
1727 };
1728 let result = match request.method.as_str() {
1729 "mobkit/access/get" => {
1730 let (config, revision) = controller.snapshot();
1731 Ok(json!({ "config": &*config, "revision": revision }))
1732 }
1733 "mobkit/access/set" => {
1734 match serde_json::from_value(request.params.get("config").cloned().unwrap_or_default())
1735 {
1736 Ok(config) => controller
1737 .replace_config(config)
1738 .map(|revision| json!({ "revision": revision })),
1739 Err(err) => {
1740 return Some(invalid_params(
1741 response_id,
1742 format!("invalid access config: {err}"),
1743 ));
1744 }
1745 }
1746 }
1747 "mobkit/access/rules/upsert" => {
1748 match serde_json::from_value::<AccessRule>(
1749 request.params.get("rule").cloned().unwrap_or_default(),
1750 ) {
1751 Ok(rule) => controller
1752 .upsert_rule(rule)
1753 .map(|revision| json!({ "revision": revision })),
1754 Err(err) => {
1755 return Some(invalid_params(
1756 response_id,
1757 format!("invalid access rule: {err}"),
1758 ));
1759 }
1760 }
1761 }
1762 "mobkit/access/rules/delete" => {
1763 let Some(rule_id) = request.params.get("id").and_then(Value::as_str) else {
1764 return Some(invalid_params(response_id, "id required"));
1765 };
1766 controller
1767 .delete_rule(rule_id)
1768 .map(|revision| json!({ "revision": revision }))
1769 }
1770 "mobkit/access/groups/set" => {
1771 let Some(name) = request.params.get("name").and_then(Value::as_str) else {
1772 return Some(invalid_params(response_id, "name required"));
1773 };
1774 match serde_json::from_value::<AccessGroup>(
1775 request.params.get("group").cloned().unwrap_or_default(),
1776 ) {
1777 Ok(group) => controller
1778 .set_group(name, group)
1779 .map(|revision| json!({ "revision": revision })),
1780 Err(err) => {
1781 return Some(invalid_params(
1782 response_id,
1783 format!("invalid access group: {err}"),
1784 ));
1785 }
1786 }
1787 }
1788 "mobkit/access/groups/delete" => {
1789 let Some(name) = request.params.get("name").and_then(Value::as_str) else {
1790 return Some(invalid_params(response_id, "name required"));
1791 };
1792 controller
1793 .delete_group(name)
1794 .map(|revision| json!({ "revision": revision }))
1795 }
1796 "mobkit/access/enable" => {
1797 let Some(enabled) = request.params.get("enabled").and_then(Value::as_bool) else {
1798 return Some(invalid_params(response_id, "enabled (bool) required"));
1799 };
1800 controller
1801 .set_enabled(enabled)
1802 .map(|revision| json!({ "revision": revision }))
1803 }
1804 "mobkit/access/preview" => {
1805 let subject = request.params.get("subject").and_then(Value::as_str);
1806 let Some(action) = request.params.get("action").and_then(Value::as_str) else {
1807 return Some(invalid_params(response_id, "action required"));
1808 };
1809 let identity = request.params.get("identity").and_then(Value::as_str);
1810 let preview_view = controller.view_for_subject(subject);
1811 let decision = match identity {
1812 Some(identity) => preview_view.decide_agent(action, identity),
1813 None => preview_view.decide(action, &AccessResource::none()),
1814 };
1815 Ok(json!({
1816 "subject": subject,
1817 "action": action,
1818 "identity": identity,
1819 "allowed": decision.is_allow(),
1820 "reason": decision.reason(),
1821 "groups": preview_view.groups().iter().collect::<Vec<_>>(),
1822 "is_admin": preview_view.is_admin(),
1823 }))
1824 }
1825 _ => {
1826 return Some(response_value(
1827 response_id,
1828 None,
1829 Some(JsonRpcError {
1830 code: -32601,
1831 message: "Method not found".to_string(),
1832 data: None,
1833 }),
1834 ));
1835 }
1836 };
1837 Some(match result {
1838 Ok(value) => response_value(response_id, Some(value), None),
1839 Err(err) => access_config_rpc_error(response_id, err),
1840 })
1841}
1842
1843fn cursor_is_after(candidate: &ConsoleCursor, current: &ConsoleCursor) -> bool {
1844 match (candidate.seq(), current.seq()) {
1845 (Some(candidate), Some(current)) => candidate > current,
1846 _ => candidate > current,
1847 }
1848}
1849
1850fn sse_event_from_timeline_event(event: &ConsoleTimelineEvent) -> Option<Event> {
1851 let (event_name, id) = match event {
1852 ConsoleTimelineEvent::SnapshotStarted { .. } => ("snapshot_started", None),
1853 ConsoleTimelineEvent::ConsoleFrame { frame } => (
1854 if frame.kind == "frame_updated" {
1855 "frame_updated"
1856 } else {
1857 "console_frame"
1858 },
1859 Some(frame.cursor.to_string()),
1860 ),
1861 ConsoleTimelineEvent::FrameUpdated { frame } => {
1862 ("frame_updated", Some(frame.cursor.to_string()))
1863 }
1864 ConsoleTimelineEvent::SnapshotComplete { cursor } => (
1865 "snapshot_complete",
1866 cursor.as_ref().map(ToString::to_string),
1867 ),
1868 ConsoleTimelineEvent::ReplayUnavailable { .. } => ("replay_unavailable", None),
1869 };
1870 let data = match serde_json::to_string(event) {
1871 Ok(value) => value,
1872 Err(_) => return None,
1873 };
1874 let mut sse = Event::default().event(event_name).data(data);
1875 if let Some(id) = id {
1876 sse = sse.id(id);
1877 }
1878 Some(sse)
1879}
1880
1881pub async fn console_rpc_multipart_handler(
1882 State(state): State<ConsoleJsonState>,
1883 headers: HeaderMap,
1884 uri: Uri,
1885 mut multipart: Multipart,
1886) -> impl IntoResponse {
1887 let auth_context = match console_request_auth_context(&state, &headers, &uri) {
1888 Some(context) => context,
1889 None => {
1890 return (
1891 StatusCode::UNAUTHORIZED,
1892 Json::<Value>(serde_json::json!({
1893 "jsonrpc": JSONRPC_VERSION,
1894 "id": Value::Null,
1895 "error": {
1896 "code": -32600,
1897 "message": "unauthorized: console rpc requires a valid auth token",
1898 }
1899 })),
1900 );
1901 }
1902 };
1903
1904 let mut payload: Option<String> = None;
1905 let mut files: std::collections::BTreeMap<String, MultipartImageUpload> =
1906 std::collections::BTreeMap::new();
1907
1908 while let Some(mut field) = match multipart.next_field().await {
1909 Ok(field) => field,
1910 Err(err) => {
1911 return (
1912 StatusCode::BAD_REQUEST,
1913 Json::<Value>(json_rpc_error_value(
1914 Value::Null,
1915 -32602,
1916 format!("invalid multipart body: {err}"),
1917 )),
1918 );
1919 }
1920 } {
1921 let name = field.name().unwrap_or("").to_string();
1922 if name == "payload" {
1923 if payload.is_some() {
1924 return (
1925 StatusCode::BAD_REQUEST,
1926 Json::<Value>(json_rpc_error_value(
1927 Value::Null,
1928 -32602,
1929 "duplicate payload part",
1930 )),
1931 );
1932 }
1933 payload = match field.text().await {
1934 Ok(text) => Some(text),
1935 Err(err) => {
1936 return (
1937 StatusCode::BAD_REQUEST,
1938 Json::<Value>(json_rpc_error_value(
1939 Value::Null,
1940 -32602,
1941 format!("invalid payload part: {err}"),
1942 )),
1943 );
1944 }
1945 };
1946 continue;
1947 }
1948
1949 let Some(upload_id) = name.strip_prefix("file:").filter(|id| !id.is_empty()) else {
1950 return (
1951 StatusCode::BAD_REQUEST,
1952 Json::<Value>(json_rpc_error_value(
1953 Value::Null,
1954 -32602,
1955 format!("unexpected multipart field: {name}"),
1956 )),
1957 );
1958 };
1959 if files.len() >= MAX_MULTIPART_IMAGES {
1960 return (
1961 StatusCode::BAD_REQUEST,
1962 Json::<Value>(json_rpc_error_value(
1963 Value::Null,
1964 -32602,
1965 format!("too many image attachments; max {MAX_MULTIPART_IMAGES}"),
1966 )),
1967 );
1968 }
1969 if files.contains_key(upload_id) {
1970 return (
1971 StatusCode::BAD_REQUEST,
1972 Json::<Value>(json_rpc_error_value(
1973 Value::Null,
1974 -32602,
1975 format!("duplicate file part for upload_id {upload_id}"),
1976 )),
1977 );
1978 }
1979 let media_type = field
1980 .content_type()
1981 .map(str::to_string)
1982 .unwrap_or_else(|| "application/octet-stream".to_string());
1983 if !is_allowed_image_media_type(&media_type) {
1984 return (
1985 StatusCode::BAD_REQUEST,
1986 Json::<Value>(json_rpc_error_value(
1987 Value::Null,
1988 -32602,
1989 format!("unsupported image media type: {media_type}"),
1990 )),
1991 );
1992 }
1993 let mut bytes = bytes::BytesMut::new();
1994 loop {
1995 let chunk = match field.chunk().await {
1996 Ok(chunk) => chunk,
1997 Err(err) => {
1998 return (
1999 StatusCode::BAD_REQUEST,
2000 Json::<Value>(json_rpc_error_value(
2001 Value::Null,
2002 -32602,
2003 format!("invalid file part {upload_id}: {err}"),
2004 )),
2005 );
2006 }
2007 };
2008 let Some(chunk) = chunk else {
2009 break;
2010 };
2011 if bytes.len() + chunk.len() > MAX_MULTIPART_IMAGE_BYTES {
2012 return (
2013 StatusCode::BAD_REQUEST,
2014 Json::<Value>(json_rpc_error_value(
2015 Value::Null,
2016 -32602,
2017 format!("image attachment {upload_id} exceeds 25 MiB"),
2018 )),
2019 );
2020 }
2021 bytes.extend_from_slice(&chunk);
2022 }
2023 files.insert(
2024 upload_id.to_string(),
2025 MultipartImageUpload {
2026 media_type,
2027 bytes: bytes.freeze(),
2028 },
2029 );
2030 }
2031
2032 let payload = match payload {
2033 Some(payload) => payload,
2034 None => {
2035 return (
2036 StatusCode::BAD_REQUEST,
2037 Json::<Value>(json_rpc_error_value(
2038 Value::Null,
2039 -32602,
2040 "payload part required",
2041 )),
2042 );
2043 }
2044 };
2045 let mut parsed_request = match serde_json::from_str::<JsonRpcRequest>(&payload) {
2046 Ok(req) => req,
2047 Err(err) => {
2048 return (
2049 StatusCode::OK,
2050 Json::<Value>(json_rpc_error_value(
2051 Value::Null,
2052 -32600,
2053 format!("Invalid Request: {err}"),
2054 )),
2055 );
2056 }
2057 };
2058 let response_id = parsed_request.id.clone().unwrap_or(Value::Null);
2059 if state.decisions.console.read_only && is_console_mutating_rpc_method(&parsed_request.method) {
2060 return (
2061 StatusCode::OK,
2062 Json::<Value>(console_read_only_rpc_error(response_id)),
2063 );
2064 }
2065 match parsed_request.method.as_str() {
2066 "mobkit/console/send" => {
2067 let Some(aggregator) = &state.console_aggregator else {
2068 return (
2069 StatusCode::OK,
2070 Json::<Value>(invalid_params(
2071 response_id,
2072 "mobkit/console/send multipart requires a console aggregator",
2073 )),
2074 );
2075 };
2076 let Some(identity) = parsed_request
2077 .params
2078 .get("identity")
2079 .and_then(Value::as_str)
2080 else {
2081 return (
2082 StatusCode::OK,
2083 Json::<Value>(invalid_params(response_id, "identity required")),
2084 );
2085 };
2086 if let Some(runtime) = &state.runtime
2094 && let Some(controller) = state.access.as_ref().filter(|c| c.enabled())
2095 {
2096 prime_access_cache_from_runtime(runtime, controller).await;
2097 }
2098 if let Some(violation) = console_rpc_access_violation(
2099 auth_context.access_view.as_ref(),
2100 "mobkit/console/send",
2101 &parsed_request.params,
2102 ) {
2103 return (
2104 StatusCode::OK,
2105 Json::<Value>(response_value(response_id, None, Some(violation))),
2106 );
2107 }
2108 let binary_blob_store =
2109 match Box::pin(aggregator.binary_blob_store_for_identity(identity)).await {
2110 Ok(Some(store)) => store,
2111 Ok(None) => {
2112 return (
2113 StatusCode::OK,
2114 Json::<Value>(invalid_params(
2115 response_id,
2116 "binary blob store unavailable for identity",
2117 )),
2118 );
2119 }
2120 Err(err) => {
2121 return (
2122 StatusCode::OK,
2123 Json::<Value>(console_send_rpc_error(response_id, err)),
2124 );
2125 }
2126 };
2127 if let Err(message) = externalize_image_upload_placeholders(
2128 &mut parsed_request.params,
2129 files,
2130 binary_blob_store,
2131 )
2132 .await
2133 {
2134 return (
2135 StatusCode::OK,
2136 Json::<Value>(invalid_params(response_id, message)),
2137 );
2138 }
2139 }
2140 "mobkit/blob/upload" => {
2141 let Some(runtime) = &state.runtime else {
2142 return (
2143 StatusCode::NOT_FOUND,
2144 Json::<Value>(json_rpc_error_value(
2145 response_id,
2146 -32600,
2147 "mobkit/blob/upload multipart requires a unified runtime",
2148 )),
2149 );
2150 };
2151 let Some(binary_blob_store) = runtime.binary_blob_store() else {
2152 return (
2153 StatusCode::INTERNAL_SERVER_ERROR,
2154 Json::<Value>(json_rpc_error_value(
2155 response_id,
2156 -32000,
2157 "binary blob store unavailable",
2158 )),
2159 );
2160 };
2161 let result = match externalize_single_image_upload(
2162 &parsed_request.params,
2163 files,
2164 binary_blob_store,
2165 )
2166 .await
2167 {
2168 Ok(result) => result,
2169 Err(message) => {
2170 return (
2171 StatusCode::OK,
2172 Json::<Value>(invalid_params(response_id, message)),
2173 );
2174 }
2175 };
2176 return (
2177 StatusCode::OK,
2178 Json::<Value>(response_value(response_id, Some(result), None)),
2179 );
2180 }
2181 _ => {
2182 return (
2183 StatusCode::OK,
2184 Json::<Value>(invalid_params(
2185 response_id,
2186 "multipart RPC supports mobkit/console/send and mobkit/blob/upload only",
2187 )),
2188 );
2189 }
2190 }
2191 let response_value =
2192 if parsed_request.method == "mobkit/console/send" && state.runtime.is_none() {
2193 Box::pin(handle_console_aggregator_rpc(
2194 state.console_aggregator.clone(),
2195 parsed_request,
2196 true,
2197 state.decisions.console.read_only,
2198 state.access.as_ref(),
2199 auth_context.access_view.as_ref(),
2200 ))
2201 .await
2202 } else {
2203 let Some(runtime) = &state.runtime else {
2204 return (
2205 StatusCode::NOT_FOUND,
2206 Json::<Value>(json_rpc_error_value(
2207 response_id,
2208 -32600,
2209 "console rpc multipart requires a unified runtime",
2210 )),
2211 );
2212 };
2213 Box::pin(handle_console_runtime_rpc_with_visibility(
2214 runtime,
2215 state.module_runtime.clone(),
2216 state.contact_directory.as_ref(),
2217 state.gateway_peer_keys.as_ref(),
2218 state.console_events.clone(),
2219 state.console_aggregator.clone(),
2220 state.identity_runtime.clone(),
2221 state.metadata_table.clone(),
2222 state.mob_events.clone(),
2223 state.visibility_policy.as_ref(),
2224 parsed_request,
2225 true,
2226 state.decisions.console.read_only,
2227 auth_context.principal.as_deref(),
2228 state.access.as_ref(),
2229 auth_context.access_view.as_ref(),
2230 ))
2231 .await
2232 };
2233 (StatusCode::OK, Json::<Value>(response_value))
2234}
2235
2236pub async fn blob_get_handler(
2237 State(state): State<ConsoleJsonState>,
2238 headers: HeaderMap,
2239 uri: Uri,
2240 AxumPath(blob_id): AxumPath<String>,
2241) -> impl IntoResponse {
2242 if !console_request_authorized(&state, &headers, &uri) {
2243 return (
2244 StatusCode::UNAUTHORIZED,
2245 Json::<Value>(serde_json::json!({ "error": "unauthorized" })),
2246 )
2247 .into_response();
2248 }
2249 if !is_valid_blob_id_value(&blob_id) {
2250 return (
2251 StatusCode::BAD_REQUEST,
2252 Json::<Value>(serde_json::json!({ "error": "invalid_blob_id" })),
2253 )
2254 .into_response();
2255 }
2256 let blob_id = meerkat_core::BlobId::from(blob_id.as_str());
2257 let mut stores: Vec<std::sync::Arc<dyn BinaryBlobStore>> = Vec::new();
2258 if let Some(runtime) = &state.runtime
2259 && let Some(store) = runtime.binary_blob_store()
2260 {
2261 stores.push(store);
2262 }
2263 if let Some(aggregator) = &state.console_aggregator {
2264 stores.extend(aggregator.binary_blob_stores());
2265 }
2266 if stores.is_empty() {
2267 return (
2268 StatusCode::NOT_FOUND,
2269 Json::<Value>(serde_json::json!({ "error": "blob_store_unavailable" })),
2270 )
2271 .into_response();
2272 }
2273 for store in stores {
2274 match store.get_bytes(&blob_id).await {
2275 Ok(payload) => return blob_payload_response(payload),
2276 Err(meerkat_core::BlobStoreError::NotFound(_)) => continue,
2277 Err(err) => {
2278 return (
2279 StatusCode::INTERNAL_SERVER_ERROR,
2280 Json::<Value>(serde_json::json!({ "error": err.to_string() })),
2281 )
2282 .into_response();
2283 }
2284 }
2285 }
2286 (
2287 StatusCode::NOT_FOUND,
2288 Json::<Value>(serde_json::json!({ "error": "blob_not_found" })),
2289 )
2290 .into_response()
2291}
2292
2293fn blob_payload_response(payload: BinaryBlobPayload) -> axum::response::Response {
2294 let mut response_headers = HeaderMap::new();
2295 let content_type = HeaderValue::from_str(&payload.media_type)
2296 .unwrap_or_else(|_| HeaderValue::from_static("application/octet-stream"));
2297 response_headers.insert(header::CONTENT_TYPE, content_type);
2298 if let Ok(content_length) = HeaderValue::from_str(&payload.size.to_string()) {
2299 response_headers.insert(header::CONTENT_LENGTH, content_length);
2300 }
2301 response_headers.insert(
2302 header::CACHE_CONTROL,
2303 HeaderValue::from_static("private, max-age=31536000, immutable"),
2304 );
2305 (StatusCode::OK, response_headers, payload.data).into_response()
2306}
2307
2308fn console_request_authorized(state: &ConsoleJsonState, headers: &HeaderMap, uri: &Uri) -> bool {
2309 console_request_auth_context(state, headers, uri).is_some()
2310}
2311
2312fn console_request_auth_context(
2313 state: &ConsoleJsonState,
2314 headers: &HeaderMap,
2315 uri: &Uri,
2316) -> Option<ConsoleHttpAuthContext> {
2317 if !state.decisions.console.require_app_auth {
2318 let principal = state.access.as_ref().and_then(|_| {
2321 let token = console_request_token(headers, uri)?;
2322 resolve_authorized_console_auth_from_token(&state.decisions, &token)
2323 .map(|auth| auth.email)
2324 });
2325 return Some(ConsoleHttpAuthContext {
2326 access_view: state
2327 .access
2328 .as_ref()
2329 .map(|controller| controller.view_for_subject(principal.as_deref())),
2330 principal,
2331 });
2332 }
2333 let token = console_request_token(headers, uri)?;
2334 let auth = resolve_authorized_console_auth_from_token(&state.decisions, &token)?;
2335 Some(ConsoleHttpAuthContext {
2336 access_view: state
2337 .access
2338 .as_ref()
2339 .map(|controller| controller.view_for_subject(Some(auth.email.as_str()))),
2340 principal: Some(auth.email),
2341 })
2342}
2343
2344fn console_request_token(headers: &HeaderMap, uri: &Uri) -> Option<String> {
2345 let bearer_token = headers
2346 .get(header::AUTHORIZATION)
2347 .and_then(|v| v.to_str().ok())
2348 .and_then(extract_bearer_token_from_header)
2349 .map(String::from);
2350 let query_token = uri.query().and_then(|q| {
2353 form_urlencoded::parse(q.as_bytes())
2354 .find(|(key, _)| key == "auth_token")
2355 .map(|(_, value)| value.into_owned())
2356 });
2357 bearer_token.or(query_token)
2358}
2359
2360#[derive(Debug)]
2361struct MultipartImageUpload {
2362 media_type: String,
2363 bytes: bytes::Bytes,
2364}
2365
2366fn json_rpc_error_value(id: Value, code: i64, message: impl Into<String>) -> Value {
2367 serde_json::json!({
2368 "jsonrpc": JSONRPC_VERSION,
2369 "id": id,
2370 "error": {
2371 "code": code,
2372 "message": message.into(),
2373 }
2374 })
2375}
2376
2377fn is_allowed_image_media_type(media_type: &str) -> bool {
2378 matches!(
2379 media_type,
2380 "image/png" | "image/jpeg" | "image/webp" | "image/gif"
2381 )
2382}
2383
2384fn image_upload_part_name<'a>(
2385 object: &'a serde_json::Map<String, Value>,
2386 context: &str,
2387) -> Result<&'a str, String> {
2388 object
2389 .get("upload_id")
2390 .or_else(|| object.get("part_name"))
2391 .and_then(Value::as_str)
2392 .map(str::trim)
2393 .filter(|value| !value.is_empty())
2394 .ok_or_else(|| format!("{context}.upload_id or {context}.part_name is required"))
2395}
2396
2397async fn externalize_image_upload_placeholders(
2398 params: &mut Value,
2399 files: std::collections::BTreeMap<String, MultipartImageUpload>,
2400 blob_store: std::sync::Arc<dyn crate::blob_store::BinaryBlobStore>,
2401) -> Result<(), String> {
2402 let Some(content) = params.get_mut("content") else {
2403 return Err("multipart payload params.content is required".to_string());
2404 };
2405 let mut placeholders = std::collections::BTreeMap::<String, String>::new();
2406 collect_image_upload_placeholders(content, &mut placeholders)?;
2407 if placeholders.is_empty() {
2408 return Err(
2409 "multipart payload must contain at least one image_upload placeholder".to_string(),
2410 );
2411 }
2412 if placeholders.len() > MAX_MULTIPART_IMAGES {
2413 return Err(format!(
2414 "too many image_upload placeholders; max {MAX_MULTIPART_IMAGES}"
2415 ));
2416 }
2417 for upload_id in files.keys() {
2418 if !placeholders.contains_key(upload_id) {
2419 return Err(format!(
2420 "file part has no matching image_upload placeholder: {upload_id}"
2421 ));
2422 }
2423 }
2424 for upload_id in placeholders.keys() {
2425 if !files.contains_key(upload_id) {
2426 return Err(format!(
2427 "image_upload placeholder missing file part: {upload_id}"
2428 ));
2429 }
2430 }
2431
2432 let mut refs = std::collections::BTreeMap::<String, Value>::new();
2433 for (upload_id, file) in files {
2434 let declared_media_type = placeholders
2435 .get(&upload_id)
2436 .cloned()
2437 .unwrap_or_else(|| file.media_type.clone());
2438 if !is_allowed_image_media_type(&declared_media_type) {
2439 return Err(format!(
2440 "unsupported image media type in placeholder {upload_id}: {declared_media_type}"
2441 ));
2442 }
2443 if declared_media_type != file.media_type {
2444 return Err(format!(
2445 "media type mismatch for {upload_id}: placeholder {declared_media_type}, file {}",
2446 file.media_type
2447 ));
2448 }
2449 let blob_ref = blob_store
2450 .put_bytes(&file.media_type, file.bytes)
2451 .await
2452 .map_err(|err| format!("failed to store image {upload_id}: {err}"))?;
2453 refs.insert(
2454 upload_id,
2455 serde_json::json!({
2456 "type": "image",
2457 "media_type": blob_ref.media_type,
2458 "source": "blob",
2459 "blob_id": blob_ref.blob_id,
2460 }),
2461 );
2462 }
2463 replace_image_upload_placeholders(content, &refs)?;
2464 if let Some(object) = params.as_object_mut() {
2465 object.remove("message");
2466 }
2467 Ok(())
2468}
2469
2470async fn externalize_single_image_upload(
2471 params: &Value,
2472 files: std::collections::BTreeMap<String, MultipartImageUpload>,
2473 blob_store: std::sync::Arc<dyn crate::blob_store::BinaryBlobStore>,
2474) -> Result<Value, String> {
2475 let upload = params.get("upload").unwrap_or(params);
2476 if upload
2477 .get("type")
2478 .and_then(Value::as_str)
2479 .is_some_and(|kind| kind != "image_upload")
2480 {
2481 return Err("upload.type must be image_upload".to_string());
2482 }
2483 let upload_object = upload
2484 .as_object()
2485 .ok_or_else(|| "upload must be an object".to_string())?;
2486 let upload_id = image_upload_part_name(upload_object, "upload")?;
2487 let Some(file) = files.get(upload_id) else {
2488 return Err(format!(
2489 "image_upload placeholder missing file part: {upload_id}"
2490 ));
2491 };
2492 if files.len() != 1 {
2493 return Err("mobkit/blob/upload accepts exactly one file part".to_string());
2494 }
2495 let declared_media_type = upload
2496 .get("media_type")
2497 .and_then(Value::as_str)
2498 .unwrap_or(file.media_type.as_str());
2499 if !is_allowed_image_media_type(declared_media_type) {
2500 return Err(format!(
2501 "unsupported image media type in upload {upload_id}: {declared_media_type}"
2502 ));
2503 }
2504 if declared_media_type != file.media_type {
2505 return Err(format!(
2506 "media type mismatch for {upload_id}: placeholder {declared_media_type}, file {}",
2507 file.media_type
2508 ));
2509 }
2510 let size = file.bytes.len() as u64;
2511 let blob_ref = blob_store
2512 .put_bytes(&file.media_type, file.bytes.clone())
2513 .await
2514 .map_err(|err| format!("failed to store image {upload_id}: {err}"))?;
2515 Ok(json!({
2516 "blob_id": blob_ref.blob_id,
2517 "media_type": blob_ref.media_type,
2518 "size": size,
2519 }))
2520}
2521
2522fn collect_image_upload_placeholders(
2523 value: &Value,
2524 placeholders: &mut std::collections::BTreeMap<String, String>,
2525) -> Result<(), String> {
2526 match value {
2527 Value::Array(items) => {
2528 for item in items {
2529 collect_image_upload_placeholders(item, placeholders)?;
2530 }
2531 }
2532 Value::Object(object) => {
2533 if object.get("type").and_then(Value::as_str) == Some("image_upload") {
2534 let upload_id = image_upload_part_name(object, "image_upload")?;
2535 let media_type = object
2536 .get("media_type")
2537 .and_then(Value::as_str)
2538 .map(str::trim)
2539 .filter(|value| !value.is_empty())
2540 .ok_or_else(|| format!("image_upload {upload_id} requires media_type"))?;
2541 if placeholders
2542 .insert(upload_id.to_string(), media_type.to_string())
2543 .is_some()
2544 {
2545 return Err(format!("duplicate image_upload placeholder: {upload_id}"));
2546 }
2547 } else {
2548 for child in object.values() {
2549 collect_image_upload_placeholders(child, placeholders)?;
2550 }
2551 }
2552 }
2553 _ => {}
2554 }
2555 Ok(())
2556}
2557
2558fn replace_image_upload_placeholders(
2559 value: &mut Value,
2560 refs: &std::collections::BTreeMap<String, Value>,
2561) -> Result<(), String> {
2562 match value {
2563 Value::Array(items) => {
2564 for item in items {
2565 replace_image_upload_placeholders(item, refs)?;
2566 }
2567 }
2568 Value::Object(object) => {
2569 if object.get("type").and_then(Value::as_str) == Some("image_upload") {
2570 let upload_id = image_upload_part_name(object, "image_upload")?;
2571 let replacement = refs
2572 .get(upload_id)
2573 .ok_or_else(|| format!("missing blob replacement for {upload_id}"))?;
2574 *value = replacement.clone();
2575 } else {
2576 for child in object.values_mut() {
2577 replace_image_upload_placeholders(child, refs)?;
2578 }
2579 }
2580 }
2581 _ => {}
2582 }
2583 Ok(())
2584}
2585
2586fn response_value(id: Value, result: Option<Value>, error: Option<JsonRpcError>) -> Value {
2587 serde_json::to_value(JsonRpcResponse {
2588 jsonrpc: JSONRPC_VERSION.to_string(),
2589 id,
2590 result,
2591 error,
2592 })
2593 .unwrap_or_else(|_| {
2594 serde_json::json!({
2595 "jsonrpc": JSONRPC_VERSION,
2596 "id": Value::Null,
2597 "error": {
2598 "code": -32603,
2599 "message": "serialization failed",
2600 }
2601 })
2602 })
2603}
2604
2605fn invalid_params(id: Value, message: impl Into<String>) -> Value {
2606 response_value(
2607 id,
2608 None,
2609 Some(JsonRpcError {
2610 code: -32602,
2611 message: message.into(),
2612 data: None,
2613 }),
2614 )
2615}
2616
2617fn gating_decision_failed_error(id: Value, err: impl std::fmt::Display) -> Value {
2618 tracing::warn!(
2619 target: "mobkit::console",
2620 error = %err,
2621 "console gating decision failed"
2622 );
2623 invalid_params(id, "gating decision failed")
2624}
2625
2626fn runtime_binding_from_wire(
2627 binding: WireRuntimeBinding,
2628) -> Result<meerkat_mob::RuntimeBinding, String> {
2629 match binding {
2630 WireRuntimeBinding::Session => Ok(meerkat_mob::RuntimeBinding::Session),
2631 WireRuntimeBinding::External {
2632 address,
2633 bootstrap_token,
2634 identity,
2635 } => {
2636 let resolved = identity.resolve().map_err(|err| err.to_string())?;
2637 Ok(meerkat_mob::RuntimeBinding::External {
2638 peer_id: resolved.peer_id.to_string(),
2639 address,
2640 bootstrap_token,
2641 pubkey: resolved.pubkey,
2642 })
2643 }
2644 }
2645}
2646
2647fn parse_optional_runtime_mode(params: &Value) -> Result<Option<MobRuntimeMode>, String> {
2648 match params.get("runtime_mode") {
2649 None | Some(Value::Null) => Ok(None),
2650 Some(value) => serde_json::from_value::<MobRuntimeMode>(value.clone())
2651 .map(Some)
2652 .map_err(|err| {
2653 format!("runtime_mode must be \"autonomous_host\" or \"turn_driven\": {err}")
2654 }),
2655 }
2656}
2657
2658fn parse_optional_backend(params: &Value) -> Result<Option<MobBackendKind>, String> {
2659 match params.get("backend") {
2660 None | Some(Value::Null) => Ok(None),
2661 Some(value) => serde_json::from_value::<MobBackendKind>(value.clone())
2662 .map(Some)
2663 .map_err(|err| format!("backend must be \"session\" or \"external\": {err}")),
2664 }
2665}
2666
2667fn parse_optional_runtime_binding(
2668 params: &Value,
2669) -> Result<Option<meerkat_mob::RuntimeBinding>, String> {
2670 match params.get("binding") {
2671 None | Some(Value::Null) => Ok(None),
2672 Some(value) => {
2673 let wire = serde_json::from_value::<WireRuntimeBinding>(value.clone())
2674 .map_err(|err| format!("binding: {err}"))?;
2675 runtime_binding_from_wire(wire).map(Some)
2676 }
2677 }
2678}
2679
2680async fn member_entry_to_console_json(
2681 runtime: &MobRuntime,
2682 entry: &meerkat_mob::runtime::MobMemberListEntry,
2683) -> Value {
2684 let mut value = member_entry_to_json(entry);
2685 if let Some(object) = value.as_object_mut() {
2686 object.insert(
2687 "model_capabilities".to_string(),
2688 serde_json::to_value(model_capabilities_for_member_entry(
2689 runtime.handle().definition(),
2690 entry,
2691 ))
2692 .unwrap_or(Value::Null),
2693 );
2694 }
2695 value
2696}
2697
2698fn internal_error(id: Value, message: impl Into<String>) -> Value {
2699 let message = message.into();
2700 tracing::warn!(
2701 target: "mobkit::console",
2702 error = %message,
2703 "console JSON-RPC internal error"
2704 );
2705 response_value(
2706 id,
2707 None,
2708 Some(JsonRpcError {
2709 code: -32000,
2710 message: "internal error".to_string(),
2711 data: Some(json!({ "error": "internal_error" })),
2712 }),
2713 )
2714}
2715
2716fn stale_event_cursor_response(id: Value, after_cursor: u64, latest_cursor: u64) -> Value {
2721 response_value(
2722 id,
2723 None,
2724 Some(JsonRpcError {
2725 code: crate::rpc::MOB_EVENTS_STALE_CURSOR_CODE,
2726 message: format!(
2727 "stale mob event cursor: requested {after_cursor}, latest {latest_cursor}"
2728 ),
2729 data: Some(serde_json::json!({
2730 "error": "event_query_stale",
2731 "after_cursor": after_cursor,
2732 "latest_cursor": latest_cursor,
2733 })),
2734 }),
2735 )
2736}
2737
2738fn console_timeline_replay_unavailable_response(
2739 id: Value,
2740 err: ConsoleLogError,
2741 requested_cursor: Option<&ConsoleCursor>,
2742 latest_cursor: Option<ConsoleCursor>,
2743) -> Value {
2744 tracing::warn!(
2745 target: "mobkit::console",
2746 error = %err,
2747 "console timeline replay unavailable"
2748 );
2749 response_value(
2750 id,
2751 None,
2752 Some(JsonRpcError {
2753 code: crate::rpc::CONSOLE_TIMELINE_REPLAY_UNAVAILABLE_CODE,
2754 message: "timeline replay unavailable".to_string(),
2755 data: Some(json!({
2756 "error": "replay_unavailable",
2757 "stream": "timeline",
2758 "requested_cursor": requested_cursor.map(ToString::to_string),
2759 "latest_cursor": latest_cursor.map(|cursor| cursor.to_string()),
2760 })),
2761 }),
2762 )
2763}
2764
2765fn parse_console_helper_options(
2766 options_val: Option<&Value>,
2767) -> Result<meerkat_mob::HelperOptions, String> {
2768 crate::rpc::mob_methods::parse_helper_options(options_val)
2769}
2770
2771fn member_is_addressable(member: &meerkat_mob::runtime::MobMemberListEntry) -> bool {
2772 member
2773 .labels
2774 .get("addressable")
2775 .map(|value: &String| !value.eq_ignore_ascii_case("false"))
2776 .unwrap_or(true)
2777}
2778
2779fn member_addressability(member: &meerkat_mob::runtime::MobMemberListEntry) -> &'static str {
2780 if member_is_addressable(member) {
2781 "addressable"
2782 } else {
2783 "internal_only"
2784 }
2785}
2786
2787fn console_identity_status_json_for_identity(
2788 identity: &str,
2789 member: &meerkat_mob::runtime::MobMemberListEntry,
2790 session_id: Option<String>,
2791 response_phase: Option<String>,
2792) -> Value {
2793 json!({
2794 "identity": identity,
2795 "state": member_status_state_string(member.status),
2796 "role": member.role.to_string(),
2797 "addressability": member_addressability(member),
2798 "display_name": member.labels.get("display_name"),
2799 "labels": member.labels,
2800 "agent_runtime_id": crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str()),
2801 "session_id": session_id,
2802 "generation": Value::Null,
2803 "checkpoint_version": Value::Null,
2804 "continuity_health": Value::Null,
2805 "lease_healthy": Value::Null,
2806 "lease": Value::Null,
2807 "response_phase": response_phase,
2808 })
2809}
2810
2811fn console_identity_inspect_json_for_identity(
2812 identity: &str,
2813 member: &meerkat_mob::runtime::MobMemberListEntry,
2814 session_id: Option<String>,
2815 response_phase: Option<String>,
2816) -> Value {
2817 let peers: Vec<String> = member.wired_to.iter().map(ToString::to_string).collect();
2818 json!({
2819 "identity": identity,
2820 "state": member_status_state_string(member.status),
2821 "role": member.role.to_string(),
2822 "addressability": member_addressability(member),
2823 "display_name": member.labels.get("display_name"),
2824 "labels": member.labels,
2825 "continuity_health": Value::Null,
2826 "lease_healthy": Value::Null,
2827 "lease": Value::Null,
2828 "continuity": {
2829 "generation": Value::Null,
2830 "checkpoint_version": Value::Null,
2831 "session_id": session_id,
2832 "agent_runtime_id": crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str()),
2833 },
2834 "topology_peers": peers,
2835 "output_preview": Value::Null,
2836 "response_phase": response_phase,
2837 })
2838}
2839
2840async fn lookup_member_with_session(
2844 handle: &MobHandle,
2845 identity: &AgentIdentity,
2846) -> Option<(meerkat_mob::runtime::MobMemberListEntry, Option<String>)> {
2847 let entries = handle.list_members_including_retiring().await;
2848 let entry = entries
2849 .into_iter()
2850 .find(|e| &e.agent_identity == identity)?;
2851 let session_id = handle
2852 .resolve_bridge_session_id_observation(identity)
2853 .await
2854 .map(|s| s.to_string());
2855 Some((entry, session_id))
2856}
2857
2858#[derive(Debug, Clone)]
2859struct ConsoleRuntimeIdentityAlias {
2860 identity: String,
2861 runtime_member_id: String,
2862 member: meerkat_mob::runtime::MobMemberListEntry,
2863 session_id: Option<String>,
2864}
2865
2866fn durable_identity_for_member(member: &meerkat_mob::runtime::MobMemberListEntry) -> String {
2867 member
2868 .labels
2869 .get("agent_identity")
2870 .filter(|value| !value.trim().is_empty())
2871 .cloned()
2872 .unwrap_or_else(|| {
2874 crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str()).into_owned()
2875 })
2876}
2877
2878async fn lookup_member_alias_with_session(
2879 handle: &MobHandle,
2880 visibility_policy: &dyn ConsoleVisibilityPolicy,
2881 requested_identity: &str,
2882) -> Result<Option<ConsoleRuntimeIdentityAlias>, JsonRpcError> {
2883 let all_matches = lookup_member_alias_candidates_with_session(handle, requested_identity).await;
2884 let mut visible_matches = Vec::new();
2885 for alias in &all_matches {
2886 if runtime_alias_visible_to_console(handle, visibility_policy, alias) {
2887 visible_matches.push(alias.clone());
2888 }
2889 }
2890 let member = if visible_matches.len() > 1 {
2891 return Err(ambiguous_live_identity_alias_error(
2892 requested_identity,
2893 &visible_matches
2894 .iter()
2895 .map(|alias| alias.runtime_member_id.clone())
2896 .collect::<Vec<_>>(),
2897 ));
2898 } else if let Some(alias) = visible_matches.into_iter().next() {
2899 Some(alias)
2900 } else {
2901 all_matches.into_iter().next()
2902 };
2903 Ok(member)
2904}
2905
2906async fn lookup_visible_member_alias_candidates_with_session(
2907 handle: &MobHandle,
2908 visibility_policy: &dyn ConsoleVisibilityPolicy,
2909 requested_identity: &str,
2910) -> Vec<ConsoleRuntimeIdentityAlias> {
2911 let mut visible = Vec::new();
2912 for alias in lookup_member_alias_candidates_with_session(handle, requested_identity).await {
2913 if runtime_alias_visible_to_console(handle, visibility_policy, &alias) {
2914 visible.push(alias);
2915 }
2916 }
2917 visible
2918}
2919
2920async fn lookup_member_alias_candidates_with_session(
2921 handle: &MobHandle,
2922 requested_identity: &str,
2923) -> Vec<ConsoleRuntimeIdentityAlias> {
2924 let requested_member_id = crate::member_comms_id::mob_member_id(requested_identity);
2927 let entries = handle.list_members_including_retiring().await;
2928 let exact_matches = entries
2929 .iter()
2930 .filter(|entry| entry.agent_identity == requested_member_id)
2931 .cloned()
2932 .collect::<Vec<_>>();
2933 let label_matches = entries
2934 .iter()
2935 .filter(|entry| {
2936 entry
2937 .labels
2938 .get("agent_identity")
2939 .is_some_and(|identity| identity == requested_identity)
2940 })
2941 .cloned()
2942 .collect::<Vec<_>>();
2943 let mut matches = exact_matches;
2944 matches.extend(label_matches);
2945 let mut seen_member_ids = BTreeSet::new();
2946 matches.retain(|entry| seen_member_ids.insert(entry.agent_identity.to_string()));
2947 let mut aliases = Vec::with_capacity(matches.len());
2948 for member in matches {
2949 let runtime_member_id =
2950 crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str()).into_owned();
2951 let identity = durable_identity_for_member(&member);
2952 let session_id = handle
2953 .resolve_bridge_session_id_observation(&member.agent_identity)
2954 .await
2955 .map(|s| s.to_string());
2956 aliases.push(ConsoleRuntimeIdentityAlias {
2957 identity,
2958 runtime_member_id,
2959 member,
2960 session_id,
2961 });
2962 }
2963 aliases
2964}
2965
2966async fn reject_ambiguous_projected_live_identity(
2967 handle: &MobHandle,
2968 visibility_policy: &dyn ConsoleVisibilityPolicy,
2969 alias: &ConsoleRuntimeIdentityAlias,
2970) -> Result<(), JsonRpcError> {
2971 let candidates = lookup_visible_member_alias_candidates_with_session(
2972 handle,
2973 visibility_policy,
2974 &alias.identity,
2975 )
2976 .await;
2977 if candidates.len() > 1 {
2978 return Err(ambiguous_live_identity_alias_error(
2979 &alias.identity,
2980 &candidates
2981 .iter()
2982 .map(|candidate| candidate.runtime_member_id.clone())
2983 .collect::<Vec<_>>(),
2984 ));
2985 }
2986 Ok(())
2987}
2988
2989fn ambiguous_live_identity_alias_error(
2990 requested_identity: &str,
2991 candidates: &[String],
2992) -> JsonRpcError {
2993 JsonRpcError {
2994 code: -32602,
2995 message: format!(
2996 "ambiguous live identity alias {requested_identity}: candidates [{}]",
2997 candidates.join(", ")
2998 ),
2999 data: Some(json!({
3000 "kind": "ambiguous_live_identity_alias",
3001 "identity": requested_identity,
3002 "candidates": candidates,
3003 })),
3004 }
3005}
3006
3007async fn lookup_member_runtime_alias_with_session(
3008 handle: &MobHandle,
3009 runtime_member_id: &str,
3010) -> Option<ConsoleRuntimeIdentityAlias> {
3011 let requested_member_id = crate::member_comms_id::mob_member_id(runtime_member_id);
3012 let entries = handle.list_members_including_retiring().await;
3013 let member = entries
3014 .into_iter()
3015 .find(|entry| entry.agent_identity == requested_member_id)?;
3016 let runtime_member_id =
3017 crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str()).into_owned();
3018 let identity = durable_identity_for_member(&member);
3019 let session_id = handle
3020 .resolve_bridge_session_id_observation(&member.agent_identity)
3021 .await
3022 .map(|s| s.to_string());
3023 Some(ConsoleRuntimeIdentityAlias {
3024 identity,
3025 runtime_member_id,
3026 member,
3027 session_id,
3028 })
3029}
3030
3031async fn identity_runtime_alias(
3032 identity_runtime: &crate::identity_first::IdentityRuntime,
3033 requested_identity: &str,
3034) -> Result<Option<(crate::identity_first::AgentIdentity, bool)>, String> {
3035 if requested_identity.starts_with("rt:") {
3036 for status in identity_runtime.statuses().await {
3037 if status
3038 .agent_runtime_id
3039 .as_ref()
3040 .is_some_and(|runtime_id| runtime_id.as_str() == requested_identity)
3041 {
3042 return Ok(Some((status.identity, false)));
3043 }
3044 }
3045 return Ok(None);
3046 }
3047 if let Ok(identity) = crate::identity_first::AgentIdentity::parse(requested_identity) {
3048 match identity_runtime.status(&identity).await {
3049 Ok(_) => return Ok(Some((identity, true))),
3050 Err(crate::identity_first::IdentityRuntimeError::UnknownIdentity(_)) => {}
3051 Err(err) => return Err(err.to_string()),
3052 }
3053 }
3054
3055 for status in identity_runtime.statuses().await {
3056 if status
3057 .agent_runtime_id
3058 .as_ref()
3059 .is_some_and(|runtime_id| runtime_id.as_str() == requested_identity)
3060 {
3061 return Ok(Some((status.identity, false)));
3062 }
3063 }
3064 Ok(None)
3065}
3066
3067async fn resolve_console_identity_control_target(
3068 handle: &MobHandle,
3069 identity_runtime: Option<&Arc<crate::identity_first::IdentityRuntime>>,
3070 visibility_policy: &dyn ConsoleVisibilityPolicy,
3071 requested_identity: &str,
3072) -> Result<
3073 Option<(
3074 crate::identity_first::AgentIdentity,
3075 bool,
3076 Option<ConsoleRuntimeIdentityAlias>,
3077 )>,
3078 JsonRpcError,
3079> {
3080 if let Some(identity_runtime) = identity_runtime {
3081 match identity_runtime_alias(identity_runtime, requested_identity).await {
3082 Ok(Some((identity, exact))) => {
3083 let live = if exact {
3084 let registered_live = match identity_runtime.status(&identity).await {
3085 Ok(status) => match status.agent_runtime_id.as_ref() {
3086 Some(runtime_id) => {
3087 lookup_member_runtime_alias_with_session(
3088 handle,
3089 runtime_id.as_str(),
3090 )
3091 .await
3092 }
3093 None => None,
3094 },
3095 Err(_) => None,
3096 };
3097 if let Some(alias) = registered_live.as_ref()
3098 && !runtime_alias_visible_to_console(handle, visibility_policy, alias)
3099 {
3100 return Err(identity_hidden_by_policy_error(requested_identity));
3101 }
3102 if let Some(registered) = registered_live {
3103 return Ok(Some((identity, exact, Some(registered))));
3104 }
3105 let requested_live_candidates =
3106 lookup_visible_member_alias_candidates_with_session(
3107 handle,
3108 visibility_policy,
3109 requested_identity,
3110 )
3111 .await;
3112 let requested_live = if requested_live_candidates.len() > 1 {
3113 return Err(ambiguous_live_identity_alias_error(
3114 requested_identity,
3115 &requested_live_candidates
3116 .iter()
3117 .map(|alias| alias.runtime_member_id.clone())
3118 .collect::<Vec<_>>(),
3119 ));
3120 } else {
3121 requested_live_candidates.into_iter().next()
3122 };
3123 match (registered_live, requested_live) {
3124 (Some(registered), Some(requested))
3125 if registered.runtime_member_id == requested.runtime_member_id =>
3126 {
3127 Some(registered)
3128 }
3129 (Some(registered), None) => Some(registered),
3130 (Some(_registered), Some(requested)) => Some(requested),
3131 (None, requested) => requested,
3132 }
3133 } else {
3134 let registered_live =
3135 lookup_member_runtime_alias_with_session(handle, requested_identity).await;
3136 if let Some(alias) = registered_live.as_ref()
3137 && !runtime_alias_visible_to_console(handle, visibility_policy, alias)
3138 {
3139 return Err(identity_hidden_by_policy_error(requested_identity));
3140 }
3141 if let Some(registered) = registered_live {
3142 return Ok(Some((identity, exact, Some(registered))));
3143 }
3144 let durable_live_candidates =
3145 lookup_visible_member_alias_candidates_with_session(
3146 handle,
3147 visibility_policy,
3148 identity.as_str(),
3149 )
3150 .await;
3151 let durable_live = if durable_live_candidates.len() > 1 {
3152 return Err(ambiguous_live_identity_alias_error(
3153 identity.as_str(),
3154 &durable_live_candidates
3155 .iter()
3156 .map(|alias| alias.runtime_member_id.clone())
3157 .collect::<Vec<_>>(),
3158 ));
3159 } else {
3160 durable_live_candidates.into_iter().next()
3161 };
3162 match (registered_live, durable_live) {
3163 (Some(registered), Some(durable))
3164 if registered.runtime_member_id == durable.runtime_member_id =>
3165 {
3166 Some(registered)
3167 }
3168 (Some(registered), None) => Some(registered),
3169 (Some(_registered), Some(durable)) => Some(durable),
3170 (None, durable) => durable,
3171 }
3172 };
3173 return Ok(Some((identity, exact, live)));
3174 }
3175 Ok(None) => {}
3176 Err(err) => {
3177 return Err(JsonRpcError {
3178 code: -32000,
3179 message: err,
3180 data: None,
3181 });
3182 }
3183 }
3184 }
3185
3186 let live_alias =
3187 lookup_member_alias_with_session(handle, visibility_policy, requested_identity).await?;
3188 let Some(alias) = live_alias else {
3189 return Ok(None);
3190 };
3191 if let Some(identity_runtime) = identity_runtime
3192 && let Some(bound_status) = identity_runtime
3193 .statuses()
3194 .await
3195 .into_iter()
3196 .find(|status| {
3197 status
3198 .agent_runtime_id
3199 .as_ref()
3200 .is_some_and(|runtime_id| runtime_id.as_str() == alias.runtime_member_id)
3201 })
3202 && bound_status.identity.as_str() != alias.identity
3203 {
3204 return Err(JsonRpcError {
3205 code: -32000,
3206 message: format!(
3207 "stale live identity alias: live console alias {} resolves to {}, but identity runtime binding belongs to {}",
3208 alias.identity,
3209 alias.runtime_member_id,
3210 bound_status.identity.as_str(),
3211 ),
3212 data: Some(json!({
3213 "kind": "stale_live_identity_alias",
3214 "identity": alias.identity,
3215 "runtime_member_id": alias.runtime_member_id,
3216 "registered_identity": bound_status.identity.as_str(),
3217 })),
3218 });
3219 }
3220 let identity = crate::identity_first::AgentIdentity::parse(&alias.identity).map_err(|err| {
3221 JsonRpcError {
3222 code: -32602,
3223 message: format!("invalid identity: {err}"),
3224 data: None,
3225 }
3226 })?;
3227 let durable_live_candidates = lookup_visible_member_alias_candidates_with_session(
3228 handle,
3229 visibility_policy,
3230 identity.as_str(),
3231 )
3232 .await;
3233 if durable_live_candidates.len() > 1 {
3234 return Err(ambiguous_live_identity_alias_error(
3235 identity.as_str(),
3236 &durable_live_candidates
3237 .iter()
3238 .map(|alias| alias.runtime_member_id.clone())
3239 .collect::<Vec<_>>(),
3240 ));
3241 }
3242 Ok(Some((identity, false, Some(alias))))
3243}
3244
3245fn live_alias_matches_status_runtime(
3246 alias: Option<&ConsoleRuntimeIdentityAlias>,
3247 status: &crate::identity_first::IdentityStatus,
3248) -> bool {
3249 let Some(alias) = alias else {
3250 return true;
3251 };
3252 let session_matches = match (
3253 status.session_id.as_ref().map(ToString::to_string),
3254 alias.session_id.as_deref(),
3255 ) {
3256 (Some(status_session), Some(live_session)) => status_session == live_session,
3257 _ => true,
3258 };
3259 status
3260 .agent_runtime_id
3261 .as_ref()
3262 .is_some_and(|runtime_id| runtime_id.as_str() == alias.runtime_member_id)
3263 && alias.identity == status.identity.as_str()
3264 && session_matches
3265}
3266
3267async fn stale_live_alias_json_rpc_error(
3268 operation: &str,
3269 identity_runtime: &crate::identity_first::IdentityRuntime,
3270 identity: &crate::identity_first::AgentIdentity,
3271 live_alias: Option<&ConsoleRuntimeIdentityAlias>,
3272) -> Option<JsonRpcError> {
3273 let live_alias = live_alias?;
3274 let Ok(status) = identity_runtime.status(identity).await else {
3275 return None;
3276 };
3277 if live_alias_matches_status_runtime(Some(live_alias), &status) {
3278 return None;
3279 }
3280 let registered_runtime_member_id = status
3281 .agent_runtime_id
3282 .as_ref()
3283 .map(crate::identity_first::AgentRuntimeId::as_str);
3284 Some(JsonRpcError {
3285 code: -32000,
3286 message: format!(
3287 "{operation} failed: identity runtime binding for {} points at {}, but requested live member is {}",
3288 identity.as_str(),
3289 registered_runtime_member_id.unwrap_or("<none>"),
3290 live_alias.runtime_member_id
3291 ),
3292 data: Some(json!({
3293 "kind": "stale_identity_runtime_binding",
3294 "identity": identity.as_str(),
3295 "registered_runtime_member_id": registered_runtime_member_id,
3296 "live_runtime_member_id": live_alias.runtime_member_id,
3297 "registered_session_id": status.session_id.as_ref().map(ToString::to_string),
3298 "live_session_id": live_alias.session_id,
3299 })),
3300 })
3301}
3302
3303fn reset_requires_session_bridge_json_rpc_error() -> JsonRpcError {
3304 JsonRpcError {
3305 code: -32602,
3306 message: "reset requires an identity runtime with a session bridge".to_string(),
3307 data: Some(json!({
3308 "kind": "identity_reset_requires_session_bridge",
3309 })),
3310 }
3311}
3312
3313fn console_identity_status_json_from_record(
3314 record: &crate::console_aggregator::ConsoleIdentityRecord,
3315 response_phase: Option<String>,
3316) -> Value {
3317 json!({
3318 "identity": record.identity,
3319 "state": record.health,
3320 "role": record.labels.get("role"),
3321 "addressability": if record.addressable { "addressable" } else { "internal_only" },
3322 "display_name": record.display_name,
3323 "labels": record.labels,
3324 "agent_runtime_id": record.runtime_member_id,
3325 "session_id": record.session_id,
3326 "generation": Value::Null,
3327 "checkpoint_version": Value::Null,
3328 "continuity_health": Value::Null,
3329 "lease_healthy": Value::Null,
3330 "lease": Value::Null,
3331 "response_phase": response_phase,
3332 })
3333}
3334
3335fn console_addressability_json(
3336 addressability: crate::identity_first::AgentAddressability,
3337) -> &'static str {
3338 match addressability {
3339 crate::identity_first::AgentAddressability::Addressable => "addressable",
3340 crate::identity_first::AgentAddressability::InternalOnly => "internal_only",
3341 }
3342}
3343
3344fn console_identity_record_from_identity_status(
3345 status: &crate::identity_first::IdentityStatus,
3346) -> ConsoleIdentityRecord {
3347 let mut labels = status.labels.clone();
3348 if let Some(profile) = status.profile.as_ref() {
3349 labels
3350 .entry("role".to_string())
3351 .or_insert_with(|| profile.as_str().to_string());
3352 }
3353 let runtime_member_id = status
3354 .agent_runtime_id
3355 .as_ref()
3356 .map(crate::identity_first::AgentRuntimeId::as_str)
3357 .unwrap_or_else(|| status.identity.as_str())
3358 .to_string();
3359 let addressable = status.addressability
3360 == crate::identity_first::AgentAddressability::Addressable
3361 && matches!(
3362 status.state,
3363 crate::identity_first::IdentityLifecycleState::Active
3364 | crate::identity_first::IdentityLifecycleState::Dormant
3365 | crate::identity_first::IdentityLifecycleState::Uninitialized
3366 );
3367 let visibility = match status.state {
3368 crate::identity_first::IdentityLifecycleState::Retiring => {
3369 ConsoleVisibility::RetiredReadable
3370 }
3371 crate::identity_first::IdentityLifecycleState::Broken
3372 | crate::identity_first::IdentityLifecycleState::Suspended => {
3373 ConsoleVisibility::Unreachable
3374 }
3375 _ if addressable => ConsoleVisibility::Addressable,
3376 _ => ConsoleVisibility::Hidden,
3377 };
3378 let health = match status.state {
3379 crate::identity_first::IdentityLifecycleState::Active => "ready",
3380 crate::identity_first::IdentityLifecycleState::Dormant => "dormant",
3381 crate::identity_first::IdentityLifecycleState::Uninitialized => "uninitialized",
3382 crate::identity_first::IdentityLifecycleState::Broken => "broken",
3383 crate::identity_first::IdentityLifecycleState::Suspended => "suspended",
3384 crate::identity_first::IdentityLifecycleState::Retiring => "retired",
3385 }
3386 .to_string();
3387 ConsoleIdentityRecord {
3388 identity: status.identity.as_str().to_string(),
3389 display_name: status
3390 .display_name
3391 .as_ref()
3392 .map(crate::identity_first::DisplayName::as_str)
3393 .unwrap_or_else(|| status.identity.as_str())
3394 .to_string(),
3395 runtime_key: "identity-first".to_string(),
3396 runtime_member_id,
3397 session_id: status.session_id.as_ref().map(ToString::to_string),
3398 visibility,
3399 addressable,
3400 health,
3401 topology_peers: Vec::new(),
3402 labels,
3403 }
3404}
3405
3406fn identity_hidden_by_policy_response(response_id: Value, identity: &str) -> Value {
3407 response_value(
3408 response_id,
3409 None,
3410 Some(identity_hidden_by_policy_error(identity)),
3411 )
3412}
3413
3414fn identity_hidden_by_policy_error(identity: &str) -> JsonRpcError {
3415 JsonRpcError {
3416 code: -32001,
3417 message: format!("unknown identity: {identity}"),
3418 data: Some(json!({
3419 "kind": "identity_hidden_by_policy",
3420 "identity": identity,
3421 })),
3422 }
3423}
3424
3425fn identity_status_visible_to_console(
3426 visibility_policy: &dyn ConsoleVisibilityPolicy,
3427 status: &crate::identity_first::IdentityStatus,
3428) -> bool {
3429 visibility_policy.identity_visible(&console_identity_record_from_identity_status(status))
3430}
3431
3432fn console_member_from_runtime_alias(
3433 handle: &MobHandle,
3434 alias: &ConsoleRuntimeIdentityAlias,
3435) -> ConsoleMember {
3436 ConsoleMember {
3437 agent_identity: alias.runtime_member_id.clone(),
3438 role: alias.member.role.to_string(),
3439 state: member_status_state_string(alias.member.status),
3440 model_capabilities: model_capabilities_for_member_entry(handle.definition(), &alias.member),
3441 runtime_mode: Some(alias.member.runtime_mode.to_string()),
3442 session_id: alias.session_id.clone(),
3443 wired_to: alias
3444 .member
3445 .wired_to
3446 .iter()
3447 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
3448 .collect(),
3449 labels: alias.member.labels.clone(),
3450 }
3451}
3452
3453fn console_identity_record_from_runtime_alias(
3454 alias: &ConsoleRuntimeIdentityAlias,
3455) -> ConsoleIdentityRecord {
3456 let addressable = alias
3457 .member
3458 .labels
3459 .get("addressable")
3460 .map(|value| !value.eq_ignore_ascii_case("false"))
3461 .unwrap_or(true)
3462 && alias.member.status == meerkat_mob::MobMemberStatus::Active;
3463 let visibility = match alias.member.status {
3464 meerkat_mob::MobMemberStatus::Retiring | meerkat_mob::MobMemberStatus::Completed => {
3465 ConsoleVisibility::RetiredReadable
3466 }
3467 meerkat_mob::MobMemberStatus::Broken | meerkat_mob::MobMemberStatus::Unknown => {
3470 ConsoleVisibility::Unreachable
3471 }
3472 _ if addressable => ConsoleVisibility::Addressable,
3473 _ => ConsoleVisibility::Hidden,
3474 };
3475 ConsoleIdentityRecord {
3476 identity: alias.identity.clone(),
3477 display_name: alias
3478 .member
3479 .labels
3480 .get("display_name")
3481 .cloned()
3482 .unwrap_or_else(|| alias.identity.clone()),
3483 runtime_key: "runtime".to_string(),
3484 runtime_member_id: alias.runtime_member_id.clone(),
3485 session_id: alias.session_id.clone(),
3486 visibility,
3487 addressable,
3488 health: match alias.member.status {
3489 meerkat_mob::MobMemberStatus::Active => "ready".to_string(),
3490 meerkat_mob::MobMemberStatus::Retiring => "retired".to_string(),
3491 other => format!("{other:?}").to_ascii_lowercase(),
3494 },
3495 topology_peers: alias
3496 .member
3497 .wired_to
3498 .iter()
3499 .map(ToString::to_string)
3500 .collect(),
3501 labels: alias.member.labels.clone(),
3502 }
3503}
3504
3505fn runtime_alias_visible_to_console(
3506 handle: &MobHandle,
3507 visibility_policy: &dyn ConsoleVisibilityPolicy,
3508 alias: &ConsoleRuntimeIdentityAlias,
3509) -> bool {
3510 let member = console_member_from_runtime_alias(handle, alias);
3511 if !visibility_policy.member_visible(&member) {
3512 return false;
3513 }
3514 visibility_policy.identity_visible(&console_identity_record_from_runtime_alias(alias))
3515}
3516
3517fn console_identity_status_json_from_identity_status(
3518 status: &crate::identity_first::IdentityStatus,
3519 response_phase: Option<String>,
3520) -> Value {
3521 json!({
3522 "identity": status.identity.as_str(),
3523 "state": status.state.wire_str(),
3524 "role": status.profile.as_ref().map(ProfileName::as_str),
3525 "addressability": console_addressability_json(status.addressability),
3526 "display_name": status.display_name.as_ref().map(crate::identity_first::DisplayName::as_str),
3527 "labels": status.labels,
3528 "agent_runtime_id": status.agent_runtime_id.as_ref().map(crate::identity_first::AgentRuntimeId::as_str),
3529 "session_id": status.session_id.as_ref().map(ToString::to_string),
3530 "generation": status.generation.map(crate::identity_first::ContinuityGeneration::get),
3531 "checkpoint_version": status.checkpoint_version.map(crate::identity_first::CheckpointVersion::get),
3532 "continuity_health": status.continuity_health,
3533 "lease_healthy": status.lease.as_ref().map(|lease| lease.healthy),
3534 "lease": status.lease.as_ref().map(|lease| json!({
3535 "fencing_token": lease.fencing_token.get(),
3536 "ttl_remaining_ms": lease.ttl_remaining.as_millis() as u64,
3537 "healthy": lease.healthy,
3538 })),
3539 "response_phase": response_phase,
3540 })
3541}
3542
3543fn console_identity_inspect_json_from_identity_status(
3544 status: &crate::identity_first::IdentityStatus,
3545 live_alias: Option<&ConsoleRuntimeIdentityAlias>,
3546 response_phase: Option<String>,
3547) -> Value {
3548 let topology_peers = live_alias
3549 .map(|alias| {
3550 alias
3551 .member
3552 .wired_to
3553 .iter()
3554 .map(ToString::to_string)
3555 .map(Value::String)
3556 .collect::<Vec<_>>()
3557 })
3558 .unwrap_or_default();
3559 let session_id = status
3560 .session_id
3561 .as_ref()
3562 .map(ToString::to_string)
3563 .or_else(|| live_alias.and_then(|alias| alias.session_id.clone()));
3564 let agent_runtime_id = status
3565 .agent_runtime_id
3566 .as_ref()
3567 .map(crate::identity_first::AgentRuntimeId::as_str)
3568 .map(ToString::to_string)
3569 .or_else(|| live_alias.map(|alias| alias.runtime_member_id.clone()));
3570 json!({
3571 "identity": status.identity.as_str(),
3572 "state": status.state.wire_str(),
3573 "role": status.profile.as_ref().map(ProfileName::as_str),
3574 "addressability": console_addressability_json(status.addressability),
3575 "display_name": status.display_name.as_ref().map(crate::identity_first::DisplayName::as_str),
3576 "labels": status.labels,
3577 "continuity_health": status.continuity_health,
3578 "lease_healthy": status.lease.as_ref().map(|lease| lease.healthy),
3579 "lease": status.lease.as_ref().map(|lease| json!({
3580 "fencing_token": lease.fencing_token.get(),
3581 "ttl_remaining_ms": lease.ttl_remaining.as_millis() as u64,
3582 "healthy": lease.healthy,
3583 })),
3584 "continuity": {
3585 "generation": status.generation.map(crate::identity_first::ContinuityGeneration::get),
3586 "checkpoint_version": status.checkpoint_version.map(crate::identity_first::CheckpointVersion::get),
3587 "session_id": session_id,
3588 "agent_runtime_id": agent_runtime_id,
3589 },
3590 "topology_peers": topology_peers,
3591 "output_preview": Value::Null,
3592 "response_phase": response_phase,
3593 })
3594}
3595
3596fn console_identity_inspect_json_from_record(
3597 inspection: &crate::console_aggregator::ConsoleIdentityInspection,
3598 response_phase: Option<String>,
3599) -> Value {
3600 let record = &inspection.identity;
3601 json!({
3602 "identity": record.identity,
3603 "state": record.health,
3604 "role": record.labels.get("role"),
3605 "addressability": if record.addressable { "addressable" } else { "internal_only" },
3606 "display_name": record.display_name,
3607 "labels": record.labels,
3608 "continuity_health": Value::Null,
3609 "lease_healthy": Value::Null,
3610 "lease": Value::Null,
3611 "continuity": {
3612 "generation": Value::Null,
3613 "checkpoint_version": Value::Null,
3614 "session_id": record.session_id,
3615 "agent_runtime_id": record.runtime_member_id,
3616 },
3617 "topology_peers": inspection.peers,
3618 "output_preview": Value::Null,
3619 "response_phase": response_phase,
3620 })
3621}
3622
3623fn lifecycle_archive_cleanup_completed(error: &str) -> bool {
3624 is_recoverable_lifecycle_cleanup_error(error)
3625}
3626
3627async fn respawn_console_member(
3628 handle: &MobHandle,
3629 runtime_member_id: &AgentIdentity,
3630) -> Result<Option<Value>, String> {
3631 let entry_before_respawn = handle.get_member(runtime_member_id).await.ok().flatten();
3634 match handle.respawn(runtime_member_id.clone(), None).await {
3635 Ok(_receipt) => Ok(None),
3636 Err(err) => {
3637 if let Some(failed_peer_ids) = topology_restore_failed_peer_ids(&err) {
3638 tracing::warn!(
3639 member_id = %runtime_member_id,
3640 failed_peer_count = failed_peer_ids.len(),
3641 failed_peer_ids = ?failed_peer_ids,
3642 "console member respawn restored member with isolated peer edges; continuing degraded respawn"
3643 );
3644 return Ok(Some(topology_restore_warning_json(&failed_peer_ids)));
3645 }
3646
3647 if lifecycle_archive_cleanup_completed(&err.to_string()) {
3648 if handle
3651 .get_member(runtime_member_id)
3652 .await
3653 .map_err(|lookup_err| lookup_err.to_string())?
3654 .is_none()
3655 && let Some(entry) = entry_before_respawn
3656 {
3657 let mut spec =
3658 SpawnMemberSpec::new(entry.role.clone(), runtime_member_id.clone());
3659 if !entry.labels.is_empty() {
3660 spec = spec.with_labels(entry.labels.clone());
3661 }
3662 handle
3663 .ensure_member(spec)
3664 .await
3665 .map_err(|ensure_err| ensure_err.to_string())?;
3666 }
3667 return Ok(None);
3668 }
3669
3670 Err(err.to_string())
3671 }
3672 }
3673}
3674
3675async fn respawn_live_console_member_response(
3679 handle: &MobHandle,
3680 console_events: Option<&ConsoleEventStore>,
3681 alias: &ConsoleRuntimeIdentityAlias,
3682 lifecycle_kind: &str,
3683 action: &str,
3684 response_id: Value,
3685) -> Value {
3686 let mid = crate::member_comms_id::mob_member_id(alias.runtime_member_id.as_str());
3687 match Box::pin(respawn_console_member(handle, &mid)).await {
3688 Ok(topology_restore_warning) => {
3689 if let Some(store) = console_events {
3690 store
3691 .record_lifecycle(
3692 &alias.identity,
3693 lifecycle_kind,
3694 json!({ "topology_restore_warning": topology_restore_warning.clone() }),
3695 )
3696 .await;
3697 }
3698 let mut body = match lookup_member_with_session(handle, &mid).await {
3699 Some((entry, session_id)) => console_identity_status_json_for_identity(
3700 &alias.identity,
3701 &entry,
3702 session_id,
3703 None,
3704 ),
3705 None => json!({ "identity": alias.identity }),
3706 };
3707 if let Some(warning) = topology_restore_warning {
3708 body["topology_restore_warning"] = warning;
3709 }
3710 response_value(response_id, Some(body), None)
3711 }
3712 Err(err) => internal_error(response_id, format!("{action} failed: {err}")),
3713 }
3714}
3715
3716async fn retire_console_member(
3717 handle: &MobHandle,
3718 runtime_member_id: &AgentIdentity,
3719) -> Result<(), String> {
3720 match handle.retire(runtime_member_id.clone()).await {
3721 Ok(()) => Ok(()),
3722 Err(err) if lifecycle_archive_cleanup_completed(&err.to_string()) => Ok(()),
3723 Err(err) => Err(err.to_string()),
3724 }
3725}
3726
3727#[cfg(test)]
3728fn member_id_matches_durable_identity(member_id: &str, durable_identity: &str) -> bool {
3729 member_id == durable_identity
3730}
3731
3732async fn retire_stale_console_members_for_identity(
3733 handle: &MobHandle,
3734 visibility_policy: &dyn ConsoleVisibilityPolicy,
3735 durable_identity: &str,
3736 keep_runtime_member_id: Option<&str>,
3737) -> Result<(), String> {
3738 let stale_members = lookup_member_alias_candidates_with_session(handle, durable_identity)
3739 .await
3740 .into_iter()
3741 .filter(|alias| {
3742 runtime_alias_visible_to_console(handle, visibility_policy, alias)
3743 && keep_runtime_member_id
3744 .map(|keep| alias.runtime_member_id != keep)
3745 .unwrap_or(true)
3746 })
3747 .map(|alias| crate::member_comms_id::mob_member_id(alias.runtime_member_id.as_str()))
3748 .collect::<Vec<_>>();
3749 for member_id in stale_members {
3750 retire_console_member(handle, &member_id).await?;
3751 }
3752 Ok(())
3753}
3754
3755fn console_identity_error_response(
3756 response_id: Value,
3757 operation: &str,
3758 err: crate::identity_first::IdentityRuntimeError,
3759) -> Value {
3760 match err {
3761 crate::identity_first::IdentityRuntimeError::UnknownIdentity(identity) => {
3762 invalid_params(response_id, format!("identity not found: {identity}"))
3763 }
3764 other => internal_error(response_id, format!("{operation} failed: {other}")),
3765 }
3766}
3767
3768#[allow(clippy::too_many_arguments)]
3769async fn handle_console_aggregator_rpc(
3770 console_aggregator: Option<MobKitConsoleAggregator>,
3771 request: JsonRpcRequest,
3772 is_authenticated: bool,
3773 read_only: bool,
3774 access: Option<&AccessController>,
3775 access_view: Option<&AccessView>,
3776) -> Value {
3777 let response_id = request.id.clone().unwrap_or(Value::Null);
3778 let can_mutate = is_authenticated && !read_only;
3779 if is_console_mutating_rpc_method(request.method.as_str()) && !can_mutate {
3780 return console_read_only_rpc_error(response_id);
3781 }
3782 if let Some(response) = handle_access_admin_rpc(access, access_view, &request) {
3783 return response;
3784 }
3785 if let Some(error) =
3786 console_rpc_access_violation(access_view, request.method.as_str(), &request.params)
3787 {
3788 return response_value(response_id, None, Some(error));
3789 }
3790 match request.method.as_str() {
3791 "mobkit/capabilities" => {
3792 let mut methods = vec![
3793 "mobkit/capabilities",
3794 "mobkit/console/list_identities",
3795 "mobkit/console/inspect_identity",
3796 "mobkit/console/query_timeline",
3797 ];
3798 if can_mutate {
3799 methods.extend_from_slice(&["mobkit/retire", "mobkit/console/send"]);
3800 }
3801 if access.is_some() {
3802 methods.push("mobkit/access/status");
3803 if access_view.is_some_and(AccessView::can_administer) {
3804 methods.extend_from_slice(&["mobkit/access/get", "mobkit/access/preview"]);
3805 if can_mutate {
3806 methods.extend_from_slice(&[
3807 "mobkit/access/set",
3808 "mobkit/access/enable",
3809 "mobkit/access/rules/upsert",
3810 "mobkit/access/rules/delete",
3811 "mobkit/access/groups/set",
3812 "mobkit/access/groups/delete",
3813 ]);
3814 }
3815 }
3816 }
3817 response_value(
3818 response_id,
3819 Some(json!({
3820 "methods": methods,
3821 "authenticated": is_authenticated,
3822 "read_only": read_only,
3823 "runtime_capabilities": {
3824 "can_send_messages": can_mutate,
3825 "can_retire_members": can_mutate,
3826 "can_spawn_members": false,
3827 },
3828 "features": {
3829 "console_aggregator": console_aggregator.is_some(),
3830 "multi_runtime_console": console_aggregator.is_some(),
3831 }
3832 })),
3833 None,
3834 )
3835 }
3836 "mobkit/console/list_identities" => {
3837 let Some(aggregator) = &console_aggregator else {
3838 return console_aggregator_unavailable(response_id);
3839 };
3840 match Box::pin(aggregator.list_identities()).await {
3841 Ok(mut identities) => {
3842 retain_visible_identity_records(&mut identities, access_view);
3843 response_value(response_id, Some(json!({ "identities": identities })), None)
3844 }
3845 Err(err) => internal_error(response_id, format!("list_identities failed: {err}")),
3846 }
3847 }
3848 "mobkit/console/inspect_identity" => {
3849 let Some(identity) = request.params.get("identity").and_then(Value::as_str) else {
3850 return invalid_params(response_id, "identity required");
3851 };
3852 let Some(aggregator) = &console_aggregator else {
3853 return console_aggregator_unavailable(response_id);
3854 };
3855 match Box::pin(aggregator.inspect_identity(identity)).await {
3856 Ok(Some(inspection)) => response_value(
3857 response_id,
3858 Some(serde_json::to_value(inspection).unwrap_or(Value::Null)),
3859 None,
3860 ),
3861 Ok(None) => response_value(
3862 response_id,
3863 None,
3864 Some(JsonRpcError {
3865 code: -32001,
3866 message: format!("unknown identity: {identity}"),
3867 data: None,
3868 }),
3869 ),
3870 Err(err) => internal_error(response_id, format!("inspect_identity failed: {err}")),
3871 }
3872 }
3873 "mobkit/console/query_timeline" => {
3874 let query: ConsoleTimelineWindowQuery =
3875 match serde_json::from_value(request.params.clone()) {
3876 Ok(query) => query,
3877 Err(err) => {
3878 return invalid_params(response_id, format!("invalid query params: {err}"));
3879 }
3880 };
3881 let Some(aggregator) = &console_aggregator else {
3882 return console_aggregator_unavailable(response_id);
3883 };
3884 match Box::pin(aggregator.query_timeline_windowed(query.clone())).await {
3885 Ok(mut page) => {
3886 retain_visible_timeline_frames(&mut page, access_view);
3887 response_value(
3888 response_id,
3889 Some(serde_json::to_value(page).unwrap_or(Value::Null)),
3890 None,
3891 )
3892 }
3893 Err(err) => {
3894 let latest_cursor = aggregator.latest_cursor().await.ok().flatten();
3895 console_timeline_replay_unavailable_response(
3896 response_id,
3897 err,
3898 query.after.as_ref(),
3899 latest_cursor,
3900 )
3901 }
3902 }
3903 }
3904 "mobkit/console/send" => {
3905 let send_request: ConsoleSendRequest =
3906 match serde_json::from_value(request.params.clone()) {
3907 Ok(request) => request,
3908 Err(err) => {
3909 return invalid_params(response_id, format!("invalid send params: {err}"));
3910 }
3911 };
3912 let Some(aggregator) = &console_aggregator else {
3913 return console_aggregator_unavailable(response_id);
3914 };
3915 match Box::pin(aggregator.send(send_request)).await {
3916 Ok(accepted) => response_value(
3917 response_id,
3918 Some(serde_json::to_value(accepted).unwrap_or(Value::Null)),
3919 None,
3920 ),
3921 Err(err) => {
3922 response_value(response_id, None, Some(console_send_json_rpc_error(err)))
3923 }
3924 }
3925 }
3926 "mobkit/retire" => {
3927 let Some(identity) = request.params.get("identity").and_then(Value::as_str) else {
3928 return invalid_params(response_id, "identity required");
3929 };
3930 let Some(aggregator) = &console_aggregator else {
3931 return console_aggregator_unavailable(response_id);
3932 };
3933 match Box::pin(aggregator.retire_identity(identity)).await {
3934 Ok(true) => {
3935 response_value(response_id, Some(json!({ "identity": identity })), None)
3936 }
3937 Ok(false) => response_value(
3938 response_id,
3939 None,
3940 Some(JsonRpcError {
3941 code: -32001,
3942 message: format!("unknown identity: {identity}"),
3943 data: None,
3944 }),
3945 ),
3946 Err(err) => internal_error(response_id, format!("retire failed: {err}")),
3947 }
3948 }
3949 "mobkit/reset_all" => {
3950 let Some(_aggregator) = &console_aggregator else {
3951 return console_aggregator_unavailable(response_id);
3952 };
3953 response_value(
3954 response_id,
3955 None,
3956 Some(JsonRpcError {
3957 code: -32002,
3958 message: "reset_all is not supported on the aggregator-only RPC surface"
3959 .to_string(),
3960 data: Some(json!({
3961 "kind": "unsupported_reset_all_surface",
3962 "reason": "aggregator reset_all cannot preserve baseline identity semantics",
3963 })),
3964 }),
3965 )
3966 }
3967 _ => response_value(
3968 response_id,
3969 None,
3970 Some(JsonRpcError {
3971 code: -32601,
3972 message: "Method not found".to_string(),
3973 data: None,
3974 }),
3975 ),
3976 }
3977}
3978
3979fn console_aggregator_unavailable(response_id: Value) -> Value {
3980 response_value(
3981 response_id,
3982 None,
3983 Some(JsonRpcError {
3984 code: -32004,
3985 message: "console aggregator unavailable".to_string(),
3986 data: None,
3987 }),
3988 )
3989}
3990
3991fn resolve_gating_approver_id(
3992 params: &Value,
3993 authenticated_principal: Option<&str>,
3994) -> Result<String, &'static str> {
3995 if let Some(principal) = authenticated_principal.and_then(|value| {
3996 let trimmed = value.trim();
3997 (!trimmed.is_empty()).then_some(trimmed)
3998 }) {
3999 return Ok(principal.to_string());
4000 }
4001 let Some(approver_id) = params.get("approver_id").and_then(Value::as_str) else {
4002 return Err("approver_id required");
4003 };
4004 let trimmed = approver_id.trim();
4005 if trimmed.is_empty() {
4006 return Err("approver_id required");
4007 }
4008 Ok(trimmed.to_string())
4009}
4010
4011#[allow(clippy::large_futures, clippy::too_many_arguments)]
4012#[cfg(test)]
4013async fn handle_console_runtime_rpc(
4014 runtime: &MobRuntime,
4015 module_runtime: Option<std::sync::Arc<tokio::sync::Mutex<MobkitRuntimeHandle>>>,
4016 contact_directory: Option<&ContactDirectory>,
4017 gateway_peer_keys: Option<&crate::auth::peer_keys::GatewayPeerKeys>,
4018 console_events: Option<ConsoleEventStore>,
4019 console_aggregator: Option<MobKitConsoleAggregator>,
4020 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
4021 metadata_table: Option<std::sync::Arc<RuntimeMetadataTable>>,
4022 mob_events: Option<MobEventsStore>,
4023 request: JsonRpcRequest,
4024 is_authenticated: bool,
4025) -> Value {
4026 handle_console_runtime_rpc_with_visibility(
4027 runtime,
4028 module_runtime,
4029 contact_directory,
4030 gateway_peer_keys,
4031 console_events,
4032 console_aggregator,
4033 identity_runtime,
4034 metadata_table,
4035 mob_events,
4036 &crate::console_aggregator::AllowAllConsoleVisibilityPolicy,
4037 request,
4038 is_authenticated,
4039 false,
4040 None,
4041 None,
4042 None,
4043 )
4044 .await
4045}
4046
4047#[allow(clippy::too_many_arguments)]
4048async fn handle_console_runtime_rpc_with_visibility(
4049 runtime: &MobRuntime,
4050 module_runtime: Option<std::sync::Arc<tokio::sync::Mutex<MobkitRuntimeHandle>>>,
4051 contact_directory: Option<&ContactDirectory>,
4052 gateway_peer_keys: Option<&crate::auth::peer_keys::GatewayPeerKeys>,
4053 console_events: Option<ConsoleEventStore>,
4054 console_aggregator: Option<MobKitConsoleAggregator>,
4055 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
4056 metadata_table: Option<std::sync::Arc<RuntimeMetadataTable>>,
4057 mob_events: Option<MobEventsStore>,
4058 visibility_policy: &dyn ConsoleVisibilityPolicy,
4059 request: JsonRpcRequest,
4060 is_authenticated: bool,
4061 read_only: bool,
4062 authenticated_principal: Option<&str>,
4063 access: Option<&AccessController>,
4064 access_view: Option<&AccessView>,
4065) -> Value {
4066 let response_id = request.id.clone().unwrap_or(Value::Null);
4067 let can_mutate = is_authenticated && !read_only;
4068 if is_console_mutating_rpc_method(request.method.as_str()) && !can_mutate {
4069 return console_read_only_rpc_error(response_id);
4070 }
4071 if let Some(controller) = access.filter(|controller| controller.enabled()) {
4077 prime_access_cache_from_runtime(runtime, controller).await;
4078 }
4079 if let Some(response) = handle_access_admin_rpc(access, access_view, &request) {
4080 return response;
4081 }
4082 if let Some(error) =
4083 console_rpc_access_violation(access_view, request.method.as_str(), &request.params)
4084 {
4085 return response_value(response_id, None, Some(error));
4086 }
4087
4088 match request.method.as_str() {
4089 "mobkit/capabilities" => {
4090 let mut methods = vec![
4091 "mobkit/status",
4092 "mobkit/capabilities",
4093 "mobkit/list_members",
4094 "mobkit/get_member",
4095 "mobkit/find_members",
4096 "mobkit/member_status",
4097 "mobkit/blob/get",
4098 "mobkit/wait_ready",
4099 "mobkit/flow_status",
4100 "mobkit/list_flows",
4101 "mobkit/list_runs",
4102 "mobkit/console/list_identities",
4103 "mobkit/console/inspect_identity",
4104 "mobkit/console/query_timeline",
4105 "mobkit/mob_events/query",
4106 "mobkit/mob_events/subscribe",
4107 "mobkit/cross_mob/peer_info",
4108 "mobkit/cross_mob/directory",
4109 "mobkit/peer_pubkey",
4110 ];
4111 if identity_runtime.is_some() {
4112 methods.extend_from_slice(&["mobkit/status_identity", "mobkit/inspect_identity"]);
4113 if can_mutate {
4114 methods.push("mobkit/delete_identity");
4115 }
4116 } else if console_aggregator.is_some() {
4117 methods.extend_from_slice(&["mobkit/status_identity", "mobkit/inspect_identity"]);
4118 }
4119 if module_runtime.is_some() {
4120 methods.extend_from_slice(&[
4121 "mobkit/routing/routes/list",
4122 "mobkit/delivery/history",
4123 "mobkit/gating/pending",
4124 "mobkit/gating/audit",
4125 ]);
4126 if can_mutate {
4127 methods.push("mobkit/gating/decide");
4128 }
4129 }
4130 if can_mutate {
4131 methods.extend_from_slice(&[
4132 "mobkit/retire",
4133 "mobkit/respawn",
4140 "mobkit/reset",
4141 "mobkit/reset_all",
4142 "mobkit/console/send",
4143 "mobkit/blob/upload",
4144 "mobkit/ensure_member",
4145 "mobkit/retire_member",
4146 "mobkit/respawn_member",
4147 "mobkit/force_cancel_member",
4148 "mobkit/cancel_flow",
4149 "mobkit/collect_completed",
4150 "mobkit/run_flow",
4151 "mobkit/spawn_helper",
4152 "mobkit/fork_helper",
4153 "mobkit/attach_existing_session",
4154 "mobkit/reconcile_edges",
4155 "mobkit/cross_mob/wire_local",
4156 "mobkit/cross_mob/unwire_local",
4157 ]);
4158 }
4159 if metadata_table.is_some() {
4160 methods.extend_from_slice(&["mobkit/mob_labels/get", "mobkit/run_labels/get"]);
4161 if can_mutate {
4162 methods.extend_from_slice(&[
4163 "mobkit/mob_labels/set",
4164 "mobkit/mob_labels/delete",
4165 "mobkit/run_labels/set",
4166 "mobkit/run_labels/delete",
4167 ]);
4168 }
4169 }
4170 if access.is_some() {
4171 methods.push("mobkit/access/status");
4172 if access_view.is_some_and(AccessView::can_administer) {
4173 methods.extend_from_slice(&["mobkit/access/get", "mobkit/access/preview"]);
4174 if can_mutate {
4175 methods.extend_from_slice(&[
4176 "mobkit/access/set",
4177 "mobkit/access/enable",
4178 "mobkit/access/rules/upsert",
4179 "mobkit/access/rules/delete",
4180 "mobkit/access/groups/set",
4181 "mobkit/access/groups/delete",
4182 ]);
4183 }
4184 }
4185 }
4186 if let Some(view) = access_view.filter(|view| view.enforced()) {
4194 let probe = serde_json::json!({ "identity": "\u{0}cap-probe" });
4195 methods.retain(
4196 |method| match console_rpc_access_requirement(method, &probe) {
4197 Some((action, None)) => view.allows(action),
4198 _ => true,
4199 },
4200 );
4201 }
4202 let cap = |action: &str| -> bool {
4205 can_mutate
4206 && access_view
4207 .filter(|view| view.enforced())
4208 .is_none_or(|view| view.may_perform_anywhere(action))
4209 };
4210 response_value(
4211 response_id,
4212 Some(serde_json::json!({
4213 "contract_version": crate::rpc::MOBKIT_CONTRACT_VERSION,
4214 "methods": methods,
4215 "read_only": read_only,
4216 "loaded_modules": serde_json::json!([]),
4219 "runtime_capabilities": {
4220 "can_send_messages": cap(ACTION_AGENT_SEND),
4221 "can_retire_members": cap(ACTION_AGENT_RETIRE),
4222 "can_spawn_members": cap(ACTION_AGENT_SPAWN),
4223 }
4224 })),
4225 None,
4226 )
4227 }
4228 "mobkit/status" => {
4229 let mob_state = runtime.handle().status_observation_snapshot();
4230 response_value(
4231 response_id,
4232 Some(serde_json::json!({
4233 "contract_version": crate::rpc::MOBKIT_CONTRACT_VERSION,
4234 "running": matches!(mob_state, MobState::Creating | MobState::Running),
4235 "loaded_modules": serde_json::json!([]),
4238 })),
4239 None,
4240 )
4241 }
4242 "mobkit/console/list_identities" => {
4243 let Some(aggregator) = &console_aggregator else {
4244 return response_value(
4245 response_id,
4246 None,
4247 Some(JsonRpcError {
4248 code: -32004,
4249 message: "console aggregator unavailable".to_string(),
4250 data: None,
4251 }),
4252 );
4253 };
4254 match Box::pin(aggregator.list_identities()).await {
4255 Ok(mut identities) => {
4256 retain_visible_identity_records(&mut identities, access_view);
4257 response_value(response_id, Some(json!({ "identities": identities })), None)
4258 }
4259 Err(err) => internal_error(response_id, format!("list_identities failed: {err}")),
4260 }
4261 }
4262 "mobkit/console/inspect_identity" => {
4263 let Some(identity) = request.params.get("identity").and_then(Value::as_str) else {
4264 return invalid_params(response_id, "identity required");
4265 };
4266 let Some(aggregator) = &console_aggregator else {
4267 return response_value(
4268 response_id,
4269 None,
4270 Some(JsonRpcError {
4271 code: -32004,
4272 message: "console aggregator unavailable".to_string(),
4273 data: None,
4274 }),
4275 );
4276 };
4277 match Box::pin(aggregator.inspect_identity(identity)).await {
4278 Ok(Some(inspection)) => response_value(
4279 response_id,
4280 Some(serde_json::to_value(inspection).unwrap_or(Value::Null)),
4281 None,
4282 ),
4283 Ok(None) => response_value(
4284 response_id,
4285 None,
4286 Some(JsonRpcError {
4287 code: -32001,
4288 message: format!("unknown identity: {identity}"),
4289 data: None,
4290 }),
4291 ),
4292 Err(err) => internal_error(response_id, format!("inspect_identity failed: {err}")),
4293 }
4294 }
4295 "mobkit/console/query_timeline" => {
4296 let query: ConsoleTimelineWindowQuery =
4297 match serde_json::from_value(request.params.clone()) {
4298 Ok(query) => query,
4299 Err(err) => {
4300 return invalid_params(response_id, format!("invalid query params: {err}"));
4301 }
4302 };
4303 let Some(aggregator) = &console_aggregator else {
4304 return response_value(
4305 response_id,
4306 None,
4307 Some(JsonRpcError {
4308 code: -32004,
4309 message: "console aggregator unavailable".to_string(),
4310 data: None,
4311 }),
4312 );
4313 };
4314 match Box::pin(aggregator.query_timeline_windowed(query.clone())).await {
4315 Ok(mut page) => {
4316 retain_visible_timeline_frames(&mut page, access_view);
4317 response_value(
4318 response_id,
4319 Some(serde_json::to_value(page).unwrap_or(Value::Null)),
4320 None,
4321 )
4322 }
4323 Err(err) => {
4324 let latest_cursor = aggregator.latest_cursor().await.ok().flatten();
4325 console_timeline_replay_unavailable_response(
4326 response_id,
4327 err,
4328 query.after.as_ref(),
4329 latest_cursor,
4330 )
4331 }
4332 }
4333 }
4334 "mobkit/console/send" => {
4335 let send_request: ConsoleSendRequest =
4336 match serde_json::from_value(request.params.clone()) {
4337 Ok(request) => request,
4338 Err(err) => {
4339 return invalid_params(response_id, format!("invalid send params: {err}"));
4340 }
4341 };
4342 let Some(aggregator) = &console_aggregator else {
4343 return response_value(
4344 response_id,
4345 None,
4346 Some(JsonRpcError {
4347 code: -32004,
4348 message: "console aggregator unavailable".to_string(),
4349 data: None,
4350 }),
4351 );
4352 };
4353 if let Some(identity_runtime) = &identity_runtime {
4354 return match Box::pin(console_send_with_identity_first_fallback(
4355 aggregator,
4356 identity_runtime.clone(),
4357 Some(runtime),
4358 console_events.as_ref(),
4359 send_request,
4360 ))
4361 .await
4362 {
4363 Ok(accepted) => response_value(
4364 response_id,
4365 Some(serde_json::to_value(accepted).unwrap_or(Value::Null)),
4366 None,
4367 ),
4368 Err(err) => {
4369 response_value(response_id, None, Some(console_send_json_rpc_error(err)))
4370 }
4371 };
4372 }
4373 match Box::pin(aggregator.send(send_request)).await {
4374 Ok(accepted) => response_value(
4375 response_id,
4376 Some(serde_json::to_value(accepted).unwrap_or(Value::Null)),
4377 None,
4378 ),
4379 Err(err) => {
4380 response_value(response_id, None, Some(console_send_json_rpc_error(err)))
4381 }
4382 }
4383 }
4384 "mobkit/blob/get" => {
4385 let Some(blob_id) = request
4386 .params
4387 .get("blob_id")
4388 .or_else(|| request.params.get("id"))
4389 .and_then(Value::as_str)
4390 else {
4391 return invalid_params(response_id, "blob_id required");
4392 };
4393 if !is_valid_blob_id_value(blob_id) {
4394 return invalid_params(response_id, "invalid blob_id");
4395 }
4396 let Some(store) = runtime.binary_blob_store() else {
4397 return internal_error(response_id, "binary blob store unavailable");
4398 };
4399 match store.get_bytes(&meerkat_core::BlobId::from(blob_id)).await {
4400 Ok(payload) => response_value(
4401 response_id,
4402 Some(serde_json::json!({
4403 "blob_id": payload.blob_id,
4404 "media_type": payload.media_type,
4405 "size": payload.size,
4406 "data": base64::engine::general_purpose::STANDARD.encode(payload.data.as_ref()),
4407 })),
4408 None,
4409 ),
4410 Err(meerkat_core::BlobStoreError::NotFound(_)) => response_value(
4411 response_id,
4412 None,
4413 Some(JsonRpcError {
4414 code: -32001,
4415 message: format!("blob not found: {blob_id}"),
4416 data: Some(json!({ "kind": "not_found", "blob_id": blob_id })),
4417 }),
4418 ),
4419 Err(err) => internal_error(response_id, format!("blob get failed: {err}")),
4420 }
4421 }
4422 "mobkit/list_members" => {
4423 let handle = runtime.handle();
4424 let entries = handle.list_members_including_retiring().await;
4425 let mut members = Vec::with_capacity(entries.len());
4426 for entry in &entries {
4427 members.push(member_entry_to_console_json(runtime, entry).await);
4428 }
4429 retain_visible_member_rows(&mut members, access_view);
4430 response_value(response_id, Some(Value::Array(members)), None)
4431 }
4432 "mobkit/get_member" => {
4433 let Some(member_id) = request.params.get("member_id").and_then(Value::as_str) else {
4434 return invalid_params(response_id, "member_id required");
4435 };
4436 let handle = runtime.handle();
4437 let identity = crate::member_comms_id::mob_member_id(member_id);
4438 let entries = handle.list_members_including_retiring().await;
4439 match entries.into_iter().find(|e| e.agent_identity == identity) {
4440 Some(entry) => response_value(
4441 response_id,
4442 Some(member_entry_to_console_json(runtime, &entry).await),
4443 None,
4444 ),
4445 None => invalid_params(response_id, format!("member not found: {member_id}")),
4446 }
4447 }
4448 "mobkit/find_members" => {
4449 let Some(label_key) = request.params.get("label_key").and_then(Value::as_str) else {
4450 return invalid_params(response_id, "label_key required");
4451 };
4452 let Some(label_value) = request.params.get("label_value").and_then(Value::as_str)
4453 else {
4454 return invalid_params(response_id, "label_value required");
4455 };
4456 let handle = runtime.handle();
4457 let filter = MemberFilter {
4458 labels: std::collections::BTreeMap::from([(
4459 label_key.to_string(),
4460 label_value.to_string(),
4461 )]),
4462 role: None,
4463 status: None,
4464 };
4465 let entries = match handle.list_members_matching(filter).await {
4466 Ok(entries) => entries,
4467 Err(err) => {
4468 return invalid_params(response_id, format!("member lookup failed: {err}"));
4469 }
4470 };
4471 let mut matches = Vec::with_capacity(entries.len());
4472 for entry in &entries {
4473 matches.push(member_entry_to_console_json(runtime, entry).await);
4474 }
4475 retain_visible_member_rows(&mut matches, access_view);
4476 response_value(response_id, Some(Value::Array(matches)), None)
4477 }
4478 "mobkit/status_identity" => {
4479 let Some(identity) = request.params.get("identity").and_then(Value::as_str) else {
4480 return invalid_params(response_id, "identity required");
4481 };
4482 let handle = runtime.handle();
4483 if let Some(identity_runtime) = &identity_runtime {
4484 let (parsed_identity, _requested_exact_identity, live_alias) =
4485 match resolve_console_identity_control_target(
4486 &handle,
4487 Some(identity_runtime),
4488 visibility_policy,
4489 identity,
4490 )
4491 .await
4492 {
4493 Ok(Some(target)) => target,
4494 Ok(None) => {
4495 return invalid_params(
4496 response_id,
4497 format!("identity not found: {identity}"),
4498 );
4499 }
4500 Err(err) => return response_value(response_id, None, Some(err)),
4501 };
4502 match identity_runtime.status(&parsed_identity).await {
4503 Ok(status) => {
4504 if !identity_status_visible_to_console(visibility_policy, &status) {
4505 return identity_hidden_by_policy_response(response_id, identity);
4506 }
4507 let phase = if let Some(store) = &console_events {
4508 store
4509 .response_phase_for_identity(status.identity.as_str())
4510 .await
4511 } else {
4512 None
4513 };
4514 if let Some(error) = stale_live_alias_json_rpc_error(
4515 "status_identity",
4516 identity_runtime,
4517 &parsed_identity,
4518 live_alias.as_ref(),
4519 )
4520 .await
4521 {
4522 return response_value(response_id, None, Some(error));
4523 }
4524 return response_value(
4525 response_id,
4526 Some(console_identity_status_json_from_identity_status(
4527 &status, phase,
4528 )),
4529 None,
4530 );
4531 }
4532 Err(crate::identity_first::IdentityRuntimeError::UnknownIdentity(_)) => {}
4533 Err(err) => {
4534 return console_identity_error_response(
4535 response_id,
4536 "status_identity",
4537 err,
4538 );
4539 }
4540 }
4541 }
4542 if let Some(aggregator) = &console_aggregator {
4543 return match Box::pin(aggregator.inspect_identity(identity)).await {
4544 Ok(Some(inspection)) => {
4545 let phase = if let Some(store) = &console_events {
4546 store
4547 .response_phase_for_identity(&inspection.identity.identity)
4548 .await
4549 } else {
4550 None
4551 };
4552 response_value(
4553 response_id,
4554 Some(console_identity_status_json_from_record(
4555 &inspection.identity,
4556 phase,
4557 )),
4558 None,
4559 )
4560 }
4561 Ok(None) => response_value(
4562 response_id,
4563 None,
4564 Some(JsonRpcError {
4565 code: -32001,
4566 message: format!("unknown identity: {identity}"),
4567 data: None,
4568 }),
4569 ),
4570 Err(err) => {
4571 internal_error(response_id, format!("status_identity failed: {err}"))
4572 }
4573 };
4574 }
4575 let live_alias = match lookup_member_alias_with_session(
4576 &handle,
4577 visibility_policy,
4578 identity,
4579 )
4580 .await
4581 {
4582 Ok(alias) => alias,
4583 Err(err) => return response_value(response_id, None, Some(err)),
4584 };
4585 let Some(alias) = live_alias else {
4586 return invalid_params(response_id, format!("identity not found: {identity}"));
4587 };
4588 if !runtime_alias_visible_to_console(&handle, visibility_policy, &alias) {
4589 return identity_hidden_by_policy_response(response_id, identity);
4590 }
4591 if let Err(err) =
4592 reject_ambiguous_projected_live_identity(&handle, visibility_policy, &alias).await
4593 {
4594 return response_value(response_id, None, Some(err));
4595 }
4596 let phase = if let Some(store) = &console_events {
4597 store.response_phase_for_identity(&alias.identity).await
4598 } else {
4599 None
4600 };
4601 response_value(
4602 response_id,
4603 Some(console_identity_status_json_for_identity(
4604 &alias.identity,
4605 &alias.member,
4606 alias.session_id,
4607 phase,
4608 )),
4609 None,
4610 )
4611 }
4612 "mobkit/inspect_identity" => {
4613 let Some(identity) = request.params.get("identity").and_then(Value::as_str) else {
4614 return invalid_params(response_id, "identity required");
4615 };
4616 let handle = runtime.handle();
4617 if let Some(identity_runtime) = &identity_runtime {
4618 let (parsed_identity, _requested_exact_identity, live_alias) =
4619 match resolve_console_identity_control_target(
4620 &handle,
4621 Some(identity_runtime),
4622 visibility_policy,
4623 identity,
4624 )
4625 .await
4626 {
4627 Ok(Some(target)) => target,
4628 Ok(None) => {
4629 return invalid_params(
4630 response_id,
4631 format!("identity not found: {identity}"),
4632 );
4633 }
4634 Err(err) => return response_value(response_id, None, Some(err)),
4635 };
4636 match identity_runtime.status(&parsed_identity).await {
4637 Ok(status) => {
4638 if !identity_status_visible_to_console(visibility_policy, &status) {
4639 return identity_hidden_by_policy_response(response_id, identity);
4640 }
4641 let phase = if let Some(store) = &console_events {
4642 store
4643 .response_phase_for_identity(status.identity.as_str())
4644 .await
4645 } else {
4646 None
4647 };
4648 if let Some(error) = stale_live_alias_json_rpc_error(
4649 "inspect_identity",
4650 identity_runtime,
4651 &parsed_identity,
4652 live_alias.as_ref(),
4653 )
4654 .await
4655 {
4656 return response_value(response_id, None, Some(error));
4657 }
4658 return response_value(
4659 response_id,
4660 Some(console_identity_inspect_json_from_identity_status(
4661 &status,
4662 live_alias.as_ref(),
4663 phase,
4664 )),
4665 None,
4666 );
4667 }
4668 Err(crate::identity_first::IdentityRuntimeError::UnknownIdentity(_)) => {}
4669 Err(err) => {
4670 return console_identity_error_response(
4671 response_id,
4672 "inspect_identity",
4673 err,
4674 );
4675 }
4676 }
4677 }
4678 if let Some(aggregator) = &console_aggregator {
4679 return match Box::pin(aggregator.inspect_identity(identity)).await {
4680 Ok(Some(inspection)) => {
4681 let phase = if let Some(store) = &console_events {
4682 store
4683 .response_phase_for_identity(&inspection.identity.identity)
4684 .await
4685 } else {
4686 None
4687 };
4688 response_value(
4689 response_id,
4690 Some(console_identity_inspect_json_from_record(
4691 &inspection,
4692 phase,
4693 )),
4694 None,
4695 )
4696 }
4697 Ok(None) => response_value(
4698 response_id,
4699 None,
4700 Some(JsonRpcError {
4701 code: -32001,
4702 message: format!("unknown identity: {identity}"),
4703 data: None,
4704 }),
4705 ),
4706 Err(err) => {
4707 internal_error(response_id, format!("inspect_identity failed: {err}"))
4708 }
4709 };
4710 }
4711 let live_alias = match lookup_member_alias_with_session(
4712 &handle,
4713 visibility_policy,
4714 identity,
4715 )
4716 .await
4717 {
4718 Ok(alias) => alias,
4719 Err(err) => return response_value(response_id, None, Some(err)),
4720 };
4721 let Some(alias) = live_alias else {
4722 return invalid_params(response_id, format!("identity not found: {identity}"));
4723 };
4724 if !runtime_alias_visible_to_console(&handle, visibility_policy, &alias) {
4725 return identity_hidden_by_policy_response(response_id, identity);
4726 }
4727 if let Err(err) =
4728 reject_ambiguous_projected_live_identity(&handle, visibility_policy, &alias).await
4729 {
4730 return response_value(response_id, None, Some(err));
4731 }
4732 let phase = if let Some(store) = &console_events {
4733 store.response_phase_for_identity(&alias.identity).await
4734 } else {
4735 None
4736 };
4737 response_value(
4738 response_id,
4739 Some(console_identity_inspect_json_for_identity(
4740 &alias.identity,
4741 &alias.member,
4742 alias.session_id,
4743 phase,
4744 )),
4745 None,
4746 )
4747 }
4748 "mobkit/retire" => {
4749 let Some(identity) = request.params.get("identity").and_then(Value::as_str) else {
4750 return invalid_params(response_id, "identity required");
4751 };
4752 let handle = runtime.handle();
4753 if let Some(identity_runtime) = &identity_runtime {
4754 let (parsed_identity, _requested_exact_identity, live_alias) =
4755 match resolve_console_identity_control_target(
4756 &handle,
4757 Some(identity_runtime),
4758 visibility_policy,
4759 identity,
4760 )
4761 .await
4762 {
4763 Ok(Some(target)) => target,
4764 Ok(None) => {
4765 return invalid_params(
4766 response_id,
4767 format!("identity not found: {identity}"),
4768 );
4769 }
4770 Err(err) => return response_value(response_id, None, Some(err)),
4771 };
4772 let registered_status = match identity_runtime.status(&parsed_identity).await {
4773 Ok(status) => Some(status),
4774 Err(crate::identity_first::IdentityRuntimeError::UnknownIdentity(_)) => None,
4775 Err(err) => {
4776 return console_identity_error_response(response_id, "retire", err);
4777 }
4778 };
4779 if let Some(status) = registered_status.as_ref() {
4780 if !identity_status_visible_to_console(visibility_policy, status) {
4781 return identity_hidden_by_policy_response(response_id, identity);
4782 }
4783 if let Some(error) = stale_live_alias_json_rpc_error(
4784 "retire",
4785 identity_runtime,
4786 &parsed_identity,
4787 live_alias.as_ref(),
4788 )
4789 .await
4790 {
4791 return response_value(response_id, None, Some(error));
4792 }
4793 }
4794 match identity_runtime.retire(&parsed_identity).await {
4795 Ok(token) => {
4796 let keep_runtime_member_id = registered_status
4797 .as_ref()
4798 .and_then(|status| status.agent_runtime_id.as_ref())
4799 .filter(|_| identity_runtime.has_session_bridge())
4800 .map(crate::identity_first::AgentRuntimeId::as_str);
4801 let cleanup_warning = if registered_status.is_some()
4802 && let Err(err) = retire_stale_console_members_for_identity(
4803 &handle,
4804 visibility_policy,
4805 parsed_identity.as_str(),
4806 keep_runtime_member_id,
4807 )
4808 .await
4809 {
4810 Some(json!({
4811 "kind": "stale_member_cleanup_failed_after_identity_retire",
4812 "identity": parsed_identity.as_str(),
4813 "message": err,
4814 }))
4815 } else {
4816 None
4817 };
4818 if let Some(store) = &console_events {
4819 store
4820 .record_lifecycle(
4821 parsed_identity.as_str(),
4822 "identity_retired",
4823 json!({
4824 "fencing_token": token.get(),
4825 "cleanup_warning": cleanup_warning.clone(),
4826 }),
4827 )
4828 .await;
4829 }
4830 return response_value(
4831 response_id,
4832 Some(json!({
4833 "identity": parsed_identity.as_str(),
4834 "fencing_token": token.get(),
4835 "cleanup_warning": cleanup_warning,
4836 })),
4837 None,
4838 );
4839 }
4840 Err(crate::identity_first::IdentityRuntimeError::UnknownIdentity(_)) => {
4841 if let Some(alias) = live_alias.as_ref() {
4842 if !runtime_alias_visible_to_console(&handle, visibility_policy, alias)
4843 {
4844 return identity_hidden_by_policy_response(response_id, identity);
4845 }
4846 let mid = crate::member_comms_id::mob_member_id(
4847 alias.runtime_member_id.as_str(),
4848 );
4849 return match retire_console_member(&handle, &mid).await {
4850 Ok(()) => {
4851 if let Some(store) = &console_events {
4852 store
4853 .record_lifecycle(
4854 &alias.identity,
4855 "identity_retired",
4856 json!({}),
4857 )
4858 .await;
4859 }
4860 response_value(
4861 response_id,
4862 Some(json!({ "identity": alias.identity })),
4863 None,
4864 )
4865 }
4866 Err(err) => {
4867 internal_error(response_id, format!("retire failed: {err}"))
4868 }
4869 };
4870 }
4871 }
4872 Err(err) => return console_identity_error_response(response_id, "retire", err),
4873 }
4874 }
4875 if let Some(aggregator) = &console_aggregator {
4876 let canonical_identity = match Box::pin(aggregator.inspect_identity(identity)).await
4877 {
4878 Ok(Some(inspection)) => inspection.identity.identity,
4879 Ok(None) => identity.to_string(),
4880 Err(_) => identity.to_string(),
4881 };
4882 return match Box::pin(aggregator.retire_identity(identity)).await {
4883 Ok(true) => {
4884 if let Some(store) = &console_events {
4885 store
4886 .record_lifecycle(
4887 &canonical_identity,
4888 "identity_retired",
4889 json!({}),
4890 )
4891 .await;
4892 }
4893 response_value(
4894 response_id,
4895 Some(json!({ "identity": canonical_identity })),
4896 None,
4897 )
4898 }
4899 Ok(false) => response_value(
4900 response_id,
4901 None,
4902 Some(JsonRpcError {
4903 code: -32001,
4904 message: format!("unknown identity: {identity}"),
4905 data: None,
4906 }),
4907 ),
4908 Err(err) => internal_error(response_id, format!("retire failed: {err}")),
4909 };
4910 }
4911 let live_alias = match lookup_member_alias_with_session(
4912 &handle,
4913 visibility_policy,
4914 identity,
4915 )
4916 .await
4917 {
4918 Ok(alias) => alias,
4919 Err(err) => return response_value(response_id, None, Some(err)),
4920 };
4921 let Some(alias) = live_alias else {
4922 return invalid_params(response_id, format!("identity not found: {identity}"));
4923 };
4924 if !runtime_alias_visible_to_console(&handle, visibility_policy, &alias) {
4925 return identity_hidden_by_policy_response(response_id, identity);
4926 }
4927 if let Err(err) =
4928 reject_ambiguous_projected_live_identity(&handle, visibility_policy, &alias).await
4929 {
4930 return response_value(response_id, None, Some(err));
4931 }
4932 let mid = crate::member_comms_id::mob_member_id(alias.runtime_member_id.as_str());
4933 match retire_console_member(&handle, &mid).await {
4934 Ok(()) => {
4935 if let Some(store) = &console_events {
4936 store
4937 .record_lifecycle(&alias.identity, "identity_retired", json!({}))
4938 .await;
4939 }
4940 response_value(
4941 response_id,
4942 Some(json!({ "identity": alias.identity })),
4943 None,
4944 )
4945 }
4946 Err(err) => internal_error(response_id, format!("retire failed: {err}")),
4947 }
4948 }
4949 "mobkit/respawn" => {
4950 let Some(identity) = request.params.get("identity").and_then(Value::as_str) else {
4951 return invalid_params(response_id, "identity required");
4952 };
4953 let handle = runtime.handle();
4954 if let Some(identity_runtime) = &identity_runtime {
4955 let (parsed_identity, _requested_exact_identity, live_alias) =
4956 match resolve_console_identity_control_target(
4957 &handle,
4958 Some(identity_runtime),
4959 visibility_policy,
4960 identity,
4961 )
4962 .await
4963 {
4964 Ok(Some(target)) => target,
4965 Ok(None) => {
4966 return invalid_params(
4967 response_id,
4968 format!("identity not found: {identity}"),
4969 );
4970 }
4971 Err(err) => return response_value(response_id, None, Some(err)),
4972 };
4973 let registered_status = match identity_runtime.status(&parsed_identity).await {
4974 Ok(status) => Some(status),
4975 Err(crate::identity_first::IdentityRuntimeError::UnknownIdentity(_)) => None,
4976 Err(err) => {
4977 return console_identity_error_response(response_id, "respawn", err);
4978 }
4979 };
4980 if let Some(status) = registered_status.as_ref() {
4981 if !identity_status_visible_to_console(visibility_policy, status) {
4982 return identity_hidden_by_policy_response(response_id, identity);
4983 }
4984 if let Some(error) = stale_live_alias_json_rpc_error(
4985 "respawn",
4986 identity_runtime,
4987 &parsed_identity,
4988 live_alias.as_ref(),
4989 )
4990 .await
4991 {
4992 return response_value(response_id, None, Some(error));
4993 }
4994 }
4995 match identity_runtime.respawn(&parsed_identity).await {
4996 Ok(mut record) => {
4997 let live_respawn_warning = match Box::pin(respawn_console_member(
4998 &handle,
4999 &crate::member_comms_id::mob_member_id(
5000 record.agent_runtime_id.as_str(),
5001 ),
5002 ))
5003 .await
5004 {
5005 Ok(topology_restore_warning) => {
5006 let live_session_id = handle
5007 .resolve_bridge_session_id_observation(
5008 &crate::member_comms_id::mob_member_id(
5009 record.agent_runtime_id.as_str(),
5010 ),
5011 )
5012 .await;
5013 if let Some(live_session_id) = live_session_id {
5014 match identity_runtime
5015 .rebind_session_after_live_respawn(
5016 &parsed_identity,
5017 live_session_id.clone(),
5018 )
5019 .await
5020 {
5021 Ok(updated_record) => {
5022 record = updated_record;
5023 topology_restore_warning
5024 }
5025 Err(err) => Some(json!({
5026 "kind": "identity_rebind_failed_after_member_respawn",
5027 "identity": record.identity.as_str(),
5028 "agent_runtime_id": record.agent_runtime_id.as_str(),
5029 "live_session_id": live_session_id.to_string(),
5030 "message": err.to_string(),
5031 })),
5032 }
5033 } else {
5034 topology_restore_warning
5035 }
5036 }
5037 Err(err) => Some(json!({
5038 "kind": "member_respawn_failed_after_identity_refresh",
5039 "identity": record.identity.as_str(),
5040 "agent_runtime_id": record.agent_runtime_id.as_str(),
5041 "message": err,
5042 })),
5043 };
5044 let cleanup_warning = if registered_status.is_some()
5045 && let Err(err) = retire_stale_console_members_for_identity(
5046 &handle,
5047 visibility_policy,
5048 parsed_identity.as_str(),
5049 Some(record.agent_runtime_id.as_str()),
5050 )
5051 .await
5052 {
5053 Some(json!({
5054 "kind": "stale_member_cleanup_failed_after_identity_respawn",
5055 "identity": parsed_identity.as_str(),
5056 "agent_runtime_id": record.agent_runtime_id.as_str(),
5057 "message": err,
5058 }))
5059 } else {
5060 None
5061 };
5062 if let Some(store) = &console_events {
5063 store
5064 .record_lifecycle(
5065 parsed_identity.as_str(),
5066 "identity_respawned",
5067 json!({
5068 "generation": record.generation.get(),
5069 "checkpoint_version": record.checkpoint_version.get(),
5070 "live_respawn_warning": live_respawn_warning.clone(),
5071 "cleanup_warning": cleanup_warning.clone(),
5072 }),
5073 )
5074 .await;
5075 }
5076 return response_value(
5077 response_id,
5078 Some(json!({
5079 "identity": record.identity.as_str(),
5080 "agent_runtime_id": record.agent_runtime_id.as_str(),
5081 "session_id": record.session_id.to_string(),
5082 "generation": record.generation.get(),
5083 "checkpoint_version": record.checkpoint_version.get(),
5084 "live_respawn_warning": live_respawn_warning,
5085 "cleanup_warning": cleanup_warning,
5086 })),
5087 None,
5088 );
5089 }
5090 Err(crate::identity_first::IdentityRuntimeError::UnknownIdentity(_)) => {
5091 if live_alias.is_none() {
5092 return invalid_params(
5093 response_id,
5094 format!("identity not found: {identity}"),
5095 );
5096 }
5097 }
5098 Err(err) => {
5099 return console_identity_error_response(response_id, "respawn", err);
5100 }
5101 }
5102 }
5103 let live_alias = match lookup_member_alias_with_session(
5104 &handle,
5105 visibility_policy,
5106 identity,
5107 )
5108 .await
5109 {
5110 Ok(alias) => alias,
5111 Err(err) => return response_value(response_id, None, Some(err)),
5112 };
5113 let Some(alias) = live_alias else {
5114 return invalid_params(response_id, format!("identity not found: {identity}"));
5115 };
5116 if !runtime_alias_visible_to_console(&handle, visibility_policy, &alias) {
5117 return identity_hidden_by_policy_response(response_id, identity);
5118 }
5119 if let Err(err) =
5120 reject_ambiguous_projected_live_identity(&handle, visibility_policy, &alias).await
5121 {
5122 return response_value(response_id, None, Some(err));
5123 }
5124 Box::pin(respawn_live_console_member_response(
5125 &handle,
5126 console_events.as_ref(),
5127 &alias,
5128 "identity_respawned",
5129 "respawn",
5130 response_id,
5131 ))
5132 .await
5133 }
5134 "mobkit/reset" => {
5135 let Some(identity) = request.params.get("identity").and_then(Value::as_str) else {
5136 return invalid_params(response_id, "identity required");
5137 };
5138 let handle = runtime.handle();
5139 let Some(identity_runtime) = &identity_runtime else {
5140 let live_alias =
5146 match lookup_member_alias_with_session(&handle, visibility_policy, identity)
5147 .await
5148 {
5149 Ok(alias) => alias,
5150 Err(err) => return response_value(response_id, None, Some(err)),
5151 };
5152 let Some(alias) = live_alias else {
5153 return invalid_params(response_id, format!("identity not found: {identity}"));
5154 };
5155 if !runtime_alias_visible_to_console(&handle, visibility_policy, &alias) {
5156 return identity_hidden_by_policy_response(response_id, identity);
5157 }
5158 if let Err(err) =
5159 reject_ambiguous_projected_live_identity(&handle, visibility_policy, &alias)
5160 .await
5161 {
5162 return response_value(response_id, None, Some(err));
5163 }
5164 return Box::pin(respawn_live_console_member_response(
5165 &handle,
5166 console_events.as_ref(),
5167 &alias,
5168 "identity_reset",
5169 "reset",
5170 response_id,
5171 ))
5172 .await;
5173 };
5174 let (parsed_identity, _requested_exact_identity, live_alias) =
5175 match resolve_console_identity_control_target(
5176 &handle,
5177 Some(identity_runtime),
5178 visibility_policy,
5179 identity,
5180 )
5181 .await
5182 {
5183 Ok(Some(target)) => target,
5184 Ok(None) => {
5185 return invalid_params(
5186 response_id,
5187 format!("identity not found: {identity}"),
5188 );
5189 }
5190 Err(err) => return response_value(response_id, None, Some(err)),
5191 };
5192 match identity_runtime.status(&parsed_identity).await {
5193 Ok(status) => {
5194 if !identity_status_visible_to_console(visibility_policy, &status) {
5195 return identity_hidden_by_policy_response(response_id, identity);
5196 }
5197 if let Some(error) = stale_live_alias_json_rpc_error(
5198 "reset",
5199 identity_runtime,
5200 &parsed_identity,
5201 live_alias.as_ref(),
5202 )
5203 .await
5204 {
5205 return response_value(response_id, None, Some(error));
5206 }
5207 if !identity_runtime.has_session_bridge() {
5208 return response_value(
5209 response_id,
5210 None,
5211 Some(reset_requires_session_bridge_json_rpc_error()),
5212 );
5213 }
5214 }
5215 Err(crate::identity_first::IdentityRuntimeError::UnknownIdentity(_)) => {
5216 if let Some(alias) = live_alias.as_ref() {
5217 if !runtime_alias_visible_to_console(&handle, visibility_policy, alias) {
5218 return identity_hidden_by_policy_response(response_id, identity);
5219 }
5220 return Box::pin(respawn_live_console_member_response(
5221 &handle,
5222 console_events.as_ref(),
5223 alias,
5224 "identity_reset",
5225 "reset",
5226 response_id,
5227 ))
5228 .await;
5229 }
5230 return invalid_params(response_id, format!("identity not found: {identity}"));
5231 }
5232 Err(err) => return console_identity_error_response(response_id, "reset", err),
5233 }
5234 match identity_runtime.reset(&parsed_identity).await {
5235 Ok(record) => {
5236 let cleanup_warning = if let Err(err) =
5237 retire_stale_console_members_for_identity(
5238 &handle,
5239 visibility_policy,
5240 parsed_identity.as_str(),
5241 Some(record.agent_runtime_id.as_str()),
5242 )
5243 .await
5244 {
5245 Some(json!({
5246 "kind": "stale_member_cleanup_failed_after_identity_reset",
5247 "identity": parsed_identity.as_str(),
5248 "agent_runtime_id": record.agent_runtime_id.as_str(),
5249 "message": err,
5250 }))
5251 } else {
5252 None
5253 };
5254 if let Some(store) = &console_events {
5255 store
5256 .record_lifecycle(
5257 parsed_identity.as_str(),
5258 "identity_reset",
5259 json!({
5260 "generation": record.generation.get(),
5261 "checkpoint_version": record.checkpoint_version.get(),
5262 "cleanup_warning": cleanup_warning.clone(),
5263 }),
5264 )
5265 .await;
5266 }
5267 response_value(
5268 response_id,
5269 Some(json!({
5270 "identity": record.identity.as_str(),
5271 "agent_runtime_id": record.agent_runtime_id.as_str(),
5272 "session_id": record.session_id.to_string(),
5273 "generation": record.generation.get(),
5274 "checkpoint_version": record.checkpoint_version.get(),
5275 "cleanup_warning": cleanup_warning,
5276 })),
5277 None,
5278 )
5279 }
5280 Err(crate::identity_first::IdentityRuntimeError::UnknownIdentity(_)) => {
5281 invalid_params(response_id, format!("identity not found: {identity}"))
5282 }
5283 Err(err) => console_identity_error_response(response_id, "reset", err),
5284 }
5285 }
5286 "mobkit/delete_identity" => {
5287 let Some(identity) = request.params.get("identity").and_then(Value::as_str) else {
5288 return invalid_params(response_id, "identity required");
5289 };
5290 let handle = runtime.handle();
5291 let Some(identity_runtime) = &identity_runtime else {
5292 return invalid_params(
5293 response_id,
5294 "identity-first runtime required for delete_identity",
5295 );
5296 };
5297 let (parsed_identity, _requested_exact_identity, live_alias) =
5298 match resolve_console_identity_control_target(
5299 &handle,
5300 Some(identity_runtime),
5301 visibility_policy,
5302 identity,
5303 )
5304 .await
5305 {
5306 Ok(Some(target)) => target,
5307 Ok(None) => {
5308 return invalid_params(
5309 response_id,
5310 format!("identity not found: {identity}"),
5311 );
5312 }
5313 Err(err) => return response_value(response_id, None, Some(err)),
5314 };
5315 let registered_status = match identity_runtime.status(&parsed_identity).await {
5316 Ok(status) => status,
5317 Err(crate::identity_first::IdentityRuntimeError::UnknownIdentity(_)) => {
5318 if let Some(alias) = live_alias.as_ref() {
5319 if !runtime_alias_visible_to_console(&handle, visibility_policy, alias) {
5320 return identity_hidden_by_policy_response(response_id, identity);
5321 }
5322 return response_value(
5323 response_id,
5324 None,
5325 Some(JsonRpcError {
5326 code: -32602,
5327 message: format!(
5328 "delete_identity requires durable identity: {} is live-only",
5329 parsed_identity.as_str()
5330 ),
5331 data: Some(json!({
5332 "kind": "live_only_identity_delete_unsupported",
5333 "identity": parsed_identity.as_str(),
5334 })),
5335 }),
5336 );
5337 }
5338 return invalid_params(response_id, format!("identity not found: {identity}"));
5339 }
5340 Err(err) => {
5341 return console_identity_error_response(response_id, "delete_identity", err);
5342 }
5343 };
5344 if !identity_status_visible_to_console(visibility_policy, ®istered_status) {
5345 return identity_hidden_by_policy_response(response_id, identity);
5346 }
5347 if let Some(error) = stale_live_alias_json_rpc_error(
5348 "delete_identity",
5349 identity_runtime,
5350 &parsed_identity,
5351 live_alias.as_ref(),
5352 )
5353 .await
5354 {
5355 return response_value(response_id, None, Some(error));
5356 }
5357 match identity_runtime.delete_identity(&parsed_identity).await {
5358 Ok(()) => {
5359 let keep_runtime_member_id = registered_status
5360 .agent_runtime_id
5361 .as_ref()
5362 .filter(|_| identity_runtime.has_session_bridge())
5363 .map(crate::identity_first::AgentRuntimeId::as_str);
5364 let cleanup_warning = if let Err(err) =
5365 retire_stale_console_members_for_identity(
5366 &handle,
5367 visibility_policy,
5368 parsed_identity.as_str(),
5369 keep_runtime_member_id,
5370 )
5371 .await
5372 {
5373 Some(json!({
5374 "kind": "stale_member_cleanup_failed_after_identity_delete",
5375 "identity": parsed_identity.as_str(),
5376 "message": err,
5377 }))
5378 } else {
5379 None
5380 };
5381 if let Some(store) = &console_events {
5382 store
5383 .record_lifecycle(
5384 parsed_identity.as_str(),
5385 "identity_deleted",
5386 json!({
5387 "cleanup_warning": cleanup_warning.clone(),
5388 }),
5389 )
5390 .await;
5391 }
5392 response_value(
5393 response_id,
5394 Some(json!({
5395 "identity": parsed_identity.as_str(),
5396 "cleanup_warning": cleanup_warning,
5397 })),
5398 None,
5399 )
5400 }
5401 Err(err) => console_identity_error_response(response_id, "delete_identity", err),
5402 }
5403 }
5404 "mobkit/reset_all" => {
5405 match Box::pin(reset_all_live_console_agents(
5406 runtime,
5407 console_events.as_ref(),
5408 console_aggregator.as_ref(),
5409 identity_runtime.as_ref(),
5410 visibility_policy,
5411 ))
5412 .await
5413 {
5414 Ok(body) => {
5415 if body
5416 .get("failed")
5417 .and_then(Value::as_array)
5418 .is_some_and(|failed| !failed.is_empty())
5419 {
5420 response_value(
5421 response_id,
5422 None,
5423 Some(JsonRpcError {
5424 code: -32000,
5425 message: "reset_all failed for one or more identities".to_string(),
5426 data: Some(body),
5427 }),
5428 )
5429 } else {
5430 response_value(response_id, Some(body), None)
5431 }
5432 }
5433 Err(err) => internal_error(response_id, format!("reset_all failed: {err}")),
5434 }
5435 }
5436 "mobkit/routing/routes/list" => {
5437 let Some(module_runtime) = &module_runtime else {
5438 return response_value(
5439 response_id,
5440 None,
5441 Some(JsonRpcError {
5442 code: -32601,
5443 message: "Method not found".to_string(),
5444 data: None,
5445 }),
5446 );
5447 };
5448 let routes = module_runtime.lock().await.list_runtime_routes();
5449 response_value(response_id, Some(json!({ "routes": routes })), None)
5450 }
5451 "mobkit/delivery/history" => {
5452 let Some(module_runtime) = &module_runtime else {
5453 return response_value(
5454 response_id,
5455 None,
5456 Some(JsonRpcError {
5457 code: -32601,
5458 message: "Method not found".to_string(),
5459 data: None,
5460 }),
5461 );
5462 };
5463 let limit = request
5464 .params
5465 .get("limit")
5466 .and_then(Value::as_u64)
5467 .unwrap_or(50) as usize;
5468 let history = module_runtime
5469 .lock()
5470 .await
5471 .delivery_history(DeliveryHistoryRequest {
5472 recipient: None,
5473 sink: None,
5474 limit,
5475 });
5476 response_value(
5477 response_id,
5478 Some(serde_json::to_value(history).unwrap_or(Value::Null)),
5479 None,
5480 )
5481 }
5482 "mobkit/gating/pending" => {
5483 let Some(module_runtime) = &module_runtime else {
5484 return response_value(
5485 response_id,
5486 None,
5487 Some(JsonRpcError {
5488 code: -32601,
5489 message: "Method not found".to_string(),
5490 data: None,
5491 }),
5492 );
5493 };
5494 let pending = module_runtime.lock().await.list_gating_pending();
5495 response_value(response_id, Some(json!({ "pending": pending })), None)
5496 }
5497 "mobkit/gating/audit" => {
5498 let Some(module_runtime) = &module_runtime else {
5499 return response_value(
5500 response_id,
5501 None,
5502 Some(JsonRpcError {
5503 code: -32601,
5504 message: "Method not found".to_string(),
5505 data: None,
5506 }),
5507 );
5508 };
5509 let limit = request
5510 .params
5511 .get("limit")
5512 .and_then(Value::as_u64)
5513 .unwrap_or(50) as usize;
5514 let entries = module_runtime.lock().await.gating_audit_entries(limit);
5515 response_value(response_id, Some(json!({ "entries": entries })), None)
5516 }
5517 "mobkit/gating/decide" => {
5518 let Some(module_runtime) = &module_runtime else {
5519 return response_value(
5520 response_id,
5521 None,
5522 Some(JsonRpcError {
5523 code: -32601,
5524 message: "Method not found".to_string(),
5525 data: None,
5526 }),
5527 );
5528 };
5529 let Some(pending_id) = request.params.get("pending_id").and_then(Value::as_str) else {
5530 return invalid_params(response_id, "pending_id required");
5531 };
5532 let approver_id =
5533 match resolve_gating_approver_id(&request.params, authenticated_principal) {
5534 Ok(approver_id) => approver_id,
5535 Err(message) => return invalid_params(response_id, message),
5536 };
5537 let Some(raw_decision) = request.params.get("decision").and_then(Value::as_str) else {
5538 return invalid_params(response_id, "decision required");
5539 };
5540 let decision = match raw_decision {
5541 "approve" => GatingDecision::Approve,
5542 "reject" | "deny" => GatingDecision::Reject,
5543 "escalate" => GatingDecision::Escalate,
5544 _ => {
5545 return invalid_params(
5546 response_id,
5547 format!("unsupported decision: {raw_decision}"),
5548 );
5549 }
5550 };
5551 let reason = request
5552 .params
5553 .get("reason")
5554 .and_then(Value::as_str)
5555 .map(ToString::to_string);
5556 match module_runtime
5557 .lock()
5558 .await
5559 .decide_gating_action(GatingDecideRequest {
5560 pending_id: pending_id.to_string(),
5561 approver_id,
5562 decision,
5563 reason,
5564 }) {
5565 Ok(result) => response_value(
5566 response_id,
5567 Some(serde_json::to_value(result).unwrap_or(Value::Null)),
5568 None,
5569 ),
5570 Err(err) => gating_decision_failed_error(response_id, err),
5571 }
5572 }
5573 "mobkit/ensure_member" => {
5574 let Some(role) = request.params.get("role").and_then(Value::as_str) else {
5575 return invalid_params(response_id, "role required");
5576 };
5577 let Some(agent_identity) = request.params.get("agent_identity").and_then(Value::as_str)
5578 else {
5579 return invalid_params(response_id, "agent_identity required");
5580 };
5581 let labels = match request.params.get("labels") {
5582 None | Some(Value::Null) => std::collections::BTreeMap::new(),
5583 Some(value) => match serde_json::from_value(value.clone()) {
5584 Ok(map) => map,
5585 Err(err) => {
5586 return invalid_params(response_id, format!("invalid labels: {err}"));
5587 }
5588 },
5589 };
5590 let context = request.params.get("context").cloned();
5591 let resume_session_id = match request.params.get("resume_session_id") {
5592 None => None,
5593 Some(Value::Null) => None,
5594 Some(v) => match v.as_str() {
5595 Some(s) => match meerkat_core::types::SessionId::parse(s) {
5596 Ok(sid) => Some(sid),
5597 Err(_) => {
5598 return invalid_params(
5599 response_id,
5600 format!("invalid resume_session_id: {s:?}"),
5601 );
5602 }
5603 },
5604 None => {
5605 return invalid_params(
5606 response_id,
5607 "resume_session_id must be a string".to_string(),
5608 );
5609 }
5610 },
5611 };
5612 let additional_instructions = match request.params.get("additional_instructions") {
5613 None | Some(Value::Null) => None,
5614 Some(Value::Array(arr)) => {
5615 let mut strs = Vec::with_capacity(arr.len());
5616 for (i, entry) in arr.iter().enumerate() {
5617 match entry.as_str() {
5618 Some(s) => strs.push(s.to_string()),
5619 None => {
5620 return invalid_params(
5621 response_id,
5622 format!("additional_instructions[{i}] must be a string"),
5623 );
5624 }
5625 }
5626 }
5627 if strs.is_empty() { None } else { Some(strs) }
5628 }
5629 Some(_) => {
5630 return invalid_params(
5631 response_id,
5632 "additional_instructions must be an array of strings",
5633 );
5634 }
5635 };
5636 let runtime_mode = match parse_optional_runtime_mode(&request.params) {
5637 Ok(value) => value,
5638 Err(message) => return invalid_params(response_id, message),
5639 };
5640 let backend = match parse_optional_backend(&request.params) {
5641 Ok(value) => value,
5642 Err(message) => return invalid_params(response_id, message),
5643 };
5644 let binding = match parse_optional_runtime_binding(&request.params) {
5645 Ok(value) => value,
5646 Err(message) => return invalid_params(response_id, message),
5647 };
5648 let mut spec = SpawnMemberSpec::new(
5649 ProfileName::from(role),
5650 crate::member_comms_id::mob_member_id(agent_identity),
5651 );
5652 if let Some(runtime_mode) = runtime_mode {
5653 spec = spec.with_runtime_mode(runtime_mode);
5654 }
5655 if let Some(backend) = backend {
5656 spec = spec.with_backend(backend);
5657 }
5658 if let Some(binding) = binding {
5659 spec.binding = Some(binding);
5660 }
5661 if !labels.is_empty() {
5662 spec = spec.with_labels(labels);
5663 }
5664 if let Some(ctx) = context {
5665 spec = spec.with_context(ctx);
5666 }
5667 if let Some(sid) = resume_session_id {
5668 spec = spec.with_resume_bridge_session_id(sid);
5669 }
5670 if let Some(instructions) = additional_instructions {
5671 spec = spec.with_additional_instructions(instructions);
5672 }
5673 let handle = runtime.handle();
5674 let mid = spec.identity.clone();
5675 match handle.ensure_member(spec).await {
5676 Ok(_outcome) => {
5677 let body = match lookup_member_with_session(&handle, &mid).await {
5678 Some((entry, _sid)) => member_entry_to_json(&entry),
5679 None => Value::Null,
5680 };
5681 response_value(response_id, Some(body), None)
5682 }
5683 Err(err) => internal_error(response_id, format!("ensure_member failed: {err}")),
5684 }
5685 }
5686 "mobkit/retire_member" => {
5687 let Some(member_id) = request.params.get("member_id").and_then(Value::as_str) else {
5688 return invalid_params(response_id, "member_id required");
5689 };
5690 if let Some(aggregator) = &console_aggregator {
5691 return match Box::pin(aggregator.retire_identity(member_id)).await {
5692 Ok(true) => response_value(
5693 response_id,
5694 Some(serde_json::json!({ "accepted": true })),
5695 None,
5696 ),
5697 Ok(false) => response_value(
5698 response_id,
5699 None,
5700 Some(JsonRpcError {
5701 code: -32001,
5702 message: format!("unknown identity: {member_id}"),
5703 data: None,
5704 }),
5705 ),
5706 Err(err) => internal_error(response_id, format!("retire_member failed: {err}")),
5707 };
5708 }
5709 match runtime
5710 .handle()
5711 .retire(crate::member_comms_id::mob_member_id(member_id))
5712 .await
5713 {
5714 Ok(()) => response_value(
5715 response_id,
5716 Some(serde_json::json!({ "accepted": true })),
5717 None,
5718 ),
5719 Err(err) => internal_error(response_id, format!("retire_member failed: {err}")),
5720 }
5721 }
5722 "mobkit/respawn_member" => {
5723 let Some(member_id) = request.params.get("member_id").and_then(Value::as_str) else {
5724 return invalid_params(response_id, "member_id required");
5725 };
5726 match runtime
5727 .handle()
5728 .respawn(crate::member_comms_id::mob_member_id(member_id), None)
5729 .await
5730 {
5731 Ok(_receipt) => response_value(
5732 response_id,
5733 Some(serde_json::json!({ "accepted": true })),
5734 None,
5735 ),
5736 Err(err) => {
5737 if let Some(failed_peer_ids) = topology_restore_failed_peer_ids(&err) {
5738 tracing::warn!(
5739 member_id = %member_id,
5740 failed_peer_count = failed_peer_ids.len(),
5741 failed_peer_ids = ?failed_peer_ids,
5742 "console member respawn restored member with isolated peer edges; accepting degraded respawn"
5743 );
5744 response_value(
5745 response_id,
5746 Some(serde_json::json!({
5747 "accepted": true,
5748 "topology_restore_warning": topology_restore_warning_json(&failed_peer_ids),
5749 })),
5750 None,
5751 )
5752 } else {
5753 internal_error(response_id, format!("respawn_member failed: {err}"))
5754 }
5755 }
5756 }
5757 }
5758 "mobkit/reconcile_edges" => response_value(
5759 response_id,
5760 Some(serde_json::json!({
5761 "status": "noop",
5762 "reason": "console runtime routes directly to MobRuntime",
5763 })),
5764 None,
5765 ),
5766 "mobkit/mob_events/query" | "mobkit/mob_events/subscribe" => {
5767 let query: EventQuery = if request.params.is_null() {
5768 EventQuery::default()
5769 } else {
5770 match serde_json::from_value(request.params.clone()) {
5771 Ok(q) => q,
5772 Err(err) => {
5773 return invalid_params(response_id, format!("invalid query params: {err}"));
5774 }
5775 }
5776 };
5777 let Some(store) = mob_events.as_ref() else {
5778 return response_value(
5779 response_id,
5780 Some(serde_json::json!({
5781 "events": [],
5782 "next_after_seq": Value::Null,
5783 })),
5784 None,
5785 );
5786 };
5787 let events_view = runtime.handle().events();
5788 let latest_at_handshake = events_view.latest_cursor().await.unwrap_or(0);
5792 let result = crate::unified_runtime::mob_events::query_ledger_with_filter(
5793 &events_view,
5794 store,
5795 &query,
5796 )
5797 .await;
5798 match result {
5799 Ok(mut events) => {
5800 if let Some(view) = access_view.filter(|view| view.enforced()) {
5806 events.retain(|event| {
5807 event
5808 .agent_identity
5809 .as_deref()
5810 .is_none_or(|identity| view.can_view_agent(identity))
5811 });
5812 }
5813 let last_cursor = events.last().map(|event| event.cursor);
5814 let body = if request.method == "mobkit/mob_events/subscribe" {
5815 let subscribe_url = crate::unified_runtime::mob_events::build_subscribe_url(
5816 &query,
5817 last_cursor,
5818 latest_at_handshake,
5819 );
5820 serde_json::json!({
5821 "stream": "mob_events",
5822 "events": events,
5823 "next_after_seq": last_cursor,
5824 "subscribe_url": subscribe_url,
5825 "keep_alive": {
5826 "interval_ms": 15_000_u64,
5827 "event": "keep_alive",
5828 },
5829 })
5830 } else {
5831 serde_json::json!({
5832 "events": events,
5833 "next_after_seq": last_cursor,
5834 })
5835 };
5836 response_value(response_id, Some(body), None)
5837 }
5838 Err(crate::unified_runtime::mob_events::MobEventsQueryError::Stale {
5839 after_cursor,
5840 latest_cursor,
5841 }) => stale_event_cursor_response(response_id, after_cursor, latest_cursor),
5842 Err(err) => internal_error(response_id, format!("mob_events query failed: {err}")),
5843 }
5844 }
5845 "mobkit/member_status" => {
5847 let Some(member_id) = request.params.get("member_id").and_then(Value::as_str) else {
5848 return invalid_params(response_id, "member_id required");
5849 };
5850 match runtime
5851 .handle()
5852 .member_status(&crate::member_comms_id::mob_member_id(member_id))
5853 .await
5854 {
5855 Ok(snapshot) => response_value(
5856 response_id,
5857 Some(serde_json::to_value(&snapshot).unwrap_or(Value::Null)),
5858 None,
5859 ),
5860 Err(err) => internal_error(response_id, format!("member_status failed: {err}")),
5861 }
5862 }
5863 "mobkit/force_cancel_member" => {
5864 let Some(member_id) = request.params.get("member_id").and_then(Value::as_str) else {
5865 return invalid_params(response_id, "member_id required");
5866 };
5867 match runtime
5868 .handle()
5869 .force_cancel_member(crate::member_comms_id::mob_member_id(member_id))
5870 .await
5871 {
5872 Ok(()) => response_value(
5873 response_id,
5874 Some(serde_json::json!({ "accepted": true })),
5875 None,
5876 ),
5877 Err(err) => {
5878 internal_error(response_id, format!("force_cancel_member failed: {err}"))
5879 }
5880 }
5881 }
5882 "mobkit/wait_ready" => {
5883 let timeout = Some(
5887 request
5888 .params
5889 .get("timeout_ms")
5890 .and_then(Value::as_u64)
5891 .map(std::time::Duration::from_millis)
5892 .unwrap_or(crate::unified_runtime::mob_ops::DEFAULT_WAIT_READY_TIMEOUT),
5893 );
5894 match runtime.handle().wait_for_ready(timeout).await {
5895 Ok(ready) => {
5896 let entries: Vec<Value> = ready
5897 .into_iter()
5898 .filter_map(|(identity, snapshot)| {
5902 let identity = identity.to_string();
5903 let visible = access_view
5904 .filter(|view| view.enforced())
5905 .is_none_or(|view| view.can_view_agent(&identity));
5906 visible.then(|| {
5907 serde_json::json!({
5908 "agent_identity": identity,
5909 "snapshot": serde_json::to_value(&snapshot)
5910 .unwrap_or(Value::Null),
5911 })
5912 })
5913 })
5914 .collect();
5915 response_value(
5916 response_id,
5917 Some(serde_json::json!({
5918 "ready": entries,
5919 "timeout": false,
5920 })),
5921 None,
5922 )
5923 }
5924 Err(err) => {
5925 if crate::unified_runtime::mob_ops::is_ready_wait_timeout(&err) {
5926 response_value(
5927 response_id,
5928 Some(serde_json::json!({
5929 "ready": Vec::<Value>::new(),
5930 "timeout": true,
5931 })),
5932 None,
5933 )
5934 } else {
5935 internal_error(response_id, format!("wait_for_ready failed: {err}"))
5936 }
5937 }
5938 }
5939 }
5940 "mobkit/collect_completed" => {
5941 let completed = runtime.handle().collect_completed().await;
5942 let entries: Vec<Value> = completed
5943 .into_iter()
5944 .filter_map(|(mid, snapshot)| {
5947 let mid = mid.to_string();
5948 let visible = access_view
5949 .filter(|view| view.enforced())
5950 .is_none_or(|view| view.can_view_agent(&mid));
5951 visible.then(|| {
5952 serde_json::json!({
5953 "member_id": mid,
5954 "snapshot": serde_json::to_value(&snapshot).unwrap_or(Value::Null),
5955 })
5956 })
5957 })
5958 .collect();
5959 response_value(
5960 response_id,
5961 Some(serde_json::json!({ "completed": entries })),
5962 None,
5963 )
5964 }
5965 "mobkit/cancel_flow" => {
5966 let Some(run_id) = request.params.get("run_id").and_then(Value::as_str) else {
5967 return invalid_params(response_id, "run_id required");
5968 };
5969 let run_id: meerkat_mob::RunId = match run_id.parse() {
5970 Ok(id) => id,
5971 Err(_) => return invalid_params(response_id, "invalid run_id format"),
5972 };
5973 match runtime.handle().cancel_flow(run_id).await {
5974 Ok(()) => response_value(
5975 response_id,
5976 Some(serde_json::json!({ "accepted": true })),
5977 None,
5978 ),
5979 Err(err) => internal_error(response_id, format!("cancel_flow failed: {err}")),
5980 }
5981 }
5982 "mobkit/flow_status" => {
5983 let Some(run_id) = request.params.get("run_id").and_then(Value::as_str) else {
5984 return invalid_params(response_id, "run_id required");
5985 };
5986 let run_id: meerkat_mob::RunId = match run_id.parse() {
5987 Ok(id) => id,
5988 Err(_) => return invalid_params(response_id, "invalid run_id format"),
5989 };
5990 match runtime.handle().flow_status(run_id).await {
5991 Ok(Some(mob_run)) => response_value(
5992 response_id,
5993 Some(serde_json::to_value(&mob_run).unwrap_or(Value::Null)),
5994 None,
5995 ),
5996 Ok(None) => response_value(response_id, Some(Value::Null), None),
5997 Err(err) => internal_error(response_id, format!("flow_status failed: {err}")),
5998 }
5999 }
6000 "mobkit/list_flows" => {
6001 let flows: Vec<String> = runtime
6002 .handle()
6003 .list_flows()
6004 .into_iter()
6005 .map(|id| id.to_string())
6006 .collect();
6007 response_value(
6008 response_id,
6009 Some(serde_json::json!({ "flows": flows })),
6010 None,
6011 )
6012 }
6013 "mobkit/list_runs" => {
6014 let flow_id = request
6015 .params
6016 .get("flow_id")
6017 .and_then(Value::as_str)
6018 .filter(|value| !value.is_empty())
6019 .map(meerkat_mob::FlowId::from);
6020 match runtime.handle().list_runs(flow_id.as_ref()).await {
6021 Ok(runs) => response_value(
6022 response_id,
6023 Some(serde_json::json!({
6024 "runs": serde_json::to_value(&runs).unwrap_or(Value::Null),
6025 })),
6026 None,
6027 ),
6028 Err(err) => internal_error(response_id, format!("list_runs failed: {err}")),
6029 }
6030 }
6031 "mobkit/run_flow" => {
6032 let Some(flow_id_str) = request.params.get("flow_id").and_then(Value::as_str) else {
6033 return invalid_params(response_id, "flow_id required");
6034 };
6035 if flow_id_str.is_empty() {
6036 return invalid_params(response_id, "flow_id required");
6037 }
6038 let flow_id = meerkat_mob::FlowId::from(flow_id_str);
6039 let flow_params = request.params.get("params").cloned().unwrap_or(Value::Null);
6040 if let Some(identity_runtime) = &identity_runtime
6041 && let Err(err) = identity_runtime.materialize_all_required().await
6042 {
6043 return internal_error(
6044 response_id,
6045 format!("identity-first flow materialization failed: {err}"),
6046 );
6047 }
6048 match Box::pin(runtime.handle().run_flow(flow_id, flow_params)).await {
6049 Ok(run_id) => response_value(
6050 response_id,
6051 Some(serde_json::json!({ "run_id": run_id.to_string() })),
6052 None,
6053 ),
6054 Err(err) => invalid_params(response_id, format!("run_flow failed: {err}")),
6055 }
6056 }
6057 "mobkit/spawn_helper" => {
6058 let Some(agent_identity) = request.params.get("agent_identity").and_then(Value::as_str)
6059 else {
6060 return invalid_params(response_id, "agent_identity required");
6061 };
6062 let Some(task) = request.params.get("task").and_then(Value::as_str) else {
6063 return invalid_params(response_id, "task required");
6064 };
6065 let options = match parse_console_helper_options(request.params.get("options")) {
6066 Ok(opts) => opts,
6067 Err(msg) => return invalid_params(response_id, msg),
6068 };
6069 let handle = runtime.handle();
6070 match Box::pin(handle.spawn_helper(
6071 crate::member_comms_id::mob_member_id(agent_identity),
6072 task,
6073 options,
6074 ))
6075 .await
6076 {
6077 Ok(result) => {
6078 response_value(
6083 response_id,
6084 Some(serde_json::json!({
6085 "output": result.output,
6086 "tokens_used": result.tokens_used,
6087 })),
6088 None,
6089 )
6090 }
6091 Err(err) => internal_error(response_id, format!("spawn_helper failed: {err}")),
6092 }
6093 }
6094 "mobkit/fork_helper" => {
6095 let Some(source) = request
6096 .params
6097 .get("source_member_id")
6098 .and_then(Value::as_str)
6099 else {
6100 return invalid_params(response_id, "source_member_id required");
6101 };
6102 let Some(agent_identity) = request.params.get("agent_identity").and_then(Value::as_str)
6103 else {
6104 return invalid_params(response_id, "agent_identity required");
6105 };
6106 let Some(task) = request.params.get("task").and_then(Value::as_str) else {
6107 return invalid_params(response_id, "task required");
6108 };
6109 let fork_context = match request.params.get("fork_context") {
6110 Some(v) if !v.is_null() => {
6111 match serde_json::from_value::<meerkat_mob::launch::ForkContext>(v.clone()) {
6112 Ok(ctx) => ctx,
6113 Err(err) => {
6114 return invalid_params(
6115 response_id,
6116 format!("invalid fork_context: {err}"),
6117 );
6118 }
6119 }
6120 }
6121 _ => meerkat_mob::launch::ForkContext::default(),
6122 };
6123 let options = match parse_console_helper_options(request.params.get("options")) {
6124 Ok(opts) => opts,
6125 Err(msg) => return invalid_params(response_id, msg),
6126 };
6127 let handle = runtime.handle();
6128 match Box::pin(handle.fork_helper(
6129 &crate::member_comms_id::mob_member_id(source),
6130 crate::member_comms_id::mob_member_id(agent_identity),
6131 task,
6132 fork_context,
6133 options,
6134 ))
6135 .await
6136 {
6137 Ok(result) => {
6138 response_value(
6142 response_id,
6143 Some(serde_json::json!({
6144 "output": result.output,
6145 "tokens_used": result.tokens_used,
6146 })),
6147 None,
6148 )
6149 }
6150 Err(err) => internal_error(response_id, format!("fork_helper failed: {err}")),
6151 }
6152 }
6153 "mobkit/attach_existing_session" => {
6154 let Some(role) = request.params.get("role").and_then(Value::as_str) else {
6155 return invalid_params(response_id, "role required");
6156 };
6157 let Some(agent_identity) = request.params.get("agent_identity").and_then(Value::as_str)
6158 else {
6159 return invalid_params(response_id, "agent_identity required");
6160 };
6161 let Some(session_id_str) = request.params.get("session_id").and_then(Value::as_str)
6162 else {
6163 return invalid_params(response_id, "session_id required");
6164 };
6165 let bridge_session_id = match meerkat_core::types::SessionId::parse(session_id_str) {
6166 Ok(s) => s,
6167 Err(_) => return invalid_params(response_id, "invalid session_id format"),
6168 };
6169 let mid = crate::member_comms_id::mob_member_id(agent_identity);
6170 let spec = SpawnMemberSpec::new(ProfileName::from(role), mid.clone())
6171 .with_launch_mode(MemberLaunchMode::Resume { bridge_session_id });
6172 let handle = runtime.handle();
6173 match Box::pin(handle.spawn_spec(spec)).await {
6174 Ok(_) => match handle.member_status(&mid).await {
6175 Ok(snapshot) => response_value(
6176 response_id,
6177 Some(serde_json::to_value(&snapshot).unwrap_or(Value::Null)),
6178 None,
6179 ),
6180 Err(err) => internal_error(
6181 response_id,
6182 format!("attach_existing_session status lookup failed: {err}"),
6183 ),
6184 },
6185 Err(err) => internal_error(
6186 response_id,
6187 format!("attach_existing_session failed: {err}"),
6188 ),
6189 }
6190 }
6191 "mobkit/cross_mob/wire_local" => {
6192 handle_console_wire_local(runtime, &request.params, response_id, true).await
6193 }
6194 "mobkit/cross_mob/unwire_local" => {
6195 handle_console_wire_local(runtime, &request.params, response_id, false).await
6196 }
6197 "mobkit/peer_pubkey" => match gateway_peer_keys {
6198 Some(keys) => response_value(
6199 response_id,
6200 Some(serde_json::json!({ "pubkey_b64": keys.pubkey_b64() })),
6201 None,
6202 ),
6203 None => response_value(
6204 response_id,
6205 None,
6206 Some(JsonRpcError {
6207 code: -32004,
6208 message: "gateway has no signing keypair configured".to_string(),
6209 data: None,
6210 }),
6211 ),
6212 },
6213 "mobkit/cross_mob/peer_info" => {
6214 let member_id = request.params.get("member_id").and_then(Value::as_str);
6215 match member_id {
6216 Some(mid) if !mid.is_empty() => {
6217 let handle = runtime.handle();
6218 let mob_id = handle.mob_id().to_string();
6219 let meerkat_id = crate::member_comms_id::mob_member_id(mid);
6220 match handle.get_member(&meerkat_id).await {
6221 Err(err) => response_value(
6222 response_id,
6223 None,
6224 Some(JsonRpcError {
6225 code: -32000,
6226 message: format!("member lookup failed: {err}"),
6227 data: None,
6228 }),
6229 ),
6230 Ok(Some(entry)) => match entry.peer_id() {
6231 Some(peer_id) => {
6232 let comms_name = format!("{}/{}/{}", mob_id, entry.role, mid);
6233 let address = format!("inproc://{comms_name}");
6234 response_value(
6235 response_id,
6236 Some(serde_json::json!({
6237 "member_id": mid,
6238 "mob_id": mob_id,
6239 "comms_name": comms_name,
6240 "peer_id": peer_id,
6241 "address": address,
6242 })),
6243 None,
6244 )
6245 }
6246 None => response_value(
6247 response_id,
6248 None,
6249 Some(JsonRpcError {
6250 code: -32000,
6251 message: format!("member {mid:?} has no comms runtime"),
6252 data: None,
6253 }),
6254 ),
6255 },
6256 Ok(None) => response_value(
6257 response_id,
6258 None,
6259 Some(JsonRpcError {
6260 code: -32000,
6261 message: format!("member {mid:?} not found"),
6262 data: None,
6263 }),
6264 ),
6265 }
6266 }
6267 _ => invalid_params(response_id, "member_id required".to_string()),
6268 }
6269 }
6270 "mobkit/cross_mob/directory" => {
6271 let entries: Vec<Value> = contact_directory
6272 .map(|dir| {
6273 dir.list()
6274 .into_iter()
6275 .filter_map(|e| serde_json::to_value(e).ok())
6276 .collect()
6277 })
6278 .unwrap_or_default();
6279 response_value(
6280 response_id,
6281 Some(serde_json::json!({ "mobs": entries })),
6282 None,
6283 )
6284 }
6285 method
6286 if matches!(
6287 method,
6288 "mobkit/mob_labels/set"
6289 | "mobkit/mob_labels/get"
6290 | "mobkit/mob_labels/delete"
6291 | "mobkit/run_labels/set"
6292 | "mobkit/run_labels/get"
6293 | "mobkit/run_labels/delete",
6294 ) =>
6295 {
6296 dispatch_label_method(
6297 method,
6298 metadata_table.as_deref(),
6299 runtime.handle().mob_id().as_str(),
6300 response_id,
6301 &request.params,
6302 )
6303 .await
6304 }
6305 _ => response_value(
6306 response_id,
6307 None,
6308 Some(JsonRpcError {
6309 code: -32601,
6310 message: "Method not found".to_string(),
6311 data: None,
6312 }),
6313 ),
6314 }
6315}
6316
6317async fn dispatch_label_method(
6324 method: &str,
6325 metadata_table: Option<&RuntimeMetadataTable>,
6326 mob_id: &str,
6327 response_id: Value,
6328 params: &Value,
6329) -> Value {
6330 let Some(table) = metadata_table else {
6331 return invalid_params(
6332 response_id,
6333 "metadata table not configured for this runtime",
6334 );
6335 };
6336
6337 let scope = match method {
6338 "mobkit/mob_labels/set" | "mobkit/mob_labels/get" | "mobkit/mob_labels/delete" => {
6339 MetadataScope::Mob(mob_id.to_string())
6340 }
6341 _ => match crate::runtime::parse_run_id_param(params) {
6342 Ok(run_id) => MetadataScope::Run(mob_id.to_string(), run_id.to_string()),
6343 Err(message) => return invalid_params(response_id, message),
6344 },
6345 };
6346
6347 let outcome = match method {
6348 "mobkit/mob_labels/set" | "mobkit/run_labels/set" => {
6349 crate::runtime::dispatch_labels_set(table, scope, params).await
6350 }
6351 "mobkit/mob_labels/get" | "mobkit/run_labels/get" => {
6352 crate::runtime::dispatch_labels_get(table, scope).await
6353 }
6354 "mobkit/mob_labels/delete" | "mobkit/run_labels/delete" => {
6355 crate::runtime::dispatch_labels_delete(table, scope).await
6356 }
6357 _ => unreachable!("dispatch_label_method called with non-label method: {method}"),
6358 };
6359
6360 match outcome {
6361 crate::runtime::LabelRpcResult::Accepted => response_value(
6362 response_id,
6363 Some(serde_json::json!({"accepted": true})),
6364 None,
6365 ),
6366 crate::runtime::LabelRpcResult::Labels(labels) => response_value(
6367 response_id,
6368 Some(serde_json::json!({"labels": labels_to_json_value(&labels)})),
6369 None,
6370 ),
6371 crate::runtime::LabelRpcResult::InvalidParams(message) => {
6372 invalid_params(response_id, message)
6373 }
6374 }
6375}
6376
6377async fn handle_console_wire_local(
6386 runtime: &MobRuntime,
6387 params: &Value,
6388 response_id: Value,
6389 wire: bool,
6390) -> Value {
6391 let local = params.get("local_member_id").and_then(Value::as_str);
6392 let comms_name = params.get("remote_comms_name").and_then(Value::as_str);
6393 let peer_id = params.get("remote_peer_id").and_then(Value::as_str);
6394 let addr = params.get("remote_address").and_then(Value::as_str);
6395
6396 let remote_pubkey = match params.get("remote_pubkey_b64") {
6397 None => None,
6398 Some(v) if v.is_null() => None,
6399 Some(v) => match v.as_str() {
6400 Some(s) if !s.is_empty() => match crate::auth::peer_keys::decode_pubkey_b64(s) {
6401 Ok(bytes) => Some(bytes),
6402 Err(err) => {
6403 return invalid_params(response_id, format!("remote_pubkey_b64: {err}"));
6404 }
6405 },
6406 _ => None,
6407 },
6408 };
6409
6410 let (local_id, cname, pid, address) = match (local, comms_name, peer_id, addr) {
6411 (Some(l), Some(c), Some(p), Some(a))
6412 if !l.is_empty() && !c.is_empty() && !p.is_empty() && !a.is_empty() =>
6413 {
6414 (l, c, p, a)
6415 }
6416 _ => {
6417 return invalid_params(
6418 response_id,
6419 "local_member_id, remote_comms_name, remote_peer_id, and remote_address required",
6420 );
6421 }
6422 };
6423
6424 let is_inproc = address.starts_with("inproc://");
6425 let spec_result = match (is_inproc, remote_pubkey) {
6426 (true, None) => TrustedPeerDescriptor::test_only_unsigned(cname, pid, address),
6427 (true, Some(bytes)) => {
6428 TrustedPeerDescriptor::unsigned_with_pubkey(cname, pid, bytes, address)
6429 }
6430 (false, None) => {
6431 return invalid_params(
6432 response_id,
6433 "remote_pubkey_b64 is required for non-inproc transports",
6434 );
6435 }
6436 (false, Some(bytes)) => {
6437 if bytes == [0u8; 32] {
6438 return invalid_params(
6439 response_id,
6440 "remote_pubkey_b64 must be non-zero for non-inproc transports",
6441 );
6442 }
6443 TrustedPeerDescriptor::unsigned_with_pubkey(cname, pid, bytes, address)
6444 }
6445 };
6446
6447 let spec = match spec_result {
6448 Ok(spec) => spec,
6449 Err(err) => {
6450 return invalid_params(response_id, format!("invalid peer spec: {err}"));
6451 }
6452 };
6453
6454 let result = if wire {
6455 runtime
6456 .handle()
6457 .wire(
6458 crate::member_comms_id::mob_member_id(local_id),
6459 PeerTarget::External(spec),
6460 )
6461 .await
6462 } else {
6463 runtime
6464 .handle()
6465 .unwire(
6466 crate::member_comms_id::mob_member_id(local_id),
6467 PeerTarget::External(spec),
6468 )
6469 .await
6470 };
6471
6472 let action = if wire { "wire_local" } else { "unwire_local" };
6473 match result {
6474 Ok(()) => response_value(
6475 response_id,
6476 Some(serde_json::json!({
6477 "accepted": true,
6478 "local_member_id": local_id,
6479 "remote_comms_name": cname,
6480 })),
6481 None,
6482 ),
6483 Err(err) => internal_error(response_id, format!("cross_mob/{action} failed: {err}")),
6484 }
6485}
6486
6487async fn build_live_snapshot(
6488 runtime: &MobRuntime,
6489 config_module_ids: &[String],
6490 console_events: Option<&ConsoleEventStore>,
6491 visibility_policy: &dyn ConsoleVisibilityPolicy,
6492 read_model: &ConsoleSnapshotReadModel,
6493) -> ConsoleLiveSnapshot {
6494 let read_model_state = Box::pin(read_model.snapshot(runtime)).await;
6495 let running = read_model_state.running.unwrap_or(true);
6496 let mut members = read_model_state.primary_members.clone();
6503 if visibility_policy.include_implicit_delegate_members() {
6504 for group in &read_model_state.delegate_member_groups {
6505 members.extend(group.iter().cloned());
6506 }
6507 }
6508 dedupe_console_members_by_identity(&mut members);
6509 members.retain(|member| {
6510 visibility_policy.member_visible(member)
6511 && visibility_policy
6512 .identity_visible(&console_identity_record_from_console_member(member))
6513 });
6514
6515 let loaded_modules = if config_module_ids.is_empty() {
6519 let mut mods: Vec<String> = members
6520 .iter()
6521 .filter(|member| member.state != MEMBER_STATE_RETIRING)
6522 .map(|member| member.agent_identity.clone())
6523 .collect();
6524 mods.sort();
6525 mods
6526 } else {
6527 let mut mods = config_module_ids.to_vec();
6528 mods.sort();
6529 mods
6530 };
6531
6532 let agents = members
6533 .iter()
6534 .map(|member| async move {
6535 let console_identity = console_member_console_identity(member);
6536 let label = member
6537 .labels
6538 .get("display_name")
6539 .cloned()
6540 .unwrap_or_else(|| member.agent_identity.clone());
6541 let watched = member
6542 .labels
6543 .get("console_watched")
6544 .map(|value: &String| value == "true");
6545 let alert_level = member
6546 .labels
6547 .get("console_alert_level")
6548 .filter(|value: &&String| matches!(value.as_str(), "elevated" | "critical"))
6549 .cloned();
6550 let degraded = member
6551 .labels
6552 .get("console_degraded")
6553 .map(|value: &String| value == "true");
6554 let degraded_reason = member.labels.get("console_degraded_reason").cloned();
6555 let response_phase = match console_events {
6556 Some(store) => store.response_phase_for_identity(console_identity).await,
6557 None => None,
6558 };
6559 ConsoleAgentLiveSnapshot {
6560 agent_id: member.agent_identity.clone(),
6561 member_id: member.agent_identity.clone(),
6562 label,
6563 kind: "meerkat".to_string(),
6564 identity: Some(console_identity.to_string()),
6565 role: Some(member.role.clone()),
6566 state: Some(member.state.clone()),
6567 session_id: member.session_id.clone(),
6568 model_capabilities: member.model_capabilities.clone(),
6569 response_phase,
6570 watched,
6571 alert_level,
6572 degraded,
6573 degraded_reason,
6574 }
6575 })
6576 .collect::<Vec<_>>();
6577 let mut agents = join_all(agents).await;
6578 agents.sort_by(|left, right| left.label.cmp(&right.label));
6579 ConsoleLiveSnapshot::new(
6580 Some(runtime.handle().mob_id().to_string()),
6581 running,
6582 loaded_modules,
6583 agents,
6584 members,
6585 true,
6586 )
6587}
6588
6589async fn collect_console_snapshot_read_model(
6590 runtime: &MobRuntime,
6591) -> ConsoleSnapshotReadModelState {
6592 let handle = runtime.handle();
6593 let mut state = ConsoleSnapshotReadModelState {
6594 running: Some(matches!(
6595 handle.status_observation_snapshot(),
6596 MobState::Creating | MobState::Running
6597 )),
6598 ..ConsoleSnapshotReadModelState::default()
6599 };
6600 collect_console_session_index_for_handle(&handle, &mut state).await;
6601 let registered_labels = runtime.console_identity_labels().await;
6602
6603 let (primary_members, _primary_owner_index) =
6609 project_console_members_from_handle(&handle, None, None, &state, ®istered_labels).await;
6610 state.primary_members = primary_members;
6611
6612 let Some(mcp_state) = runtime.agent_mob_mcp_state() else {
6613 return state;
6614 };
6615 let primary_mob_id = handle.mob_id().to_string();
6616 let mut processed = BTreeSet::from([primary_mob_id]);
6617 let mut delegate_groups: Vec<Vec<ConsoleMember>> = Vec::new();
6618 loop {
6619 let mut progressed = false;
6620 for (mob_id, delegate_handle) in Box::pin(mcp_state.mob_handles_snapshot())
6621 .await
6622 .unwrap_or_default()
6623 {
6624 if processed.contains(mob_id.as_str()) {
6625 continue;
6626 }
6627 let Some(owner_authority) = delegate_handle.owner_bridge_session_lifecycle_authority()
6630 else {
6631 processed.insert(mob_id.to_string());
6632 continue;
6633 };
6634 let owner_session_id = owner_authority.bridge_session_id.to_string();
6635 let Some(host_identity) = state
6636 .session_owner_by_id
6637 .get(owner_session_id.as_str())
6638 .cloned()
6639 else {
6640 continue;
6641 };
6642 collect_console_session_index_for_handle(&delegate_handle, &mut state).await;
6643 let (delegate_members, _delegate_owner_index) = project_console_members_from_handle(
6644 &delegate_handle,
6645 Some(&host_identity),
6646 Some(mob_id.as_str()),
6647 &state,
6648 ®istered_labels,
6649 )
6650 .await;
6651 delegate_groups.push(delegate_members);
6652 processed.insert(mob_id.to_string());
6653 progressed = true;
6654 }
6655 if !progressed {
6656 break;
6657 }
6658 }
6659 state.delegate_member_groups = delegate_groups;
6660 state
6661}
6662
6663async fn collect_console_session_index_for_handle(
6664 handle: &MobHandle,
6665 state: &mut ConsoleSnapshotReadModelState,
6666) {
6667 for entry in handle.list_members_observation_snapshot().await {
6668 let identity =
6671 crate::member_comms_id::runtime_alias_str(entry.agent_identity.as_str()).into_owned();
6672 let Some(session_id) = handle
6673 .resolve_bridge_session_id_observation(&entry.agent_identity)
6674 .await
6675 .map(|session_id| session_id.to_string())
6676 else {
6677 state.session_id_by_identity.remove(&identity);
6678 continue;
6679 };
6680 state
6681 .session_owner_by_id
6682 .insert(session_id.clone(), identity.clone());
6683 state.session_id_by_identity.insert(identity, session_id);
6684 }
6685}
6686
6687fn apply_console_visibility_policy(
6688 snapshot: &mut ConsoleLiveSnapshot,
6689 visibility_policy: &dyn ConsoleVisibilityPolicy,
6690) {
6691 let mut hidden = BTreeSet::new();
6692 snapshot.members.retain(|member| {
6693 let visible = visibility_policy.member_visible(member)
6694 && visibility_policy
6695 .identity_visible(&console_identity_record_from_console_member(member));
6696 if !visible {
6697 hidden.insert(member.agent_identity.clone());
6698 }
6699 visible
6700 });
6701 snapshot
6702 .agents
6703 .retain(|agent| !hidden.contains(&agent.agent_id));
6704 snapshot
6705 .loaded_modules
6706 .retain(|module_id| !hidden.contains(module_id));
6707}
6708
6709async fn reset_all_live_console_agents(
6710 runtime: &MobRuntime,
6711 console_events: Option<&ConsoleEventStore>,
6712 console_aggregator: Option<&MobKitConsoleAggregator>,
6713 identity_runtime: Option<&Arc<crate::identity_first::IdentityRuntime>>,
6714 visibility_policy: &dyn ConsoleVisibilityPolicy,
6715) -> Result<Value, Box<dyn std::error::Error + Send + Sync>> {
6716 let read_model = ConsoleSnapshotReadModel::default();
6717 *read_model.inner.write().await = Box::pin(collect_console_snapshot_read_model(runtime)).await;
6718 read_model
6721 .primed
6722 .store(true, std::sync::atomic::Ordering::Release);
6723 let snapshot = Box::pin(build_live_snapshot(
6724 runtime,
6725 &[],
6726 console_events,
6727 visibility_policy,
6728 &read_model,
6729 ))
6730 .await;
6731 let raw_snapshot = Box::pin(build_live_snapshot(
6732 runtime,
6733 &[],
6734 console_events,
6735 &crate::console_aggregator::AllowAllConsoleVisibilityPolicy,
6736 &read_model,
6737 ))
6738 .await;
6739 let identity_runtime_statuses = if let Some(identity_runtime) = identity_runtime {
6740 identity_runtime.statuses().await
6741 } else {
6742 Vec::new()
6743 };
6744 let identity_by_runtime_member_id = identity_runtime_statuses
6745 .iter()
6746 .filter_map(|status| {
6747 status
6748 .agent_runtime_id
6749 .as_ref()
6750 .map(|runtime_id| (runtime_id.as_str().to_string(), status.identity.to_string()))
6751 })
6752 .collect::<BTreeMap<_, _>>();
6753 let mut durable_identity_runtime_identities = identity_runtime_statuses
6754 .iter()
6755 .filter(|status| identity_status_visible_to_console(visibility_policy, status))
6756 .map(|status| status.identity.to_string())
6757 .collect::<BTreeSet<_>>();
6758 let mut main_identities = BTreeSet::new();
6759 let mut runtime_member_id_by_identity = BTreeMap::new();
6760 let mut runtime_member_ids_by_identity: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
6761 let mut session_id_by_identity_runtime_member: BTreeMap<(String, String), Option<String>> =
6762 BTreeMap::new();
6763 let mut live_alias_by_runtime_member_id: BTreeMap<String, (String, Option<String>)> =
6764 BTreeMap::new();
6765 let mut visible_runtime_member_ids = BTreeSet::new();
6766 let mut duplicate_live_identities = BTreeSet::new();
6767 let mut delegate_members = BTreeSet::new();
6768 for member in snapshot.members {
6769 if member.state == MEMBER_STATE_RETIRING {
6770 continue;
6771 }
6772 if let Some(source_mob_id) = member.labels.get("source_mob_id").cloned() {
6773 delegate_members.insert((source_mob_id, member.agent_identity));
6774 } else {
6775 let identity = member
6776 .labels
6777 .get("agent_identity")
6778 .filter(|value| !value.trim().is_empty())
6779 .cloned()
6780 .or_else(|| {
6781 identity_by_runtime_member_id
6782 .get(&member.agent_identity)
6783 .cloned()
6784 })
6785 .unwrap_or_else(|| member.agent_identity.clone());
6786 if let Some(existing) = runtime_member_id_by_identity.get(&identity)
6787 && existing != &member.agent_identity
6788 {
6789 duplicate_live_identities.insert(identity.clone());
6790 }
6791 runtime_member_ids_by_identity
6792 .entry(identity.clone())
6793 .or_default()
6794 .insert(member.agent_identity.clone());
6795 session_id_by_identity_runtime_member.insert(
6796 (identity.clone(), member.agent_identity.clone()),
6797 member.session_id.clone(),
6798 );
6799 live_alias_by_runtime_member_id.insert(
6800 member.agent_identity.clone(),
6801 (identity.clone(), member.session_id.clone()),
6802 );
6803 visible_runtime_member_ids.insert(member.agent_identity.clone());
6804 runtime_member_id_by_identity
6805 .entry(identity.clone())
6806 .or_insert(member.agent_identity);
6807 main_identities.insert(identity);
6808 }
6809 }
6810 let mut raw_runtime_member_ids_by_identity: BTreeMap<String, BTreeSet<String>> =
6811 BTreeMap::new();
6812 let mut raw_session_id_by_identity_runtime_member: BTreeMap<(String, String), Option<String>> =
6813 BTreeMap::new();
6814 let mut raw_live_alias_by_runtime_member_id: BTreeMap<String, (String, Option<String>)> =
6815 BTreeMap::new();
6816 for member in raw_snapshot.members {
6817 if member.state == MEMBER_STATE_RETIRING || member.labels.contains_key("source_mob_id") {
6818 continue;
6819 }
6820 let identity = member
6821 .labels
6822 .get("agent_identity")
6823 .filter(|value| !value.trim().is_empty())
6824 .cloned()
6825 .or_else(|| {
6826 identity_by_runtime_member_id
6827 .get(&member.agent_identity)
6828 .cloned()
6829 })
6830 .unwrap_or_else(|| member.agent_identity.clone());
6831 raw_runtime_member_ids_by_identity
6832 .entry(identity.clone())
6833 .or_default()
6834 .insert(member.agent_identity.clone());
6835 raw_session_id_by_identity_runtime_member.insert(
6836 (identity.clone(), member.agent_identity.clone()),
6837 member.session_id.clone(),
6838 );
6839 raw_live_alias_by_runtime_member_id
6840 .insert(member.agent_identity, (identity, member.session_id));
6841 }
6842 durable_identity_runtime_identities.retain(|identity| {
6843 identity_runtime_statuses
6844 .iter()
6845 .find(|status| status.identity.as_str() == identity)
6846 .and_then(|status| status.agent_runtime_id.as_ref())
6847 .is_none_or(|runtime_id| {
6848 let runtime_id = runtime_id.as_str();
6849 !raw_live_alias_by_runtime_member_id.contains_key(runtime_id)
6850 || visible_runtime_member_ids.contains(runtime_id)
6851 })
6852 });
6853 let current_main_identities = main_identities.clone();
6854 let baseline_specs = runtime.baseline_member_specs().await;
6855 let baseline_identities = baseline_specs
6856 .iter()
6857 .filter(|spec| baseline_spec_visible_to_console(visibility_policy, spec))
6858 .map(|spec| spec.identity.to_string())
6859 .collect::<BTreeSet<_>>();
6860 main_identities.extend(baseline_identities.iter().cloned());
6861 main_identities.extend(durable_identity_runtime_identities.iter().cloned());
6862
6863 let mut retired_delegates = Vec::new();
6864 let mut reset_main = Vec::new();
6865 let mut retired_delegate_details = Vec::new();
6866 let mut reset_details = Vec::new();
6867 let mut failures = Vec::new();
6868 let mut warnings = Vec::new();
6869
6870 for identity in &main_identities {
6871 let parsed_identity = crate::identity_first::AgentIdentity::parse(identity).ok();
6872 let registered_status = if let (Some(identity_runtime), Some(parsed_identity)) =
6873 (identity_runtime, parsed_identity.as_ref())
6874 {
6875 identity_runtime.status(parsed_identity).await.ok()
6876 } else {
6877 None
6878 };
6879 let baseline_identity_runtime_registered = registered_status.is_some();
6880 if baseline_identities.contains(identity)
6881 && !current_main_identities.contains(identity)
6882 && !baseline_identity_runtime_registered
6883 {
6884 continue;
6885 }
6886 let registered_runtime_id = registered_status
6887 .as_ref()
6888 .and_then(|status| status.agent_runtime_id.as_ref())
6889 .map(crate::identity_first::AgentRuntimeId::as_str);
6890 let registered_visible = registered_runtime_id
6891 .is_some_and(|runtime_id| visible_runtime_member_ids.contains(runtime_id));
6892 let registered_hidden = registered_runtime_id.is_some_and(|runtime_id| {
6893 raw_live_alias_by_runtime_member_id.contains_key(runtime_id)
6894 && !visible_runtime_member_ids.contains(runtime_id)
6895 });
6896 if registered_hidden {
6897 continue;
6898 }
6899 if duplicate_live_identities.contains(identity) && !registered_visible {
6900 failures.push(json!({
6901 "identity": identity,
6902 "error": "ambiguous live identity alias",
6903 }));
6904 continue;
6905 }
6906 if let Some(status) = registered_status.as_ref() {
6907 if let Some(registered_runtime_id) = registered_runtime_id
6908 && let Some((live_identity, _live_session_id)) =
6909 raw_live_alias_by_runtime_member_id.get(registered_runtime_id)
6910 && live_identity != identity
6911 {
6912 failures.push(json!({
6913 "identity": identity,
6914 "error": format!(
6915 "stale live identity alias: identity runtime binding points at {registered_runtime_id}, but live console alias projects identity {live_identity}"
6916 ),
6917 "kind": "stale_live_identity_alias",
6918 }));
6919 continue;
6920 }
6921 if let Some(live_runtime_ids) = raw_runtime_member_ids_by_identity.get(identity) {
6922 if !registered_runtime_id
6923 .is_some_and(|runtime_id| live_runtime_ids.contains(runtime_id))
6924 {
6925 failures.push(json!({
6926 "identity": identity,
6927 "error": format!(
6928 "stale live identity alias: identity runtime binding points at {}, but live console alias resolves to [{}]",
6929 registered_runtime_id.unwrap_or("<none>"),
6930 live_runtime_ids.iter().cloned().collect::<Vec<_>>().join(", ")
6931 ),
6932 "kind": "stale_live_identity_alias",
6933 }));
6934 continue;
6935 }
6936 if let Some(registered_runtime_id) = registered_runtime_id
6937 && let Some(registered_session_id) =
6938 status.session_id.as_ref().map(ToString::to_string)
6939 && let Some(Some(live_session_id)) = raw_session_id_by_identity_runtime_member
6940 .get(&(identity.clone(), registered_runtime_id.to_string()))
6941 && live_session_id != ®istered_session_id
6942 {
6943 failures.push(json!({
6944 "identity": identity,
6945 "error": format!(
6946 "stale live identity alias: identity runtime binding points at {registered_runtime_id} session {registered_session_id}, but live console alias resolves to session {live_session_id}"
6947 ),
6948 "kind": "stale_live_identity_alias",
6949 }));
6950 continue;
6951 }
6952 }
6953 if baseline_identities.contains(identity)
6954 && !identity_runtime
6955 .is_some_and(|identity_runtime| identity_runtime.has_session_bridge())
6956 {
6957 failures.push(json!({
6958 "identity": identity,
6959 "error": "reset requires an identity runtime with a session bridge",
6960 "kind": "identity_reset_requires_session_bridge",
6961 }));
6962 }
6963 continue;
6964 }
6965
6966 let runtime_member_id = runtime_member_id_by_identity
6967 .get(identity)
6968 .map(String::as_str)
6969 .unwrap_or(identity.as_str());
6970 if let Some(bound_identity) = identity_by_runtime_member_id.get(runtime_member_id)
6971 && bound_identity != identity
6972 {
6973 failures.push(json!({
6974 "identity": identity,
6975 "error": format!(
6976 "stale live identity alias: live console alias resolves to {runtime_member_id}, but identity runtime binding belongs to {bound_identity}"
6977 ),
6978 "kind": "stale_live_identity_alias",
6979 }));
6980 }
6981 }
6982
6983 if !failures.is_empty() {
6984 return Ok(json!({
6985 "reset": reset_main,
6986 "retired_delegates": retired_delegates,
6987 "reset_details": reset_details,
6988 "retired_delegate_details": retired_delegate_details,
6989 "warnings": warnings,
6990 "failed": failures,
6991 "startup_history": Value::Null,
6992 }));
6993 }
6994
6995 if let Some(state) = runtime.agent_mob_mcp_state() {
6996 for (mob_id, identity) in delegate_members {
6997 match Box::pin(state.handle_for(&MobId::from(mob_id.as_str()))).await {
6998 Ok(handle) => {
6999 match retire_console_member(
7000 &handle,
7001 &crate::member_comms_id::mob_member_id(identity.as_str()),
7002 )
7003 .await
7004 {
7005 Ok(()) => {
7006 let detail = json!({
7007 "identity": identity,
7008 "mob_id": mob_id,
7009 });
7010 retired_delegates.push(detail.clone());
7011 retired_delegate_details.push(detail);
7012 }
7013 Err(err) => failures.push(json!({
7014 "identity": identity,
7015 "mob_id": mob_id,
7016 "error": err,
7017 })),
7018 }
7019 }
7020 Err(err) => failures.push(json!({
7021 "identity": identity,
7022 "mob_id": mob_id,
7023 "error": err.to_string(),
7024 })),
7025 }
7026 }
7027 } else if let Some(aggregator) = console_aggregator {
7028 let identities = delegate_members
7029 .into_iter()
7030 .map(|(_, identity)| identity)
7031 .collect::<BTreeSet<_>>();
7032 for identity in identities {
7033 match Box::pin(aggregator.retire_identity(&identity)).await {
7034 Ok(true) => {
7035 let detail = json!({ "identity": identity });
7036 retired_delegates.push(detail.clone());
7037 retired_delegate_details.push(detail);
7038 }
7039 Ok(false) => failures.push(json!({
7040 "identity": identity,
7041 "error": "unknown identity",
7042 })),
7043 Err(err) => failures.push(json!({
7044 "identity": identity,
7045 "error": err.to_string(),
7046 })),
7047 }
7048 }
7049 }
7050
7051 let handle = runtime.handle();
7052 for spec in baseline_specs {
7053 let identity = spec.identity.to_string();
7054 if !baseline_spec_visible_to_console(visibility_policy, &spec) {
7055 continue;
7056 }
7057 if current_main_identities.contains(&identity) {
7058 continue;
7059 }
7060 if let Some(identity_runtime) = identity_runtime
7061 && let Ok(parsed_identity) = crate::identity_first::AgentIdentity::parse(&identity)
7062 && identity_runtime.status(&parsed_identity).await.is_ok()
7063 {
7064 continue;
7065 }
7066 match handle.ensure_member(spec).await {
7067 Ok(_outcome) => {
7068 if let Some(store) = console_events {
7069 store
7070 .record_lifecycle(
7071 &identity,
7072 "identity_reset",
7073 json!({ "scope": "reset_all", "restored": true }),
7074 )
7075 .await;
7076 }
7077 reset_main.push(identity.clone());
7078 reset_details.push(json!({ "identity": identity }));
7079 }
7080 Err(err) => failures.push(json!({
7081 "identity": identity,
7082 "error": err.to_string(),
7083 })),
7084 }
7085 }
7086 for identity in main_identities {
7087 let baseline_identity_runtime_registered = if let Some(identity_runtime) = identity_runtime
7088 && let Ok(parsed_identity) = crate::identity_first::AgentIdentity::parse(&identity)
7089 {
7090 identity_runtime.status(&parsed_identity).await.is_ok()
7091 } else {
7092 false
7093 };
7094 if baseline_identities.contains(&identity)
7095 && !current_main_identities.contains(&identity)
7096 && !baseline_identity_runtime_registered
7097 {
7098 continue;
7099 }
7100 let registered_status = if let Some(identity_runtime) = identity_runtime
7101 && let Ok(parsed_identity) = crate::identity_first::AgentIdentity::parse(&identity)
7102 {
7103 identity_runtime.status(&parsed_identity).await.ok()
7104 } else {
7105 None
7106 };
7107 let registered_runtime_id = registered_status
7108 .as_ref()
7109 .and_then(|status| status.agent_runtime_id.as_ref())
7110 .map(crate::identity_first::AgentRuntimeId::as_str);
7111 let registered_visible = registered_runtime_id
7112 .is_some_and(|runtime_id| visible_runtime_member_ids.contains(runtime_id));
7113 let registered_hidden = registered_runtime_id.is_some_and(|runtime_id| {
7114 raw_live_alias_by_runtime_member_id.contains_key(runtime_id)
7115 && !visible_runtime_member_ids.contains(runtime_id)
7116 });
7117 if registered_hidden {
7118 continue;
7119 }
7120 if duplicate_live_identities.contains(&identity) && !registered_visible {
7121 failures.push(json!({
7122 "identity": identity,
7123 "error": "ambiguous live identity alias",
7124 }));
7125 continue;
7126 }
7127 if baseline_identities.contains(&identity) {
7128 if let Some(identity_runtime) = identity_runtime
7129 && let Ok(parsed_identity) = crate::identity_first::AgentIdentity::parse(&identity)
7130 && let Ok(status) = identity_runtime.status(&parsed_identity).await
7131 {
7132 let registered_runtime_id = status
7133 .agent_runtime_id
7134 .as_ref()
7135 .map(crate::identity_first::AgentRuntimeId::as_str);
7136 if let Some(registered_runtime_id) = registered_runtime_id
7137 && let Some((live_identity, _live_session_id)) =
7138 raw_live_alias_by_runtime_member_id.get(registered_runtime_id)
7139 && live_identity != &identity
7140 {
7141 failures.push(json!({
7142 "identity": identity,
7143 "error": format!(
7144 "stale live identity alias: identity runtime binding points at {registered_runtime_id}, but live console alias projects identity {live_identity}"
7145 ),
7146 "kind": "stale_live_identity_alias",
7147 }));
7148 continue;
7149 }
7150 if let Some(live_runtime_ids) = raw_runtime_member_ids_by_identity.get(&identity) {
7151 if !registered_runtime_id
7152 .is_some_and(|runtime_id| live_runtime_ids.contains(runtime_id))
7153 {
7154 failures.push(json!({
7155 "identity": identity,
7156 "error": format!(
7157 "stale live identity alias: identity runtime binding points at {}, but live console alias resolves to [{}]",
7158 registered_runtime_id.unwrap_or("<none>"),
7159 live_runtime_ids.iter().cloned().collect::<Vec<_>>().join(", ")
7160 ),
7161 "kind": "stale_live_identity_alias",
7162 }));
7163 continue;
7164 }
7165 if let Some(registered_runtime_id) = registered_runtime_id
7166 && let Some(registered_session_id) =
7167 status.session_id.as_ref().map(ToString::to_string)
7168 && let Some(Some(live_session_id)) =
7169 raw_session_id_by_identity_runtime_member
7170 .get(&(identity.clone(), registered_runtime_id.to_string()))
7171 && live_session_id != ®istered_session_id
7172 {
7173 failures.push(json!({
7174 "identity": identity,
7175 "error": format!(
7176 "stale live identity alias: identity runtime binding points at {registered_runtime_id} session {registered_session_id}, but live console alias resolves to session {live_session_id}"
7177 ),
7178 "kind": "stale_live_identity_alias",
7179 }));
7180 continue;
7181 }
7182 }
7183 if !identity_runtime.has_session_bridge() {
7184 failures.push(json!({
7185 "identity": identity,
7186 "error": "reset requires an identity runtime with a session bridge",
7187 "kind": "identity_reset_requires_session_bridge",
7188 }));
7189 continue;
7190 }
7191 match identity_runtime.reset(&parsed_identity).await {
7192 Ok(record) => {
7193 match retire_stale_console_members_for_identity(
7194 &handle,
7195 visibility_policy,
7196 parsed_identity.as_str(),
7197 Some(record.agent_runtime_id.as_str()),
7198 )
7199 .await
7200 {
7201 Ok(()) => {
7202 reset_details.push(json!({ "identity": identity }));
7203 reset_main.push(identity);
7204 if let Some(store) = console_events {
7205 store
7206 .record_lifecycle(
7207 parsed_identity.as_str(),
7208 "identity_reset",
7209 json!({
7210 "scope": "reset_all",
7211 "generation": record.generation.get(),
7212 "checkpoint_version": record.checkpoint_version.get(),
7213 }),
7214 )
7215 .await;
7216 }
7217 }
7218 Err(err) => {
7219 warnings.push(json!({
7220 "identity": identity,
7221 "kind": "stale_member_cleanup_failed_after_identity_reset",
7222 "message": err,
7223 }));
7224 reset_details.push(json!({
7225 "identity": identity,
7226 "cleanup_warning": warnings.last().cloned(),
7227 }));
7228 reset_main.push(identity);
7229 if let Some(store) = console_events {
7230 store
7231 .record_lifecycle(
7232 parsed_identity.as_str(),
7233 "identity_reset",
7234 json!({
7235 "scope": "reset_all",
7236 "generation": record.generation.get(),
7237 "checkpoint_version": record.checkpoint_version.get(),
7238 "cleanup_warning": warnings.last().cloned(),
7239 }),
7240 )
7241 .await;
7242 }
7243 }
7244 }
7245 }
7246 Err(err) => failures.push(json!({
7247 "identity": identity,
7248 "error": err.to_string(),
7249 })),
7250 }
7251 continue;
7252 }
7253 let runtime_member_id = runtime_member_id_by_identity
7254 .get(&identity)
7255 .map(String::as_str)
7256 .unwrap_or(identity.as_str());
7257 if let Some(bound_identity) = identity_by_runtime_member_id.get(runtime_member_id)
7258 && bound_identity != &identity
7259 {
7260 failures.push(json!({
7261 "identity": identity,
7262 "error": format!(
7263 "stale live identity alias: live console alias resolves to {runtime_member_id}, but identity runtime binding belongs to {bound_identity}"
7264 ),
7265 "kind": "stale_live_identity_alias",
7266 }));
7267 continue;
7268 }
7269 match Box::pin(respawn_console_member(
7270 &handle,
7271 &crate::member_comms_id::mob_member_id(runtime_member_id),
7272 ))
7273 .await
7274 {
7275 Ok(topology_restore_warning) => {
7276 if let Some(store) = console_events {
7277 store
7278 .record_lifecycle(
7279 &identity,
7280 "identity_reset",
7281 json!({
7282 "scope": "reset_all",
7283 "topology_restore_warning": topology_restore_warning.clone(),
7284 }),
7285 )
7286 .await;
7287 }
7288 reset_main.push(identity.clone());
7289 let mut detail = json!({ "identity": identity });
7290 if let Some(warning) = topology_restore_warning {
7291 detail["topology_restore_warning"] = warning;
7292 }
7293 reset_details.push(detail);
7294 }
7295 Err(err) => failures.push(json!({
7296 "identity": identity,
7297 "error": err,
7298 })),
7299 }
7300 } else {
7301 if let Some(identity_runtime) = identity_runtime
7302 && let Ok(parsed_identity) = crate::identity_first::AgentIdentity::parse(&identity)
7303 && let Ok(registered_status) = identity_runtime.status(&parsed_identity).await
7304 {
7305 let registered_runtime_id = registered_status
7306 .agent_runtime_id
7307 .as_ref()
7308 .map(crate::identity_first::AgentRuntimeId::as_str);
7309 let registered_visible = registered_runtime_id
7310 .is_some_and(|runtime_id| visible_runtime_member_ids.contains(runtime_id));
7311 if duplicate_live_identities.contains(&identity) && !registered_visible {
7312 failures.push(json!({
7313 "identity": identity,
7314 "error": "ambiguous live identity alias",
7315 }));
7316 continue;
7317 }
7318 if let Some(registered_runtime_id) = registered_runtime_id
7319 && let Some((live_identity, _live_session_id)) =
7320 raw_live_alias_by_runtime_member_id.get(registered_runtime_id)
7321 && live_identity != &identity
7322 {
7323 failures.push(json!({
7324 "identity": identity,
7325 "error": format!(
7326 "stale live identity alias: identity runtime binding points at {registered_runtime_id}, but live console alias projects identity {live_identity}"
7327 ),
7328 "kind": "stale_live_identity_alias",
7329 }));
7330 continue;
7331 }
7332 if let Some(live_runtime_ids) = raw_runtime_member_ids_by_identity.get(&identity) {
7333 if !registered_runtime_id
7334 .is_some_and(|runtime_id| live_runtime_ids.contains(runtime_id))
7335 {
7336 failures.push(json!({
7337 "identity": identity,
7338 "error": format!(
7339 "stale live identity alias: identity runtime binding points at {}, but live console alias resolves to [{}]",
7340 registered_runtime_id.unwrap_or("<none>"),
7341 live_runtime_ids.iter().cloned().collect::<Vec<_>>().join(", ")
7342 ),
7343 "kind": "stale_live_identity_alias",
7344 }));
7345 continue;
7346 }
7347 if let Some(registered_runtime_id) = registered_runtime_id
7348 && let Some(registered_session_id) = registered_status
7349 .session_id
7350 .as_ref()
7351 .map(ToString::to_string)
7352 && let Some(Some(live_session_id)) =
7353 raw_session_id_by_identity_runtime_member
7354 .get(&(identity.clone(), registered_runtime_id.to_string()))
7355 && live_session_id != ®istered_session_id
7356 {
7357 failures.push(json!({
7358 "identity": identity,
7359 "error": format!(
7360 "stale live identity alias: identity runtime binding points at {registered_runtime_id} session {registered_session_id}, but live console alias resolves to session {live_session_id}"
7361 ),
7362 "kind": "stale_live_identity_alias",
7363 }));
7364 continue;
7365 }
7366 }
7367 match identity_runtime.retire(&parsed_identity).await {
7368 Ok(token) => {
7369 let keep_runtime_member_id = registered_status
7370 .agent_runtime_id
7371 .as_ref()
7372 .filter(|_| identity_runtime.has_session_bridge())
7373 .map(crate::identity_first::AgentRuntimeId::as_str);
7374 match retire_stale_console_members_for_identity(
7375 &handle,
7376 visibility_policy,
7377 parsed_identity.as_str(),
7378 keep_runtime_member_id,
7379 )
7380 .await
7381 {
7382 Ok(()) => {
7383 retired_delegate_details.push(json!({ "identity": identity }));
7384 retired_delegates.push(json!({ "identity": identity }));
7385 if let Some(store) = console_events {
7386 store
7387 .record_lifecycle(
7388 parsed_identity.as_str(),
7389 "identity_retired",
7390 json!({
7391 "scope": "reset_all",
7392 "dynamic": true,
7393 "fencing_token": token.get(),
7394 }),
7395 )
7396 .await;
7397 }
7398 }
7399 Err(err) => {
7400 warnings.push(json!({
7401 "identity": identity,
7402 "kind": "stale_member_cleanup_failed_after_identity_retire",
7403 "message": err,
7404 }));
7405 retired_delegate_details.push(json!({
7406 "identity": identity,
7407 "cleanup_warning": warnings.last().cloned(),
7408 }));
7409 retired_delegates.push(json!({ "identity": identity }));
7410 if let Some(store) = console_events {
7411 store
7412 .record_lifecycle(
7413 parsed_identity.as_str(),
7414 "identity_retired",
7415 json!({
7416 "scope": "reset_all",
7417 "dynamic": true,
7418 "fencing_token": token.get(),
7419 "cleanup_warning": warnings.last().cloned(),
7420 }),
7421 )
7422 .await;
7423 }
7424 }
7425 }
7426 }
7427 Err(err) => failures.push(json!({
7428 "identity": identity,
7429 "error": err.to_string(),
7430 })),
7431 }
7432 continue;
7433 }
7434 let runtime_member_id = runtime_member_id_by_identity
7435 .get(&identity)
7436 .map(String::as_str)
7437 .unwrap_or(identity.as_str());
7438 if let Some(bound_identity) = identity_by_runtime_member_id.get(runtime_member_id)
7439 && bound_identity != &identity
7440 {
7441 failures.push(json!({
7442 "identity": identity,
7443 "error": format!(
7444 "stale live identity alias: live console alias resolves to {runtime_member_id}, but identity runtime binding belongs to {bound_identity}"
7445 ),
7446 "kind": "stale_live_identity_alias",
7447 }));
7448 continue;
7449 }
7450 match retire_console_member(
7451 &handle,
7452 &crate::member_comms_id::mob_member_id(runtime_member_id),
7453 )
7454 .await
7455 {
7456 Ok(()) => {
7457 if let Some(store) = console_events {
7458 store
7459 .record_lifecycle(
7460 &identity,
7461 "identity_retired",
7462 json!({ "scope": "reset_all", "dynamic": true }),
7463 )
7464 .await;
7465 }
7466 retired_delegates.push(json!({ "identity": identity }));
7467 retired_delegate_details.push(json!({ "identity": identity }));
7468 }
7469 Err(err) => failures.push(json!({
7470 "identity": identity,
7471 "error": err,
7472 })),
7473 }
7474 }
7475 }
7476
7477 let startup_history = if failures.is_empty() {
7478 if let Some(aggregator) = console_aggregator {
7479 Box::pin(wait_for_reset_startup_history(
7480 aggregator,
7481 reset_main.iter().cloned().collect::<BTreeSet<_>>(),
7482 Duration::from_secs(10),
7483 ))
7484 .await
7485 .unwrap_or_else(|err| json!({ "error": err.to_string() }))
7486 } else {
7487 Value::Null
7488 }
7489 } else {
7490 Value::Null
7491 };
7492
7493 Ok(json!({
7494 "reset": reset_main,
7495 "retired_delegates": retired_delegates,
7496 "reset_details": reset_details,
7497 "retired_delegate_details": retired_delegate_details,
7498 "warnings": warnings,
7499 "failed": failures,
7500 "startup_history": startup_history,
7501 }))
7502}
7503
7504async fn wait_for_reset_startup_history(
7505 aggregator: &MobKitConsoleAggregator,
7506 identities: BTreeSet<String>,
7507 timeout: Duration,
7508) -> ConsoleLogResult<Value> {
7509 if identities.is_empty() {
7510 return Ok(json!({
7511 "timeout": false,
7512 "ready": Vec::<String>::new(),
7513 "pending": Vec::<String>::new(),
7514 }));
7515 }
7516
7517 let deadline = Instant::now() + timeout;
7518 let mut pending = identities;
7519 let mut ready = BTreeSet::new();
7520 while !pending.is_empty() {
7521 for identity in pending.clone() {
7522 let page = Box::pin(aggregator.query_timeline(ConsoleTimelineQuery {
7523 identity: Some(identity.clone()),
7524 limit: 1000,
7525 ..ConsoleTimelineQuery::default()
7526 }))
7527 .await?;
7528 let startup_completed = page.frames.iter().any(|frame| {
7529 matches!(
7530 frame.kind.as_str(),
7531 "interaction_complete" | "turn_completed"
7532 )
7533 });
7534 if startup_completed {
7535 pending.remove(&identity);
7536 ready.insert(identity);
7537 }
7538 }
7539
7540 if pending.is_empty() {
7541 break;
7542 }
7543 if Instant::now() >= deadline {
7544 return Ok(json!({
7545 "timeout": true,
7546 "ready": ready.into_iter().collect::<Vec<_>>(),
7547 "pending": pending.into_iter().collect::<Vec<_>>(),
7548 }));
7549 }
7550 tokio::time::sleep(Duration::from_millis(250)).await;
7551 }
7552
7553 Ok(json!({
7554 "timeout": false,
7555 "ready": ready.into_iter().collect::<Vec<_>>(),
7556 "pending": Vec::<String>::new(),
7557 }))
7558}
7559
7560fn dedupe_console_members_by_identity(members: &mut Vec<ConsoleMember>) {
7561 let mut seen_member_ids = BTreeSet::new();
7562 members.retain(|member| seen_member_ids.insert(member.agent_identity.clone()));
7563}
7564
7565fn console_member_console_identity(member: &ConsoleMember) -> &str {
7566 member
7567 .labels
7568 .get("agent_identity")
7569 .filter(|value| !value.trim().is_empty())
7570 .map_or(member.agent_identity.as_str(), String::as_str)
7571}
7572
7573fn console_identity_record_from_console_member(member: &ConsoleMember) -> ConsoleIdentityRecord {
7574 let identity = console_member_console_identity(member).to_string();
7575 let addressable = member
7576 .labels
7577 .get("addressable")
7578 .map(|value| !value.eq_ignore_ascii_case("false"))
7579 .unwrap_or(true)
7580 && member.state == MEMBER_STATE_ACTIVE;
7581 let visibility = if member.state == MEMBER_STATE_RETIRING {
7582 ConsoleVisibility::RetiredReadable
7583 } else if addressable {
7584 ConsoleVisibility::Addressable
7585 } else {
7586 ConsoleVisibility::Hidden
7587 };
7588 ConsoleIdentityRecord {
7589 identity: identity.clone(),
7590 display_name: member
7591 .labels
7592 .get("display_name")
7593 .cloned()
7594 .unwrap_or(identity),
7595 runtime_key: "runtime".to_string(),
7596 runtime_member_id: member.agent_identity.clone(),
7597 session_id: member.session_id.clone(),
7598 visibility,
7599 addressable,
7600 health: member.state.clone(),
7601 topology_peers: member.wired_to.clone(),
7602 labels: member.labels.clone(),
7603 }
7604}
7605
7606fn baseline_spec_visible_to_console(
7607 visibility_policy: &dyn ConsoleVisibilityPolicy,
7608 spec: &SpawnMemberSpec,
7609) -> bool {
7610 let mut labels = spec.labels.clone().unwrap_or_default();
7611 labels
7612 .entry("role".to_string())
7613 .or_insert_with(|| spec.role_name.to_string());
7614 let record = ConsoleIdentityRecord {
7615 identity: spec.identity.to_string(),
7616 display_name: spec.identity.to_string(),
7617 runtime_key: "baseline".to_string(),
7618 runtime_member_id: spec.identity.to_string(),
7619 session_id: None,
7620 visibility: ConsoleVisibility::Addressable,
7621 addressable: true,
7622 health: "baseline".to_string(),
7623 topology_peers: Vec::new(),
7624 labels,
7625 };
7626 let member = ConsoleMember {
7627 agent_identity: spec.identity.to_string(),
7628 role: spec.role_name.to_string(),
7629 state: MEMBER_STATE_ACTIVE.to_string(),
7630 model_capabilities: ConsoleModelCapabilities::default(),
7631 runtime_mode: spec
7632 .runtime_mode
7633 .as_ref()
7634 .map(std::string::ToString::to_string),
7635 session_id: None,
7636 wired_to: Vec::new(),
7637 labels: record.labels.clone(),
7638 };
7639 visibility_policy.member_visible(&member) && visibility_policy.identity_visible(&record)
7640}
7641
7642pub(crate) async fn prime_access_cache_from_runtime(
7656 runtime: &MobRuntime,
7657 access: &AccessController,
7658) {
7659 if !access.enabled() {
7660 return;
7661 }
7662 let registered = runtime.console_identity_labels().await;
7663 prime_access_cache_from_handle_with_registry(&runtime.handle(), access, ®istered).await;
7664}
7665
7666pub(crate) async fn prime_access_cache_from_handle(handle: &MobHandle, access: &AccessController) {
7667 if !access.enabled() {
7668 return;
7669 }
7670 prime_access_cache_from_handle_with_registry(handle, access, &BTreeMap::new()).await;
7671}
7672
7673async fn prime_access_cache_from_handle_with_registry(
7674 handle: &MobHandle,
7675 access: &AccessController,
7676 registered: &BTreeMap<String, BTreeMap<String, String>>,
7677) {
7678 let mut registry_only: BTreeMap<&String, &BTreeMap<String, String>> =
7679 registered.iter().collect();
7680 for entry in handle.list_all_members().await {
7681 let member_identity =
7684 crate::member_comms_id::runtime_alias_str(entry.agent_identity.as_str()).into_owned();
7685 let console_identity = entry
7688 .labels
7689 .get("agent_identity")
7690 .map(|value| value.trim())
7691 .filter(|value| !value.is_empty())
7692 .map_or_else(|| member_identity.clone(), ToString::to_string);
7693 let mut labels = entry.labels.clone();
7694 crate::console_spawn::sanitize_unverified_lineage_labels(&mut labels);
7698 if let Some(spawn_labels) = registry_only.remove(&console_identity) {
7699 crate::console_spawn::merge_registered_labels(&mut labels, spawn_labels);
7700 }
7701 access.record_agent_attributes(AgentResourceAttributes {
7702 identity: console_identity,
7703 agent_id: Some(member_identity),
7704 role: Some(entry.role.to_string()),
7705 labels,
7706 });
7707 }
7708 for (identity, spawn_labels) in registry_only {
7709 access.record_agent_attributes(AgentResourceAttributes {
7710 identity: identity.clone(),
7711 agent_id: None,
7712 role: spawn_labels.get("role").cloned(),
7713 labels: spawn_labels.clone(),
7714 });
7715 }
7716}
7717
7718async fn project_console_members_from_handle(
7719 handle: &MobHandle,
7720 host_identity: Option<&str>,
7721 source_mob_id: Option<&str>,
7722 read_model: &ConsoleSnapshotReadModelState,
7723 registered_labels: &BTreeMap<String, BTreeMap<String, String>>,
7724) -> (Vec<ConsoleMember>, BTreeMap<String, String>) {
7725 let entries = handle.list_members_including_retiring().await;
7729 let mut members = Vec::with_capacity(entries.len());
7730 let mut session_owner_by_id = BTreeMap::new();
7731 for entry in &entries {
7732 let identity =
7735 crate::member_comms_id::runtime_alias_str(entry.agent_identity.as_str()).into_owned();
7736 let session_id = read_model.session_id_by_identity.get(&identity).cloned();
7737 if let Some(session_id) = session_id.as_ref() {
7738 session_owner_by_id.insert(session_id.clone(), identity.clone());
7739 }
7740 let model_capabilities =
7741 model_capabilities_for_role(handle.definition(), entry.role.as_str());
7742 let mut labels = entry.labels.clone();
7743 crate::console_spawn::sanitize_unverified_lineage_labels(&mut labels);
7748 let console_identity_key = entry
7749 .labels
7750 .get("agent_identity")
7751 .map(|value| value.trim())
7752 .filter(|value| !value.is_empty())
7753 .map_or(identity.as_str(), |value| value);
7754 if let Some(spawn_labels) = registered_labels.get(console_identity_key) {
7755 crate::console_spawn::merge_registered_labels(&mut labels, spawn_labels);
7756 }
7757 if let Some(host_identity) = host_identity {
7758 labels
7759 .entry("delegate_host_identity".to_string())
7760 .or_insert_with(|| host_identity.to_string());
7761 labels
7762 .entry("group".to_string())
7763 .or_insert_with(|| "Coordinators".to_string());
7764 }
7765 if let Some(source_mob_id) = source_mob_id {
7766 labels
7767 .entry("source_mob_id".to_string())
7768 .or_insert_with(|| source_mob_id.to_string());
7769 }
7770 let mut wired_to: Vec<String> = entry
7771 .wired_to
7772 .iter()
7773 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
7774 .collect();
7775 if let Some(host_identity) = host_identity
7776 && !wired_to.iter().any(|peer| peer == host_identity)
7777 {
7778 wired_to.push(host_identity.to_string());
7779 }
7780 members.push(ConsoleMember {
7781 agent_identity: identity,
7782 role: entry.role.to_string(),
7783 state: member_status_state_string(entry.status),
7784 model_capabilities,
7785 runtime_mode: Some(entry.runtime_mode.to_string()),
7786 session_id,
7787 wired_to,
7788 labels,
7789 });
7790 }
7791 (members, session_owner_by_id)
7792}
7793
7794async fn build_aggregator_live_snapshot(
7795 aggregator: &MobKitConsoleAggregator,
7796 config_module_ids: &[String],
7797) -> Result<ConsoleLiveSnapshot, Box<dyn std::error::Error + Send + Sync>> {
7798 let identities = Box::pin(aggregator.list_identities()).await?;
7799 let mut members = Vec::with_capacity(identities.len());
7800 for identity in &identities {
7801 let mut labels = identity.labels.clone();
7802 labels
7803 .entry("display_name".to_string())
7804 .or_insert_with(|| identity.display_name.clone());
7805 labels
7806 .entry("addressable".to_string())
7807 .or_insert_with(|| identity.addressable.to_string());
7808 let wired_to = identity.topology_peers.clone();
7812 members.push(ConsoleMember {
7813 agent_identity: identity.identity.clone(),
7814 role: labels
7815 .get("role")
7816 .cloned()
7817 .unwrap_or_else(|| "identity".to_string()),
7818 state: identity.health.clone(),
7819 model_capabilities: ConsoleModelCapabilities::default(),
7820 runtime_mode: Some("console_aggregator".to_string()),
7821 session_id: identity.session_id.clone(),
7822 wired_to,
7823 labels,
7824 });
7825 }
7826 members.sort_by(|left, right| left.agent_identity.cmp(&right.agent_identity));
7827 let agents = members
7828 .iter()
7829 .map(|member| ConsoleAgentLiveSnapshot {
7830 agent_id: member.agent_identity.clone(),
7831 member_id: member.agent_identity.clone(),
7832 label: member
7833 .labels
7834 .get("display_name")
7835 .cloned()
7836 .unwrap_or_else(|| member.agent_identity.clone()),
7837 kind: "meerkat".to_string(),
7838 identity: Some(member.agent_identity.clone()),
7839 role: Some(member.role.clone()),
7840 state: Some(member.state.clone()),
7841 session_id: member.session_id.clone(),
7842 model_capabilities: member.model_capabilities.clone(),
7843 response_phase: None,
7844 watched: None,
7845 alert_level: None,
7846 degraded: None,
7847 degraded_reason: None,
7848 })
7849 .collect::<Vec<_>>();
7850 let loaded_modules = if config_module_ids.is_empty() {
7851 members
7852 .iter()
7853 .map(|member| member.agent_identity.clone())
7854 .collect()
7855 } else {
7856 config_module_ids.to_vec()
7857 };
7858 Ok(ConsoleLiveSnapshot::new(
7859 Some("console-aggregator".to_string()),
7860 true,
7861 loaded_modules,
7862 agents,
7863 members,
7864 true,
7865 ))
7866}
7867
7868pub async fn console_frontend_index_handler() -> impl IntoResponse {
7869 (
7870 [
7871 (header::CONTENT_TYPE, "text/html; charset=utf-8"),
7872 (header::CACHE_CONTROL, "no-store"),
7873 ],
7874 CONSOLE_FRONTEND_INDEX_HTML,
7875 )
7876}
7877
7878pub async fn console_frontend_app_js_handler() -> impl IntoResponse {
7879 (
7880 [
7881 (
7882 header::CONTENT_TYPE,
7883 "application/javascript; charset=utf-8",
7884 ),
7885 (header::CACHE_CONTROL, "no-store"),
7886 ],
7887 CONSOLE_FRONTEND_APP_JS,
7888 )
7889}
7890
7891pub async fn console_frontend_app_css_handler() -> impl IntoResponse {
7892 (
7893 [
7894 (header::CONTENT_TYPE, "text/css; charset=utf-8"),
7895 (header::CACHE_CONTROL, "no-store"),
7896 ],
7897 CONSOLE_FRONTEND_APP_CSS,
7898 )
7899}
7900
7901#[cfg(test)]
7902#[allow(clippy::expect_used, clippy::large_futures)]
7903mod tests {
7904 use super::ConsoleTimelineHttpQuery;
7905 use super::{
7906 ConsoleSnapshotReadModel, ConsoleSnapshotReadModelState, MAX_MULTIPART_BODY_BYTES,
7907 MAX_MULTIPART_IMAGE_BYTES, MultipartImageUpload, apply_console_visibility_policy,
7908 build_aggregator_live_snapshot, collect_console_snapshot_read_model,
7909 console_send_identity_first, console_send_with_identity_first_fallback,
7910 console_timeline_replay_unavailable_response, cursor_is_after,
7911 dedupe_console_members_by_identity, externalize_image_upload_placeholders,
7912 externalize_single_image_upload, handle_console_aggregator_rpc, handle_console_runtime_rpc,
7913 handle_console_runtime_rpc_with_visibility, member_id_matches_durable_identity,
7914 project_console_members_from_handle, query_timeline_snapshot, timeline_query_from_http,
7915 };
7916 use crate::blob_store::{BinaryBlobStore, ObjectStoreBlobStore};
7917 use crate::console_aggregator::{
7918 AllowAllConsoleVisibilityPolicy, ConsoleIdentityRecord,
7919 HideImplicitDelegateMembersConsoleVisibilityPolicy,
7920 };
7921 use crate::console_aggregator::{
7922 ConsoleCursor, ConsoleFrameSource, ConsoleFrameSourceKind, ConsoleFrameStatus,
7923 ConsoleTimelineQuery, ConsoleTimelineWindowQuery, ConsoleVisibilityPolicy,
7924 MobKitConsoleAggregator, NewConsoleFrame,
7925 };
7926 use crate::identity_first::contracts::{ContinuityStore, LeaseProvider};
7927 use crate::identity_first::{
7928 AgentAddressability, AgentBuildDraft, AgentIdentity, AgentRuntimeId, BridgeError,
7929 CheckpointVersion, ContinuityGeneration, ContinuityRecord, DurabilityPolicy,
7930 DurableAgentSpec, FencingToken, IdentityLifecycleState, IdentityRuntime,
7931 IdentityRuntimeConfig, LeaseAcquireResult, LeaseGrant, LocalContinuityStore,
7932 LocalLeaseProvider, ManagedPeerEdge, ResumeSessionOutcome, SessionBridge, SessionSnapshot,
7933 };
7934 use crate::mob_handle_runtime::{MobRuntime, model_capabilities_for_role};
7935 use crate::rpc::{JSONRPC_VERSION, JsonRpcRequest};
7936 use crate::runtime::{ConsoleAgentLiveSnapshot, ConsoleLiveSnapshot, ConsoleMember};
7937 use crate::unified_runtime::ConsoleEventStore;
7938 use crate::{MobBootstrapOptions, MobBootstrapSpec};
7939 use bytes::Bytes;
7940 use meerkat::{AgentFactory, Config, build_ephemeral_service};
7941 use meerkat_client::TestClient;
7942 use meerkat_core::types::HandlingMode;
7943 use meerkat_mob::ProfileName;
7944 use meerkat_mob::{MobDefinition, MobStorage, SpawnMemberSpec};
7945 use serde_json::{Value, json};
7946 use std::collections::BTreeMap;
7947 use std::sync::atomic::{AtomicUsize, Ordering};
7948 use std::sync::{Arc, Mutex};
7949 use std::time::Duration;
7950
7951 struct BlockingIdentityBridge {
7952 deliver_calls: Arc<AtomicUsize>,
7953 }
7954
7955 struct RecordingIdentityBridge {
7956 session_id: meerkat_core::types::SessionId,
7957 handling_modes: Arc<Mutex<Vec<HandlingMode>>>,
7958 }
7959
7960 #[async_trait::async_trait]
7961 impl SessionBridge for BlockingIdentityBridge {
7962 async fn create_session(
7963 &self,
7964 _identity: &AgentIdentity,
7965 _runtime_id: &AgentRuntimeId,
7966 _spec: &DurableAgentSpec,
7967 _draft: &AgentBuildDraft,
7968 session_id: &meerkat_core::types::SessionId,
7969 ) -> Result<meerkat_core::types::SessionId, BridgeError> {
7970 Ok(session_id.clone())
7971 }
7972
7973 async fn resume_session(
7974 &self,
7975 _identity: &AgentIdentity,
7976 _runtime_id: &AgentRuntimeId,
7977 _spec: &DurableAgentSpec,
7978 _draft: &AgentBuildDraft,
7979 session_id: &meerkat_core::types::SessionId,
7980 _snapshot: &SessionSnapshot,
7981 ) -> Result<ResumeSessionOutcome, BridgeError> {
7982 Ok(ResumeSessionOutcome::Resumed {
7983 session_id: session_id.clone(),
7984 })
7985 }
7986
7987 async fn deliver(
7988 &self,
7989 _runtime_id: &AgentRuntimeId,
7990 _content: &meerkat_core::ContentInput,
7991 ) -> Result<meerkat_core::types::SessionId, BridgeError> {
7992 self.deliver_calls.fetch_add(1, Ordering::SeqCst);
7993 std::future::pending().await
7994 }
7995
7996 async fn checkpoint_session(
7997 &self,
7998 _runtime_id: &AgentRuntimeId,
7999 _session_id: &meerkat_core::types::SessionId,
8000 ) -> Result<SessionSnapshot, BridgeError> {
8001 Err(BridgeError::Mob("checkpoint not used in test".to_string()))
8002 }
8003
8004 async fn retire_member(&self, _runtime_id: &AgentRuntimeId) -> Result<(), BridgeError> {
8005 Ok(())
8006 }
8007 }
8008
8009 #[async_trait::async_trait]
8010 impl SessionBridge for RecordingIdentityBridge {
8011 async fn create_session(
8012 &self,
8013 _identity: &AgentIdentity,
8014 _runtime_id: &AgentRuntimeId,
8015 _spec: &DurableAgentSpec,
8016 _draft: &AgentBuildDraft,
8017 session_id: &meerkat_core::types::SessionId,
8018 ) -> Result<meerkat_core::types::SessionId, BridgeError> {
8019 Ok(session_id.clone())
8020 }
8021
8022 async fn resume_session(
8023 &self,
8024 _identity: &AgentIdentity,
8025 _runtime_id: &AgentRuntimeId,
8026 _spec: &DurableAgentSpec,
8027 _draft: &AgentBuildDraft,
8028 session_id: &meerkat_core::types::SessionId,
8029 _snapshot: &SessionSnapshot,
8030 ) -> Result<ResumeSessionOutcome, BridgeError> {
8031 Ok(ResumeSessionOutcome::Resumed {
8032 session_id: session_id.clone(),
8033 })
8034 }
8035
8036 async fn deliver(
8037 &self,
8038 runtime_id: &AgentRuntimeId,
8039 content: &meerkat_core::ContentInput,
8040 ) -> Result<meerkat_core::types::SessionId, BridgeError> {
8041 self.deliver_with_mode(runtime_id, content, HandlingMode::Queue)
8042 .await
8043 }
8044
8045 async fn deliver_with_mode(
8046 &self,
8047 _runtime_id: &AgentRuntimeId,
8048 _content: &meerkat_core::ContentInput,
8049 handling_mode: HandlingMode,
8050 ) -> Result<meerkat_core::types::SessionId, BridgeError> {
8051 self.handling_modes
8052 .lock()
8053 .map_err(|_| BridgeError::Mob("handling modes mutex poisoned".to_string()))?
8054 .push(handling_mode);
8055 Ok(self.session_id.clone())
8056 }
8057
8058 async fn checkpoint_session(
8059 &self,
8060 _runtime_id: &AgentRuntimeId,
8061 _session_id: &meerkat_core::types::SessionId,
8062 ) -> Result<SessionSnapshot, BridgeError> {
8063 Err(BridgeError::Mob("checkpoint not used in test".to_string()))
8064 }
8065
8066 async fn retire_member(&self, _runtime_id: &AgentRuntimeId) -> Result<(), BridgeError> {
8067 Ok(())
8068 }
8069 }
8070
8071 async fn build_empty_console_test_runtime(
8072 mob_id: &str,
8073 ) -> Result<(tempfile::TempDir, MobRuntime), Box<dyn std::error::Error + Send + Sync>> {
8074 let temp_dir = tempfile::tempdir()?;
8075 let session_path = temp_dir.path().join("sessions");
8076 std::fs::create_dir_all(&session_path)?;
8077 let factory = AgentFactory::new(&session_path).comms(true);
8078 let session_service = Arc::new(build_ephemeral_service(factory, Config::default(), 16));
8079 let definition = MobDefinition::from_toml(&format!(
8080 r#"
8081[mob]
8082id = "{mob_id}"
8083
8084[profiles.worker]
8085model = "gpt-5.5"
8086external_addressable = true
8087
8088[profiles.worker.tools]
8089comms = true
8090"#
8091 ))?;
8092 let runtime = MobRuntime::bootstrap(
8093 MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
8094 .with_options(MobBootstrapOptions {
8095 allow_ephemeral_sessions: true,
8096 notify_orchestrator_on_resume: true,
8097 default_llm_client: Some(Arc::new(TestClient::default())),
8098 }),
8099 )
8100 .await?;
8101 Ok((temp_dir, runtime))
8102 }
8103
8104 fn rpc_request(method: &str) -> JsonRpcRequest {
8105 JsonRpcRequest {
8106 jsonrpc: JSONRPC_VERSION.to_string(),
8107 id: Some(json!(1)),
8108 method: method.to_string(),
8109 params: json!({}),
8110 }
8111 }
8112
8113 fn rpc_request_with_params(method: &str, params: Value) -> JsonRpcRequest {
8114 JsonRpcRequest {
8115 jsonrpc: JSONRPC_VERSION.to_string(),
8116 id: Some(json!(1)),
8117 method: method.to_string(),
8118 params,
8119 }
8120 }
8121
8122 #[tokio::test]
8123 async fn read_only_aggregator_capabilities_omit_mutating_methods() {
8124 let response = handle_console_aggregator_rpc(
8125 None,
8126 rpc_request("mobkit/capabilities"),
8127 true,
8128 true,
8129 None,
8130 None,
8131 )
8132 .await;
8133
8134 let methods = response["result"]["methods"]
8135 .as_array()
8136 .expect("capabilities methods");
8137 assert!(
8138 methods.iter().all(|method| method != "mobkit/console/send"),
8139 "read-only capabilities must omit send: {methods:#?}"
8140 );
8141 assert_eq!(response["result"]["read_only"], json!(true));
8142 assert_eq!(
8143 response["result"]["runtime_capabilities"]["can_send_messages"],
8144 json!(false)
8145 );
8146 }
8147
8148 #[tokio::test]
8149 async fn read_only_aggregator_denies_direct_mutating_rpc() {
8150 let response = handle_console_aggregator_rpc(
8151 None,
8152 rpc_request_with_params(
8153 "mobkit/console/send",
8154 json!({
8155 "identity": "worker",
8156 "content": "hello",
8157 "origin": "test",
8158 "idempotency_key": "read-only-send",
8159 }),
8160 ),
8161 true,
8162 true,
8163 None,
8164 None,
8165 )
8166 .await;
8167
8168 assert_eq!(response["result"], Value::Null);
8169 assert_eq!(response["error"]["code"], json!(-32010));
8170 assert_eq!(response["error"]["data"]["kind"], json!("read_only"));
8171 }
8172
8173 #[test]
8174 fn read_only_mutating_methods_include_state_draining_collect_completed() {
8175 assert!(super::is_console_mutating_rpc_method(
8176 "mobkit/collect_completed"
8177 ));
8178 }
8179
8180 #[test]
8181 fn authenticated_gating_approver_comes_from_console_principal() {
8182 let forged_params = json!({ "approver_id": "forged-browser-value" });
8183
8184 assert_eq!(
8185 super::resolve_gating_approver_id(&forged_params, Some("admin@example.com"),),
8186 Ok("admin@example.com".to_string()),
8187 );
8188 }
8189
8190 #[test]
8191 fn unauthenticated_local_gating_approver_requires_request_param() {
8192 assert_eq!(
8193 super::resolve_gating_approver_id(&json!({ "approver_id": " local-operator " }), None,),
8194 Ok("local-operator".to_string()),
8195 );
8196 assert_eq!(
8197 super::resolve_gating_approver_id(&json!({}), None),
8198 Err("approver_id required"),
8199 );
8200 }
8201
8202 #[test]
8203 fn internal_error_does_not_disclose_backend_details() {
8204 let response = super::internal_error(json!(7), "secret backend DSN");
8205
8206 assert_eq!(response["error"]["code"], json!(-32000));
8207 assert_eq!(response["error"]["message"], json!("internal error"));
8208 assert_eq!(response["error"]["data"]["error"], json!("internal_error"));
8209 assert!(!response.to_string().contains("secret backend DSN"));
8210 }
8211
8212 #[test]
8213 fn console_send_public_message_hides_dispatch_details() {
8214 let message = super::console_send_public_message(
8215 &crate::console_aggregator::ConsoleSendError::Dispatch(
8216 "secret backend DSN".to_string(),
8217 ),
8218 );
8219
8220 assert_eq!(message, "console send failed");
8221 assert!(!message.contains("secret backend DSN"));
8222 }
8223
8224 #[test]
8225 fn console_send_json_rpc_error_hides_backend_details() {
8226 let response = super::console_send_rpc_error(
8227 json!(7),
8228 crate::console_aggregator::ConsoleSendError::Dispatch("secret backend DSN".to_string()),
8229 );
8230
8231 assert_eq!(response["error"]["code"], json!(-32000));
8232 assert_eq!(response["error"]["message"], json!("console send failed"));
8233 assert!(!response.to_string().contains("secret backend DSN"));
8234 }
8235
8236 #[test]
8237 fn console_timeline_replay_error_hides_backend_details() {
8238 let response = super::console_timeline_replay_unavailable_response(
8239 json!(7),
8240 Box::new(std::io::Error::other("secret backend DSN")),
8241 None,
8242 None,
8243 );
8244
8245 assert_eq!(
8246 response["error"]["code"],
8247 json!(crate::rpc::CONSOLE_TIMELINE_REPLAY_UNAVAILABLE_CODE)
8248 );
8249 assert_eq!(
8250 response["error"]["message"],
8251 json!("timeline replay unavailable")
8252 );
8253 assert!(!response.to_string().contains("secret backend DSN"));
8254 }
8255
8256 #[test]
8257 fn gating_decision_error_hides_backend_details() {
8258 let response = super::gating_decision_failed_error(json!(7), "secret backend DSN");
8259
8260 assert_eq!(response["error"]["code"], json!(-32602));
8261 assert_eq!(
8262 response["error"]["message"],
8263 json!("gating decision failed")
8264 );
8265 assert!(!response.to_string().contains("secret backend DSN"));
8266 }
8267
8268 #[tokio::test]
8269 async fn console_runtime_identity_controls_resolve_durable_member_aliases()
8270 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
8271 let (_temp_dir, runtime) =
8272 build_empty_console_test_runtime("console-identity-control-alias").await?;
8273 let mut labels = BTreeMap::new();
8274 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
8275 labels.insert("display_name".to_string(), "Review Agent".to_string());
8276 runtime
8277 .handle()
8278 .spawn_spec(
8279 SpawnMemberSpec::from_wire(
8280 "worker".to_string(),
8281 crate::member_comms_id::mob_member_id_str("rt:review:singleton:0").into_owned(),
8284 Some("You are Review Agent.".into()),
8285 None,
8286 None,
8287 )
8288 .with_labels(labels),
8289 )
8290 .await?;
8291
8292 let durable_status = Box::pin(handle_console_runtime_rpc(
8293 &runtime,
8294 None,
8295 None,
8296 None,
8297 None,
8298 None,
8299 None,
8300 None,
8301 None,
8302 rpc_request_with_params(
8303 "mobkit/status_identity",
8304 json!({ "identity": "review:singleton" }),
8305 ),
8306 true,
8307 ))
8308 .await;
8309 assert_eq!(durable_status["error"], Value::Null);
8310 assert_eq!(
8311 durable_status["result"]["identity"],
8312 json!("review:singleton")
8313 );
8314 assert_eq!(
8315 durable_status["result"]["agent_runtime_id"],
8316 json!("rt:review:singleton:0")
8317 );
8318
8319 let runtime_id_status = Box::pin(handle_console_runtime_rpc(
8320 &runtime,
8321 None,
8322 None,
8323 None,
8324 None,
8325 None,
8326 None,
8327 None,
8328 None,
8329 rpc_request_with_params(
8330 "mobkit/status_identity",
8331 json!({ "identity": "rt:review:singleton:0" }),
8332 ),
8333 true,
8334 ))
8335 .await;
8336 assert_eq!(runtime_id_status["error"], Value::Null);
8337 assert_eq!(
8338 runtime_id_status["result"]["identity"],
8339 json!("review:singleton")
8340 );
8341
8342 let runtime_id_inspect = Box::pin(handle_console_runtime_rpc(
8343 &runtime,
8344 None,
8345 None,
8346 None,
8347 None,
8348 None,
8349 None,
8350 None,
8351 None,
8352 rpc_request_with_params(
8353 "mobkit/inspect_identity",
8354 json!({ "identity": "rt:review:singleton:0" }),
8355 ),
8356 true,
8357 ))
8358 .await;
8359 assert_eq!(runtime_id_inspect["error"], Value::Null);
8360 assert_eq!(
8361 runtime_id_inspect["result"]["identity"],
8362 json!("review:singleton")
8363 );
8364
8365 let respawn = Box::pin(handle_console_runtime_rpc(
8366 &runtime,
8367 None,
8368 None,
8369 None,
8370 Some(ConsoleEventStore::new()),
8371 None,
8372 None,
8373 None,
8374 None,
8375 rpc_request_with_params("mobkit/respawn", json!({ "identity": "review:singleton" })),
8376 true,
8377 ))
8378 .await;
8379 assert_eq!(respawn["error"], Value::Null);
8380 assert_eq!(respawn["result"]["identity"], json!("review:singleton"));
8381 assert_eq!(
8382 respawn["result"]["agent_runtime_id"],
8383 json!("rt:review:singleton:0")
8384 );
8385
8386 let reset_without_identity_runtime = Box::pin(handle_console_runtime_rpc(
8390 &runtime,
8391 None,
8392 None,
8393 None,
8394 Some(ConsoleEventStore::new()),
8395 None,
8396 None,
8397 None,
8398 None,
8399 rpc_request_with_params("mobkit/reset", json!({ "identity": "review:singleton" })),
8400 true,
8401 ))
8402 .await;
8403 assert_eq!(
8404 reset_without_identity_runtime["error"],
8405 Value::Null,
8406 "{reset_without_identity_runtime:#?}"
8407 );
8408 assert_eq!(
8409 reset_without_identity_runtime["result"]["identity"],
8410 json!("review:singleton")
8411 );
8412 assert_eq!(
8413 reset_without_identity_runtime["result"]["agent_runtime_id"],
8414 json!("rt:review:singleton:0")
8415 );
8416
8417 let reset_unknown_identity = Box::pin(handle_console_runtime_rpc(
8418 &runtime,
8419 None,
8420 None,
8421 None,
8422 Some(ConsoleEventStore::new()),
8423 None,
8424 None,
8425 None,
8426 None,
8427 rpc_request_with_params("mobkit/reset", json!({ "identity": "missing:member" })),
8428 true,
8429 ))
8430 .await;
8431 assert_ne!(reset_unknown_identity["error"], Value::Null);
8432 assert!(
8433 reset_unknown_identity["error"]["message"]
8434 .as_str()
8435 .unwrap_or_default()
8436 .contains("identity not found")
8437 );
8438
8439 let _ = runtime.handle().stop().await;
8440 Ok(())
8441 }
8442
8443 #[tokio::test]
8444 async fn console_runtime_identity_controls_reject_ambiguous_live_label_aliases()
8445 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
8446 let (_temp_dir, runtime) =
8447 build_empty_console_test_runtime("console-identity-ambiguous-live-alias").await?;
8448 for runtime_id in ["rt:review:singleton:0", "rt:review:singleton:1"] {
8449 let mut labels = BTreeMap::new();
8450 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
8451 runtime
8452 .handle()
8453 .spawn_spec(
8454 SpawnMemberSpec::from_wire(
8455 "worker".to_string(),
8456 crate::member_comms_id::mob_member_id_str(runtime_id).into_owned(),
8459 Some("You are a duplicate Review Agent.".into()),
8460 None,
8461 None,
8462 )
8463 .with_labels(labels),
8464 )
8465 .await?;
8466 }
8467
8468 for requested_identity in ["review:singleton", "rt:review:singleton:0"] {
8469 for method in [
8470 "mobkit/status_identity",
8471 "mobkit/inspect_identity",
8472 "mobkit/retire",
8473 "mobkit/respawn",
8474 "mobkit/reset",
8475 ] {
8476 let response = Box::pin(handle_console_runtime_rpc(
8477 &runtime,
8478 None,
8479 None,
8480 None,
8481 Some(ConsoleEventStore::new()),
8482 None,
8483 None,
8484 None,
8485 None,
8486 rpc_request_with_params(method, json!({ "identity": requested_identity })),
8487 true,
8488 ))
8489 .await;
8490 assert_ne!(
8491 response["error"],
8492 Value::Null,
8493 "{method} must reject ambiguous live alias for {requested_identity}"
8494 );
8495 assert_eq!(
8496 response["error"]["data"]["kind"],
8497 json!("ambiguous_live_identity_alias"),
8498 "unexpected response for {method}/{requested_identity}: {response:#?}"
8499 );
8500 }
8501 }
8502
8503 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
8504 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
8505 lease_provider: Arc::new(LocalLeaseProvider::new()),
8506 runtime_instance_id: "console-identity-ambiguous-live-alias".to_string(),
8507 has_runtime_store: true,
8508 durability_policy: DurabilityPolicy::SyncWriteThrough,
8509 bridge: None,
8510 default_timeout: None,
8511 }));
8512 for requested_identity in ["review:singleton", "rt:review:singleton:0"] {
8513 for method in ["mobkit/reset", "mobkit/delete_identity"] {
8514 let response = Box::pin(handle_console_runtime_rpc(
8515 &runtime,
8516 None,
8517 None,
8518 None,
8519 Some(ConsoleEventStore::new()),
8520 None,
8521 Some(identity_runtime.clone()),
8522 None,
8523 None,
8524 rpc_request_with_params(method, json!({ "identity": requested_identity })),
8525 true,
8526 ))
8527 .await;
8528 assert_ne!(
8529 response["error"],
8530 Value::Null,
8531 "{method} must reject ambiguous live alias for {requested_identity}"
8532 );
8533 assert_eq!(
8534 response["error"]["data"]["kind"],
8535 json!("ambiguous_live_identity_alias"),
8536 "unexpected response for {method}/{requested_identity}: {response:#?}"
8537 );
8538 }
8539 }
8540
8541 let _ = runtime.handle().stop().await;
8542 Ok(())
8543 }
8544
8545 #[tokio::test]
8546 async fn console_runtime_durable_identity_prefers_registered_live_over_duplicate_labels()
8547 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
8548 let (_temp_dir, runtime) =
8549 build_empty_console_test_runtime("console-durable-wins-duplicate-live-labels").await?;
8550 for runtime_id in ["rt:review:singleton:0", "rt:review:singleton:1"] {
8551 let mut labels = BTreeMap::new();
8552 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
8553 runtime
8554 .handle()
8555 .spawn_spec(
8556 SpawnMemberSpec::from_wire(
8557 "worker".to_string(),
8558 crate::member_comms_id::mob_member_id_str(runtime_id).into_owned(),
8561 Some("You are a Review Agent candidate.".into()),
8562 None,
8563 None,
8564 )
8565 .with_labels(labels),
8566 )
8567 .await?;
8568 }
8569
8570 let store = Arc::new(LocalContinuityStore::in_memory()?);
8571 let lease_provider = Arc::new(LocalLeaseProvider::new());
8572 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
8573 continuity_store: store.clone(),
8574 lease_provider: lease_provider.clone(),
8575 runtime_instance_id: "test-runtime".to_string(),
8576 has_runtime_store: true,
8577 durability_policy: DurabilityPolicy::SyncWriteThrough,
8578 bridge: None,
8579 default_timeout: None,
8580 }));
8581 let identity = AgentIdentity::parse("review:singleton")?;
8582 let registered_session_id = runtime
8583 .handle()
8584 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
8585 "rt:review:singleton:0",
8586 ))
8587 .await
8588 .unwrap_or_else(meerkat_core::types::SessionId::new);
8589 let record = ContinuityRecord {
8590 identity: identity.clone(),
8591 agent_runtime_id: AgentRuntimeId::parse("rt:review:singleton:0")?,
8592 session_id: registered_session_id,
8593 generation: ContinuityGeneration::new(0),
8594 checkpoint_version: CheckpointVersion::new(0),
8595 };
8596 let grants = lease_provider
8597 .acquire_leases(std::slice::from_ref(&identity), "test-runtime")
8598 .await?;
8599 let grant = match grants.get(&identity).cloned() {
8600 Some(LeaseAcquireResult::Acquired(grant)) => grant,
8601 other => return Err(format!("expected acquired lease, got {other:?}").into()),
8602 };
8603 store
8604 .upsert_continuity_record(&record, grant.fencing_token)
8605 .await?;
8606 identity_runtime
8607 .register(
8608 DurableAgentSpec {
8609 identity: identity.clone(),
8610 profile: ProfileName::from("worker"),
8611 addressability: AgentAddressability::Addressable,
8612 display_name: None,
8613 labels: BTreeMap::new(),
8614 context: None,
8615 additional_instructions: Vec::new(),
8616 initial_message: None,
8617 runtime_mode_override: None,
8618 backend: None,
8619 binding: None,
8620 },
8621 IdentityLifecycleState::Active,
8622 Some(record),
8623 Some(grant),
8624 )
8625 .await;
8626
8627 for requested_identity in ["review:singleton", "rt:review:singleton:0"] {
8628 for method in ["mobkit/status_identity", "mobkit/inspect_identity"] {
8629 let response = Box::pin(handle_console_runtime_rpc(
8630 &runtime,
8631 None,
8632 None,
8633 None,
8634 Some(ConsoleEventStore::new()),
8635 None,
8636 Some(identity_runtime.clone()),
8637 None,
8638 None,
8639 rpc_request_with_params(method, json!({ "identity": requested_identity })),
8640 true,
8641 ))
8642 .await;
8643 assert_eq!(
8644 response["error"],
8645 Value::Null,
8646 "{method} must use durable registered live binding despite duplicate labels for {requested_identity}: {response:#?}"
8647 );
8648 }
8649 }
8650 let reset_all_response = Box::pin(handle_console_runtime_rpc(
8651 &runtime,
8652 None,
8653 None,
8654 None,
8655 Some(ConsoleEventStore::new()),
8656 None,
8657 Some(identity_runtime.clone()),
8658 None,
8659 None,
8660 rpc_request("mobkit/reset_all"),
8661 true,
8662 ))
8663 .await;
8664 assert_eq!(
8665 reset_all_response["error"],
8666 Value::Null,
8667 "reset_all must also prefer the durable registered live binding despite duplicate labels: {reset_all_response:#?}"
8668 );
8669 assert!(
8670 reset_all_response["result"]["failed"]
8671 .as_array()
8672 .is_some_and(Vec::is_empty),
8673 "reset_all should not report duplicate-label failure for durable registered binding: {reset_all_response:#?}"
8674 );
8675
8676 let _ = runtime.handle().stop().await;
8677 Ok(())
8678 }
8679
8680 #[tokio::test]
8681 async fn console_runtime_identity_controls_reject_wrong_projected_live_only_alias()
8682 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
8683 let (_temp_dir, runtime) =
8684 build_empty_console_test_runtime("console-identity-wrong-projected-live-only").await?;
8685
8686 let mut labels = BTreeMap::new();
8687 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
8688 runtime
8689 .handle()
8690 .spawn_spec(
8691 SpawnMemberSpec::from_wire(
8692 "worker".to_string(),
8693 crate::member_comms_id::mob_member_id_str("rt:review:singleton:0").into_owned(),
8696 Some("You are a wrong-projected Review Agent.".into()),
8697 None,
8698 None,
8699 )
8700 .with_labels(labels),
8701 )
8702 .await?;
8703
8704 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
8705 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
8706 lease_provider: Arc::new(LocalLeaseProvider::new()),
8707 runtime_instance_id: "console-identity-wrong-projected-live-only".to_string(),
8708 has_runtime_store: true,
8709 durability_policy: DurabilityPolicy::SyncWriteThrough,
8710 bridge: None,
8711 default_timeout: None,
8712 }));
8713 let identity = AgentIdentity::parse("review:singleton")?;
8714 let record = ContinuityRecord {
8715 identity: identity.clone(),
8716 agent_runtime_id: AgentRuntimeId::parse("rt:review:singleton:0")?,
8717 session_id: meerkat_core::types::SessionId::new(),
8718 generation: ContinuityGeneration::new(0),
8719 checkpoint_version: CheckpointVersion::new(0),
8720 };
8721 identity_runtime
8722 .register(
8723 DurableAgentSpec {
8724 identity: identity.clone(),
8725 profile: ProfileName::from("worker"),
8726 addressability: AgentAddressability::Addressable,
8727 display_name: None,
8728 labels: BTreeMap::new(),
8729 context: None,
8730 additional_instructions: Vec::new(),
8731 initial_message: None,
8732 runtime_mode_override: None,
8733 backend: None,
8734 binding: None,
8735 },
8736 IdentityLifecycleState::Active,
8737 Some(record),
8738 Some(LeaseGrant {
8739 identity,
8740 fencing_token: FencingToken::new(1),
8741 ttl: Duration::from_mins(1),
8742 }),
8743 )
8744 .await;
8745
8746 for method in [
8747 "mobkit/status_identity",
8748 "mobkit/inspect_identity",
8749 "mobkit/retire",
8750 ] {
8751 let response = Box::pin(handle_console_runtime_rpc(
8752 &runtime,
8753 None,
8754 None,
8755 None,
8756 Some(ConsoleEventStore::new()),
8757 None,
8758 Some(identity_runtime.clone()),
8759 None,
8760 None,
8761 rpc_request_with_params(method, json!({ "identity": "other:singleton" })),
8762 true,
8763 ))
8764 .await;
8765 assert_ne!(
8766 response["error"],
8767 Value::Null,
8768 "{method} must reject wrong-projected live-only alias"
8769 );
8770 assert_eq!(
8771 response["error"]["data"]["kind"],
8772 json!("stale_live_identity_alias"),
8773 "unexpected response for {method}: {response:#?}"
8774 );
8775 }
8776 assert!(
8777 runtime
8778 .handle()
8779 .get_member(&crate::member_comms_id::mob_member_id(
8780 "rt:review:singleton:0"
8781 ))
8782 .await
8783 .ok()
8784 .flatten()
8785 .is_some(),
8786 "wrong-projected durable runtime member must not be retired through projected alias"
8787 );
8788
8789 let _ = runtime.handle().stop().await;
8790 Ok(())
8791 }
8792
8793 #[derive(Debug)]
8794 struct HideIdentityPolicy(&'static str);
8795
8796 impl ConsoleVisibilityPolicy for HideIdentityPolicy {
8797 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
8798 record.identity != self.0
8799 }
8800 }
8801
8802 #[derive(Debug)]
8803 struct HideMemberPolicy(&'static str);
8804
8805 impl ConsoleVisibilityPolicy for HideMemberPolicy {
8806 fn member_visible(&self, member: &ConsoleMember) -> bool {
8807 member.agent_identity != self.0
8808 && member
8809 .labels
8810 .get("agent_identity")
8811 .is_none_or(|identity| identity != self.0)
8812 }
8813
8814 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
8815 record.runtime_member_id != self.0
8816 }
8817 }
8818
8819 #[derive(Debug)]
8820 struct HideOnlyMemberPolicy(&'static str);
8821
8822 impl ConsoleVisibilityPolicy for HideOnlyMemberPolicy {
8823 fn member_visible(&self, member: &ConsoleMember) -> bool {
8824 member.agent_identity != self.0
8825 }
8826 }
8827
8828 #[tokio::test]
8829 async fn console_runtime_identity_controls_respect_visibility_policy()
8830 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
8831 let (_temp_dir, runtime) =
8832 build_empty_console_test_runtime("console-identity-hidden-controls").await?;
8833 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
8834 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
8835 lease_provider: Arc::new(LocalLeaseProvider::new()),
8836 runtime_instance_id: "console-identity-hidden-controls".to_string(),
8837 has_runtime_store: true,
8838 durability_policy: DurabilityPolicy::SyncWriteThrough,
8839 bridge: None,
8840 default_timeout: None,
8841 }));
8842 let identity = AgentIdentity::parse("review:singleton")?;
8843 let record = ContinuityRecord {
8844 identity: identity.clone(),
8845 agent_runtime_id: AgentRuntimeId::parse("rt:review:singleton:0")?,
8846 session_id: meerkat_core::types::SessionId::new(),
8847 generation: ContinuityGeneration::new(0),
8848 checkpoint_version: CheckpointVersion::new(0),
8849 };
8850 identity_runtime
8851 .register(
8852 DurableAgentSpec {
8853 identity: identity.clone(),
8854 profile: ProfileName::from("worker"),
8855 addressability: AgentAddressability::Addressable,
8856 display_name: None,
8857 labels: BTreeMap::new(),
8858 context: None,
8859 additional_instructions: Vec::new(),
8860 initial_message: None,
8861 runtime_mode_override: None,
8862 backend: None,
8863 binding: None,
8864 },
8865 IdentityLifecycleState::Active,
8866 Some(record),
8867 Some(LeaseGrant {
8868 identity: identity.clone(),
8869 fencing_token: FencingToken::new(7),
8870 ttl: Duration::from_mins(1),
8871 }),
8872 )
8873 .await;
8874
8875 for method in [
8876 "mobkit/status_identity",
8877 "mobkit/inspect_identity",
8878 "mobkit/retire",
8879 "mobkit/respawn",
8880 "mobkit/reset",
8881 "mobkit/delete_identity",
8882 ] {
8883 let response = Box::pin(handle_console_runtime_rpc_with_visibility(
8884 &runtime,
8885 None,
8886 None,
8887 None,
8888 Some(ConsoleEventStore::new()),
8889 None,
8890 Some(identity_runtime.clone()),
8891 None,
8892 None,
8893 &HideIdentityPolicy("review:singleton"),
8894 rpc_request_with_params(method, json!({ "identity": "review:singleton" })),
8895 true,
8896 false,
8897 None,
8898 None,
8899 None,
8900 ))
8901 .await;
8902 assert_ne!(
8903 response["error"],
8904 Value::Null,
8905 "{method} must reject hidden durable identity"
8906 );
8907 assert_eq!(
8908 response["error"]["data"]["kind"],
8909 json!("identity_hidden_by_policy"),
8910 "unexpected hidden response for {method}: {response:#?}"
8911 );
8912 }
8913 identity_runtime
8914 .status(&AgentIdentity::parse("review:singleton")?)
8915 .await
8916 .expect("hidden control RPCs must not mutate the durable identity");
8917
8918 let _ = runtime.handle().stop().await;
8919 Ok(())
8920 }
8921
8922 #[tokio::test]
8923 async fn console_runtime_durable_identity_controls_reject_hidden_bound_member()
8924 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
8925 let (_temp_dir, runtime) =
8926 build_empty_console_test_runtime("console-durable-hidden-bound-member").await?;
8927 runtime
8928 .handle()
8929 .spawn_spec(SpawnMemberSpec::from_wire(
8930 "worker".to_string(),
8931 crate::member_comms_id::mob_member_id_str("rt:review:singleton:0").into_owned(),
8934 Some("You are the live Review Agent.".into()),
8935 None,
8936 None,
8937 ))
8938 .await?;
8939
8940 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
8941 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
8942 lease_provider: Arc::new(LocalLeaseProvider::new()),
8943 runtime_instance_id: "console-durable-hidden-bound-member".to_string(),
8944 has_runtime_store: true,
8945 durability_policy: DurabilityPolicy::SyncWriteThrough,
8946 bridge: None,
8947 default_timeout: None,
8948 }));
8949 let identity = AgentIdentity::parse("review:singleton")?;
8950 let record = ContinuityRecord {
8951 identity: identity.clone(),
8952 agent_runtime_id: AgentRuntimeId::parse("rt:review:singleton:0")?,
8953 session_id: meerkat_core::types::SessionId::new(),
8954 generation: ContinuityGeneration::new(0),
8955 checkpoint_version: CheckpointVersion::new(0),
8956 };
8957 identity_runtime
8958 .register(
8959 DurableAgentSpec {
8960 identity: identity.clone(),
8961 profile: ProfileName::from("worker"),
8962 addressability: AgentAddressability::Addressable,
8963 display_name: None,
8964 labels: BTreeMap::new(),
8965 context: None,
8966 additional_instructions: Vec::new(),
8967 initial_message: None,
8968 runtime_mode_override: None,
8969 backend: None,
8970 binding: None,
8971 },
8972 IdentityLifecycleState::Active,
8973 Some(record),
8974 Some(LeaseGrant {
8975 identity: identity.clone(),
8976 fencing_token: FencingToken::new(7),
8977 ttl: Duration::from_mins(1),
8978 }),
8979 )
8980 .await;
8981
8982 for requested_identity in ["review:singleton", "rt:review:singleton:0"] {
8983 for method in [
8984 "mobkit/status_identity",
8985 "mobkit/inspect_identity",
8986 "mobkit/retire",
8987 "mobkit/respawn",
8988 "mobkit/reset",
8989 "mobkit/delete_identity",
8990 ] {
8991 let response = Box::pin(handle_console_runtime_rpc_with_visibility(
8992 &runtime,
8993 None,
8994 None,
8995 None,
8996 Some(ConsoleEventStore::new()),
8997 None,
8998 Some(identity_runtime.clone()),
8999 None,
9000 None,
9001 &HideOnlyMemberPolicy("rt:review:singleton:0"),
9002 rpc_request_with_params(method, json!({ "identity": requested_identity })),
9003 true,
9004 false,
9005 None,
9006 None,
9007 None,
9008 ))
9009 .await;
9010 assert_eq!(
9011 response["error"]["data"]["kind"],
9012 json!("identity_hidden_by_policy"),
9013 "durable {method} must reject hidden bound live member for {requested_identity}: {response:#?}"
9014 );
9015 }
9016 }
9017
9018 let _ = runtime.handle().stop().await;
9019 Ok(())
9020 }
9021
9022 #[tokio::test]
9023 async fn console_runtime_live_only_identity_controls_respect_visibility_policy()
9024 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
9025 let (_temp_dir, runtime) =
9026 build_empty_console_test_runtime("console-live-only-hidden-controls").await?;
9027
9028 let mut labels = BTreeMap::new();
9029 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
9030 runtime
9031 .handle()
9032 .spawn_spec(
9033 SpawnMemberSpec::from_wire(
9034 "worker".to_string(),
9035 crate::member_comms_id::mob_member_id_str("rt:review:singleton:0").into_owned(),
9038 Some("You are the live Review Agent.".into()),
9039 None,
9040 None,
9041 )
9042 .with_labels(labels),
9043 )
9044 .await?;
9045
9046 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
9047 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
9048 lease_provider: Arc::new(LocalLeaseProvider::new()),
9049 runtime_instance_id: "console-live-only-hidden-controls".to_string(),
9050 has_runtime_store: true,
9051 durability_policy: DurabilityPolicy::SyncWriteThrough,
9052 bridge: None,
9053 default_timeout: None,
9054 }));
9055
9056 for method in [
9057 "mobkit/status_identity",
9058 "mobkit/inspect_identity",
9059 "mobkit/retire",
9060 "mobkit/respawn",
9061 "mobkit/reset",
9062 "mobkit/delete_identity",
9063 ] {
9064 let response = Box::pin(handle_console_runtime_rpc_with_visibility(
9065 &runtime,
9066 None,
9067 None,
9068 None,
9069 Some(ConsoleEventStore::new()),
9070 None,
9071 Some(identity_runtime.clone()),
9072 None,
9073 None,
9074 &HideMemberPolicy("rt:review:singleton:0"),
9075 rpc_request_with_params(method, json!({ "identity": "review:singleton" })),
9076 true,
9077 false,
9078 None,
9079 None,
9080 None,
9081 ))
9082 .await;
9083 assert_ne!(
9084 response["error"],
9085 Value::Null,
9086 "{method} must reject hidden live-only identity"
9087 );
9088 assert_eq!(
9089 response["error"]["data"]["kind"],
9090 json!("identity_hidden_by_policy"),
9091 "unexpected hidden live-only response for {method}: {response:#?}"
9092 );
9093 }
9094 assert!(
9095 runtime
9096 .handle()
9097 .get_member(&crate::member_comms_id::mob_member_id(
9098 "rt:review:singleton:0"
9099 ))
9100 .await
9101 .ok()
9102 .flatten()
9103 .is_some(),
9104 "hidden live-only controls must not mutate the live member"
9105 );
9106
9107 let _ = runtime.handle().stop().await;
9108 Ok(())
9109 }
9110
9111 #[tokio::test]
9112 async fn console_runtime_reset_live_only_alias_without_session_bridge_uses_live_fallback()
9113 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
9114 let (_temp_dir, runtime) =
9115 build_empty_console_test_runtime("console-reset-live-only-no-bridge").await?;
9116
9117 let mut labels = BTreeMap::new();
9118 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
9119 runtime
9120 .handle()
9121 .spawn_spec(
9122 SpawnMemberSpec::from_wire(
9123 "worker".to_string(),
9124 crate::member_comms_id::mob_member_id_str("rt:review:singleton:0").into_owned(),
9127 Some("You are the live Review Agent.".into()),
9128 None,
9129 None,
9130 )
9131 .with_labels(labels),
9132 )
9133 .await?;
9134
9135 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
9136 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
9137 lease_provider: Arc::new(LocalLeaseProvider::new()),
9138 runtime_instance_id: "console-reset-live-only-no-bridge".to_string(),
9139 has_runtime_store: true,
9140 durability_policy: DurabilityPolicy::SyncWriteThrough,
9141 bridge: None,
9142 default_timeout: None,
9143 }));
9144
9145 let response = Box::pin(handle_console_runtime_rpc(
9146 &runtime,
9147 None,
9148 None,
9149 None,
9150 Some(ConsoleEventStore::new()),
9151 None,
9152 Some(identity_runtime),
9153 None,
9154 None,
9155 rpc_request_with_params("mobkit/reset", json!({ "identity": "review:singleton" })),
9156 true,
9157 ))
9158 .await;
9159 assert_eq!(
9160 response["error"],
9161 Value::Null,
9162 "live-only reset should use live fallback instead of requiring session bridge: {response:#?}"
9163 );
9164 assert_eq!(response["result"]["identity"], json!("review:singleton"));
9165
9166 let _ = runtime.handle().stop().await;
9167 Ok(())
9168 }
9169
9170 #[tokio::test]
9171 async fn reset_all_rejects_registered_runtime_projected_under_wrong_identity()
9172 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
9173 let (_temp_dir, runtime) =
9174 build_empty_console_test_runtime("console-reset-all-stale-projection").await?;
9175
9176 let mut labels = BTreeMap::new();
9177 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
9178 runtime
9179 .handle()
9180 .spawn_spec(
9181 SpawnMemberSpec::from_wire(
9182 "worker".to_string(),
9183 crate::member_comms_id::mob_member_id_str("rt:review:singleton:0").into_owned(),
9186 Some("You are a mislabeled Review Agent.".into()),
9187 None,
9188 None,
9189 )
9190 .with_labels(labels),
9191 )
9192 .await?;
9193
9194 let store = Arc::new(LocalContinuityStore::in_memory()?);
9195 let lease_provider = Arc::new(LocalLeaseProvider::new());
9196 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
9197 continuity_store: store.clone(),
9198 lease_provider: lease_provider.clone(),
9199 runtime_instance_id: "test-runtime".to_string(),
9200 has_runtime_store: true,
9201 durability_policy: DurabilityPolicy::SyncWriteThrough,
9202 bridge: None,
9203 default_timeout: None,
9204 }));
9205 let identity = AgentIdentity::parse("review:singleton")?;
9206 let record = ContinuityRecord {
9207 identity: identity.clone(),
9208 agent_runtime_id: AgentRuntimeId::parse("rt:review:singleton:0")?,
9209 session_id: meerkat_core::types::SessionId::new(),
9210 generation: ContinuityGeneration::new(0),
9211 checkpoint_version: CheckpointVersion::new(0),
9212 };
9213 let grants = lease_provider
9214 .acquire_leases(std::slice::from_ref(&identity), "test-runtime")
9215 .await?;
9216 let grant = match grants.get(&identity).cloned() {
9217 Some(LeaseAcquireResult::Acquired(grant)) => grant,
9218 other => return Err(format!("expected acquired lease, got {other:?}").into()),
9219 };
9220 store
9221 .upsert_continuity_record(&record, grant.fencing_token)
9222 .await?;
9223 identity_runtime
9224 .register(
9225 DurableAgentSpec {
9226 identity: identity.clone(),
9227 profile: ProfileName::from("worker"),
9228 addressability: AgentAddressability::Addressable,
9229 display_name: None,
9230 labels: BTreeMap::new(),
9231 context: None,
9232 additional_instructions: Vec::new(),
9233 initial_message: None,
9234 runtime_mode_override: None,
9235 backend: None,
9236 binding: None,
9237 },
9238 IdentityLifecycleState::Active,
9239 Some(record),
9240 Some(grant),
9241 )
9242 .await;
9243
9244 let response = Box::pin(handle_console_runtime_rpc(
9245 &runtime,
9246 None,
9247 None,
9248 None,
9249 Some(ConsoleEventStore::new()),
9250 None,
9251 Some(identity_runtime),
9252 None,
9253 None,
9254 rpc_request("mobkit/reset_all"),
9255 true,
9256 ))
9257 .await;
9258 assert_ne!(response["error"], Value::Null);
9259 let failed = response["error"]["data"]["failed"]
9260 .as_array()
9261 .expect("reset_all should report failed identities");
9262 let stale_failure = failed
9263 .iter()
9264 .find(|failure| failure["identity"] == json!("review:singleton"))
9265 .expect("review identity should fail stale alias validation");
9266 assert_eq!(
9267 stale_failure["kind"],
9268 json!("stale_live_identity_alias"),
9269 "unexpected reset_all response: {response:#?}"
9270 );
9271 assert!(
9272 stale_failure["error"]
9273 .as_str()
9274 .unwrap_or_default()
9275 .contains("projects identity other:singleton"),
9276 "unexpected stale failure: {stale_failure:#?}"
9277 );
9278 let retired = response["error"]["data"]["retired_delegates"]
9279 .as_array()
9280 .expect("reset_all should return retired delegates");
9281 assert!(
9282 !retired
9283 .iter()
9284 .any(|entry| entry["identity"] == json!("other:singleton")),
9285 "wrong-projected live alias must not be destructively retired before stale validation; response: {response:#?}"
9286 );
9287 assert!(
9288 runtime
9289 .handle()
9290 .get_member(&crate::member_comms_id::mob_member_id(
9291 "rt:review:singleton:0"
9292 ))
9293 .await
9294 .ok()
9295 .flatten()
9296 .is_some(),
9297 "wrong-projected durable runtime member must remain present after reset_all rejection"
9298 );
9299
9300 let _ = runtime.handle().stop().await;
9301 Ok(())
9302 }
9303
9304 #[tokio::test]
9305 async fn reset_all_respects_console_visibility_policy_for_live_members()
9306 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
9307 let (_temp_dir, runtime) =
9308 build_empty_console_test_runtime("console-reset-all-hidden-live").await?;
9309
9310 runtime
9311 .handle()
9312 .spawn_spec(
9313 SpawnMemberSpec::from_wire(
9314 "worker".to_string(),
9315 crate::member_comms_id::mob_member_id_str("hidden:singleton").into_owned(),
9318 Some("You are hidden from console lifecycle controls.".into()),
9319 None,
9320 None,
9321 )
9322 .with_labels(BTreeMap::from([(
9323 "agent_identity".to_string(),
9324 "hidden:singleton".to_string(),
9325 )])),
9326 )
9327 .await?;
9328
9329 let response = Box::pin(handle_console_runtime_rpc_with_visibility(
9330 &runtime,
9331 None,
9332 None,
9333 None,
9334 Some(ConsoleEventStore::new()),
9335 None,
9336 None,
9337 None,
9338 None,
9339 &HideMemberPolicy("hidden:singleton"),
9340 rpc_request("mobkit/reset_all"),
9341 true,
9342 false,
9343 None,
9344 None,
9345 None,
9346 ))
9347 .await;
9348 assert_eq!(
9349 response["error"],
9350 Value::Null,
9351 "hidden live member should be outside reset_all target set: {response:#?}"
9352 );
9353 assert!(
9354 runtime
9355 .handle()
9356 .get_member(&crate::member_comms_id::mob_member_id("hidden:singleton"))
9357 .await
9358 .ok()
9359 .flatten()
9360 .is_some(),
9361 "reset_all must not retire hidden live members"
9362 );
9363
9364 let _ = runtime.handle().stop().await;
9365 Ok(())
9366 }
9367
9368 #[tokio::test]
9369 async fn reset_all_skips_durable_identity_with_hidden_bound_member()
9370 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
9371 let (_temp_dir, runtime) =
9372 build_empty_console_test_runtime("console-reset-all-hidden-durable-bound").await?;
9373 runtime
9374 .handle()
9375 .spawn_spec(SpawnMemberSpec::from_wire(
9376 "worker".to_string(),
9377 crate::member_comms_id::mob_member_id_str("rt:review:singleton:0").into_owned(),
9380 Some("You are the hidden Review Agent.".into()),
9381 None,
9382 None,
9383 ))
9384 .await?;
9385
9386 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
9387 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
9388 lease_provider: Arc::new(LocalLeaseProvider::new()),
9389 runtime_instance_id: "console-reset-all-hidden-durable-bound".to_string(),
9390 has_runtime_store: true,
9391 durability_policy: DurabilityPolicy::SyncWriteThrough,
9392 bridge: None,
9393 default_timeout: None,
9394 }));
9395 let identity = AgentIdentity::parse("review:singleton")?;
9396 identity_runtime
9397 .register(
9398 DurableAgentSpec {
9399 identity: identity.clone(),
9400 profile: ProfileName::from("worker"),
9401 addressability: AgentAddressability::Addressable,
9402 display_name: None,
9403 labels: BTreeMap::new(),
9404 context: None,
9405 additional_instructions: Vec::new(),
9406 initial_message: None,
9407 runtime_mode_override: None,
9408 backend: None,
9409 binding: None,
9410 },
9411 IdentityLifecycleState::Active,
9412 Some(ContinuityRecord {
9413 identity: identity.clone(),
9414 agent_runtime_id: AgentRuntimeId::parse("rt:review:singleton:0")?,
9415 session_id: meerkat_core::types::SessionId::new(),
9416 generation: ContinuityGeneration::new(0),
9417 checkpoint_version: CheckpointVersion::new(0),
9418 }),
9419 Some(LeaseGrant {
9420 identity: identity.clone(),
9421 fencing_token: FencingToken::new(9),
9422 ttl: Duration::from_mins(1),
9423 }),
9424 )
9425 .await;
9426
9427 let response = Box::pin(handle_console_runtime_rpc_with_visibility(
9428 &runtime,
9429 None,
9430 None,
9431 None,
9432 Some(ConsoleEventStore::new()),
9433 None,
9434 Some(identity_runtime.clone()),
9435 None,
9436 None,
9437 &HideOnlyMemberPolicy("rt:review:singleton:0"),
9438 rpc_request("mobkit/reset_all"),
9439 true,
9440 false,
9441 None,
9442 None,
9443 None,
9444 ))
9445 .await;
9446 assert_eq!(
9447 response["error"],
9448 Value::Null,
9449 "hidden durable bound member should be outside reset_all target set: {response:#?}"
9450 );
9451 assert_eq!(
9452 identity_runtime.status(&identity).await?.state,
9453 IdentityLifecycleState::Active
9454 );
9455
9456 let _ = runtime.handle().stop().await;
9457 Ok(())
9458 }
9459
9460 #[tokio::test]
9461 async fn identity_lifecycle_cleanup_skips_hidden_projected_duplicates()
9462 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
9463 let (_temp_dir, runtime) =
9464 build_empty_console_test_runtime("console-hidden-stale-duplicate-cleanup").await?;
9465 for runtime_id in ["rt:review:singleton:0", "rt:review:singleton:1"] {
9466 runtime
9467 .handle()
9468 .spawn_spec(
9469 SpawnMemberSpec::from_wire(
9470 "worker".to_string(),
9471 crate::member_comms_id::mob_member_id_str(runtime_id).into_owned(),
9474 Some("You are a Review Agent candidate.".into()),
9475 None,
9476 None,
9477 )
9478 .with_labels(BTreeMap::from([(
9479 "agent_identity".to_string(),
9480 "review:singleton".to_string(),
9481 )])),
9482 )
9483 .await?;
9484 }
9485
9486 let store = Arc::new(LocalContinuityStore::in_memory()?);
9487 let lease_provider = Arc::new(LocalLeaseProvider::new());
9488 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
9489 continuity_store: store.clone(),
9490 lease_provider: lease_provider.clone(),
9491 runtime_instance_id: "console-hidden-stale-duplicate-cleanup".to_string(),
9492 has_runtime_store: true,
9493 durability_policy: DurabilityPolicy::SyncWriteThrough,
9494 bridge: None,
9495 default_timeout: None,
9496 }));
9497 let identity = AgentIdentity::parse("review:singleton")?;
9498 let record = ContinuityRecord {
9499 identity: identity.clone(),
9500 agent_runtime_id: AgentRuntimeId::parse("rt:review:singleton:0")?,
9501 session_id: runtime
9502 .handle()
9503 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
9504 "rt:review:singleton:0",
9505 ))
9506 .await
9507 .unwrap_or_else(meerkat_core::types::SessionId::new),
9508 generation: ContinuityGeneration::new(0),
9509 checkpoint_version: CheckpointVersion::new(0),
9510 };
9511 let grants = lease_provider
9512 .acquire_leases(
9513 std::slice::from_ref(&identity),
9514 "console-hidden-stale-duplicate-cleanup",
9515 )
9516 .await?;
9517 let grant = match grants.get(&identity).cloned() {
9518 Some(LeaseAcquireResult::Acquired(grant)) => grant,
9519 other => return Err(format!("expected acquired lease, got {other:?}").into()),
9520 };
9521 store
9522 .upsert_continuity_record(&record, grant.fencing_token)
9523 .await?;
9524 identity_runtime
9525 .register(
9526 DurableAgentSpec {
9527 identity: identity.clone(),
9528 profile: ProfileName::from("worker"),
9529 addressability: AgentAddressability::Addressable,
9530 display_name: None,
9531 labels: BTreeMap::new(),
9532 context: None,
9533 additional_instructions: Vec::new(),
9534 initial_message: None,
9535 runtime_mode_override: None,
9536 backend: None,
9537 binding: None,
9538 },
9539 IdentityLifecycleState::Active,
9540 Some(record),
9541 Some(grant),
9542 )
9543 .await;
9544
9545 let response = Box::pin(handle_console_runtime_rpc_with_visibility(
9546 &runtime,
9547 None,
9548 None,
9549 None,
9550 Some(ConsoleEventStore::new()),
9551 None,
9552 Some(identity_runtime),
9553 None,
9554 None,
9555 &HideOnlyMemberPolicy("rt:review:singleton:1"),
9556 rpc_request_with_params("mobkit/retire", json!({ "identity": "review:singleton" })),
9557 true,
9558 false,
9559 None,
9560 None,
9561 None,
9562 ))
9563 .await;
9564 assert_eq!(
9565 response["error"],
9566 Value::Null,
9567 "visible durable retire should succeed without touching hidden duplicate: {response:#?}"
9568 );
9569 assert!(
9570 runtime
9571 .handle()
9572 .get_member(&crate::member_comms_id::mob_member_id(
9573 "rt:review:singleton:1"
9574 ))
9575 .await
9576 .ok()
9577 .flatten()
9578 .is_some(),
9579 "post-mutation stale cleanup must not retire member-hidden projected duplicates"
9580 );
9581
9582 let _ = runtime.handle().stop().await;
9583 Ok(())
9584 }
9585
9586 #[tokio::test]
9587 async fn console_runtime_capabilities_advertise_identity_controls_when_identity_runtime_exists()
9588 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
9589 let (_temp_dir, runtime) =
9590 build_empty_console_test_runtime("console-identity-capabilities").await?;
9591 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
9592 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
9593 lease_provider: Arc::new(LocalLeaseProvider::new()),
9594 runtime_instance_id: "console-identity-capabilities".to_string(),
9595 has_runtime_store: true,
9596 durability_policy: DurabilityPolicy::SyncWriteThrough,
9597 bridge: None,
9598 default_timeout: None,
9599 }));
9600
9601 let response = Box::pin(handle_console_runtime_rpc(
9602 &runtime,
9603 None,
9604 None,
9605 None,
9606 None,
9607 None,
9608 Some(identity_runtime.clone()),
9609 None,
9610 None,
9611 rpc_request("mobkit/capabilities"),
9612 true,
9613 ))
9614 .await;
9615
9616 assert_eq!(response["error"], Value::Null, "{response:#?}");
9617 let methods = response["result"]["methods"]
9618 .as_array()
9619 .ok_or("capabilities methods should be an array")?;
9620 for method in [
9621 "mobkit/status_identity",
9622 "mobkit/inspect_identity",
9623 "mobkit/respawn",
9624 "mobkit/reset",
9625 "mobkit/delete_identity",
9626 ] {
9627 assert!(
9628 methods.iter().any(|candidate| candidate == method),
9629 "identity runtime capabilities should advertise {method}: {methods:#?}"
9630 );
9631 }
9632 for method in crate::rpc::MOBPACK_AUTHORING_METHODS {
9633 assert!(
9634 !methods.iter().any(|candidate| candidate == method),
9635 "console runtime capabilities must not advertise flow-editor method {method}: {methods:#?}"
9636 );
9637 }
9638
9639 let mobpack_response = Box::pin(handle_console_runtime_rpc(
9640 &runtime,
9641 None,
9642 None,
9643 None,
9644 None,
9645 None,
9646 Some(identity_runtime),
9647 None,
9648 None,
9649 rpc_request("mobkit/mobpacks/schema"),
9650 true,
9651 ))
9652 .await;
9653 assert_eq!(
9654 mobpack_response["error"]["code"],
9655 json!(-32601),
9656 "console runtime RPC must not handle flow-editor authoring methods: {mobpack_response:#?}"
9657 );
9658
9659 let _ = runtime.handle().stop().await;
9660 Ok(())
9661 }
9662
9663 #[tokio::test]
9664 async fn console_runtime_capabilities_advertise_respawn_and_reset_without_identity_runtime()
9665 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
9666 let (_temp_dir, runtime) =
9667 build_empty_console_test_runtime("console-plain-lifecycle-capabilities").await?;
9668
9669 let response = Box::pin(handle_console_runtime_rpc(
9673 &runtime,
9674 None,
9675 None,
9676 None,
9677 None,
9678 None,
9679 None,
9680 None,
9681 None,
9682 rpc_request("mobkit/capabilities"),
9683 true,
9684 ))
9685 .await;
9686 assert_eq!(response["error"], Value::Null, "{response:#?}");
9687 let methods = response["result"]["methods"]
9688 .as_array()
9689 .ok_or("capabilities methods should be an array")?;
9690 for method in ["mobkit/retire", "mobkit/respawn", "mobkit/reset"] {
9691 assert!(
9692 methods.iter().any(|candidate| candidate == method),
9693 "plain runtime capabilities should advertise {method}: {methods:#?}"
9694 );
9695 }
9696 assert!(
9697 !methods
9698 .iter()
9699 .any(|candidate| candidate == "mobkit/delete_identity"),
9700 "delete_identity requires an identity runtime: {methods:#?}"
9701 );
9702
9703 let read_only_response = Box::pin(handle_console_runtime_rpc(
9706 &runtime,
9707 None,
9708 None,
9709 None,
9710 None,
9711 None,
9712 None,
9713 None,
9714 None,
9715 rpc_request("mobkit/capabilities"),
9716 false,
9717 ))
9718 .await;
9719 assert_eq!(
9720 read_only_response["error"],
9721 Value::Null,
9722 "{read_only_response:#?}"
9723 );
9724 let read_only_methods = read_only_response["result"]["methods"]
9725 .as_array()
9726 .ok_or("capabilities methods should be an array")?;
9727 for method in ["mobkit/retire", "mobkit/respawn", "mobkit/reset"] {
9728 assert!(
9729 !read_only_methods
9730 .iter()
9731 .any(|candidate| candidate == method),
9732 "unauthenticated capabilities must omit {method}: {read_only_methods:#?}"
9733 );
9734 }
9735
9736 let _ = runtime.handle().stop().await;
9737 Ok(())
9738 }
9739
9740 #[tokio::test]
9741 async fn console_aggregator_rpc_does_not_expose_flow_editor_authoring_methods() {
9742 let response = Box::pin(handle_console_aggregator_rpc(
9743 None,
9744 rpc_request("mobkit/capabilities"),
9745 true,
9746 false,
9747 None,
9748 None,
9749 ))
9750 .await;
9751
9752 assert_eq!(response["error"], Value::Null, "{response:#?}");
9753 let methods = response["result"]["methods"]
9754 .as_array()
9755 .expect("capabilities methods should be an array");
9756 for method in crate::rpc::MOBPACK_AUTHORING_METHODS {
9757 assert!(
9758 !methods.iter().any(|candidate| candidate == method),
9759 "console aggregator capabilities must not advertise flow-editor method {method}: {methods:#?}"
9760 );
9761 }
9762
9763 let mobpack_response = Box::pin(handle_console_aggregator_rpc(
9764 None,
9765 rpc_request("mobkit/mobpacks/schema"),
9766 true,
9767 false,
9768 None,
9769 None,
9770 ))
9771 .await;
9772 assert_eq!(
9773 mobpack_response["error"]["code"],
9774 json!(-32601),
9775 "console aggregator RPC must not handle flow-editor authoring methods: {mobpack_response:#?}"
9776 );
9777 }
9778
9779 #[tokio::test]
9780 async fn console_runtime_identity_reads_reject_stale_runtime_aliases()
9781 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
9782 let (_temp_dir, runtime) =
9783 build_empty_console_test_runtime("console-identity-stale-read-alias").await?;
9784 let mut labels = BTreeMap::new();
9785 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
9786 runtime
9787 .handle()
9788 .spawn_spec(
9789 SpawnMemberSpec::from_wire(
9790 "worker".to_string(),
9791 crate::member_comms_id::mob_member_id_str("rt:review:singleton:0").into_owned(),
9794 Some("You are the stale Review Agent.".into()),
9795 None,
9796 None,
9797 )
9798 .with_labels(labels),
9799 )
9800 .await?;
9801
9802 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
9803 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
9804 lease_provider: Arc::new(LocalLeaseProvider::new()),
9805 runtime_instance_id: "console-identity-stale-read-alias".to_string(),
9806 has_runtime_store: true,
9807 durability_policy: DurabilityPolicy::SyncWriteThrough,
9808 bridge: None,
9809 default_timeout: None,
9810 }));
9811 let identity = AgentIdentity::parse("review:singleton")?;
9812 let record = ContinuityRecord {
9813 identity: identity.clone(),
9814 agent_runtime_id: AgentRuntimeId::parse("rt:review:singleton:1")?,
9815 session_id: meerkat_core::types::SessionId::new(),
9816 generation: ContinuityGeneration::new(1),
9817 checkpoint_version: CheckpointVersion::new(0),
9818 };
9819 identity_runtime
9820 .register(
9821 DurableAgentSpec {
9822 identity: identity.clone(),
9823 profile: ProfileName::from("worker"),
9824 addressability: AgentAddressability::Addressable,
9825 display_name: None,
9826 labels: BTreeMap::new(),
9827 context: None,
9828 additional_instructions: Vec::new(),
9829 initial_message: None,
9830 runtime_mode_override: None,
9831 backend: None,
9832 binding: None,
9833 },
9834 IdentityLifecycleState::Active,
9835 Some(record),
9836 Some(LeaseGrant {
9837 identity,
9838 fencing_token: FencingToken::new(7),
9839 ttl: Duration::from_mins(1),
9840 }),
9841 )
9842 .await;
9843
9844 for requested_identity in ["rt:review:singleton:0", "review:singleton"] {
9845 for method in ["mobkit/status_identity", "mobkit/inspect_identity"] {
9846 let response = Box::pin(handle_console_runtime_rpc(
9847 &runtime,
9848 None,
9849 None,
9850 None,
9851 None,
9852 None,
9853 Some(identity_runtime.clone()),
9854 None,
9855 None,
9856 rpc_request_with_params(method, json!({ "identity": requested_identity })),
9857 true,
9858 ))
9859 .await;
9860 assert_ne!(
9861 response["error"],
9862 Value::Null,
9863 "{method} must reject stale alias for {requested_identity}"
9864 );
9865 let message = response["error"]["message"].as_str().unwrap_or_default();
9866 assert!(
9867 message.contains(
9868 "identity runtime binding for review:singleton points at rt:review:singleton:1"
9869 ),
9870 "unexpected stale-alias message for {method}/{requested_identity}: {message}"
9871 );
9872 assert_eq!(
9873 response["error"]["data"]["kind"],
9874 json!("stale_identity_runtime_binding")
9875 );
9876 assert_eq!(
9877 response["error"]["data"]["registered_runtime_member_id"],
9878 json!("rt:review:singleton:1")
9879 );
9880 assert_eq!(
9881 response["error"]["data"]["live_runtime_member_id"],
9882 json!("rt:review:singleton:0")
9883 );
9884 }
9885 }
9886
9887 let (_temp_dir_without_stale, runtime_without_stale) =
9888 build_empty_console_test_runtime("console-identity-no-live-stale-alias").await?;
9889
9890 for method in [
9891 "mobkit/status_identity",
9892 "mobkit/inspect_identity",
9893 "mobkit/retire",
9894 "mobkit/respawn",
9895 "mobkit/reset",
9896 ] {
9897 let response = Box::pin(handle_console_runtime_rpc(
9898 &runtime_without_stale,
9899 None,
9900 None,
9901 None,
9902 Some(ConsoleEventStore::new()),
9903 None,
9904 Some(identity_runtime.clone()),
9905 None,
9906 None,
9907 rpc_request_with_params(method, json!({ "identity": "rt:review:singleton:0" })),
9908 true,
9909 ))
9910 .await;
9911 assert_ne!(
9912 response["error"],
9913 Value::Null,
9914 "{method} must reject stale synthetic runtime alias"
9915 );
9916 assert!(
9917 response["error"]["message"]
9918 .as_str()
9919 .unwrap_or_default()
9920 .contains("identity not found: rt:review:singleton:0"),
9921 "unexpected no-live stale-alias response for {method}: {response:#?}"
9922 );
9923 }
9924 let _ = runtime_without_stale.handle().stop().await;
9925
9926 let _ = runtime.handle().stop().await;
9927 Ok(())
9928 }
9929
9930 #[tokio::test]
9931 async fn aggregator_live_snapshot_projects_identity_first_topology_peers()
9932 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
9933 let (_temp_dir, mob_runtime) =
9934 build_empty_console_test_runtime("identity-topology-snapshot-test").await?;
9935 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
9936 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
9937 lease_provider: Arc::new(LocalLeaseProvider::new()),
9938 runtime_instance_id: "console-topology-snapshot-test".to_string(),
9939 has_runtime_store: true,
9940 durability_policy: DurabilityPolicy::SyncWriteThrough,
9941 bridge: None,
9942 default_timeout: None,
9943 }));
9944
9945 for name in ["agent:alpha", "agent:beta"] {
9946 let identity = AgentIdentity::parse(name)?;
9947 let record = ContinuityRecord {
9948 identity: identity.clone(),
9949 agent_runtime_id: AgentRuntimeId::parse(&format!("rt:{name}:0"))?,
9950 session_id: meerkat_core::types::SessionId::new(),
9951 generation: ContinuityGeneration::new(0),
9952 checkpoint_version: CheckpointVersion::new(0),
9953 };
9954 identity_runtime
9955 .register(
9956 DurableAgentSpec {
9957 identity: identity.clone(),
9958 profile: ProfileName::from("default"),
9959 addressability: AgentAddressability::Addressable,
9960 display_name: None,
9961 labels: BTreeMap::new(),
9962 context: None,
9963 additional_instructions: Vec::new(),
9964 initial_message: None,
9965 runtime_mode_override: None,
9966 backend: None,
9967 binding: None,
9968 },
9969 IdentityLifecycleState::Active,
9970 Some(record),
9971 Some(LeaseGrant {
9972 identity,
9973 fencing_token: FencingToken::new(7),
9974 ttl: Duration::from_mins(1),
9975 }),
9976 )
9977 .await;
9978 }
9979 identity_runtime
9980 .set_desired_peer_edges(vec![ManagedPeerEdge::new(
9981 AgentIdentity::parse("agent:alpha")?,
9982 AgentIdentity::parse("agent:beta")?,
9983 )?])
9984 .await;
9985
9986 let aggregator = MobKitConsoleAggregator::in_memory();
9987 aggregator.register_runtime_handles_with_policy(
9988 "identity-first",
9989 "",
9990 mob_runtime.clone(),
9991 Some(identity_runtime),
9992 ConsoleEventStore::new(),
9993 Arc::new(AllowAllConsoleVisibilityPolicy),
9994 );
9995
9996 let snapshot = build_aggregator_live_snapshot(&aggregator, &[]).await?;
9997 let alpha = snapshot
9998 .members
9999 .iter()
10000 .find(|member| member.agent_identity == "agent:alpha")
10001 .ok_or("agent:alpha missing from live snapshot")?;
10002 assert_eq!(alpha.wired_to, vec!["agent:beta".to_string()]);
10003
10004 let _ = mob_runtime.handle().stop().await;
10005 Ok(())
10006 }
10007
10008 #[tokio::test]
10009 async fn identity_first_console_send_reserves_timeline_and_uses_identity_runtime()
10010 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
10011 let (_temp_dir, mob_runtime) =
10012 build_empty_console_test_runtime("identity-send-runtime-key-test").await?;
10013 let identity = AgentIdentity::parse("agent:console")?;
10014 let record = ContinuityRecord {
10015 identity: identity.clone(),
10016 agent_runtime_id: AgentRuntimeId::parse("rt:agent:console:0")?,
10017 session_id: meerkat_core::types::SessionId::new(),
10018 generation: ContinuityGeneration::new(0),
10019 checkpoint_version: CheckpointVersion::new(0),
10020 };
10021 let runtime = IdentityRuntime::new(IdentityRuntimeConfig {
10022 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10023 lease_provider: Arc::new(LocalLeaseProvider::new()),
10024 runtime_instance_id: "console-test".to_string(),
10025 has_runtime_store: true,
10026 durability_policy: DurabilityPolicy::SyncWriteThrough,
10027 bridge: None,
10028 default_timeout: None,
10029 });
10030 runtime
10031 .register(
10032 DurableAgentSpec {
10033 identity: identity.clone(),
10034 profile: ProfileName::from("default"),
10035 addressability: AgentAddressability::Addressable,
10036 display_name: None,
10037 labels: BTreeMap::new(),
10038 context: None,
10039 additional_instructions: Vec::new(),
10040 initial_message: None,
10041 runtime_mode_override: None,
10042 backend: None,
10043 binding: None,
10044 },
10045 IdentityLifecycleState::Active,
10046 Some(record.clone()),
10047 Some(LeaseGrant {
10048 identity: identity.clone(),
10049 fencing_token: FencingToken::new(7),
10050 ttl: Duration::from_mins(1),
10051 }),
10052 )
10053 .await;
10054
10055 let aggregator = MobKitConsoleAggregator::in_memory();
10056 let events = ConsoleEventStore::new();
10057 let runtime = Arc::new(runtime);
10058 aggregator.register_runtime_handles_with_policy(
10059 "default",
10060 "",
10061 mob_runtime.clone(),
10062 Some(runtime.clone()),
10063 events.clone(),
10064 Arc::new(AllowAllConsoleVisibilityPolicy),
10065 );
10066 let accepted = console_send_identity_first(
10067 &aggregator,
10068 runtime.clone(),
10069 Some(&mob_runtime),
10070 Some(&events),
10071 crate::console_aggregator::ConsoleSendRequest {
10072 identity: identity.as_str().to_string(),
10073 content: serde_json::to_value(meerkat_core::ContentInput::Text(
10074 "hello".to_string(),
10075 ))?,
10076 origin: "test".to_string(),
10077 idempotency_key: "idem-1".to_string(),
10078 handling_mode: None,
10079 },
10080 )
10081 .await?;
10082
10083 assert_eq!(accepted.identity, identity.as_str());
10084 assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
10085 assert_eq!(accepted.session_id, Some(record.session_id.to_string()));
10086
10087 let page = aggregator
10088 .query_timeline(ConsoleTimelineQuery {
10089 identity: Some(identity.as_str().to_string()),
10090 ..ConsoleTimelineQuery::default()
10091 })
10092 .await?;
10093 assert_eq!(page.frames.len(), 1);
10094 assert_eq!(page.frames[0].runtime_key, "default");
10095 assert_eq!(page.frames[0].status, ConsoleFrameStatus::Accepted);
10096 assert_eq!(
10097 page.frames[0].session_id,
10098 Some(record.session_id.to_string())
10099 );
10100 let _ = mob_runtime.handle().stop().await;
10101 Ok(())
10102 }
10103
10104 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10105 async fn identity_first_console_send_falls_back_to_member_only_spawned_worker()
10106 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
10107 let (_temp_dir, mob_runtime) =
10108 build_empty_console_test_runtime("identity-send-member-only-test").await?;
10109 let identity_runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10110 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10111 lease_provider: Arc::new(LocalLeaseProvider::new()),
10112 runtime_instance_id: "console-member-only-send-test".to_string(),
10113 has_runtime_store: true,
10114 durability_policy: DurabilityPolicy::SyncWriteThrough,
10115 bridge: None,
10116 default_timeout: None,
10117 }));
10118
10119 let aggregator = MobKitConsoleAggregator::in_memory();
10120 let events = ConsoleEventStore::new();
10121 aggregator.register_runtime_handles_with_policy(
10122 "identity-first",
10123 "",
10124 mob_runtime.clone(),
10125 Some(identity_runtime.clone()),
10126 events.clone(),
10127 Arc::new(AllowAllConsoleVisibilityPolicy),
10128 );
10129
10130 mob_runtime
10131 .handle()
10132 .spawn_spec(SpawnMemberSpec::from_wire(
10133 "worker".to_string(),
10134 crate::member_comms_id::mob_member_id_str("agent:member-only").into_owned(),
10137 Some("You are a member-only spawned worker.".into()),
10138 None,
10139 None,
10140 ))
10141 .await?;
10142
10143 let accepted = console_send_with_identity_first_fallback(
10144 &aggregator,
10145 identity_runtime,
10146 Some(&mob_runtime),
10147 Some(&events),
10148 crate::console_aggregator::ConsoleSendRequest {
10149 identity: "agent:member-only".to_string(),
10150 content: serde_json::to_value(meerkat_core::ContentInput::Text(
10151 "hello spawned worker".to_string(),
10152 ))?,
10153 origin: "test".to_string(),
10154 idempotency_key: "member-only-idem-1".to_string(),
10155 handling_mode: None,
10156 },
10157 )
10158 .await?;
10159
10160 assert_eq!(accepted.identity, "agent:member-only");
10161 assert!(accepted.session_id.is_some());
10162
10163 let page = aggregator
10164 .query_timeline(ConsoleTimelineQuery {
10165 identity: Some("agent:member-only".to_string()),
10166 ..ConsoleTimelineQuery::default()
10167 })
10168 .await?;
10169 assert!(
10170 page.frames.iter().any(|frame| frame.kind == "user_input"),
10171 "fallback send should persist a user input frame for the member-only worker: {page:#?}"
10172 );
10173
10174 let _ = mob_runtime.handle().stop().await;
10175 Ok(())
10176 }
10177
10178 #[tokio::test]
10179 async fn identity_first_console_send_returns_before_bridge_delivery_completes()
10180 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
10181 let identity = AgentIdentity::parse("agent:slow-console")?;
10182 let record = ContinuityRecord {
10183 identity: identity.clone(),
10184 agent_runtime_id: AgentRuntimeId::parse("rt:agent:slow-console:0")?,
10185 session_id: meerkat_core::types::SessionId::new(),
10186 generation: ContinuityGeneration::new(0),
10187 checkpoint_version: CheckpointVersion::new(0),
10188 };
10189 let deliver_calls = Arc::new(AtomicUsize::new(0));
10190 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10191 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10192 lease_provider: Arc::new(LocalLeaseProvider::new()),
10193 runtime_instance_id: "console-slow-send-test".to_string(),
10194 has_runtime_store: true,
10195 durability_policy: DurabilityPolicy::SyncWriteThrough,
10196 bridge: Some(Arc::new(BlockingIdentityBridge {
10197 deliver_calls: deliver_calls.clone(),
10198 })),
10199 default_timeout: None,
10200 }));
10201 runtime
10202 .register(
10203 DurableAgentSpec {
10204 identity: identity.clone(),
10205 profile: ProfileName::from("default"),
10206 addressability: AgentAddressability::Addressable,
10207 display_name: None,
10208 labels: BTreeMap::new(),
10209 context: None,
10210 additional_instructions: Vec::new(),
10211 initial_message: None,
10212 runtime_mode_override: None,
10213 backend: None,
10214 binding: None,
10215 },
10216 IdentityLifecycleState::Active,
10217 Some(record.clone()),
10218 Some(LeaseGrant {
10219 identity: identity.clone(),
10220 fencing_token: FencingToken::new(7),
10221 ttl: Duration::from_mins(1),
10222 }),
10223 )
10224 .await;
10225
10226 let aggregator = MobKitConsoleAggregator::in_memory();
10227 let accepted = match tokio::time::timeout(
10228 Duration::from_millis(100),
10229 console_send_identity_first(
10230 &aggregator,
10231 runtime,
10232 None,
10233 None,
10234 crate::console_aggregator::ConsoleSendRequest {
10235 identity: identity.as_str().to_string(),
10236 content: serde_json::to_value(meerkat_core::ContentInput::Text(
10237 "hello slow bridge".to_string(),
10238 ))?,
10239 origin: "test".to_string(),
10240 idempotency_key: "idem-slow-bridge".to_string(),
10241 handling_mode: None,
10242 },
10243 ),
10244 )
10245 .await
10246 {
10247 Ok(Ok(accepted)) => accepted,
10248 Ok(Err(err)) => return Err(format!("send should be accepted: {err}").into()),
10249 Err(err) => {
10250 return Err(
10251 format!("console send should not wait for bridge delivery: {err}").into(),
10252 );
10253 }
10254 };
10255
10256 assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
10257 if tokio::time::timeout(Duration::from_millis(100), async {
10258 while deliver_calls.load(Ordering::SeqCst) == 0 {
10259 tokio::time::sleep(Duration::from_millis(5)).await;
10260 }
10261 })
10262 .await
10263 .is_err()
10264 {
10265 return Err("delivery should be spawned in the background".into());
10266 }
10267 Ok(())
10268 }
10269
10270 #[tokio::test]
10271 async fn identity_first_console_steer_waits_for_bridge_delivery()
10272 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
10273 let identity = AgentIdentity::parse("agent:slow-steer-console")?;
10274 let record = ContinuityRecord {
10275 identity: identity.clone(),
10276 agent_runtime_id: AgentRuntimeId::parse("rt:agent:slow-steer-console:0")?,
10277 session_id: meerkat_core::types::SessionId::new(),
10278 generation: ContinuityGeneration::new(0),
10279 checkpoint_version: CheckpointVersion::new(0),
10280 };
10281 let deliver_calls = Arc::new(AtomicUsize::new(0));
10282 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10283 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10284 lease_provider: Arc::new(LocalLeaseProvider::new()),
10285 runtime_instance_id: "console-slow-steer-send-test".to_string(),
10286 has_runtime_store: true,
10287 durability_policy: DurabilityPolicy::SyncWriteThrough,
10288 bridge: Some(Arc::new(BlockingIdentityBridge {
10289 deliver_calls: deliver_calls.clone(),
10290 })),
10291 default_timeout: None,
10292 }));
10293 runtime
10294 .register(
10295 DurableAgentSpec {
10296 identity: identity.clone(),
10297 profile: ProfileName::from("default"),
10298 addressability: AgentAddressability::Addressable,
10299 display_name: None,
10300 labels: BTreeMap::new(),
10301 context: None,
10302 additional_instructions: Vec::new(),
10303 initial_message: None,
10304 runtime_mode_override: None,
10305 backend: None,
10306 binding: None,
10307 },
10308 IdentityLifecycleState::Active,
10309 Some(record),
10310 Some(LeaseGrant {
10311 identity: identity.clone(),
10312 fencing_token: FencingToken::new(7),
10313 ttl: Duration::from_mins(1),
10314 }),
10315 )
10316 .await;
10317
10318 let aggregator = MobKitConsoleAggregator::in_memory();
10319 let result = tokio::time::timeout(
10320 Duration::from_millis(100),
10321 console_send_identity_first(
10322 &aggregator,
10323 runtime,
10324 None,
10325 None,
10326 crate::console_aggregator::ConsoleSendRequest {
10327 identity: identity.as_str().to_string(),
10328 content: serde_json::to_value(meerkat_core::ContentInput::Text(
10329 "hello slow steer bridge".to_string(),
10330 ))?,
10331 origin: "test".to_string(),
10332 idempotency_key: "idem-slow-steer-bridge".to_string(),
10333 handling_mode: Some("steer".to_string()),
10334 },
10335 ),
10336 )
10337 .await;
10338
10339 if result.is_ok() {
10340 return Err("steer send must wait for bridge delivery admission".into());
10341 }
10342 assert_eq!(
10343 deliver_calls.load(Ordering::SeqCst),
10344 1,
10345 "steer delivery should have reached the bridge before the console response waits"
10346 );
10347 Ok(())
10348 }
10349
10350 #[tokio::test]
10351 async fn identity_first_console_send_forwards_handling_mode()
10352 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
10353 let identity = AgentIdentity::parse("agent:mode-console")?;
10354 let record = ContinuityRecord {
10355 identity: identity.clone(),
10356 agent_runtime_id: AgentRuntimeId::parse("rt:agent:mode-console:0")?,
10357 session_id: meerkat_core::types::SessionId::new(),
10358 generation: ContinuityGeneration::new(0),
10359 checkpoint_version: CheckpointVersion::new(0),
10360 };
10361 let handling_modes = Arc::new(Mutex::new(Vec::new()));
10362 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10363 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10364 lease_provider: Arc::new(LocalLeaseProvider::new()),
10365 runtime_instance_id: "console-mode-send-test".to_string(),
10366 has_runtime_store: true,
10367 durability_policy: DurabilityPolicy::SyncWriteThrough,
10368 bridge: Some(Arc::new(RecordingIdentityBridge {
10369 session_id: record.session_id.clone(),
10370 handling_modes: handling_modes.clone(),
10371 })),
10372 default_timeout: None,
10373 }));
10374 runtime
10375 .register(
10376 DurableAgentSpec {
10377 identity: identity.clone(),
10378 profile: ProfileName::from("default"),
10379 addressability: AgentAddressability::Addressable,
10380 display_name: None,
10381 labels: BTreeMap::new(),
10382 context: None,
10383 additional_instructions: Vec::new(),
10384 initial_message: None,
10385 runtime_mode_override: None,
10386 backend: None,
10387 binding: None,
10388 },
10389 IdentityLifecycleState::Active,
10390 Some(record),
10391 Some(LeaseGrant {
10392 identity: identity.clone(),
10393 fencing_token: FencingToken::new(7),
10394 ttl: Duration::from_mins(1),
10395 }),
10396 )
10397 .await;
10398
10399 let aggregator = MobKitConsoleAggregator::in_memory();
10400 let accepted = console_send_identity_first(
10401 &aggregator,
10402 runtime,
10403 None,
10404 None,
10405 crate::console_aggregator::ConsoleSendRequest {
10406 identity: identity.as_str().to_string(),
10407 content: serde_json::to_value(meerkat_core::ContentInput::Text(
10408 "hello steer bridge".to_string(),
10409 ))?,
10410 origin: "test".to_string(),
10411 idempotency_key: "idem-steer-bridge".to_string(),
10412 handling_mode: Some("steer".to_string()),
10413 },
10414 )
10415 .await?;
10416
10417 if tokio::time::timeout(Duration::from_millis(100), async {
10418 loop {
10419 if handling_modes
10420 .lock()
10421 .map(|modes| modes.contains(&HandlingMode::Steer))
10422 .unwrap_or(false)
10423 {
10424 break;
10425 }
10426 tokio::time::sleep(Duration::from_millis(5)).await;
10427 }
10428 })
10429 .await
10430 .is_err()
10431 {
10432 return Err("identity-first console send should forward steer mode".into());
10433 }
10434
10435 let terminal_frame = tokio::time::timeout(Duration::from_millis(500), async {
10436 loop {
10437 let page = aggregator
10438 .query_timeline(ConsoleTimelineQuery {
10439 identity: Some(identity.as_str().to_string()),
10440 ..ConsoleTimelineQuery::default()
10441 })
10442 .await
10443 .map_err(|err| format!("query timeline: {err}"))?;
10444 if page.frames.iter().any(|frame| {
10445 frame.kind == "interaction_complete"
10446 && frame.interaction_id.as_deref() == Some(accepted.interaction_id.as_str())
10447 && frame.payload.get("reason").and_then(Value::as_str)
10448 == Some("steer_delivered")
10449 }) {
10450 return Ok::<(), String>(());
10451 }
10452 tokio::time::sleep(Duration::from_millis(5)).await;
10453 }
10454 })
10455 .await;
10456 match terminal_frame {
10457 Ok(Ok(())) => {}
10458 Ok(Err(err)) => return Err(err.into()),
10459 Err(_) => {
10460 return Err(
10461 "identity-first steer send should terminalize its console interaction".into(),
10462 );
10463 }
10464 }
10465 Ok(())
10466 }
10467
10468 #[test]
10469 fn multipart_body_limit_covers_configured_image_limit() {
10470 const _: () = assert!(MAX_MULTIPART_BODY_BYTES > MAX_MULTIPART_IMAGE_BYTES);
10471 const _: () = assert!(MAX_MULTIPART_BODY_BYTES > 2 * 1024 * 1024);
10472 }
10473
10474 #[tokio::test]
10487 async fn cold_cache_waiter_resumes_when_refresh_lock_drops()
10488 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
10489 use std::sync::atomic::Ordering;
10490 use tokio::time::Duration;
10491
10492 let model = ConsoleSnapshotReadModel::default();
10493 let guard = model
10494 .refresh_lock
10495 .clone()
10496 .try_lock_owned()
10497 .map_err(|_| "refresh_lock unexpectedly contended at test start")?;
10498
10499 let model_for_waiter = model.clone();
10500 let waiter = tokio::spawn(async move {
10501 if model_for_waiter
10504 .primed
10505 .load(std::sync::atomic::Ordering::Acquire)
10506 {
10507 return;
10508 }
10509 let _wait_guard = model_for_waiter.refresh_lock.clone().lock_owned().await;
10510 assert!(
10513 model_for_waiter
10514 .primed
10515 .load(std::sync::atomic::Ordering::Acquire),
10516 "waiter acquired lock but primed is still false"
10517 );
10518 });
10519
10520 tokio::time::sleep(Duration::from_millis(20)).await;
10522
10523 model.primed.store(true, Ordering::Release);
10526 drop(guard);
10527
10528 let result = tokio::time::timeout(Duration::from_secs(1), waiter).await;
10529 assert!(
10530 result.is_ok(),
10531 "waiter should resume once the refresh lock drops"
10532 );
10533 Ok(())
10534 }
10535
10536 #[tokio::test]
10541 async fn snapshot_skips_refresh_lock_when_already_primed()
10542 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
10543 use std::sync::atomic::Ordering;
10544 use tokio::time::Duration;
10545
10546 let model = ConsoleSnapshotReadModel::default();
10547 model.primed.store(true, Ordering::Release);
10548 let _guard = model
10550 .refresh_lock
10551 .clone()
10552 .try_lock_owned()
10553 .map_err(|_| "refresh_lock unexpectedly contended at test start")?;
10554
10555 let snap_fast_path = async {
10561 assert!(
10562 model.primed.load(Ordering::Acquire),
10563 "primed precondition for hot-cache path"
10564 );
10565 };
10566 let result = tokio::time::timeout(Duration::from_millis(100), snap_fast_path).await;
10567 assert!(result.is_ok(), "hot-cache snapshot path should not block");
10568 Ok(())
10569 }
10570
10571 #[tokio::test]
10572 async fn console_aggregator_reset_all_rpc_rejects_destructive_retire_all_semantics()
10573 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
10574 let (_temp_dir, runtime) =
10575 build_empty_console_test_runtime("console-reset-fresh-identity-cache").await?;
10576 let aggregator = MobKitConsoleAggregator::in_memory();
10577 aggregator.register_runtime_handles_with_policy(
10578 "runtime-reset",
10579 "reset",
10580 runtime.clone(),
10581 None,
10582 ConsoleEventStore::new(),
10583 Arc::new(AllowAllConsoleVisibilityPolicy),
10584 );
10585 let primed_empty = aggregator.list_identities().await?;
10586 assert!(
10587 primed_empty.is_empty(),
10588 "test precondition: identity cache should be primed empty before late spawn"
10589 );
10590
10591 runtime
10592 .handle()
10593 .spawn_spec(SpawnMemberSpec::from_wire(
10594 "worker".to_string(),
10595 "agent-reset".to_string(),
10596 Some("You are agent-reset.".into()),
10597 None,
10598 None,
10599 ))
10600 .await?;
10601
10602 let response = Box::pin(handle_console_aggregator_rpc(
10603 Some(aggregator),
10604 rpc_request("mobkit/reset_all"),
10605 true,
10606 false,
10607 None,
10608 None,
10609 ))
10610 .await;
10611
10612 assert_eq!(response["result"], Value::Null);
10613 assert_eq!(
10614 response["error"]["data"]["kind"],
10615 json!("unsupported_reset_all_surface")
10616 );
10617 assert!(
10618 runtime
10619 .handle()
10620 .get_member(&meerkat_mob::ids::AgentIdentity::from("agent-reset"))
10621 .await
10622 .ok()
10623 .flatten()
10624 .is_some(),
10625 "aggregator reset_all must not retire live members while reporting unsupported"
10626 );
10627 let _ = runtime.handle().stop().await;
10628 Ok(())
10629 }
10630
10631 #[test]
10632 fn timeline_stream_cursor_filter_uses_numeric_console_sequence() {
10633 assert!(cursor_is_after(
10634 &ConsoleCursor::from("console:10"),
10635 &ConsoleCursor::from("console:9")
10636 ));
10637 assert!(!cursor_is_after(
10638 &ConsoleCursor::from("console:9"),
10639 &ConsoleCursor::from("console:10")
10640 ));
10641 }
10642
10643 #[test]
10644 fn console_live_snapshot_dedupes_repeated_delegate_identities() {
10645 let mut members = vec![
10646 ConsoleMember {
10647 agent_identity: "incident-commander".to_string(),
10648 role: "commander".to_string(),
10649 state: "active".to_string(),
10650 model_capabilities: Default::default(),
10651 runtime_mode: None,
10652 session_id: None,
10653 wired_to: Vec::new(),
10654 labels: BTreeMap::new(),
10655 },
10656 ConsoleMember {
10657 agent_identity: "qa-child".to_string(),
10658 role: "delegate".to_string(),
10659 state: "active".to_string(),
10660 model_capabilities: Default::default(),
10661 runtime_mode: None,
10662 session_id: Some("first".to_string()),
10663 wired_to: vec!["qa-parent".to_string()],
10664 labels: BTreeMap::from([(
10665 "delegate_host_identity".to_string(),
10666 "qa-parent".to_string(),
10667 )]),
10668 },
10669 ConsoleMember {
10670 agent_identity: "qa-child".to_string(),
10671 role: "delegate".to_string(),
10672 state: "active".to_string(),
10673 model_capabilities: Default::default(),
10674 runtime_mode: None,
10675 session_id: Some("second".to_string()),
10676 wired_to: vec!["qa-parent".to_string()],
10677 labels: BTreeMap::from([(
10678 "delegate_host_identity".to_string(),
10679 "qa-parent".to_string(),
10680 )]),
10681 },
10682 ];
10683
10684 dedupe_console_members_by_identity(&mut members);
10685
10686 assert_eq!(
10687 members
10688 .iter()
10689 .map(|member| member.agent_identity.as_str())
10690 .collect::<Vec<_>>(),
10691 vec!["incident-commander", "qa-child"]
10692 );
10693 assert_eq!(members[1].session_id.as_deref(), Some("first"));
10694 }
10695
10696 #[test]
10697 fn console_visibility_policy_hides_implicit_delegate_members_from_snapshot() {
10698 let mut snapshot = ConsoleLiveSnapshot::new(
10699 Some("runtime".to_string()),
10700 true,
10701 vec!["incident-commander".to_string(), "qa-child".to_string()],
10702 vec![
10703 ConsoleAgentLiveSnapshot {
10704 agent_id: "incident-commander".to_string(),
10705 member_id: "incident-commander".to_string(),
10706 label: "Incident Commander".to_string(),
10707 kind: "meerkat".to_string(),
10708 identity: Some("incident-commander".to_string()),
10709 role: Some("commander".to_string()),
10710 state: Some("active".to_string()),
10711 session_id: None,
10712 model_capabilities: Default::default(),
10713 response_phase: None,
10714 watched: None,
10715 alert_level: None,
10716 degraded: None,
10717 degraded_reason: None,
10718 },
10719 ConsoleAgentLiveSnapshot {
10720 agent_id: "qa-child".to_string(),
10721 member_id: "qa-child".to_string(),
10722 label: "QA Child".to_string(),
10723 kind: "meerkat".to_string(),
10724 identity: Some("qa-child".to_string()),
10725 role: Some("delegate".to_string()),
10726 state: Some("active".to_string()),
10727 session_id: Some("delegate-session".to_string()),
10728 model_capabilities: Default::default(),
10729 response_phase: None,
10730 watched: None,
10731 alert_level: None,
10732 degraded: None,
10733 degraded_reason: None,
10734 },
10735 ],
10736 vec![
10737 ConsoleMember {
10738 agent_identity: "incident-commander".to_string(),
10739 role: "commander".to_string(),
10740 state: "active".to_string(),
10741 model_capabilities: Default::default(),
10742 runtime_mode: None,
10743 session_id: None,
10744 wired_to: Vec::new(),
10745 labels: BTreeMap::new(),
10746 },
10747 ConsoleMember {
10748 agent_identity: "qa-child".to_string(),
10749 role: "delegate".to_string(),
10750 state: "active".to_string(),
10751 model_capabilities: Default::default(),
10752 runtime_mode: None,
10753 session_id: Some("delegate-session".to_string()),
10754 wired_to: vec!["qa-parent".to_string()],
10755 labels: BTreeMap::from([(
10756 "source_mob_id".to_string(),
10757 "implicit-qa-mob".to_string(),
10758 )]),
10759 },
10760 ],
10761 true,
10762 );
10763
10764 apply_console_visibility_policy(
10765 &mut snapshot,
10766 &HideImplicitDelegateMembersConsoleVisibilityPolicy,
10767 );
10768
10769 assert_eq!(
10770 snapshot
10771 .members
10772 .iter()
10773 .map(|member| member.agent_identity.as_str())
10774 .collect::<Vec<_>>(),
10775 vec!["incident-commander"]
10776 );
10777 assert_eq!(
10778 snapshot
10779 .agents
10780 .iter()
10781 .map(|agent| agent.agent_id.as_str())
10782 .collect::<Vec<_>>(),
10783 vec!["incident-commander"]
10784 );
10785 assert_eq!(snapshot.loaded_modules, vec!["incident-commander"]);
10786 }
10787
10788 #[tokio::test]
10789 async fn live_snapshot_member_projection_uses_roster_profile_capabilities()
10790 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
10791 let temp_dir = tempfile::tempdir()?;
10792 let session_path = temp_dir.path().join("sessions");
10793 std::fs::create_dir_all(&session_path)?;
10794 let factory = AgentFactory::new(&session_path).comms(true);
10795 let session_service = Arc::new(build_ephemeral_service(factory, Config::default(), 16));
10796 let definition = MobDefinition::from_toml(
10797 r#"
10798[mob]
10799id = "console-snapshot-test"
10800
10801[profiles.worker]
10802model = "gpt-5.5"
10803
10804[profiles.worker.tools]
10805comms = true
10806"#,
10807 )?;
10808 let expected = model_capabilities_for_role(&definition, "worker");
10809 let runtime = MobRuntime::bootstrap(
10810 MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
10811 .with_options(MobBootstrapOptions {
10812 allow_ephemeral_sessions: true,
10813 notify_orchestrator_on_resume: true,
10814 default_llm_client: Some(Arc::new(TestClient::default())),
10815 }),
10816 )
10817 .await?;
10818 runtime
10819 .handle()
10820 .spawn_spec(SpawnMemberSpec::from_wire(
10821 "worker".to_string(),
10822 crate::member_comms_id::mob_member_id_str("worker:one").into_owned(),
10825 Some("You are worker one.".into()),
10826 None,
10827 None,
10828 ))
10829 .await?;
10830
10831 let empty_read_model = ConsoleSnapshotReadModelState::default();
10832 let (members, session_owner_by_id) = project_console_members_from_handle(
10833 &runtime.handle(),
10834 None,
10835 None,
10836 &empty_read_model,
10837 &BTreeMap::new(),
10838 )
10839 .await;
10840
10841 assert_eq!(members.len(), 1);
10842 assert_eq!(members[0].model_capabilities, expected);
10843 assert_eq!(members[0].session_id, None);
10844 assert!(session_owner_by_id.is_empty());
10845
10846 let refreshed_read_model = collect_console_snapshot_read_model(&runtime).await;
10847 let (members, session_owner_by_id) = project_console_members_from_handle(
10848 &runtime.handle(),
10849 None,
10850 None,
10851 &refreshed_read_model,
10852 &BTreeMap::new(),
10853 )
10854 .await;
10855 assert_eq!(
10856 members[0].session_id.as_ref(),
10857 session_owner_by_id.keys().next()
10858 );
10859
10860 assert_eq!(
10866 refreshed_read_model.primary_members.len(),
10867 members.len(),
10868 "primary_members cache should hold the same members as live projection"
10869 );
10870 assert_eq!(
10871 refreshed_read_model.primary_members[0].agent_identity,
10872 members[0].agent_identity
10873 );
10874 assert_eq!(
10875 refreshed_read_model.primary_members[0].session_id,
10876 members[0].session_id
10877 );
10878 Ok(())
10879 }
10880
10881 #[tokio::test]
10882 async fn fresh_timeline_snapshot_reads_tail_without_full_log_replay()
10883 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
10884 let aggregator = MobKitConsoleAggregator::in_memory();
10885 for idx in 0..250_000 {
10886 aggregator
10887 .store()
10888 .append_if_absent(NewConsoleFrame {
10889 id: None,
10890 dedupe_key: format!("event-{idx}"),
10891 timestamp_ms: idx,
10892 runtime_key: "runtime-a".to_string(),
10893 identity: "agent-a".to_string(),
10894 conversation_id: Some("agent-a".to_string()),
10895 session_id: None,
10896 kind: "text_delta".to_string(),
10897 status: ConsoleFrameStatus::Completed,
10898 payload: json!({ "delta": idx }),
10899 source: ConsoleFrameSource {
10900 kind: ConsoleFrameSourceKind::ConsoleEvent,
10901 source_cursor: None,
10902 },
10903 source_event_id: Some(format!("event-{idx}")),
10904 interaction_id: None,
10905 turn_id: None,
10906 run_id: None,
10907 parent_frame_id: None,
10908 caused_by_frame_id: None,
10909 })
10910 .await?;
10911 }
10912
10913 let (frames, cursor) = query_timeline_snapshot(
10914 &aggregator,
10915 ConsoleTimelineWindowQuery {
10916 identity: Some("agent-a".to_string()),
10917 after: None,
10918 limit: 200,
10919 ..ConsoleTimelineWindowQuery::default()
10920 },
10921 )
10922 .await?;
10923
10924 assert!(!frames.is_empty());
10925 assert_eq!(cursor.as_ref().and_then(ConsoleCursor::seq), Some(250_000));
10926 assert_eq!(
10927 frames.last().and_then(|frame| frame.cursor.seq()),
10928 Some(250_000)
10929 );
10930 Ok(())
10931 }
10932
10933 #[tokio::test]
10934 async fn fresh_timeline_snapshot_keeps_sparse_identity_frames()
10935 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
10936 let aggregator = MobKitConsoleAggregator::in_memory();
10937 aggregator
10938 .store()
10939 .append_if_absent(NewConsoleFrame {
10940 id: None,
10941 dedupe_key: "sparse-event".to_string(),
10942 timestamp_ms: 1,
10943 runtime_key: "runtime-a".to_string(),
10944 identity: "sparse-agent".to_string(),
10945 conversation_id: Some("sparse-agent".to_string()),
10946 session_id: None,
10947 kind: "text_complete".to_string(),
10948 status: ConsoleFrameStatus::Completed,
10949 payload: json!({ "text": "still visible" }),
10950 source: ConsoleFrameSource {
10951 kind: ConsoleFrameSourceKind::ConsoleEvent,
10952 source_cursor: None,
10953 },
10954 source_event_id: Some("sparse-event".to_string()),
10955 interaction_id: None,
10956 turn_id: None,
10957 run_id: None,
10958 parent_frame_id: None,
10959 caused_by_frame_id: None,
10960 })
10961 .await?;
10962 for idx in 0..25_000 {
10963 aggregator
10964 .store()
10965 .append_if_absent(NewConsoleFrame {
10966 id: None,
10967 dedupe_key: format!("other-event-{idx}"),
10968 timestamp_ms: idx + 2,
10969 runtime_key: "runtime-a".to_string(),
10970 identity: "busy-agent".to_string(),
10971 conversation_id: Some("busy-agent".to_string()),
10972 session_id: None,
10973 kind: "text_delta".to_string(),
10974 status: ConsoleFrameStatus::Completed,
10975 payload: json!({ "delta": idx }),
10976 source: ConsoleFrameSource {
10977 kind: ConsoleFrameSourceKind::ConsoleEvent,
10978 source_cursor: None,
10979 },
10980 source_event_id: Some(format!("other-event-{idx}")),
10981 interaction_id: None,
10982 turn_id: None,
10983 run_id: None,
10984 parent_frame_id: None,
10985 caused_by_frame_id: None,
10986 })
10987 .await?;
10988 }
10989
10990 let (frames, cursor) = query_timeline_snapshot(
10991 &aggregator,
10992 ConsoleTimelineWindowQuery {
10993 identity: Some("sparse-agent".to_string()),
10994 after: None,
10995 limit: 200,
10996 ..ConsoleTimelineWindowQuery::default()
10997 },
10998 )
10999 .await?;
11000
11001 assert_eq!(frames.len(), 1);
11002 assert_eq!(frames[0].identity, "sparse-agent");
11003 assert_eq!(frames[0].payload["text"], json!("still visible"));
11004 assert_eq!(cursor.as_ref().and_then(ConsoleCursor::seq), Some(1));
11005 Ok(())
11006 }
11007
11008 #[tokio::test]
11009 async fn fresh_identity_snapshot_keeps_user_input_anchor_before_noisy_tail()
11010 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
11011 let aggregator = MobKitConsoleAggregator::in_memory();
11012 aggregator
11013 .store()
11014 .append_if_absent(NewConsoleFrame {
11015 id: None,
11016 dedupe_key: "worker-kickoff".to_string(),
11017 timestamp_ms: 1,
11018 runtime_key: "runtime-a".to_string(),
11019 identity: "review-worker-a".to_string(),
11020 conversation_id: Some("review-worker-a".to_string()),
11021 session_id: None,
11022 kind: "user_input".to_string(),
11023 status: ConsoleFrameStatus::Delivered,
11024 payload: json!({
11025 "content": [
11026 {
11027 "type": "text",
11028 "text": "Console chat smoke: review this initiative"
11029 }
11030 ]
11031 }),
11032 source: ConsoleFrameSource {
11033 kind: ConsoleFrameSourceKind::Synthetic,
11034 source_cursor: None,
11035 },
11036 source_event_id: Some("worker-kickoff".to_string()),
11037 interaction_id: Some("kickoff-1".to_string()),
11038 turn_id: None,
11039 run_id: None,
11040 parent_frame_id: None,
11041 caused_by_frame_id: None,
11042 })
11043 .await?;
11044 for idx in 0..1_500 {
11045 aggregator
11046 .store()
11047 .append_if_absent(NewConsoleFrame {
11048 id: None,
11049 dedupe_key: format!("worker-delta-{idx}"),
11050 timestamp_ms: idx + 2,
11051 runtime_key: "runtime-a".to_string(),
11052 identity: "review-worker-a".to_string(),
11053 conversation_id: Some("review-worker-a".to_string()),
11054 session_id: None,
11055 kind: "reasoning_delta".to_string(),
11056 status: ConsoleFrameStatus::Delivered,
11057 payload: json!({ "delta": idx }),
11058 source: ConsoleFrameSource {
11059 kind: ConsoleFrameSourceKind::ConsoleEvent,
11060 source_cursor: None,
11061 },
11062 source_event_id: Some(format!("worker-delta-{idx}")),
11063 interaction_id: Some("kickoff-1".to_string()),
11064 turn_id: None,
11065 run_id: None,
11066 parent_frame_id: None,
11067 caused_by_frame_id: None,
11068 })
11069 .await?;
11070 }
11071
11072 let (frames, cursor) = query_timeline_snapshot(
11073 &aggregator,
11074 ConsoleTimelineWindowQuery {
11075 identity: Some("review-worker-a".to_string()),
11076 after: None,
11077 limit: 200,
11078 ..ConsoleTimelineWindowQuery::default()
11079 },
11080 )
11081 .await?;
11082
11083 assert!(
11084 frames.iter().any(|frame| {
11085 frame.kind == "user_input"
11086 && frame.payload.to_string().contains("Console chat smoke")
11087 }),
11088 "identity chat snapshot must keep the worker kickoff prompt before a noisy tail: {frames:#?}",
11089 );
11090 assert_eq!(cursor.as_ref().and_then(ConsoleCursor::seq), Some(1_501));
11091 assert_eq!(
11092 frames.last().and_then(|frame| frame.cursor.seq()),
11093 Some(1_501)
11094 );
11095 Ok(())
11096 }
11097
11098 #[tokio::test]
11099 async fn timeline_snapshot_drains_since_backlog_across_store_pages()
11100 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
11101 let aggregator = MobKitConsoleAggregator::in_memory();
11102 for idx in 0..2_500 {
11103 aggregator
11104 .store()
11105 .append_if_absent(NewConsoleFrame {
11106 id: None,
11107 dedupe_key: format!("clamp-event-{idx}"),
11108 timestamp_ms: idx,
11109 runtime_key: "runtime-a".to_string(),
11110 identity: "agent-a".to_string(),
11111 conversation_id: Some("agent-a".to_string()),
11112 session_id: None,
11113 kind: "text_delta".to_string(),
11114 status: ConsoleFrameStatus::Completed,
11115 payload: json!({ "delta": idx }),
11116 source: ConsoleFrameSource {
11117 kind: ConsoleFrameSourceKind::ConsoleEvent,
11118 source_cursor: None,
11119 },
11120 source_event_id: Some(format!("clamp-event-{idx}")),
11121 interaction_id: None,
11122 turn_id: None,
11123 run_id: None,
11124 parent_frame_id: None,
11125 caused_by_frame_id: None,
11126 })
11127 .await?;
11128 }
11129
11130 let (frames, cursor) = query_timeline_snapshot(
11131 &aggregator,
11132 ConsoleTimelineWindowQuery {
11133 identity: Some("agent-a".to_string()),
11134 after: Some(ConsoleCursor::from("console:100")),
11135 limit: 5_000,
11136 ..ConsoleTimelineWindowQuery::default()
11137 },
11138 )
11139 .await?;
11140
11141 assert_eq!(frames.len(), 2_400);
11142 assert_eq!(
11143 frames.first().and_then(|frame| frame.cursor.seq()),
11144 Some(101)
11145 );
11146 assert_eq!(
11147 frames.last().and_then(|frame| frame.cursor.seq()),
11148 Some(2_500)
11149 );
11150 assert_eq!(cursor.as_ref().and_then(ConsoleCursor::seq), Some(2_500));
11151 Ok(())
11152 }
11153
11154 #[tokio::test]
11155 async fn timeline_snapshot_drains_since_backlog_beyond_old_page_budget()
11156 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
11157 let aggregator = MobKitConsoleAggregator::in_memory();
11158 for idx in 1..=150 {
11159 aggregator
11160 .store()
11161 .append_if_absent(NewConsoleFrame {
11162 id: None,
11163 dedupe_key: format!("deep-backlog-event-{idx}"),
11164 timestamp_ms: idx,
11165 runtime_key: "runtime-a".to_string(),
11166 identity: "agent-a".to_string(),
11167 conversation_id: Some("agent-a".to_string()),
11168 session_id: None,
11169 kind: "text_delta".to_string(),
11170 status: ConsoleFrameStatus::Completed,
11171 payload: json!({ "delta": idx }),
11172 source: ConsoleFrameSource {
11173 kind: ConsoleFrameSourceKind::ConsoleEvent,
11174 source_cursor: None,
11175 },
11176 source_event_id: Some(format!("deep-backlog-event-{idx}")),
11177 interaction_id: None,
11178 turn_id: None,
11179 run_id: None,
11180 parent_frame_id: None,
11181 caused_by_frame_id: None,
11182 })
11183 .await?;
11184 }
11185
11186 let (frames, cursor) = query_timeline_snapshot(
11187 &aggregator,
11188 ConsoleTimelineWindowQuery {
11189 identity: Some("agent-a".to_string()),
11190 after: Some(ConsoleCursor::from_seq(1)),
11191 limit: 1,
11192 ..ConsoleTimelineWindowQuery::default()
11193 },
11194 )
11195 .await?;
11196
11197 assert_eq!(frames.len(), 149);
11198 assert_eq!(frames.first().and_then(|frame| frame.cursor.seq()), Some(2));
11199 assert_eq!(
11200 frames.last().and_then(|frame| frame.cursor.seq()),
11201 Some(150)
11202 );
11203 assert_eq!(cursor.as_ref().and_then(ConsoleCursor::seq), Some(150));
11204 Ok(())
11205 }
11206
11207 #[tokio::test]
11208 async fn timeline_snapshot_rejects_after_cursor_beyond_store_frontier()
11209 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
11210 let aggregator = MobKitConsoleAggregator::in_memory();
11211 aggregator
11212 .store()
11213 .append_if_absent(NewConsoleFrame {
11214 id: None,
11215 dedupe_key: "stale-frontier-event".to_string(),
11216 timestamp_ms: 1,
11217 runtime_key: "runtime-a".to_string(),
11218 identity: "agent-a".to_string(),
11219 conversation_id: Some("agent-a".to_string()),
11220 session_id: None,
11221 kind: "text_delta".to_string(),
11222 status: ConsoleFrameStatus::Completed,
11223 payload: json!({ "delta": 1 }),
11224 source: ConsoleFrameSource {
11225 kind: ConsoleFrameSourceKind::ConsoleEvent,
11226 source_cursor: None,
11227 },
11228 source_event_id: Some("stale-frontier-event".to_string()),
11229 interaction_id: None,
11230 turn_id: None,
11231 run_id: None,
11232 parent_frame_id: None,
11233 caused_by_frame_id: None,
11234 })
11235 .await?;
11236
11237 let err = match query_timeline_snapshot(
11238 &aggregator,
11239 ConsoleTimelineWindowQuery {
11240 after: Some(ConsoleCursor::from("console:99")),
11241 limit: 200,
11242 ..ConsoleTimelineWindowQuery::default()
11243 },
11244 )
11245 .await
11246 {
11247 Ok(_) => {
11248 return Err(
11249 std::io::Error::other("future cursor must be replay-unavailable").into(),
11250 );
11251 }
11252 Err(err) => err,
11253 };
11254
11255 assert!(
11256 err.to_string()
11257 .contains("beyond the current store frontier"),
11258 "unexpected error: {err}"
11259 );
11260 Ok(())
11261 }
11262
11263 #[tokio::test]
11264 async fn timeline_snapshot_rejects_after_cursor_beyond_empty_store_frontier()
11265 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
11266 let aggregator = MobKitConsoleAggregator::in_memory();
11267
11268 let err = match query_timeline_snapshot(
11269 &aggregator,
11270 ConsoleTimelineWindowQuery {
11271 after: Some(ConsoleCursor::from("console:99")),
11272 limit: 200,
11273 ..ConsoleTimelineWindowQuery::default()
11274 },
11275 )
11276 .await
11277 {
11278 Ok(_) => {
11279 return Err(std::io::Error::other("empty store future cursor must fail").into());
11280 }
11281 Err(err) => err,
11282 };
11283
11284 assert!(
11285 err.to_string()
11286 .contains("beyond the current store frontier"),
11287 "unexpected error: {err}"
11288 );
11289 Ok(())
11290 }
11291
11292 #[test]
11293 fn timeline_query_prefers_last_event_id_over_url_after_cursor() {
11294 let query = timeline_query_from_http(
11295 ConsoleTimelineHttpQuery {
11296 identity: None,
11297 conversation_id: None,
11298 after: Some("console:100".to_string()),
11299 before: None,
11300 mode: None,
11301 limit: None,
11302 },
11303 Some("console:150".to_string()),
11304 );
11305
11306 assert_eq!(query.after.as_ref().and_then(ConsoleCursor::seq), Some(150));
11307 }
11308
11309 #[test]
11310 fn console_timeline_replay_unavailable_rpc_uses_dedicated_error_code() {
11311 let response = console_timeline_replay_unavailable_response(
11312 json!("rid"),
11313 std::io::Error::other("timeline replay cursor is beyond the current store frontier")
11314 .into(),
11315 Some(&ConsoleCursor::from_seq(100)),
11316 Some(ConsoleCursor::from_seq(42)),
11317 );
11318
11319 assert_eq!(response["error"]["code"], json!(-32013));
11320 assert_eq!(
11321 response["error"]["data"],
11322 json!({
11323 "error": "replay_unavailable",
11324 "stream": "timeline",
11325 "requested_cursor": "console:100",
11326 "latest_cursor": "console:42",
11327 })
11328 );
11329 }
11330
11331 #[tokio::test]
11332 async fn multipart_blob_upload_stores_one_file() -> Result<(), Box<dyn std::error::Error>> {
11333 let store: Arc<dyn BinaryBlobStore> = Arc::new(ObjectStoreBlobStore::memory());
11334 let mut files = BTreeMap::new();
11335 files.insert(
11336 "upload-1".to_string(),
11337 MultipartImageUpload {
11338 media_type: "image/png".to_string(),
11339 bytes: Bytes::from_static(b"png-data"),
11340 },
11341 );
11342 let result = externalize_single_image_upload(
11343 &json!({
11344 "upload": {
11345 "type": "image_upload",
11346 "upload_id": "upload-1",
11347 "media_type": "image/png"
11348 }
11349 }),
11350 files,
11351 store.clone(),
11352 )
11353 .await
11354 .map_err(std::io::Error::other)?;
11355
11356 assert_eq!(result["media_type"], json!("image/png"));
11357 assert_eq!(result["size"], json!(8));
11358 let Some(blob_id) = result["blob_id"].as_str() else {
11359 return Err(std::io::Error::other("blob id").into());
11360 };
11361 let payload = store
11362 .get_bytes(&meerkat_core::BlobId::from(blob_id))
11363 .await?;
11364 assert_eq!(payload.data.as_ref(), b"png-data");
11365 Ok(())
11366 }
11367
11368 #[tokio::test]
11369 async fn multipart_blob_upload_accepts_part_name_alias()
11370 -> Result<(), Box<dyn std::error::Error>> {
11371 let store: Arc<dyn BinaryBlobStore> = Arc::new(ObjectStoreBlobStore::memory());
11372 let mut files = BTreeMap::new();
11373 files.insert(
11374 "image-field".to_string(),
11375 MultipartImageUpload {
11376 media_type: "image/png".to_string(),
11377 bytes: Bytes::from_static(b"png-data"),
11378 },
11379 );
11380 let result = externalize_single_image_upload(
11381 &json!({
11382 "upload": {
11383 "type": "image_upload",
11384 "part_name": "image-field",
11385 "media_type": "image/png"
11386 }
11387 }),
11388 files,
11389 store,
11390 )
11391 .await
11392 .map_err(std::io::Error::other)?;
11393
11394 assert_eq!(result["media_type"], json!("image/png"));
11395 assert!(
11396 result["blob_id"]
11397 .as_str()
11398 .is_some_and(|value| value.starts_with("sha256:"))
11399 );
11400 Ok(())
11401 }
11402
11403 #[tokio::test]
11404 async fn multipart_blob_upload_rejects_media_mismatch() -> Result<(), Box<dyn std::error::Error>>
11405 {
11406 let store: Arc<dyn BinaryBlobStore> = Arc::new(ObjectStoreBlobStore::memory());
11407 let mut files = BTreeMap::new();
11408 files.insert(
11409 "upload-1".to_string(),
11410 MultipartImageUpload {
11411 media_type: "image/jpeg".to_string(),
11412 bytes: Bytes::from_static(b"jpeg-data"),
11413 },
11414 );
11415 let err = match externalize_single_image_upload(
11416 &json!({
11417 "upload": {
11418 "type": "image_upload",
11419 "upload_id": "upload-1",
11420 "media_type": "image/png"
11421 }
11422 }),
11423 files,
11424 store,
11425 )
11426 .await
11427 {
11428 Ok(_) => return Err(std::io::Error::other("media mismatch").into()),
11429 Err(err) => err,
11430 };
11431 assert!(
11432 err.contains("media type mismatch"),
11433 "unexpected error: {err}"
11434 );
11435 Ok(())
11436 }
11437
11438 #[tokio::test]
11439 async fn multipart_blob_upload_rejects_extra_file() -> Result<(), Box<dyn std::error::Error>> {
11440 let store: Arc<dyn BinaryBlobStore> = Arc::new(ObjectStoreBlobStore::memory());
11441 let mut files = BTreeMap::new();
11442 for id in ["upload-1", "upload-2"] {
11443 files.insert(
11444 id.to_string(),
11445 MultipartImageUpload {
11446 media_type: "image/png".to_string(),
11447 bytes: Bytes::from_static(b"png"),
11448 },
11449 );
11450 }
11451 let err = match externalize_single_image_upload(
11452 &json!({
11453 "upload": {
11454 "type": "image_upload",
11455 "upload_id": "upload-1",
11456 "media_type": "image/png"
11457 }
11458 }),
11459 files,
11460 store,
11461 )
11462 .await
11463 {
11464 Ok(_) => return Err(std::io::Error::other("one file only").into()),
11465 Err(err) => err,
11466 };
11467 assert!(
11468 err.contains("exactly one file part"),
11469 "unexpected error: {err}"
11470 );
11471 Ok(())
11472 }
11473
11474 #[tokio::test]
11475 async fn multipart_send_replaces_placeholders_and_removes_shadow_message()
11476 -> Result<(), Box<dyn std::error::Error>> {
11477 let store: Arc<dyn BinaryBlobStore> = Arc::new(ObjectStoreBlobStore::memory());
11478 let mut files = BTreeMap::new();
11479 files.insert(
11480 "upload-1".to_string(),
11481 MultipartImageUpload {
11482 media_type: "image/webp".to_string(),
11483 bytes: Bytes::from_static(b"webp-data"),
11484 },
11485 );
11486 let mut params = json!({
11487 "member_id": "artist",
11488 "message": "stale shadow text",
11489 "content": [
11490 { "type": "text", "text": "describe" },
11491 {
11492 "type": "image_upload",
11493 "upload_id": "upload-1",
11494 "media_type": "image/webp"
11495 }
11496 ]
11497 });
11498 externalize_image_upload_placeholders(&mut params, files, store)
11499 .await
11500 .map_err(std::io::Error::other)?;
11501
11502 assert!(params.get("message").is_none());
11503 assert_eq!(params["content"][1]["type"], json!("image"));
11504 assert_eq!(params["content"][1]["source"], json!("blob"));
11505 assert_eq!(params["content"][1]["media_type"], json!("image/webp"));
11506 assert!(
11507 params["content"][1]["blob_id"]
11508 .as_str()
11509 .is_some_and(|value| value.starts_with("sha256:"))
11510 );
11511 Ok(())
11512 }
11513
11514 #[tokio::test]
11515 async fn multipart_send_accepts_part_name_placeholder() -> Result<(), Box<dyn std::error::Error>>
11516 {
11517 let store: Arc<dyn BinaryBlobStore> = Arc::new(ObjectStoreBlobStore::memory());
11518 let mut files = BTreeMap::new();
11519 files.insert(
11520 "image-field".to_string(),
11521 MultipartImageUpload {
11522 media_type: "image/png".to_string(),
11523 bytes: Bytes::from_static(b"png-data"),
11524 },
11525 );
11526 let mut params = json!({
11527 "member_id": "analyst",
11528 "content": [
11529 { "type": "text", "text": "describe" },
11530 {
11531 "type": "image_upload",
11532 "part_name": "image-field",
11533 "media_type": "image/png"
11534 }
11535 ]
11536 });
11537
11538 externalize_image_upload_placeholders(&mut params, files, store)
11539 .await
11540 .map_err(std::io::Error::other)?;
11541
11542 assert_eq!(params["content"][1]["type"], json!("image"));
11543 assert_eq!(params["content"][1]["source"], json!("blob"));
11544 assert_eq!(params["content"][1]["media_type"], json!("image/png"));
11545 Ok(())
11546 }
11547
11548 #[tokio::test]
11549 async fn multipart_send_rejects_placeholder_without_file()
11550 -> Result<(), Box<dyn std::error::Error>> {
11551 let store: Arc<dyn BinaryBlobStore> = Arc::new(ObjectStoreBlobStore::memory());
11552 let mut params = json!({
11553 "content": [{
11554 "type": "image_upload",
11555 "upload_id": "missing",
11556 "media_type": "image/png"
11557 }]
11558 });
11559 let err = match externalize_image_upload_placeholders(&mut params, BTreeMap::new(), store)
11560 .await
11561 {
11562 Ok(()) => return Err(std::io::Error::other("missing file").into()),
11563 Err(err) => err,
11564 };
11565 assert!(err.contains("missing file part"), "unexpected error: {err}");
11566 Ok(())
11567 }
11568
11569 #[test]
11570 fn generated_runtime_ids_do_not_match_sibling_colon_identities() {
11571 assert!(!member_id_matches_durable_identity(
11572 "rt:review:singleton:0",
11573 "review:singleton"
11574 ));
11575 assert!(!member_id_matches_durable_identity(
11576 "review:singleton:gen1",
11577 "review:singleton"
11578 ));
11579 assert!(!member_id_matches_durable_identity(
11580 "review:singleton:1",
11581 "review:singleton"
11582 ));
11583 assert!(!member_id_matches_durable_identity(
11584 "rt:review:singleton:qa:0",
11585 "review:singleton"
11586 ));
11587 assert!(!member_id_matches_durable_identity(
11588 "review:singleton:qa",
11589 "review:singleton"
11590 ));
11591 }
11592}