1use std::collections::{BTreeMap, BTreeSet};
4use std::path::{Path, PathBuf};
5use std::sync::Arc;
6
7use async_trait::async_trait;
8use futures::StreamExt;
9use meerkat::{AgentFactory, Config, FactoryAgentBuilder, SessionStore};
10use meerkat_client::types::LlmStream;
11use meerkat_client::{LlmClient, LlmRequest};
12use meerkat_core::agent::CommsRuntime;
13use meerkat_core::service::{
14 CreateSessionRequest, SessionError, SessionHistoryPage, SessionHistoryQuery,
15 SessionServiceHistoryExt,
16};
17use meerkat_core::{AgentSessionStore, AssistantBlock, Message, Provider};
18use meerkat_mob::{
19 MobBuilder, MobDefinition, MobError, MobHandle, MobSessionService, MobStorage, Profile,
20 ProfileName, SpawnMemberSpec,
21};
22use meerkat_runtime::input_state::{InputStatePersistenceRecord, StoredInputState};
23use meerkat_runtime::store::MachineLifecycleCommit;
24use meerkat_store::StoreAdapter;
25use serde::Serialize;
26use serde_json::Value;
27
28use crate::blob_store::{
29 Base64BlobStoreAdapter, BinaryBlobStore, BlobStoreInjection, ObjectStoreBlobStore,
30};
31use crate::console_spawn::{
32 ConsoleSpawnSink, SharedConsoleSpawnSinkSlot, new_console_spawn_sink_slot,
33};
34use crate::storage_health::{
35 BlobDurability, BlobStoreResolutionError, ResolvedStorageSummary, RuntimeStoreResolutionError,
36 StorageResolutionError, StorageSlotSummary, blob_slot_summary, probe_session_store_incremental,
37 scratch_ring_buffer_slots,
38};
39
40pub(crate) const DELEGATE_IDLE_RETIRE_SECS_LABEL: &str = "implicit_delegate_idle_retire_secs";
41pub(crate) const DELEGATE_IDLE_RETIRE_DISABLED_LABEL: &str = "disabled";
42
43pub(crate) fn is_previous_member_cleanup_ambiguous_error(error: &str) -> bool {
44 error.contains("previous member cleanup ambiguous for member ")
45}
46
47pub(crate) fn is_recoverable_lifecycle_cleanup_error(error: &str) -> bool {
48 is_previous_member_cleanup_ambiguous_error(error)
49 || (error.contains("disposal completed but ArchiveSession failed")
50 && (
51 (error.contains("cancel-before-retire failed")
54 && error.contains("Runtime not ready: running"))
55 || (error.contains("guard rejected transition from Stopped")
62 && error.contains("input::Retire"))
63 || (error.contains("continuity save")
69 && (error.contains("continuity record not found")
70 || error.contains("stale fencing token")))
71 ))
72}
73
74fn is_session_owned_archive_absent_cleanup_error(error: &str) -> bool {
92 error.contains("disposal completed but ArchiveSession failed")
93 && error.contains("NotFound for registered runtime session")
94}
95
96pub(crate) fn is_recoverable_session_owned_retire_cleanup_error(error: &str) -> bool {
108 is_recoverable_lifecycle_cleanup_error(error)
109 || is_session_owned_archive_absent_cleanup_error(error)
110}
111
112pub(crate) fn is_stopped_session_archive_retire_rejection(error: &str) -> bool {
126 error.contains("machine archive retire failed")
127 && error.contains("guard rejected transition from Stopped")
128 && error.contains("input::Retire")
129}
130
131pub(crate) fn topology_restore_failed_peer_ids(
132 error: &meerkat_mob::MobRespawnError,
133) -> Option<Vec<String>> {
134 match error {
135 meerkat_mob::MobRespawnError::TopologyRestoreFailed {
136 receipt: _,
137 failed_peer_ids,
138 } => Some(failed_peer_ids.iter().map(ToString::to_string).collect()),
139 _ => None,
140 }
141}
142
143pub(crate) fn topology_restore_warning_json(failed_peer_ids: &[String]) -> Value {
144 serde_json::json!({
145 "kind": "topology_restore_degraded",
146 "failed_peer_ids": failed_peer_ids,
147 })
148}
149
150#[cfg(test)]
151use std::sync::{Mutex, OnceLock};
152
153pub const MEMBER_STATE_ACTIVE: &str = "active";
155pub const MEMBER_STATE_RETIRING: &str = "retiring";
157
158pub(crate) fn member_status_state_string(status: meerkat_mob::MobMemberStatus) -> String {
167 match status {
168 meerkat_mob::MobMemberStatus::Active => MEMBER_STATE_ACTIVE.to_string(),
169 meerkat_mob::MobMemberStatus::Retiring => MEMBER_STATE_RETIRING.to_string(),
170 other => format!("{other:?}").to_ascii_lowercase(),
171 }
172}
173
174#[derive(Clone, Default)]
176pub struct MobBootstrapOptions {
177 pub allow_ephemeral_sessions: bool,
178 pub notify_orchestrator_on_resume: bool,
179 pub default_llm_client: Option<Arc<dyn LlmClient>>,
180}
181
182pub struct ReplaySanitizingLlmClient {
186 inner: Arc<dyn LlmClient>,
187}
188
189type SharedDefaultLlmClientSlot = Arc<std::sync::RwLock<Option<Arc<dyn LlmClient>>>>;
190
191fn session_llm_reconfigure_blueprint(
192 builder: &FactoryAgentBuilder,
193 store_path: &Path,
194) -> (
195 meerkat::session_runtime::llm_reconfigure::SessionRuntimeLlmReconfigureHostBlueprint,
196 SharedDefaultLlmClientSlot,
197) {
198 let default_client_slot = Arc::new(std::sync::RwLock::new(None::<Arc<dyn LlmClient>>));
199 let blueprint =
200 meerkat::session_runtime::llm_reconfigure::SessionRuntimeLlmReconfigureHostBlueprint::new(
201 builder,
202 store_path.join("session-llm-reconfigure-config-state.json"),
203 Arc::clone(&default_client_slot),
204 );
205 (blueprint, default_client_slot)
206}
207
208impl ReplaySanitizingLlmClient {
209 pub fn new(inner: Arc<dyn LlmClient>) -> Self {
210 Self { inner }
211 }
212
213 pub fn wrap(inner: Arc<dyn LlmClient>) -> Arc<dyn LlmClient> {
214 Arc::new(Self::new(inner))
215 }
216}
217
218struct ProviderAgnosticLlmClient {
240 inner: Arc<dyn LlmClient>,
241}
242
243impl ProviderAgnosticLlmClient {
244 fn wrap(inner: Arc<dyn LlmClient>) -> Arc<dyn LlmClient> {
245 Arc::new(Self { inner })
246 }
247}
248
249#[async_trait]
250impl LlmClient for ProviderAgnosticLlmClient {
251 fn project_replay_messages(
252 &self,
253 messages: &[Message],
254 ) -> Result<Vec<Message>, meerkat_client::LlmError> {
255 self.inner.project_replay_messages(messages)
256 }
257
258 fn stream<'a>(&'a self, request: &'a LlmRequest) -> LlmStream<'a> {
259 self.inner.stream(request)
260 }
261
262 fn provider(&self) -> meerkat_core::Provider {
263 meerkat_core::Provider::Other
264 }
265
266 async fn health_check(&self) -> Result<(), meerkat_client::LlmError> {
267 self.inner.health_check().await
268 }
269
270 fn compile_schema(
271 &self,
272 output_schema: &meerkat_core::OutputSchema,
273 ) -> Result<meerkat_core::schema::CompiledSchema, meerkat_core::schema::SchemaError> {
274 self.inner.compile_schema(output_schema)
275 }
276}
277
278pub struct ReplaySanitizingAgentLlmClient {
286 inner: Arc<dyn meerkat_core::AgentLlmClient>,
287}
288
289impl ReplaySanitizingAgentLlmClient {
290 pub fn new(inner: Arc<dyn meerkat_core::AgentLlmClient>) -> Self {
291 Self { inner }
292 }
293
294 pub fn wrap(
295 inner: Arc<dyn meerkat_core::AgentLlmClient>,
296 ) -> Arc<dyn meerkat_core::AgentLlmClient> {
297 Arc::new(Self::new(inner))
298 }
299}
300
301#[async_trait]
302impl meerkat_core::AgentLlmClient for ReplaySanitizingAgentLlmClient {
303 async fn stream_response(
304 &self,
305 messages: &[Message],
306 tools: &[Arc<meerkat_core::ToolDef>],
307 max_tokens: u32,
308 temperature: Option<f32>,
309 provider_params: Option<&meerkat_core::lifecycle::run_primitive::ProviderParamsOverride>,
310 ) -> Result<meerkat_core::agent::LlmStreamResult, meerkat_core::AgentError> {
311 let sanitized: Vec<Message> = messages
312 .iter()
313 .cloned()
314 .map(sanitize_message_for_stateless_replay)
315 .collect();
316 self.inner
317 .stream_response(&sanitized, tools, max_tokens, temperature, provider_params)
318 .await
319 }
320
321 fn provider(&self) -> meerkat_core::Provider {
322 self.inner.provider()
323 }
324
325 fn model(&self) -> &str {
326 self.inner.model()
327 }
328
329 fn prepare_model_fallback(
330 &self,
331 failure: &meerkat_core::AgentError,
332 ) -> Option<meerkat_core::agent::AgentLlmFallbackSwitch> {
333 self.inner.prepare_model_fallback(failure)
334 }
335
336 fn commit_model_fallback(
337 &self,
338 previous_identity: &meerkat_core::SessionLlmIdentity,
339 target_identity: &meerkat_core::SessionLlmIdentity,
340 ) -> Result<(), meerkat_core::AgentError> {
341 self.inner
342 .commit_model_fallback(previous_identity, target_identity)
343 }
344
345 fn active_model_fallback_identity(&self) -> Option<meerkat_core::SessionLlmIdentity> {
346 self.inner.active_model_fallback_identity()
347 }
348
349 fn compile_model_fallback_schema(
350 &self,
351 target_identity: &meerkat_core::SessionLlmIdentity,
352 output_schema: &meerkat_core::OutputSchema,
353 ) -> Result<meerkat_core::schema::CompiledSchema, meerkat_core::AgentError> {
354 self.inner
355 .compile_model_fallback_schema(target_identity, output_schema)
356 }
357
358 fn begin_stream_output_observation(&self) {
359 self.inner.begin_stream_output_observation();
360 }
361
362 fn stream_output_observed(&self) -> bool {
363 self.inner.stream_output_observed()
364 }
365
366 fn compile_schema(
367 &self,
368 output_schema: &meerkat_core::OutputSchema,
369 ) -> Result<meerkat_core::schema::CompiledSchema, meerkat_core::schema::SchemaError> {
370 self.inner.compile_schema(output_schema)
371 }
372}
373
374#[async_trait]
375impl LlmClient for ReplaySanitizingLlmClient {
376 fn project_replay_messages(
377 &self,
378 messages: &[Message],
379 ) -> Result<Vec<Message>, meerkat_client::LlmError> {
380 let sanitized: Vec<Message> = messages
381 .iter()
382 .cloned()
383 .map(sanitize_message_for_stateless_replay)
384 .collect();
385 self.inner.project_replay_messages(&sanitized)
386 }
387
388 fn stream<'a>(&'a self, request: &'a LlmRequest) -> LlmStream<'a> {
389 let inner = Arc::clone(&self.inner);
390 let sanitized = sanitize_llm_request_for_stateless_replay(request);
391 Box::pin(async_stream::stream! {
392 let mut stream = inner.stream(&sanitized);
393 while let Some(event) = stream.next().await {
394 if runtime_turn_diagnostics_enabled() && event.is_err() {
395 tracing::error!("mobkit llm client stream error");
396 }
397 yield event;
398 }
399 })
400 }
401
402 fn provider(&self) -> meerkat_core::Provider {
403 self.inner.provider()
404 }
405
406 async fn health_check(&self) -> Result<(), meerkat_client::LlmError> {
407 self.inner.health_check().await
408 }
409
410 fn compile_schema(
411 &self,
412 output_schema: &meerkat_core::OutputSchema,
413 ) -> Result<meerkat_core::schema::CompiledSchema, meerkat_core::schema::SchemaError> {
414 self.inner.compile_schema(output_schema)
415 }
416}
417
418pub(crate) type PreBuildHook = Arc<
446 dyn Fn(
447 &mut CreateSessionRequest,
448 ) -> std::pin::Pin<
449 Box<dyn std::future::Future<Output = Result<(), SessionError>> + Send + '_>,
450 > + Send
451 + Sync,
452>;
453
454pub type AfterCreateHook = Arc<
456 dyn Fn(
457 meerkat_core::types::SessionId,
458 SessionCreatedContext,
459 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>>
460 + Send
461 + Sync,
462>;
463
464struct PreBuildMobSessionService {
471 inner: Arc<dyn MobSessionService>,
472 hook: PreBuildHook,
473 dispatch_taint: Option<crate::memory::dispatch_taint::DispatchTaintSlot>,
479 after_create_hook: Option<AfterCreateHook>,
480 runtime_adapter_override: Option<Arc<meerkat_runtime::MeerkatMachine>>,
481 session_read_absorber: Option<Arc<SessionDocumentReadAbsorber>>,
485 archived_terminal_authority: Option<Arc<dyn meerkat_runtime::RuntimeStore>>,
499}
500
501impl PreBuildMobSessionService {
502 fn absorbs_unsupported_boundary_acknowledgement(&self) -> bool {
508 self.runtime_adapter_override.is_some()
509 }
510
511 async fn overlay_runtime_archived_terminal(
518 &self,
519 load: meerkat_mob::ResumeSessionLoad,
520 ) -> Result<meerkat_mob::ResumeSessionLoad, SessionError> {
521 let meerkat_mob::ResumeSessionLoad::Revivable(mut session) = load else {
522 return Ok(load);
523 };
524 if session.lifecycle_terminal().is_some() {
525 return Ok(meerkat_mob::ResumeSessionLoad::Revivable(session));
526 }
527 let Some(store) = self.archived_terminal_authority.as_ref() else {
528 return Ok(meerkat_mob::ResumeSessionLoad::Revivable(session));
529 };
530 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(session.id());
531 let store_error = |detail: String| {
532 SessionError::Agent(meerkat_core::error::AgentError::InternalError(detail))
533 };
534 let archived_by_catalog = store
535 .load_runtime_session_catalog_entry(&runtime_id)
536 .await
537 .map_err(|e| {
538 store_error(format!(
539 "archived-terminal overlay: runtime catalog read for session {}: {e}",
540 session.id()
541 ))
542 })?
543 .is_some_and(|entry| {
544 entry.lifecycle_terminal() == Some(meerkat_core::SessionLifecycleTerminal::Archived)
545 });
546 let archived = archived_by_catalog
547 || matches!(
548 meerkat_runtime::store::load_runtime_state(store.as_ref(), &runtime_id)
549 .await
550 .map_err(|e| {
551 store_error(format!(
552 "archived-terminal overlay: runtime lifecycle read for session {}: {e}",
553 session.id()
554 ))
555 })?,
556 Some(
557 meerkat_runtime::RuntimeState::Retired
558 | meerkat_runtime::RuntimeState::Destroyed
559 )
560 );
561 if archived {
562 session
563 .set_lifecycle_terminal(meerkat_core::SessionLifecycleTerminal::Archived)
564 .map_err(|e| {
565 store_error(format!(
566 "archived-terminal overlay: terminal projection for session {}: {e}",
567 session.id()
568 ))
569 })?;
570 }
571 Ok(meerkat_mob::ResumeSessionLoad::Revivable(session))
572 }
573
574 async fn prepare_create_request(
575 &self,
576 mut req: CreateSessionRequest,
577 ) -> Result<(CreateSessionRequest, SessionCreatedContext), SessionError> {
578 (self.hook)(&mut req).await?;
579 ensure_shell_tooling_build_substrate(&mut req);
580 sanitize_create_session_request_llm_override(&mut req);
581 if let Some(slot) = self.dispatch_taint.as_ref() {
584 crate::memory::dispatch_taint::attach_member_taint_decorator(&mut req, slot);
585 }
586
587 let context = SessionCreatedContext {
588 model: req.model.clone(),
589 labels: req.labels.clone().unwrap_or_default(),
590 system_prompt: req.system_prompt.as_set_prompt().map(ToString::to_string),
591 };
592 Ok((req, context))
593 }
594
595 async fn complete_create(
596 &self,
597 result: meerkat_core::types::RunResult,
598 context: SessionCreatedContext,
599 ) -> meerkat_core::types::RunResult {
600 if let Some(ref after_hook) = self.after_create_hook {
601 after_hook(result.session_id.clone(), context).await;
602 }
603 result
604 }
605
606 async fn load_persisted_session_absorbed(
611 &self,
612 session_id: &meerkat_core::types::SessionId,
613 ) -> Result<Option<meerkat_core::session::Session>, SessionError> {
614 let Some(absorber) = self.session_read_absorber.as_ref() else {
615 return self.inner.load_persisted_session(session_id).await;
616 };
617 if let Some(session) = absorber.lookup(session_id) {
618 return Ok(Some(session));
619 }
620 let epoch = absorber.observe_epoch(session_id);
621 let loaded = self.inner.load_persisted_session(session_id).await?;
622 match loaded.as_ref() {
623 Some(session) => absorber.admit(session_id, epoch, session),
624 None => absorber.evict(session_id),
625 }
626 Ok(loaded)
627 }
628}
629
630fn no_op_pre_build_hook() -> PreBuildHook {
631 Arc::new(|_req: &mut CreateSessionRequest| Box::pin(async { Ok(()) }))
632}
633
634#[derive(Debug, Clone, Copy, PartialEq, Eq)]
635pub(crate) enum DelegateIdleRetireOverride {
636 Disabled,
637 Seconds(u64),
638}
639
640#[derive(Clone, Default)]
641pub(crate) struct ImplicitDelegateRetirementOverrides {
642 inner: Arc<tokio::sync::RwLock<BTreeMap<(String, String), DelegateIdleRetireOverride>>>,
643}
644
645impl ImplicitDelegateRetirementOverrides {
646 pub(crate) async fn set(
647 &self,
648 mob_id: impl Into<String>,
649 member_id: impl Into<String>,
650 override_policy: DelegateIdleRetireOverride,
651 ) {
652 self.inner
653 .write()
654 .await
655 .insert((mob_id.into(), member_id.into()), override_policy);
656 }
657
658 pub(crate) async fn get(
659 &self,
660 mob_id: &str,
661 member_id: &str,
662 ) -> Option<DelegateIdleRetireOverride> {
663 self.inner
664 .read()
665 .await
666 .get(&(mob_id.to_string(), member_id.to_string()))
667 .copied()
668 }
669}
670
671pub(crate) type SharedIdentityRuntimeSlot =
672 Arc<std::sync::RwLock<Option<Arc<crate::identity_first::IdentityRuntime>>>>;
673
674struct AutoWireParentMobToolsFactory {
675 inner: Arc<dyn meerkat_core::service::MobToolsFactory>,
676 implicit_delegate_retirement_overrides: ImplicitDelegateRetirementOverrides,
677 console_spawn_sink: SharedConsoleSpawnSinkSlot,
678 identity_runtime: SharedIdentityRuntimeSlot,
679 protected_mob_id: String,
680}
681
682#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
683#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
684impl meerkat_core::service::MobToolsFactory for AutoWireParentMobToolsFactory {
685 async fn build_mob_tools(
686 &self,
687 args: meerkat_core::service::MobToolsBuildArgs,
688 ) -> Result<Arc<dyn meerkat_core::AgentToolDispatcher>, Box<dyn std::error::Error + Send + Sync>>
689 {
690 let spawner_comms_name = args.comms_name.clone();
691 let inner = self.inner.build_mob_tools(args).await?;
692 Ok(Arc::new(AutoWireParentMobToolDispatcher {
693 inner,
694 implicit_delegate_retirement_overrides: self
695 .implicit_delegate_retirement_overrides
696 .clone(),
697 console_spawn_sink: Arc::clone(&self.console_spawn_sink),
698 identity_runtime: Arc::clone(&self.identity_runtime),
699 protected_mob_id: self.protected_mob_id.clone(),
700 spawner_comms_name,
701 }))
702 }
703}
704
705struct AutoWireParentMobToolDispatcher {
706 inner: Arc<dyn meerkat_core::AgentToolDispatcher>,
707 implicit_delegate_retirement_overrides: ImplicitDelegateRetirementOverrides,
708 console_spawn_sink: SharedConsoleSpawnSinkSlot,
711 identity_runtime: SharedIdentityRuntimeSlot,
714 protected_mob_id: String,
715 spawner_comms_name: Option<String>,
718}
719
720#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
721#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
722impl meerkat_core::AgentToolDispatcher for AutoWireParentMobToolDispatcher {
723 fn tools(&self) -> Arc<[Arc<meerkat_core::types::ToolDef>]> {
724 self.inner
725 .tools()
726 .iter()
727 .map(|tool| {
728 if tool.name == "delegate" {
729 Arc::new(delegate_tool_def_with_idle_retire_secs(tool))
730 } else if tool.name == "mob_spawn_member" {
731 Arc::new(mob_spawn_tool_def_with_idle_retire_secs(tool))
732 } else {
733 Arc::clone(tool)
734 }
735 })
736 .collect::<Vec<_>>()
737 .into()
738 }
739
740 async fn dispatch(
741 &self,
742 call: meerkat_core::types::ToolCallView<'_>,
743 ) -> Result<meerkat_core::ToolDispatchOutcome, meerkat_core::ToolError> {
744 self.dispatch_with_context(call, &meerkat_core::ToolDispatchContext::default())
745 .await
746 }
747
748 async fn dispatch_with_context(
749 &self,
750 call: meerkat_core::types::ToolCallView<'_>,
751 context: &meerkat_core::ToolDispatchContext,
752 ) -> Result<meerkat_core::ToolDispatchOutcome, meerkat_core::ToolError> {
753 if matches!(
754 call.name,
755 "delegate"
756 | "mob_spawn_member"
757 | "mob_retire_member"
758 | "mob_wire"
759 | "mob_unwire"
760 | "mob_destroy"
761 | "spawn_member"
762 | "spawn_many_members"
763 | "retire_member"
764 | "force_cancel_member"
765 | "member_status"
766 | "wire_members"
767 | "unwire_members"
768 ) {
769 let args = serde_json::from_str::<Value>(call.args.get()).map_err(|error| {
770 meerkat_core::ToolError::invalid_arguments(call.name, error.to_string())
771 })?;
772 if let Some((field, alias)) = reserved_raw_member_tool_argument(call.name, &args) {
773 return Err(meerkat_core::ToolError::invalid_arguments(
774 call.name,
775 format!(
776 "{field} '{alias}' uses MobKit's reserved rt:* / mk-- member namespace; use a public non-reserved alias or the IdentityRuntime authority"
777 ),
778 ));
779 }
780 let identity_runtime = self
781 .identity_runtime
782 .read()
783 .unwrap_or_else(std::sync::PoisonError::into_inner)
784 .clone();
785 if let Some(identity_runtime) = identity_runtime {
786 for (field, alias) in raw_member_tool_arguments(call.name, &args) {
787 let alias = crate::member_comms_id::runtime_alias_str(&alias).into_owned();
788 if identity_runtime
789 .identity_for_member_mutation(&alias)
790 .await
791 .is_some()
792 {
793 return Err(meerkat_core::ToolError::invalid_arguments(
794 call.name,
795 format!(
796 "{field} '{alias}' is owned by the attached IdentityRuntime; use the identity lifecycle/topology authority"
797 ),
798 ));
799 }
800 }
801 if call.name == "mob_destroy"
802 && args.get("mob_id").and_then(Value::as_str)
803 == Some(self.protected_mob_id.as_str())
804 {
805 return Err(meerkat_core::ToolError::invalid_arguments(
806 call.name,
807 format!(
808 "mob '{}' is owned by the attached IdentityRuntime and cannot be destroyed through raw agent tools",
809 self.protected_mob_id
810 ),
811 ));
812 }
813 }
814 }
815 if call.name == "delegate" {
816 return self.dispatch_delegate(call, context).await;
817 }
818 if call.name == "mob_spawn_member" {
819 return self.dispatch_mob_spawn_member(call, context).await;
820 }
821 if crate::console_spawn::is_console_spawn_tool(call.name) {
822 let args = serde_json::from_str::<Value>(call.args.get()).ok();
826 let name = call.name.to_string();
827 let outcome = self.inner.dispatch_with_context(call, context).await?;
828 if let Some(args) = args {
829 self.project_spawn_to_console(&name, &args, &outcome).await;
830 }
831 return Ok(outcome);
832 }
833 self.inner.dispatch_with_context(call, context).await
834 }
835
836 fn capabilities(&self) -> meerkat_core::agent::DispatcherCapabilities {
837 self.inner.capabilities()
838 }
839
840 fn bind_ops_lifecycle(
841 self: Arc<Self>,
842 registry: Arc<dyn meerkat_core::ops_lifecycle::OpsLifecycleRegistry>,
843 owner_bridge_session_id: meerkat_core::types::SessionId,
844 ) -> Result<meerkat_core::agent::BindOutcome, meerkat_core::agent::OpsLifecycleBindError> {
845 let owned = Arc::try_unwrap(self)
846 .map_err(|_| meerkat_core::agent::OpsLifecycleBindError::SharedOwnership)?;
847 let outcome = owned
848 .inner
849 .bind_ops_lifecycle(registry, owner_bridge_session_id)?;
850 let was_bound = outcome.was_bound();
851 let dispatcher = Arc::new(Self {
852 inner: outcome.into_dispatcher(),
853 implicit_delegate_retirement_overrides: owned.implicit_delegate_retirement_overrides,
854 console_spawn_sink: owned.console_spawn_sink,
855 identity_runtime: owned.identity_runtime,
856 protected_mob_id: owned.protected_mob_id,
857 spawner_comms_name: owned.spawner_comms_name,
858 });
859 Ok(if was_bound {
860 meerkat_core::agent::BindOutcome::Bound(dispatcher)
861 } else {
862 meerkat_core::agent::BindOutcome::Skipped(dispatcher)
863 })
864 }
865}
866
867fn reserved_raw_member_tool_argument(
872 tool_name: &str,
873 args: &Value,
874) -> Option<(&'static str, String)> {
875 raw_member_tool_arguments(tool_name, args)
876 .into_iter()
877 .find(|(_, value)| {
878 crate::member_comms_id::is_reserved_generated_alias(value)
879 || crate::member_comms_id::uses_reserved_roster_marker(value)
880 })
881 .map(|(field, value)| {
882 (
883 field,
884 crate::member_comms_id::runtime_alias_str(&value).into_owned(),
885 )
886 })
887}
888
889fn raw_member_tool_arguments(tool_name: &str, args: &Value) -> Vec<(&'static str, String)> {
890 let mut candidates = Vec::new();
891 match tool_name {
892 "delegate"
893 | "mob_spawn_member"
894 | "mob_retire_member"
895 | "spawn_member"
896 | "retire_member"
897 | "force_cancel_member"
898 | "member_status" => {
899 candidates.push(("member_id", args.get("member_id").and_then(Value::as_str)));
900 }
901 "spawn_many_members" => {
902 if let Some(specs) = args.get("specs").and_then(Value::as_array) {
903 for spec in specs {
904 candidates.push((
905 "specs[].member_id",
906 spec.get("member_id").and_then(Value::as_str),
907 ));
908 }
909 }
910 }
911 "mob_wire" | "mob_unwire" => {
912 candidates.push(("member_id", args.get("member_id").and_then(Value::as_str)));
913 candidates.push((
914 "peer.local",
915 args.get("peer")
916 .and_then(|peer| peer.get("local"))
917 .and_then(Value::as_str),
918 ));
919 }
920 "wire_members" | "unwire_members" => {
921 candidates.push(("member_id", args.get("member_id").and_then(Value::as_str)));
922 candidates.push((
923 "peer_member_id",
924 args.get("peer_member_id").and_then(Value::as_str),
925 ));
926 }
927 _ => return Vec::new(),
928 }
929 candidates
930 .into_iter()
931 .filter_map(|(field, value)| value.map(|value| (field, value.to_string())))
932 .collect()
933}
934
935impl AutoWireParentMobToolDispatcher {
936 async fn dispatch_mob_spawn_member(
937 &self,
938 call: meerkat_core::types::ToolCallView<'_>,
939 context: &meerkat_core::ToolDispatchContext,
940 ) -> Result<meerkat_core::ToolDispatchOutcome, meerkat_core::ToolError> {
941 let mut args = serde_json::from_str::<Value>(call.args.get()).map_err(|error| {
942 meerkat_core::ToolError::invalid_arguments(call.name, error.to_string())
943 })?;
944 let idle_retire_override = delegate_idle_retire_override_from_args(call.name, &mut args)?;
945 let idle_retire_targets = idle_retire_targets_from_spawn_args(&args);
946 if let Some(object) = args.as_object_mut() {
947 object
948 .entry("auto_wire_parent".to_string())
949 .or_insert(Value::Bool(true));
950 }
951 let name = call.name.to_string();
952 let args_for_console = args.clone();
953 let args = serde_json::value::RawValue::from_string(args.to_string()).map_err(|error| {
954 meerkat_core::ToolError::invalid_arguments(call.name, error.to_string())
955 })?;
956 let call = meerkat_core::types::ToolCallView {
957 id: call.id,
958 name: call.name,
959 args: &args,
960 };
961 let outcome = self.inner.dispatch_with_context(call, context).await?;
962 self.register_idle_retire_override_from_outcome(
963 &outcome,
964 idle_retire_override,
965 &idle_retire_targets,
966 )
967 .await;
968 self.project_spawn_to_console(&name, &args_for_console, &outcome)
969 .await;
970
971 Ok(outcome)
972 }
973
974 async fn dispatch_delegate(
975 &self,
976 call: meerkat_core::types::ToolCallView<'_>,
977 context: &meerkat_core::ToolDispatchContext,
978 ) -> Result<meerkat_core::ToolDispatchOutcome, meerkat_core::ToolError> {
979 let mut args = serde_json::from_str::<Value>(call.args.get()).map_err(|error| {
980 meerkat_core::ToolError::invalid_arguments(call.name, error.to_string())
981 })?;
982 let idle_retire_override = delegate_idle_retire_override_from_args(call.name, &mut args)?;
983 let name = call.name.to_string();
984 let args_for_console = args.clone();
985 let args = serde_json::value::RawValue::from_string(args.to_string()).map_err(|error| {
986 meerkat_core::ToolError::invalid_arguments(call.name, error.to_string())
987 })?;
988 let call = meerkat_core::types::ToolCallView {
989 id: call.id,
990 name: call.name,
991 args: &args,
992 };
993 let outcome = self.inner.dispatch_with_context(call, context).await?;
994
995 self.register_idle_retire_override_from_outcome(&outcome, idle_retire_override, &[])
996 .await;
997 self.project_spawn_to_console(&name, &args_for_console, &outcome)
998 .await;
999
1000 Ok(outcome)
1001 }
1002
1003 async fn project_spawn_to_console(
1008 &self,
1009 tool_name: &str,
1010 args: &Value,
1011 outcome: &meerkat_core::ToolDispatchOutcome,
1012 ) {
1013 if outcome.result.is_error {
1014 return;
1015 }
1016 let sink = self
1017 .console_spawn_sink
1018 .read()
1019 .unwrap_or_else(std::sync::PoisonError::into_inner)
1020 .clone();
1021 let Some(sink) = sink else {
1022 return;
1023 };
1024 let seeds = crate::console_spawn::console_spawn_seeds(
1025 tool_name,
1026 args,
1027 &outcome.result.text_content(),
1028 self.spawner_comms_name.as_deref(),
1029 );
1030 for seed in &seeds {
1031 sink.project_spawned_member(seed).await;
1032 }
1033 }
1034
1035 async fn register_idle_retire_override_from_outcome(
1036 &self,
1037 outcome: &meerkat_core::ToolDispatchOutcome,
1038 idle_retire_override: Option<DelegateIdleRetireOverride>,
1039 fallback_targets: &[IdleRetireTarget],
1040 ) {
1041 if outcome.result.is_error {
1042 return;
1043 }
1044 let Some(override_policy) = idle_retire_override else {
1045 return;
1046 };
1047 for target in
1048 idle_retire_targets_from_outcome_text(&outcome.result.text_content(), fallback_targets)
1049 {
1050 self.implicit_delegate_retirement_overrides
1051 .set(&target.mob_id, &target.member_id, override_policy)
1052 .await;
1053 }
1054 }
1055}
1056
1057#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
1058struct IdleRetireTarget {
1059 mob_id: String,
1060 member_id: String,
1061}
1062
1063fn text_field<'a>(value: &'a Value, key: &str) -> Option<&'a str> {
1064 value
1065 .get(key)
1066 .and_then(Value::as_str)
1067 .filter(|text| !text.is_empty())
1068}
1069
1070fn member_identity_field(value: &Value) -> Option<&str> {
1071 text_field(value, "agent_identity")
1072 .or_else(|| text_field(value, "member_id"))
1073 .or_else(|| text_field(value, "identity"))
1074}
1075
1076fn target_from_value(value: &Value, default_mob_id: Option<&str>) -> Option<IdleRetireTarget> {
1077 let mob_id = text_field(value, "mob_id").or(default_mob_id)?;
1078 let member_id = member_identity_field(value)?;
1079 Some(IdleRetireTarget {
1080 mob_id: mob_id.to_string(),
1081 member_id: member_id.to_string(),
1082 })
1083}
1084
1085fn idle_retire_targets_from_spawn_args(args: &Value) -> Vec<IdleRetireTarget> {
1086 let default_mob_id = text_field(args, "mob_id");
1087 let mut targets = BTreeSet::new();
1088 if let Some(target) = target_from_value(args, default_mob_id) {
1089 targets.insert(target);
1090 }
1091 for key in ["specs", "members"] {
1092 let Some(values) = args.get(key).and_then(Value::as_array) else {
1093 continue;
1094 };
1095 for value in values {
1096 if let Some(target) = target_from_value(value, default_mob_id) {
1097 targets.insert(target);
1098 }
1099 }
1100 }
1101 targets.into_iter().collect()
1102}
1103
1104fn target_from_result_value(
1105 value: &Value,
1106 fallback_targets: &[IdleRetireTarget],
1107) -> Option<IdleRetireTarget> {
1108 if let Some(target) = target_from_value(value, None) {
1109 return Some(target);
1110 }
1111 let member_id = member_identity_field(value)?;
1112 let mut matches = fallback_targets
1113 .iter()
1114 .filter(|target| target.member_id == member_id);
1115 let target = matches.next()?;
1116 if matches.next().is_some() {
1117 return None;
1118 }
1119 Some(target.clone())
1120}
1121
1122fn collect_idle_retire_result_targets(
1123 value: &Value,
1124 fallback_targets: &[IdleRetireTarget],
1125 targets: &mut BTreeSet<IdleRetireTarget>,
1126) {
1127 if let Some(target) = target_from_result_value(value, fallback_targets) {
1128 targets.insert(target);
1129 }
1130 for key in ["members", "specs", "spawned", "results"] {
1131 let Some(values) = value.get(key).and_then(Value::as_array) else {
1132 continue;
1133 };
1134 for value in values {
1135 collect_idle_retire_result_targets(value, fallback_targets, targets);
1136 }
1137 }
1138}
1139
1140fn idle_retire_targets_from_outcome_text(
1141 text: &str,
1142 fallback_targets: &[IdleRetireTarget],
1143) -> Vec<IdleRetireTarget> {
1144 let Ok(payload) = serde_json::from_str::<Value>(text) else {
1145 return fallback_targets.to_vec();
1146 };
1147 let mut targets = BTreeSet::new();
1148 collect_idle_retire_result_targets(&payload, fallback_targets, &mut targets);
1149 if targets.is_empty() {
1150 targets.extend(fallback_targets.iter().cloned());
1151 }
1152 targets.into_iter().collect()
1153}
1154
1155struct DefinitionSeededRealmProfileStore {
1156 inner: Arc<dyn meerkat_mob::RealmProfileStore>,
1157 profiles: BTreeMap<String, Profile>,
1158}
1159
1160impl DefinitionSeededRealmProfileStore {
1161 fn new(
1162 definition: &MobDefinition,
1163 inner: Arc<dyn meerkat_mob::RealmProfileStore>,
1164 ) -> Option<Self> {
1165 let profiles = definition
1166 .profiles
1167 .iter()
1168 .filter_map(|(name, binding)| {
1169 binding
1170 .as_inline()
1171 .cloned()
1172 .map(|profile| (name.to_string(), profile))
1173 })
1174 .collect::<BTreeMap<_, _>>();
1175
1176 (!profiles.is_empty()).then_some(Self { inner, profiles })
1177 }
1178
1179 fn stored(&self, name: &str, profile: &Profile) -> meerkat_mob::StoredRealmProfile {
1180 let now = chrono::Utc::now();
1181 meerkat_mob::StoredRealmProfile {
1182 name: name.to_string(),
1183 profile: profile.clone(),
1184 revision: 0,
1185 created_at: now,
1186 updated_at: now,
1187 }
1188 }
1189}
1190
1191#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
1192#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
1193impl meerkat_mob::RealmProfileStore for DefinitionSeededRealmProfileStore {
1194 async fn create(
1195 &self,
1196 name: &str,
1197 profile: &Profile,
1198 ) -> Result<meerkat_mob::StoredRealmProfile, meerkat_mob::MobStoreError> {
1199 if self.profiles.contains_key(name) {
1200 return Err(meerkat_mob::MobStoreError::CasConflict(format!(
1201 "realm profile already exists: {name}"
1202 )));
1203 }
1204 self.inner.create(name, profile).await
1205 }
1206
1207 async fn get(
1208 &self,
1209 name: &str,
1210 ) -> Result<Option<meerkat_mob::StoredRealmProfile>, meerkat_mob::MobStoreError> {
1211 if let Some(profile) = self.profiles.get(name) {
1212 return Ok(Some(self.stored(name, profile)));
1213 }
1214 self.inner.get(name).await
1215 }
1216
1217 async fn list(
1218 &self,
1219 ) -> Result<Vec<meerkat_mob::StoredRealmProfile>, meerkat_mob::MobStoreError> {
1220 let mut merged = self.inner.list().await?;
1221 merged.retain(|profile| !self.profiles.contains_key(profile.name.as_str()));
1222 merged.extend(
1223 self.profiles
1224 .iter()
1225 .map(|(name, profile)| self.stored(name, profile)),
1226 );
1227 merged.sort_by(|a, b| a.name.cmp(&b.name));
1228 Ok(merged)
1229 }
1230
1231 async fn update(
1232 &self,
1233 name: &str,
1234 profile: &Profile,
1235 expected_revision: u64,
1236 ) -> Result<meerkat_mob::StoredRealmProfile, meerkat_mob::MobStoreError> {
1237 if self.profiles.contains_key(name) {
1238 return Err(meerkat_mob::MobStoreError::CasConflict(format!(
1239 "realm profile '{name}' is provided by the mob definition"
1240 )));
1241 }
1242 self.inner.update(name, profile, expected_revision).await
1243 }
1244
1245 async fn delete(
1246 &self,
1247 name: &str,
1248 expected_revision: u64,
1249 ) -> Result<meerkat_mob::StoredRealmProfile, meerkat_mob::MobStoreError> {
1250 if self.profiles.contains_key(name) {
1251 return Err(meerkat_mob::MobStoreError::CasConflict(format!(
1252 "realm profile '{name}' is provided by the mob definition"
1253 )));
1254 }
1255 self.inner.delete(name, expected_revision).await
1256 }
1257}
1258
1259fn delegate_idle_retire_override_from_args(
1260 tool_name: &str,
1261 args: &mut Value,
1262) -> Result<Option<DelegateIdleRetireOverride>, meerkat_core::ToolError> {
1263 let Some(object) = args.as_object_mut() else {
1264 return Ok(None);
1265 };
1266 let Some(value) = object.remove("idle_retire_secs") else {
1267 return Ok(None);
1268 };
1269 if value.is_null() {
1270 return Ok(Some(DelegateIdleRetireOverride::Disabled));
1271 }
1272 value
1273 .as_u64()
1274 .map(DelegateIdleRetireOverride::Seconds)
1275 .map(Some)
1276 .ok_or_else(|| {
1277 meerkat_core::ToolError::invalid_arguments(
1278 tool_name,
1279 "idle_retire_secs must be a non-negative integer or null",
1280 )
1281 })
1282}
1283
1284fn delegate_tool_def_with_idle_retire_secs(
1285 tool: &meerkat_core::types::ToolDef,
1286) -> meerkat_core::types::ToolDef {
1287 let mut patched = tool.clone();
1288 if !patched.description.contains("IDLE RETIREMENT:") {
1289 patched.description.push_str(
1290 "\n\nIDLE RETIREMENT:\n\
1291 Omit idle_retire_secs to use the runtime default. Pass an integer \
1292 number of seconds to override idle auto-retirement for this helper. \
1293 Pass null to disable auto-retirement for this helper.",
1294 );
1295 }
1296 if let Some(properties) = patched
1297 .input_schema
1298 .get_mut("properties")
1299 .and_then(Value::as_object_mut)
1300 {
1301 properties
1302 .entry("idle_retire_secs".to_string())
1303 .or_insert_with(|| {
1304 serde_json::json!({
1305 "description": "Override idle auto-retirement for this helper. Omit to use the runtime default, use an integer number of seconds to override, or null to disable auto-retirement for this helper.",
1306 "anyOf": [
1307 {"type": "integer", "minimum": 0},
1308 {"type": "null"}
1309 ]
1310 })
1311 });
1312 }
1313 patched
1314}
1315
1316fn mob_spawn_tool_def_with_idle_retire_secs(
1317 tool: &meerkat_core::types::ToolDef,
1318) -> meerkat_core::types::ToolDef {
1319 let mut patched = tool.clone();
1320 if !patched.description.contains("IDLE RETIREMENT:") {
1321 patched.description.push_str(
1322 "\n\nIDLE RETIREMENT:\n\
1323 Omit idle_retire_secs to leave this spawned member out of auto-retirement. \
1324 Pass an integer number of seconds to retire the member after it has been \
1325 idle for that long. Pass null to explicitly disable auto-retirement.",
1326 );
1327 }
1328 if let Some(properties) = patched
1329 .input_schema
1330 .get_mut("properties")
1331 .and_then(Value::as_object_mut)
1332 {
1333 properties
1334 .entry("idle_retire_secs".to_string())
1335 .or_insert_with(|| {
1336 serde_json::json!({
1337 "description": "Opt this spawned member into idle auto-retirement. Omit to keep the member indefinitely, use an integer number of seconds to retire after that much idle time, or null to explicitly disable auto-retirement.",
1338 "anyOf": [
1339 {"type": "integer", "minimum": 0},
1340 {"type": "null"}
1341 ]
1342 })
1343 });
1344 }
1345 patched
1346}
1347
1348fn install_agent_mob_tools(
1349 definition: &MobDefinition,
1350 slot: Arc<std::sync::RwLock<Option<Arc<dyn meerkat_core::service::MobToolsFactory>>>>,
1351 session_service: Arc<dyn MobSessionService>,
1352 workgraph_service: Option<meerkat::WorkGraphService>,
1353 default_llm_client_slot: Option<SharedDefaultLlmClientSlot>,
1354) -> (
1355 Arc<meerkat_mob_mcp::MobMcpState>,
1356 ImplicitDelegateRetirementOverrides,
1357 SharedDefaultLlmClientSlot,
1358 SharedConsoleSpawnSinkSlot,
1359 SharedIdentityRuntimeSlot,
1360) {
1361 let default_llm_client_slot = default_llm_client_slot
1362 .unwrap_or_else(|| Arc::new(std::sync::RwLock::new(None::<Arc<dyn LlmClient>>)));
1363 let default_llm_client_provider_slot = Arc::clone(&default_llm_client_slot);
1364 let mut state =
1367 meerkat_mob_mcp::MobMcpState::new(session_service, meerkat_mob::MobControlPrincipal::Owner)
1368 .with_workgraph_service(workgraph_service);
1369 if let Some(base_store) = state.realm_profile_store().cloned()
1370 && let Some(store) = DefinitionSeededRealmProfileStore::new(definition, base_store)
1371 {
1372 state = state.with_realm_profile_store(Some(Arc::new(store)));
1373 }
1374 state = state
1375 .with_realm_skill_sources(definition.skills.clone())
1376 .with_default_llm_client_provider(Some(Arc::new(move || {
1377 default_llm_client_provider_slot
1378 .read()
1379 .unwrap_or_else(std::sync::PoisonError::into_inner)
1380 .clone()
1381 })));
1382 let state = Arc::new(state);
1383 let implicit_delegate_retirement_overrides = ImplicitDelegateRetirementOverrides::default();
1384 let console_spawn_sink = new_console_spawn_sink_slot();
1385 let identity_runtime = Arc::new(std::sync::RwLock::new(None));
1386 let inner = Arc::new(meerkat_mob_mcp::AgentMobToolSurfaceFactory::new(
1387 Arc::clone(&state),
1388 ));
1389 let factory = Arc::new(AutoWireParentMobToolsFactory {
1390 inner,
1391 implicit_delegate_retirement_overrides: implicit_delegate_retirement_overrides.clone(),
1392 console_spawn_sink: Arc::clone(&console_spawn_sink),
1393 identity_runtime: Arc::clone(&identity_runtime),
1394 protected_mob_id: definition.id.to_string(),
1395 });
1396 *slot
1397 .write()
1398 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(factory);
1399 (
1400 state,
1401 implicit_delegate_retirement_overrides,
1402 default_llm_client_slot,
1403 console_spawn_sink,
1404 identity_runtime,
1405 )
1406}
1407
1408#[cfg(test)]
1409#[allow(dead_code)]
1410#[derive(Debug, Clone)]
1411pub(crate) struct RuntimeTurnTrace {
1412 pub(crate) session_id: String,
1413 pub(crate) boundary: String,
1414 pub(crate) contributing_input_count: usize,
1415 pub(crate) outcome: String,
1416}
1417
1418fn is_replay_unsafe_server_tool_content(name: &str, content: &Value) -> bool {
1419 name == "web_search_annotations"
1420 || content
1421 .get("type")
1422 .and_then(Value::as_str)
1423 .is_some_and(|kind| kind.starts_with("response."))
1424}
1425
1426fn sanitize_llm_request_for_stateless_replay(request: &LlmRequest) -> LlmRequest {
1427 let mut sanitized = request.clone();
1428 sanitized.messages = request
1429 .messages
1430 .iter()
1431 .cloned()
1432 .map(sanitize_message_for_stateless_replay)
1433 .collect();
1434 sanitized
1435}
1436
1437fn sanitize_create_session_request_llm_override(req: &mut CreateSessionRequest) {
1438 let Some(build) = req.build.as_mut() else {
1439 return;
1440 };
1441 let Some(client) = build
1442 .llm_client_override
1443 .as_ref()
1444 .and_then(meerkat::decode_llm_client_override_from_service)
1445 else {
1446 return;
1447 };
1448 build.llm_client_override = Some(meerkat::encode_llm_client_override_for_service(
1449 ReplaySanitizingLlmClient::wrap(client),
1450 ));
1451}
1452
1453const SHELL_BUILTIN_TOOL_NAMES: [&str; 4] = [
1454 "shell",
1455 "shell_job_status",
1456 "shell_jobs",
1457 "shell_job_cancel",
1458];
1459const COMMS_TOOL_NAMES: [&str; 4] = ["peers", "send_message", "send_request", "send_response"];
1460
1461fn shell_and_comms_tool_filter() -> meerkat_core::ToolFilter {
1462 meerkat_core::ToolFilter::Allow(
1463 SHELL_BUILTIN_TOOL_NAMES
1464 .iter()
1465 .chain(COMMS_TOOL_NAMES.iter())
1466 .map(|name| (*name).to_string())
1467 .collect(),
1468 )
1469}
1470
1471pub fn ensure_shell_tooling_build_substrate(req: &mut CreateSessionRequest) {
1477 let Some(build) = req.build.as_mut() else {
1478 return;
1479 };
1480 if matches!(
1481 build.override_shell,
1482 meerkat_core::ToolCategoryOverride::Enable
1483 ) && matches!(
1484 build.override_builtins,
1485 meerkat_core::ToolCategoryOverride::Disable
1486 ) {
1487 build.override_builtins = meerkat_core::ToolCategoryOverride::Enable;
1488 if build.initial_tool_filter.is_none() {
1489 build.initial_tool_filter = Some(shell_and_comms_tool_filter());
1490 }
1491 }
1492}
1493
1494fn sanitize_message_for_stateless_replay(message: Message) -> Message {
1495 match message {
1496 Message::BlockAssistant(mut assistant) => {
1497 assistant.blocks = assistant
1498 .blocks
1499 .into_iter()
1500 .filter_map(|block| match block {
1501 AssistantBlock::ServerToolContent { kind, content, .. }
1504 if is_replay_unsafe_server_tool_content(kind.provider_name(), &content) =>
1505 {
1506 None
1507 }
1508 other => Some(other),
1509 })
1510 .collect();
1511 Message::BlockAssistant(assistant)
1512 }
1513 other => other,
1514 }
1515}
1516
1517fn build_persistent_runtime_store(
1529 store_path: &Path,
1530) -> Result<Arc<dyn meerkat_runtime::RuntimeStore>, RuntimeStoreResolutionError> {
1531 let runtime_db = store_path.join(crate::storage_layout::RUNTIME_DB_FILE_NAME);
1532 match meerkat_runtime::store::SqliteRuntimeStore::new(&runtime_db) {
1533 Ok(store) => Ok(Arc::new(store)),
1534 Err(err) => Err(RuntimeStoreResolutionError {
1535 path: runtime_db,
1536 message: err.to_string(),
1537 }),
1538 }
1539}
1540
1541#[derive(Default)]
1552pub(crate) struct SessionSnapshotWriteEpochs {
1553 epochs: std::sync::Mutex<BTreeMap<String, u64>>,
1554}
1555
1556impl SessionSnapshotWriteEpochs {
1557 fn advance(&self, runtime_id: &meerkat_runtime::LogicalRuntimeId) {
1561 let mut epochs = self
1562 .epochs
1563 .lock()
1564 .unwrap_or_else(std::sync::PoisonError::into_inner);
1565 *epochs.entry(runtime_id.0.clone()).or_insert(0) += 1;
1566 }
1567
1568 fn observe(&self, session_id: &meerkat_core::types::SessionId) -> u64 {
1569 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(session_id);
1570 self.epochs
1571 .lock()
1572 .unwrap_or_else(std::sync::PoisonError::into_inner)
1573 .get(&runtime_id.0)
1574 .copied()
1575 .unwrap_or(0)
1576 }
1577}
1578
1579#[derive(Clone)]
1586pub struct SessionWriteEpochsHandle {
1587 pub(crate) epochs: Arc<SessionSnapshotWriteEpochs>,
1588}
1589
1590pub fn epoch_tracking_runtime_store(
1600 inner: Arc<dyn meerkat_runtime::RuntimeStore>,
1601) -> (
1602 Arc<dyn meerkat_runtime::RuntimeStore>,
1603 SessionWriteEpochsHandle,
1604) {
1605 let epochs = Arc::new(SessionSnapshotWriteEpochs::default());
1606 let store: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
1607 SessionStoreBackedRuntimeStore::with_write_epochs(inner, Arc::clone(&epochs)),
1608 );
1609 (store, SessionWriteEpochsHandle { epochs })
1610}
1611
1612pub fn epoch_tracking_runtime_store_with_durable_projection(
1628 inner: Arc<dyn meerkat_runtime::RuntimeStore>,
1629 session_store: Arc<dyn SessionStore>,
1630) -> (
1631 Arc<dyn meerkat_runtime::RuntimeStore>,
1632 SessionWriteEpochsHandle,
1633) {
1634 let epochs = Arc::new(SessionSnapshotWriteEpochs::default());
1635 let store: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
1636 SessionStoreBackedRuntimeStore::with_write_epochs_and_durable_projection(
1637 inner,
1638 Arc::clone(&epochs),
1639 session_store,
1640 ),
1641 );
1642 (store, SessionWriteEpochsHandle { epochs })
1643}
1644
1645pub(crate) struct SessionDocumentReadAbsorber {
1661 epochs: Arc<SessionSnapshotWriteEpochs>,
1662 cache: std::sync::Mutex<BTreeMap<String, AbsorbedSessionDocument>>,
1664}
1665
1666struct AbsorbedSessionDocument {
1667 epoch: u64,
1668 session: meerkat_core::session::Session,
1669}
1670
1671impl SessionDocumentReadAbsorber {
1672 pub(crate) fn new(epochs: Arc<SessionSnapshotWriteEpochs>) -> Self {
1673 Self {
1674 epochs,
1675 cache: std::sync::Mutex::new(BTreeMap::new()),
1676 }
1677 }
1678
1679 fn observe_epoch(&self, session_id: &meerkat_core::types::SessionId) -> u64 {
1680 self.epochs.observe(session_id)
1681 }
1682
1683 fn lookup(
1686 &self,
1687 session_id: &meerkat_core::types::SessionId,
1688 ) -> Option<meerkat_core::session::Session> {
1689 let current = self.epochs.observe(session_id);
1690 let key = session_id.to_string();
1691 let mut cache = self
1692 .cache
1693 .lock()
1694 .unwrap_or_else(std::sync::PoisonError::into_inner);
1695 match cache.get(&key) {
1696 Some(entry) if entry.epoch == current => Some(entry.session.clone()),
1697 Some(_) => {
1698 cache.remove(&key);
1699 None
1700 }
1701 None => None,
1702 }
1703 }
1704
1705 fn admit(
1709 &self,
1710 session_id: &meerkat_core::types::SessionId,
1711 epoch: u64,
1712 session: &meerkat_core::session::Session,
1713 ) {
1714 self.cache
1715 .lock()
1716 .unwrap_or_else(std::sync::PoisonError::into_inner)
1717 .insert(
1718 session_id.to_string(),
1719 AbsorbedSessionDocument {
1720 epoch,
1721 session: session.clone(),
1722 },
1723 );
1724 }
1725
1726 fn evict(&self, session_id: &meerkat_core::types::SessionId) {
1727 self.cache
1728 .lock()
1729 .unwrap_or_else(std::sync::PoisonError::into_inner)
1730 .remove(&session_id.to_string());
1731 }
1732}
1733
1734struct SessionStoreBackedRuntimeStore {
1748 inner: Arc<dyn meerkat_runtime::RuntimeStore>,
1749 write_epochs: Option<Arc<SessionSnapshotWriteEpochs>>,
1750 session_store: Option<Arc<dyn SessionStore>>,
1767 mint_flights: std::sync::Mutex<
1777 std::collections::HashMap<String, std::sync::Weak<tokio::sync::Mutex<()>>>,
1778 >,
1779 projection_flights: std::sync::Mutex<
1786 std::collections::HashMap<String, std::sync::Weak<tokio::sync::Mutex<()>>>,
1787 >,
1788 freshened: std::sync::Mutex<std::collections::HashSet<String>>,
1795}
1796
1797impl SessionStoreBackedRuntimeStore {
1798 fn new(
1799 inner: Arc<dyn meerkat_runtime::RuntimeStore>,
1800 session_store: Arc<dyn SessionStore>,
1801 ) -> Self {
1802 Self {
1803 inner,
1804 write_epochs: None,
1805 session_store: Some(session_store),
1806 mint_flights: std::sync::Mutex::new(std::collections::HashMap::new()),
1807 projection_flights: std::sync::Mutex::new(std::collections::HashMap::new()),
1808 freshened: std::sync::Mutex::new(std::collections::HashSet::new()),
1809 }
1810 }
1811
1812 fn with_write_epochs(
1813 inner: Arc<dyn meerkat_runtime::RuntimeStore>,
1814 write_epochs: Arc<SessionSnapshotWriteEpochs>,
1815 ) -> Self {
1816 Self {
1817 inner,
1818 write_epochs: Some(write_epochs),
1819 session_store: None,
1820 mint_flights: std::sync::Mutex::new(std::collections::HashMap::new()),
1821 projection_flights: std::sync::Mutex::new(std::collections::HashMap::new()),
1822 freshened: std::sync::Mutex::new(std::collections::HashSet::new()),
1823 }
1824 }
1825
1826 fn with_write_epochs_and_durable_projection(
1830 inner: Arc<dyn meerkat_runtime::RuntimeStore>,
1831 write_epochs: Arc<SessionSnapshotWriteEpochs>,
1832 session_store: Arc<dyn SessionStore>,
1833 ) -> Self {
1834 Self {
1835 inner,
1836 write_epochs: Some(write_epochs),
1837 session_store: Some(session_store),
1838 mint_flights: std::sync::Mutex::new(std::collections::HashMap::new()),
1839 projection_flights: std::sync::Mutex::new(std::collections::HashMap::new()),
1840 freshened: std::sync::Mutex::new(std::collections::HashSet::new()),
1841 }
1842 }
1843
1844 async fn mint_runtime_authority_from_durable(
1866 &self,
1867 runtime_id: &meerkat_runtime::LogicalRuntimeId,
1868 ) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
1869 let Some(session_store) = self.session_store.as_ref() else {
1870 return Ok(false);
1871 };
1872 if self.inner.session_persistence_profile()
1876 != meerkat_runtime::store::RuntimeSessionPersistenceProfile::WholeBlobV1
1877 {
1878 return Ok(false);
1879 }
1880 let raw = runtime_id.0.as_str();
1881 let candidate = raw.strip_prefix("rt:session:").unwrap_or(raw);
1882 let Ok(session_id) = meerkat_core::types::SessionId::parse(candidate) else {
1883 return Ok(false);
1884 };
1885 let flight = self.mint_flight_for(runtime_id);
1891 let _flight = flight.lock().await;
1892 if self
1893 .inner
1894 .load_whole_blob_store_authority(runtime_id)
1895 .await?
1896 .is_some()
1897 {
1898 return Ok(true);
1899 }
1900 if self
1906 .inner
1907 .load_runtime_session_catalog_entry(runtime_id)
1908 .await?
1909 .is_some()
1910 {
1911 return Ok(false);
1912 }
1913 let session = session_store.load(&session_id).await.map_err(|e| {
1914 meerkat_runtime::store::RuntimeStoreError::ReadFailed(format!(
1915 "durable session read for runtime-authority mint: {e}"
1916 ))
1917 })?;
1918 let Some(mut session) = session else {
1919 return Ok(false);
1920 };
1921 if let Some(incremental) = Arc::clone(session_store).as_incremental() {
1941 let records = match incremental.load_rewrites(&session_id).await {
1942 Ok(records) => records,
1943 Err(e) => {
1944 tracing::warn!(
1945 runtime_id = %runtime_id,
1946 session_id = %session_id,
1947 error = %e,
1948 "runtime-authority mint could not reconstruct durable rewrite \
1949 records; seeding slim (legacy/import shapes take the adoption lane)"
1950 );
1951 Vec::new()
1952 }
1953 };
1954 if !records.is_empty() {
1955 let validated =
1956 meerkat_core::ValidatedTranscriptHistory::from_rewrite_records_with_proved(
1957 records, None,
1958 )
1959 .map_err(|e| {
1960 meerkat_runtime::store::RuntimeStoreError::ReadFailed(format!(
1961 "rewrite-graph proof for runtime-authority mint: {e}"
1962 ))
1963 })?;
1964 if let Some(validated) = validated {
1965 session
1966 .install_validated_audited_transcript_history_preserving_live(validated)
1967 .map_err(|e| {
1968 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
1969 "rewrite-graph install for runtime-authority mint: {e}"
1970 ))
1971 })?;
1972 tracing::info!(
1973 runtime_id = %runtime_id,
1974 session_id = %session_id,
1975 "runtime-authority mint hydrated the rewrite graph from durable records"
1976 );
1977 }
1978 }
1979 }
1980 let bytes = session.to_persisted_bytes().map_err(|e| {
1981 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
1982 "durable session encode for runtime-authority mint: {e}"
1983 ))
1984 })?;
1985 self.note_session_scoped_write(runtime_id);
1986 let result = self
1987 .inner
1988 .commit_session_snapshot(
1989 runtime_id,
1990 meerkat_runtime::store::SerializedSessionSnapshot {
1991 session_snapshot: Arc::new(bytes),
1992 },
1993 )
1994 .await;
1995 self.note_session_scoped_write(runtime_id);
1996 result?;
1997 tracing::info!(
1998 runtime_id = %runtime_id,
1999 session_id = %session_id,
2000 "runtime authority re-minted from durable session facts \
2001 (ephemeral runtime-store activation)"
2002 );
2003 Ok(true)
2004 }
2005
2006 async fn freshen_stale_runtime_authority_from_durable(
2044 &self,
2045 runtime_id: &meerkat_runtime::LogicalRuntimeId,
2046 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
2047 let Some(session_store) = self.session_store.as_ref() else {
2048 return Ok(());
2049 };
2050 if self.inner.session_persistence_profile()
2051 != meerkat_runtime::store::RuntimeSessionPersistenceProfile::WholeBlobV1
2052 {
2053 return Ok(());
2054 }
2055 if self
2056 .freshened
2057 .lock()
2058 .unwrap_or_else(std::sync::PoisonError::into_inner)
2059 .contains(runtime_id.0.as_str())
2060 {
2061 return Ok(());
2062 }
2063 let raw = runtime_id.0.as_str();
2064 let candidate = raw.strip_prefix("rt:session:").unwrap_or(raw);
2065 let Ok(session_id) = meerkat_core::types::SessionId::parse(candidate) else {
2066 return Ok(());
2067 };
2068 let flight = self.mint_flight_for(runtime_id);
2069 let _flight = flight.lock().await;
2070 let mark_fresh = || {
2071 self.freshened
2072 .lock()
2073 .unwrap_or_else(std::sync::PoisonError::into_inner)
2074 .insert(runtime_id.0.clone());
2075 };
2076 if self
2077 .freshened
2078 .lock()
2079 .unwrap_or_else(std::sync::PoisonError::into_inner)
2080 .contains(runtime_id.0.as_str())
2081 {
2082 return Ok(());
2083 }
2084 let lifecycle_terminal = self
2085 .inner
2086 .load_runtime_session_catalog_entry(runtime_id)
2087 .await?
2088 .is_some_and(|entry| entry.lifecycle_terminal().is_some());
2089 let Some(committed) = self
2090 .inner
2091 .load_committed_whole_blob_snapshot(runtime_id)
2092 .await?
2093 else {
2094 mark_fresh();
2097 return Ok(());
2098 };
2099 let durable = session_store.load(&session_id).await.map_err(|e| {
2100 meerkat_runtime::store::RuntimeStoreError::ReadFailed(format!(
2101 "durable session read for runtime-authority freshness probe: {e}"
2102 ))
2103 })?;
2104 let Some(durable) = durable else {
2105 self.project_committed_session_to_durable(runtime_id)
2113 .await?;
2114 mark_fresh();
2115 return Ok(());
2116 };
2117 let order_of = |session: &meerkat_core::Session| {
2118 session
2119 .transcript_rewrite_generation()
2120 .map(|generation| (generation, session.messages().len()))
2121 .map_err(|e| {
2122 meerkat_runtime::store::RuntimeStoreError::ReadFailed(format!(
2123 "transcript rewrite generation for runtime-authority freshness probe: {e}"
2124 ))
2125 })
2126 };
2127 let durable_order = order_of(&durable)?;
2128 let committed_order = order_of(committed.session())?;
2129 if durable_order == committed_order {
2130 let durable_revision = durable.transcript_revision().map_err(|e| {
2141 meerkat_runtime::store::RuntimeStoreError::ReadFailed(format!(
2142 "durable transcript revision for runtime-authority freshness probe: {e}"
2143 ))
2144 })?;
2145 let committed_revision = committed.session().transcript_revision().map_err(|e| {
2146 meerkat_runtime::store::RuntimeStoreError::ReadFailed(format!(
2147 "committed transcript revision for runtime-authority freshness \
2148 probe: {e}"
2149 ))
2150 })?;
2151 if durable_revision != committed_revision {
2152 return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
2153 format!(
2154 "durable session row for runtime {runtime_id} matches the committed \
2155 runtime authority on (rewrite generation {}, message count {}) but \
2156 DIVERGES in content (durable revision {durable_revision}, committed \
2157 revision {committed_revision}): a fork between lineages; refusing \
2158 to adopt either side",
2159 durable_order.0, durable_order.1
2160 ),
2161 ));
2162 }
2163 let durable_bytes = durable.to_persisted_bytes().map_err(|e| {
2175 meerkat_runtime::store::RuntimeStoreError::ReadFailed(format!(
2176 "durable session encode for envelope-currency probe: {e}"
2177 ))
2178 })?;
2179 let committed_bytes = committed.session().to_persisted_bytes().map_err(|e| {
2180 meerkat_runtime::store::RuntimeStoreError::ReadFailed(format!(
2181 "committed session encode for envelope-currency probe: {e}"
2182 ))
2183 })?;
2184 if durable_bytes != committed_bytes {
2185 self.project_committed_session_to_durable(runtime_id)
2186 .await?;
2187 }
2188 mark_fresh();
2189 return Ok(());
2190 }
2191 if durable_order < committed_order {
2192 tracing::info!(
2207 runtime_id = %runtime_id,
2208 session_id = %session_id,
2209 durable_rewrite_generation = durable_order.0,
2210 durable_message_count = durable_order.1,
2211 committed_rewrite_generation = committed_order.0,
2212 committed_message_count = committed_order.1,
2213 "durable row orders behind committed runtime authority; \
2214 running the committed->durable reconciliation"
2215 );
2216 self.project_committed_session_to_durable(runtime_id)
2217 .await?;
2218 mark_fresh();
2219 return Ok(());
2220 }
2221 if lifecycle_terminal {
2223 mark_fresh();
2227 return Ok(());
2228 }
2229 meerkat_core::session_store::run_boundary_snapshot_save_guard(
2230 &durable,
2231 Some(committed.session()),
2232 )
2233 .map_err(|e| {
2234 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2235 "durable session row for runtime {runtime_id} orders ahead of the committed \
2236 runtime authority but does not extend it; refusing to adopt either side: {e}"
2237 ))
2238 })?;
2239 let bytes = durable.to_persisted_bytes().map_err(|e| {
2240 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2241 "durable session encode for runtime-authority freshen: {e}"
2242 ))
2243 })?;
2244 self.note_session_scoped_write(runtime_id);
2245 let result = self
2246 .inner
2247 .commit_session_snapshot(
2248 runtime_id,
2249 meerkat_runtime::store::SerializedSessionSnapshot {
2250 session_snapshot: Arc::new(bytes),
2251 },
2252 )
2253 .await;
2254 self.note_session_scoped_write(runtime_id);
2255 result?;
2256 tracing::warn!(
2257 runtime_id = %runtime_id,
2258 session_id = %session_id,
2259 stale_rewrite_generation = committed_order.0,
2260 stale_message_count = committed_order.1,
2261 durable_rewrite_generation = durable_order.0,
2262 durable_message_count = durable_order.1,
2263 "stale committed runtime authority re-seeded from the durable session row \
2264 (runtime store rollback/restore detected)"
2265 );
2266 mark_fresh();
2267 Ok(())
2268 }
2269
2270 fn note_session_scoped_write(&self, runtime_id: &meerkat_runtime::LogicalRuntimeId) {
2276 if let Some(epochs) = self.write_epochs.as_ref() {
2277 epochs.advance(runtime_id);
2278 }
2279 }
2280
2281 fn mint_flight_for(
2285 &self,
2286 runtime_id: &meerkat_runtime::LogicalRuntimeId,
2287 ) -> Arc<tokio::sync::Mutex<()>> {
2288 let mut flights = self
2289 .mint_flights
2290 .lock()
2291 .unwrap_or_else(std::sync::PoisonError::into_inner);
2292 if let Some(existing) = flights
2293 .get(runtime_id.0.as_str())
2294 .and_then(std::sync::Weak::upgrade)
2295 {
2296 return existing;
2297 }
2298 flights.retain(|_, flight| flight.strong_count() > 0);
2299 let fresh = Arc::new(tokio::sync::Mutex::new(()));
2300 flights.insert(runtime_id.0.clone(), Arc::downgrade(&fresh));
2301 fresh
2302 }
2303
2304 async fn project_durable_to_exact_rewrite_parent(
2320 &self,
2321 session_store: &Arc<dyn SessionStore>,
2322 successor: &meerkat_core::Session,
2323 sealed: &meerkat_core::ValidatedTranscriptHistory,
2324 commit: &meerkat_core::TranscriptRewriteCommit,
2325 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
2326 let parent_session = successor
2327 .with_validated_transcript_rewrite_parent_projection(sealed, commit)
2328 .map_err(|e| {
2329 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2330 "exact rewrite-parent projection at generation {}: {e}",
2331 commit.rewrite_generation
2332 ))
2333 })?;
2334 session_store
2335 .save_authoritative_projection(&parent_session)
2336 .await
2337 .map_err(|e| {
2338 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2339 "exact rewrite-parent save at generation {}: {e}",
2340 commit.rewrite_generation
2341 ))
2342 })
2343 }
2344
2345 fn inverse_append_replay_chain<'a>(
2364 sealed: &'a meerkat_core::ValidatedTranscriptHistory,
2365 durable_predecessor: &meerkat_core::Session,
2366 ) -> Result<
2367 Option<Vec<&'a meerkat_core::TranscriptRewriteCommit>>,
2368 meerkat_runtime::store::RuntimeStoreError,
2369 > {
2370 let durable_messages = durable_predecessor.messages();
2371 let commits: Vec<_> = sealed.state().commits().collect();
2372 for (index, commit) in commits.iter().enumerate() {
2373 if commit.messages_before > durable_messages.len() {
2374 continue;
2375 }
2376 let prefix_digest = meerkat_core::transcript_messages_digest(
2377 &durable_messages[..commit.messages_before],
2378 )
2379 .map_err(|e| {
2380 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2381 "durable prefix digest for inverse-append admission at generation {}: {e}",
2382 commit.rewrite_generation
2383 ))
2384 })?;
2385 let matched = prefix_digest == commit.parent_revision;
2389 tracing::info!(
2390 session_id = %durable_predecessor.id(),
2391 rewrite_generation = commit.rewrite_generation,
2392 messages_before = commit.messages_before,
2393 durable_messages = durable_messages.len(),
2394 %prefix_digest,
2395 parent_revision = %commit.parent_revision,
2396 matched,
2397 "inverse-append proof verdict"
2398 );
2399 if matched {
2400 return Ok(Some(commits[index..].to_vec()));
2401 }
2402 }
2403 Ok(None)
2404 }
2405
2406 fn durable_behind_prefix_chain<'a>(
2421 successor: &meerkat_core::Session,
2422 sealed: &'a meerkat_core::ValidatedTranscriptHistory,
2423 durable_predecessor: &meerkat_core::Session,
2424 ) -> Result<
2425 Option<Vec<&'a meerkat_core::TranscriptRewriteCommit>>,
2426 meerkat_runtime::store::RuntimeStoreError,
2427 > {
2428 let durable_messages = durable_predecessor.messages();
2429 let durable_digest =
2430 meerkat_core::transcript_messages_digest(durable_messages).map_err(|e| {
2431 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2432 "durable digest for durable-behind admission: {e}"
2433 ))
2434 })?;
2435 let commits: Vec<_> = sealed.state().commits().collect();
2436 for (index, commit) in commits.iter().enumerate() {
2437 if commit.messages_before < durable_messages.len() {
2438 continue;
2439 }
2440 let parent_session = successor
2441 .with_validated_transcript_rewrite_parent_projection(sealed, commit)
2442 .map_err(|e| {
2443 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2444 "parent projection for durable-behind admission at generation {}: {e}",
2445 commit.rewrite_generation
2446 ))
2447 })?;
2448 let parent_messages = parent_session.messages();
2449 if parent_messages.len() < durable_messages.len() {
2450 continue;
2451 }
2452 let parent_prefix_digest = meerkat_core::transcript_messages_digest(
2453 &parent_messages[..durable_messages.len()],
2454 )
2455 .map_err(|e| {
2456 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2457 "parent prefix digest for durable-behind admission at generation {}: {e}",
2458 commit.rewrite_generation
2459 ))
2460 })?;
2461 let matched = parent_prefix_digest == durable_digest;
2462 tracing::info!(
2463 session_id = %durable_predecessor.id(),
2464 rewrite_generation = commit.rewrite_generation,
2465 messages_before = commit.messages_before,
2466 durable_messages = durable_messages.len(),
2467 parent_messages = parent_messages.len(),
2468 %parent_prefix_digest,
2469 %durable_digest,
2470 matched,
2471 "durable-behind proof verdict"
2472 );
2473 if matched {
2474 return Ok(Some(commits[index..].to_vec()));
2475 }
2476 }
2477 Ok(None)
2478 }
2479
2480 fn log_no_admission_diagnostic(
2486 sealed: &meerkat_core::ValidatedTranscriptHistory,
2487 durable_predecessor: &meerkat_core::Session,
2488 successor: &meerkat_core::Session,
2489 ) {
2490 if let (Ok(durable_revision), Ok(successor_revision)) = (
2496 durable_predecessor.transcript_revision(),
2497 successor.transcript_revision(),
2498 ) && durable_revision == successor_revision
2499 {
2500 tracing::debug!(
2501 session_id = %durable_predecessor.id(),
2502 "no repair admission holds because the durable row is already \
2503 converged with the committed authority; nothing to repair"
2504 );
2505 return;
2506 }
2507 let durable_messages = durable_predecessor.messages();
2508 let commits: Vec<_> = sealed.state().commits().collect();
2509 let Some(first) = commits.first() else {
2510 return;
2511 };
2512 let tested_len = first.messages_before.min(durable_messages.len());
2513 let (parent_len, first_divergence) =
2514 match successor.with_validated_transcript_rewrite_parent_projection(sealed, first) {
2515 Ok(parent) => {
2516 let parent_messages = parent.messages();
2517 let divergence = parent_messages
2518 .iter()
2519 .zip(durable_messages.iter())
2520 .position(|(parent_message, durable_message)| {
2521 serde_json::to_vec(parent_message).ok()
2522 != serde_json::to_vec(durable_message).ok()
2523 })
2524 .map_or_else(
2525 || format!("<none within first {tested_len} messages>"),
2526 |index| index.to_string(),
2527 );
2528 (parent_messages.len().to_string(), divergence)
2529 }
2530 Err(e) => (
2531 format!("<parent projection failed: {e}>"),
2532 "<unavailable>".to_string(),
2533 ),
2534 };
2535 tracing::warn!(
2536 session_id = %durable_predecessor.id(),
2537 rewrite_generation = first.rewrite_generation,
2538 messages_before = first.messages_before,
2539 durable_messages = durable_messages.len(),
2540 parent_len = %parent_len,
2541 parent_revision = %first.parent_revision,
2542 first_divergence = %first_divergence,
2543 "no repair admission holds: the durable row is neither a proven \
2544 append-extension nor a proven prefix of the sealed parent"
2545 );
2546 }
2547
2548 fn projection_flight_for(
2551 &self,
2552 runtime_id: &meerkat_runtime::LogicalRuntimeId,
2553 ) -> Arc<tokio::sync::Mutex<()>> {
2554 let mut flights = self
2555 .projection_flights
2556 .lock()
2557 .unwrap_or_else(std::sync::PoisonError::into_inner);
2558 if let Some(existing) = flights
2559 .get(runtime_id.0.as_str())
2560 .and_then(std::sync::Weak::upgrade)
2561 {
2562 return existing;
2563 }
2564 flights.retain(|_, flight| flight.strong_count() > 0);
2565 let fresh = Arc::new(tokio::sync::Mutex::new(()));
2566 flights.insert(runtime_id.0.clone(), Arc::downgrade(&fresh));
2567 fresh
2568 }
2569
2570 async fn project_committed_session_to_durable(
2607 &self,
2608 runtime_id: &meerkat_runtime::LogicalRuntimeId,
2609 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
2610 let Some(session_store) = self.session_store.as_ref() else {
2611 return Ok(());
2612 };
2613 if self.inner.session_persistence_profile()
2617 != meerkat_runtime::store::RuntimeSessionPersistenceProfile::WholeBlobV1
2618 {
2619 return Ok(());
2620 }
2621 let flight = self.projection_flight_for(runtime_id);
2628 let _flight = flight.lock().await;
2629 let Some(snapshot) = self
2630 .inner
2631 .load_committed_whole_blob_snapshot(runtime_id)
2632 .await?
2633 else {
2634 return Ok(());
2637 };
2638 let successor = snapshot.session();
2639 let durable_predecessor = session_store.load(successor.id()).await.map_err(|e| {
2651 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2652 "durable predecessor read before boundary projection: {e}"
2653 ))
2654 })?;
2655 let mut preserved_suffix: Vec<meerkat_core::Message> = Vec::new();
2660 if let Some(durable_predecessor) = durable_predecessor.as_ref() {
2661 let sealed = successor
2662 .validated_transcript_history_state()
2663 .map_err(|e| {
2664 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2665 "committed WholeBlob transcript-history seal: {e}"
2666 ))
2667 })?;
2668 if let Some(sealed) = sealed.as_ref()
2669 && sealed.commit_count() != 0
2670 {
2671 let missing_commits =
2672 meerkat_core::session_store::find_transcript_rewrite_commit_chain_extending_session(
2673 sealed,
2674 durable_predecessor,
2675 sealed.state().head(),
2676 )
2677 .map_err(|e| {
2678 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2679 "rewrite chain walk against durable predecessor: {e}"
2680 ))
2681 })?;
2682 tracing::info!(
2687 runtime_id = %runtime_id,
2688 session_id = %successor.id(),
2689 walk = match missing_commits.as_ref() {
2690 None => "none".to_string(),
2691 Some(commits) => commits.len().to_string(),
2692 },
2693 durable_revision = %durable_predecessor
2694 .transcript_revision()
2695 .unwrap_or_else(|_| "<unreadable>".to_string()),
2696 sealed_head_revision = %sealed.state().head(),
2697 sealed_commits = sealed.commit_count(),
2698 durable_messages = durable_predecessor.messages().len(),
2699 committed_messages = successor.messages().len(),
2700 "rewrite-suffix walk against the durable predecessor"
2701 );
2702 let missing_commits = match missing_commits {
2720 Some(chain) => Some(chain),
2721 None => {
2722 let inverse =
2723 Self::inverse_append_replay_chain(sealed, durable_predecessor)?;
2724 if let Some(chain) = inverse.as_ref()
2725 && let Some(first) = chain.first()
2726 {
2727 let suffix =
2728 durable_predecessor.messages()[first.messages_before..].to_vec();
2729 let lifecycle_terminal = self
2730 .inner
2731 .load_runtime_session_catalog_entry(runtime_id)
2732 .await?
2733 .is_some_and(|entry| entry.lifecycle_terminal().is_some());
2734 tracing::info!(
2735 runtime_id = %runtime_id,
2736 session_id = %successor.id(),
2737 replay_from_generation = first.rewrite_generation,
2738 suffix_rows = suffix.len(),
2739 lifecycle_terminal,
2740 "inverse-append admission: the durable row extends the \
2741 sealed commit parent; replaying the committed rewrites"
2742 );
2743 if lifecycle_terminal {
2744 tracing::warn!(
2750 runtime_id = %runtime_id,
2751 session_id = %successor.id(),
2752 dropped_suffix_rows = suffix.len(),
2753 "inverse-append repair under a lifecycle terminal \
2754 converges to the committed authority; the durable \
2755 suffix beyond the sealed parent is dropped"
2756 );
2757 } else {
2758 preserved_suffix = suffix;
2759 }
2760 }
2761 match inverse {
2762 Some(chain) => Some(chain),
2763 None => {
2764 let behind = Self::durable_behind_prefix_chain(
2765 successor,
2766 sealed,
2767 durable_predecessor,
2768 )?;
2769 match behind {
2770 Some(chain) => {
2771 tracing::info!(
2772 runtime_id = %runtime_id,
2773 session_id = %successor.id(),
2774 "durable-behind admission: the durable row is a \
2775 digest-prefix of the sealed commit parent (failed \
2776 projection tear); replaying the committed rewrites"
2777 );
2778 Some(chain)
2779 }
2780 None => {
2781 Self::log_no_admission_diagnostic(
2782 sealed,
2783 durable_predecessor,
2784 successor,
2785 );
2786 None
2787 }
2788 }
2789 }
2790 }
2791 }
2792 };
2793 if let Some(missing_commits) = missing_commits {
2794 let mut durable_head_revision =
2811 durable_predecessor.transcript_revision().map_err(|e| {
2812 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2813 "durable predecessor revision before rewrite replay: {e}"
2814 ))
2815 })?;
2816 for commit in missing_commits {
2817 let exact_parent_projected =
2818 durable_head_revision != commit.parent_revision;
2819 if exact_parent_projected {
2820 self.project_durable_to_exact_rewrite_parent(
2821 session_store,
2822 successor,
2823 sealed,
2824 commit,
2825 )
2826 .await?;
2827 durable_head_revision.clone_from(&commit.parent_revision);
2828 }
2829 tracing::info!(
2830 runtime_id = %runtime_id,
2831 session_id = %successor.id(),
2832 rewrite_generation = commit.rewrite_generation,
2833 exact_parent_projected,
2834 "replaying missing rewrite commit into the durable row"
2835 );
2836 let projected_history =
2837 sealed.project_at_rewrite_commit(commit).map_err(|e| {
2838 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2839 "rewrite replay projection at generation {}: {e}",
2840 commit.rewrite_generation
2841 ))
2842 })?;
2843 let prefix_session = successor
2844 .with_validated_transcript_history_projection(projected_history)
2845 .map_err(|e| {
2846 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2847 "rewrite replay prefix session at generation {}: {e}",
2848 commit.rewrite_generation
2849 ))
2850 })?;
2851 session_store
2852 .save_transcript_rewrite(&prefix_session, commit)
2853 .await
2854 .map_err(|e| {
2855 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2856 "rewrite replay save at generation {}: {e}",
2857 commit.rewrite_generation
2858 ))
2859 })?;
2860 durable_head_revision = commit.revision.clone();
2861 }
2862 }
2863 }
2864 }
2865 let rebased = if preserved_suffix.is_empty() {
2871 None
2872 } else {
2873 let mut rebased = successor.clone();
2874 for message in preserved_suffix {
2875 rebased.push(message);
2876 }
2877 Some(rebased)
2878 };
2879 let projected_document: &meerkat_core::Session = rebased.as_ref().unwrap_or(successor);
2880 session_store
2881 .save_authoritative_projection(projected_document)
2882 .await
2883 .map_err(|e| {
2884 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2885 "durable session projection after runtime boundary commit: {e}"
2886 ))
2887 })?;
2888 if let Some(rebased) = rebased.as_ref() {
2889 let bytes = rebased.to_persisted_bytes().map_err(|e| {
2896 meerkat_runtime::store::RuntimeStoreError::WriteFailed(format!(
2897 "rebased session encode after inverse-append repair: {e}"
2898 ))
2899 })?;
2900 self.note_session_scoped_write(runtime_id);
2901 let result = self
2902 .inner
2903 .commit_session_snapshot(
2904 runtime_id,
2905 meerkat_runtime::store::SerializedSessionSnapshot {
2906 session_snapshot: Arc::new(bytes),
2907 },
2908 )
2909 .await;
2910 self.note_session_scoped_write(runtime_id);
2911 result?;
2912 tracing::info!(
2913 runtime_id = %runtime_id,
2914 session_id = %projected_document.id(),
2915 "inverse-append repair converged: committed runtime snapshot \
2916 re-seeded at the rebased state (compacted head plus preserved \
2917 suffix)"
2918 );
2919 }
2920 Ok(())
2921 }
2922}
2923
2924#[async_trait]
2925impl meerkat_runtime::RuntimeStore for SessionStoreBackedRuntimeStore {
2926 fn session_authority_ops(&self) -> &dyn meerkat_runtime::store::RuntimeSessionAuthorityOps {
2931 self.inner.session_authority_ops()
2932 }
2933
2934 fn session_persistence_profile(
2935 &self,
2936 ) -> meerkat_runtime::store::RuntimeSessionPersistenceProfile {
2937 self.inner.session_persistence_profile()
2938 }
2939
2940 fn session_boundary_authority_read_cost(
2941 &self,
2942 ) -> meerkat_runtime::store::RuntimeSessionAuthorityReadCost {
2943 self.inner.session_boundary_authority_read_cost()
2944 }
2945
2946 fn auth_authority_key(&self) -> Option<String> {
2947 self.inner.auth_authority_key()
2948 }
2949
2950 fn persist_auth_oauth_flow_snapshot(
2951 &self,
2952 snapshot_json: &[u8],
2953 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
2954 self.inner.persist_auth_oauth_flow_snapshot(snapshot_json)
2955 }
2956
2957 fn load_auth_oauth_flow_snapshot(
2958 &self,
2959 ) -> Result<Option<Vec<u8>>, meerkat_runtime::store::RuntimeStoreError> {
2960 self.inner.load_auth_oauth_flow_snapshot()
2961 }
2962
2963 fn update_auth_oauth_flow_snapshot(
2964 &self,
2965 update: &mut meerkat_runtime::store::AuthOAuthFlowSnapshotUpdate<'_>,
2966 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
2967 self.inner.update_auth_oauth_flow_snapshot(update)
2968 }
2969
2970 async fn commit_session_snapshot(
2971 &self,
2972 runtime_id: &meerkat_runtime::LogicalRuntimeId,
2973 session_delta: meerkat_runtime::store::SerializedSessionSnapshot,
2974 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
2975 self.note_session_scoped_write(runtime_id);
2976 let result = self
2977 .inner
2978 .commit_session_snapshot(runtime_id, session_delta)
2979 .await;
2980 self.note_session_scoped_write(runtime_id);
2981 result?;
2982 self.project_committed_session_to_durable(runtime_id).await
2983 }
2984
2985 async fn commit_prepared_session_boundary(
2986 &self,
2987 runtime_id: &meerkat_runtime::LogicalRuntimeId,
2988 request: meerkat_runtime::store::PreparedRuntimeSessionCommit,
2989 ) -> Result<
2990 meerkat_runtime::store::PreparedRuntimeSessionCommitResult,
2991 meerkat_runtime::store::RuntimeStoreError,
2992 > {
2993 self.note_session_scoped_write(runtime_id);
2994 let result = self
2995 .inner
2996 .commit_prepared_session_boundary(runtime_id, request)
2997 .await;
2998 self.note_session_scoped_write(runtime_id);
2999 let result = result?;
3000 self.project_committed_session_to_durable(runtime_id)
3001 .await?;
3002 Ok(result)
3003 }
3004
3005 async fn load_session_boundary_authority(
3006 &self,
3007 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3008 ) -> Result<
3009 Option<meerkat_runtime::store::RuntimeSessionAuthority>,
3010 meerkat_runtime::store::RuntimeStoreError,
3011 > {
3012 self.freshen_stale_runtime_authority_from_durable(runtime_id)
3013 .await?;
3014 if let Some(authority) = self
3015 .inner
3016 .load_session_boundary_authority(runtime_id)
3017 .await?
3018 {
3019 return Ok(Some(authority));
3020 }
3021 if !self.mint_runtime_authority_from_durable(runtime_id).await? {
3022 return Ok(None);
3023 }
3024 self.inner.load_session_boundary_authority(runtime_id).await
3025 }
3026
3027 async fn load_whole_blob_store_authority(
3028 &self,
3029 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3030 ) -> Result<
3031 Option<meerkat_runtime::store::WholeBlobStoreAuthority>,
3032 meerkat_runtime::store::RuntimeStoreError,
3033 > {
3034 self.freshen_stale_runtime_authority_from_durable(runtime_id)
3035 .await?;
3036 if let Some(authority) = self
3037 .inner
3038 .load_whole_blob_store_authority(runtime_id)
3039 .await?
3040 {
3041 return Ok(Some(authority));
3042 }
3043 if !self.mint_runtime_authority_from_durable(runtime_id).await? {
3044 return Ok(None);
3045 }
3046 self.inner.load_whole_blob_store_authority(runtime_id).await
3047 }
3048
3049 async fn load_committed_whole_blob_snapshot(
3050 &self,
3051 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3052 ) -> Result<
3053 Option<meerkat_runtime::store::CommittedWholeBlobSnapshot>,
3054 meerkat_runtime::store::RuntimeStoreError,
3055 > {
3056 self.freshen_stale_runtime_authority_from_durable(runtime_id)
3057 .await?;
3058 if let Some(snapshot) = self
3059 .inner
3060 .load_committed_whole_blob_snapshot(runtime_id)
3061 .await?
3062 {
3063 return Ok(Some(snapshot));
3064 }
3065 if !self.mint_runtime_authority_from_durable(runtime_id).await? {
3066 return Ok(None);
3067 }
3068 self.inner
3069 .load_committed_whole_blob_snapshot(runtime_id)
3070 .await
3071 }
3072
3073 async fn commit_prepared_whole_blob_snapshot_cas(
3074 &self,
3075 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3076 prepared: meerkat_runtime::store::PreparedWholeBlobSnapshotCas,
3077 ) -> Result<
3078 meerkat_runtime::store::WholeBlobSnapshotCasOutcome,
3079 meerkat_runtime::store::RuntimeStoreError,
3080 > {
3081 self.note_session_scoped_write(runtime_id);
3082 let result = self
3083 .inner
3084 .commit_prepared_whole_blob_snapshot_cas(runtime_id, prepared)
3085 .await;
3086 self.note_session_scoped_write(runtime_id);
3087 let outcome = result?;
3088 if matches!(
3091 outcome,
3092 meerkat_runtime::store::WholeBlobSnapshotCasOutcome::Committed(_)
3093 ) {
3094 self.project_committed_session_to_durable(runtime_id)
3095 .await?;
3096 }
3097 Ok(outcome)
3098 }
3099
3100 async fn commit_prepared_whole_blob_rewrite_boundary(
3101 &self,
3102 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3103 boundary: meerkat_runtime::store::PreparedWholeBlobRewriteStoreParts,
3104 ) -> Result<
3105 meerkat_runtime::store::WholeBlobStoreAuthority,
3106 meerkat_runtime::store::RuntimeStoreError,
3107 > {
3108 self.note_session_scoped_write(runtime_id);
3109 let result = self
3110 .inner
3111 .commit_prepared_whole_blob_rewrite_boundary(runtime_id, boundary)
3112 .await;
3113 self.note_session_scoped_write(runtime_id);
3114 let authority = result?;
3115 self.project_committed_session_to_durable(runtime_id)
3116 .await?;
3117 Ok(authority)
3118 }
3119
3120 async fn delete_runtime_session_catalog_entry(
3121 &self,
3122 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3123 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
3124 self.note_session_scoped_write(runtime_id);
3125 let result = self
3126 .inner
3127 .delete_runtime_session_catalog_entry(runtime_id)
3128 .await;
3129 self.note_session_scoped_write(runtime_id);
3130 result
3131 }
3132
3133 async fn load_runtime_session_catalog_entry(
3134 &self,
3135 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3136 ) -> Result<
3137 Option<meerkat_runtime::store::RuntimeSessionCatalogEntry>,
3138 meerkat_runtime::store::RuntimeStoreError,
3139 > {
3140 self.inner
3141 .load_runtime_session_catalog_entry(runtime_id)
3142 .await
3143 }
3144
3145 async fn list_runtime_session_catalog_entries(
3146 &self,
3147 filter: meerkat_core::SessionFilter,
3148 ) -> Result<
3149 Vec<meerkat_runtime::store::RuntimeSessionCatalogEntry>,
3150 meerkat_runtime::store::RuntimeStoreError,
3151 > {
3152 self.inner
3153 .list_runtime_session_catalog_entries(filter)
3154 .await
3155 }
3156
3157 async fn write_prepared_whole_blob_provisional_tail(
3158 &self,
3159 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3160 prepared: meerkat_runtime::store::PreparedWholeBlobProvisionalTail,
3161 ) -> Result<
3162 meerkat_runtime::store::WholeBlobProvisionalTailAuthority,
3163 meerkat_runtime::store::RuntimeStoreError,
3164 > {
3165 self.note_session_scoped_write(runtime_id);
3166 let result = self
3167 .inner
3168 .write_prepared_whole_blob_provisional_tail(runtime_id, prepared)
3169 .await;
3170 self.note_session_scoped_write(runtime_id);
3171 result
3172 }
3173
3174 async fn load_whole_blob_provisional_tail(
3175 &self,
3176 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3177 ) -> Result<
3178 Option<meerkat_runtime::store::CommittedWholeBlobProvisionalTail>,
3179 meerkat_runtime::store::RuntimeStoreError,
3180 > {
3181 self.inner
3182 .load_whole_blob_provisional_tail(runtime_id)
3183 .await
3184 }
3185
3186 async fn discard_whole_blob_provisional_tail(
3187 &self,
3188 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3189 expected: &meerkat_runtime::store::WholeBlobProvisionalTailAuthority,
3190 ) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
3191 self.note_session_scoped_write(runtime_id);
3192 let result = self
3193 .inner
3194 .discard_whole_blob_provisional_tail(runtime_id, expected)
3195 .await;
3196 self.note_session_scoped_write(runtime_id);
3197 result
3198 }
3199
3200 async fn write_prepared_head_canonical_provisional_tail(
3201 &self,
3202 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3203 prepared: meerkat_runtime::store::PreparedHeadCanonicalProvisionalTail,
3204 ) -> Result<
3205 meerkat_runtime::store::HeadCanonicalProvisionalTailAuthority,
3206 meerkat_runtime::store::RuntimeStoreError,
3207 > {
3208 self.note_session_scoped_write(runtime_id);
3209 let result = self
3210 .inner
3211 .write_prepared_head_canonical_provisional_tail(runtime_id, prepared)
3212 .await;
3213 self.note_session_scoped_write(runtime_id);
3214 result
3215 }
3216
3217 async fn load_head_canonical_provisional_tail(
3218 &self,
3219 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3220 ) -> Result<
3221 Option<meerkat_runtime::store::HeadCanonicalProvisionalTailAuthority>,
3222 meerkat_runtime::store::RuntimeStoreError,
3223 > {
3224 self.inner
3225 .load_head_canonical_provisional_tail(runtime_id)
3226 .await
3227 }
3228
3229 async fn discard_head_canonical_provisional_tail(
3230 &self,
3231 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3232 expected: &meerkat_runtime::store::HeadCanonicalProvisionalTailAuthority,
3233 ) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
3234 self.note_session_scoped_write(runtime_id);
3235 let result = self
3236 .inner
3237 .discard_head_canonical_provisional_tail(runtime_id, expected)
3238 .await;
3239 self.note_session_scoped_write(runtime_id);
3240 result
3241 }
3242
3243 async fn load_durable_tail_recovery_source(
3244 &self,
3245 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3246 ) -> Result<
3247 Option<meerkat_runtime::store::PreparedDurableTailRecoverySource>,
3248 meerkat_runtime::store::RuntimeStoreError,
3249 > {
3250 self.inner
3251 .load_durable_tail_recovery_source(runtime_id)
3252 .await
3253 }
3254
3255 async fn load_durable_tail_recovery_receipts(
3256 &self,
3257 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3258 run_id: &meerkat_core::lifecycle::RunId,
3259 ) -> Result<
3260 Vec<meerkat_runtime::store::PreparedRecoveryReceiptSource>,
3261 meerkat_runtime::store::RuntimeStoreError,
3262 > {
3263 self.inner
3264 .load_durable_tail_recovery_receipts(runtime_id, run_id)
3265 .await
3266 }
3267
3268 async fn load_committed_recovery_boundary(
3269 &self,
3270 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3271 candidate_id: &str,
3272 ) -> Result<
3273 Option<meerkat_runtime::store::CommittedRecoveryBoundary>,
3274 meerkat_runtime::store::RuntimeStoreError,
3275 > {
3276 self.inner
3277 .load_committed_recovery_boundary(runtime_id, candidate_id)
3278 .await
3279 }
3280
3281 async fn load_runtime_delivery_authority(
3282 &self,
3283 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3284 ) -> Result<
3285 Option<meerkat_runtime::store::RuntimeDeliveryAuthorityRecord>,
3286 meerkat_runtime::store::RuntimeStoreError,
3287 > {
3288 self.inner.load_runtime_delivery_authority(runtime_id).await
3289 }
3290
3291 async fn load_runtime_delivery_record(
3292 &self,
3293 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3294 delivery_id: &str,
3295 ) -> Result<
3296 Option<meerkat_runtime::store::RuntimeDeliveryStoreRecord>,
3297 meerkat_runtime::store::RuntimeStoreError,
3298 > {
3299 self.inner
3300 .load_runtime_delivery_record(runtime_id, delivery_id)
3301 .await
3302 }
3303
3304 async fn compare_and_swap_runtime_delivery_authority(
3305 &self,
3306 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3307 expected_revision: Option<u64>,
3308 replacement: meerkat_runtime::store::RuntimeDeliveryAuthorityRecord,
3309 inserted_delivery: Option<meerkat_runtime::store::RuntimeDeliveryStoreRecord>,
3310 ) -> Result<
3311 meerkat_runtime::store::RuntimeDeliveryAuthorityCasOutcome,
3312 meerkat_runtime::store::RuntimeStoreError,
3313 > {
3314 self.inner
3315 .compare_and_swap_runtime_delivery_authority(
3316 runtime_id,
3317 expected_revision,
3318 replacement,
3319 inserted_delivery,
3320 )
3321 .await
3322 }
3323
3324 async fn list_runtime_delivery_records(
3325 &self,
3326 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3327 after_sequence: u64,
3328 limit: usize,
3329 ) -> Result<
3330 Vec<meerkat_runtime::store::RuntimeDeliveryStoreRecord>,
3331 meerkat_runtime::store::RuntimeStoreError,
3332 > {
3333 self.inner
3334 .list_runtime_delivery_records(runtime_id, after_sequence, limit)
3335 .await
3336 }
3337
3338 async fn atomic_apply(
3339 &self,
3340 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3341 session_delta: Option<meerkat_runtime::store::SerializedSessionSnapshot>,
3342 receipt: meerkat_core::lifecycle::RunBoundaryReceipt,
3343 input_updates: Vec<InputStatePersistenceRecord>,
3344 session_store_key: Option<meerkat_core::types::SessionId>,
3345 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
3346 let committed_snapshot = session_delta.is_some();
3347 self.note_session_scoped_write(runtime_id);
3348 let result = self
3349 .inner
3350 .atomic_apply(
3351 runtime_id,
3352 session_delta,
3353 receipt,
3354 input_updates,
3355 session_store_key,
3356 )
3357 .await;
3358 self.note_session_scoped_write(runtime_id);
3359 result?;
3360 if committed_snapshot {
3361 self.project_committed_session_to_durable(runtime_id)
3362 .await?;
3363 }
3364 Ok(())
3365 }
3366
3367 async fn atomic_apply_with_machine_lifecycle(
3368 &self,
3369 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3370 session_delta: meerkat_runtime::store::SerializedSessionSnapshot,
3371 receipt: meerkat_core::lifecycle::RunBoundaryReceipt,
3372 machine_lifecycle: MachineLifecycleCommit,
3373 input_updates: Vec<InputStatePersistenceRecord>,
3374 session_store_key: meerkat_core::types::SessionId,
3375 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
3376 self.note_session_scoped_write(runtime_id);
3377 let result = self
3378 .inner
3379 .atomic_apply_with_machine_lifecycle(
3380 runtime_id,
3381 session_delta,
3382 receipt,
3383 machine_lifecycle,
3384 input_updates,
3385 session_store_key,
3386 )
3387 .await;
3388 self.note_session_scoped_write(runtime_id);
3389 result?;
3390 self.project_committed_session_to_durable(runtime_id).await
3391 }
3392
3393 async fn load_input_states(
3394 &self,
3395 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3396 ) -> Result<Vec<meerkat_runtime::InputStateRow>, meerkat_runtime::store::RuntimeStoreError>
3397 {
3398 self.inner.load_input_states(runtime_id).await
3403 }
3404
3405 async fn load_input_states_strict(
3406 &self,
3407 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3408 ) -> Result<
3409 Vec<meerkat_runtime::input_state::StoredInputState>,
3410 meerkat_runtime::store::RuntimeStoreError,
3411 > {
3412 self.inner.load_input_states_strict(runtime_id).await
3413 }
3414
3415 async fn load_input_states_with_versions(
3416 &self,
3417 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3418 ) -> Result<
3419 meerkat_runtime::store::PreparedRecoveryInputSnapshot,
3420 meerkat_runtime::store::RuntimeStoreError,
3421 > {
3422 self.inner.load_input_states_with_versions(runtime_id).await
3423 }
3424
3425 async fn load_input_state_by_idempotency_key(
3426 &self,
3427 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3428 key: &meerkat_runtime::IdempotencyKey,
3429 ) -> Result<
3430 Option<meerkat_runtime::store::ExactInputStateObservation>,
3431 meerkat_runtime::store::RuntimeStoreError,
3432 > {
3433 self.inner
3434 .load_input_state_by_idempotency_key(runtime_id, key)
3435 .await
3436 }
3437
3438 async fn load_input_states_by_ids(
3439 &self,
3440 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3441 input_ids: &[meerkat_core::lifecycle::InputId],
3442 ) -> Result<
3443 Vec<Option<meerkat_runtime::input_state::StoredInputState>>,
3444 meerkat_runtime::store::RuntimeStoreError,
3445 > {
3446 self.inner
3447 .load_input_states_by_ids(runtime_id, input_ids)
3448 .await
3449 }
3450
3451 async fn load_pending_terminal_owner_ids_page(
3452 &self,
3453 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3454 after: Option<&meerkat_core::lifecycle::InputId>,
3455 limit: usize,
3456 ) -> Result<Vec<meerkat_core::lifecycle::InputId>, meerkat_runtime::store::RuntimeStoreError>
3457 {
3458 self.inner
3459 .load_pending_terminal_owner_ids_page(runtime_id, after, limit)
3460 .await
3461 }
3462
3463 fn input_state_batch_cas_implementation_profile(
3464 &self,
3465 ) -> meerkat_runtime::store::InputStateBatchCasImplementationProfile {
3466 self.inner.input_state_batch_cas_implementation_profile()
3467 }
3468
3469 async fn compare_and_swap_recovery_input_states_atomically(
3470 &self,
3471 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3472 expected_revision: meerkat_runtime::store::RecoveryInputSetRevision,
3473 mutations: &[meerkat_runtime::store::RecoveryInputStateMutation],
3474 ) -> Result<
3475 meerkat_runtime::store::InputStateBatchCasOutcome,
3476 meerkat_runtime::store::RuntimeStoreError,
3477 > {
3478 self.inner
3479 .compare_and_swap_recovery_input_states_atomically(
3480 runtime_id,
3481 expected_revision,
3482 mutations,
3483 )
3484 .await
3485 }
3486
3487 async fn compare_and_swap_recovery_input_states_atomically_with_fence(
3488 &self,
3489 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3490 expected_revision: meerkat_runtime::store::RecoveryInputSetRevision,
3491 mutations: &[meerkat_runtime::store::RecoveryInputStateMutation],
3492 write_fence: Arc<dyn meerkat_runtime::store::RuntimeStoreWriteFence>,
3493 ) -> Result<
3494 meerkat_runtime::store::FencedInputStateBatchCasOutcome,
3495 meerkat_runtime::store::RuntimeStoreError,
3496 > {
3497 self.inner
3498 .compare_and_swap_recovery_input_states_atomically_with_fence(
3499 runtime_id,
3500 expected_revision,
3501 mutations,
3502 write_fence,
3503 )
3504 .await
3505 }
3506
3507 async fn load_committed_boundary_receipts(
3508 &self,
3509 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3510 run_id: &meerkat_core::lifecycle::RunId,
3511 ) -> Result<
3512 Vec<meerkat_core::lifecycle::RunBoundaryReceipt>,
3513 meerkat_runtime::store::RuntimeStoreError,
3514 > {
3515 self.inner
3516 .load_committed_boundary_receipts(runtime_id, run_id)
3517 .await
3518 }
3519
3520 async fn load_boundary_receipt(
3521 &self,
3522 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3523 run_id: &meerkat_core::lifecycle::RunId,
3524 sequence: u64,
3525 ) -> Result<
3526 Option<meerkat_core::lifecycle::RunBoundaryReceipt>,
3527 meerkat_runtime::store::RuntimeStoreError,
3528 > {
3529 self.inner
3530 .load_boundary_receipt(runtime_id, run_id, sequence)
3531 .await
3532 }
3533
3534 async fn load_session_snapshot(
3535 &self,
3536 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3537 ) -> Result<Option<Arc<Vec<u8>>>, meerkat_runtime::store::RuntimeStoreError> {
3538 self.inner.load_session_snapshot(runtime_id).await
3539 }
3540
3541 async fn clear_session_snapshot(
3542 &self,
3543 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3544 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
3545 self.note_session_scoped_write(runtime_id);
3546 let result = self.inner.clear_session_snapshot(runtime_id).await;
3547 self.note_session_scoped_write(runtime_id);
3548 result
3549 }
3550
3551 async fn replace_session_snapshot_if_current(
3552 &self,
3553 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3554 expected_current: &[u8],
3555 replacement: Vec<u8>,
3556 ) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
3557 self.note_session_scoped_write(runtime_id);
3558 let result = self
3559 .inner
3560 .replace_session_snapshot_if_current(runtime_id, expected_current, replacement)
3561 .await;
3562 self.note_session_scoped_write(runtime_id);
3563 let replaced = result?;
3564 if replaced {
3565 self.project_committed_session_to_durable(runtime_id)
3566 .await?;
3567 }
3568 Ok(replaced)
3569 }
3570
3571 async fn clear_session_snapshot_if_current(
3572 &self,
3573 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3574 expected_current: &[u8],
3575 ) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
3576 self.note_session_scoped_write(runtime_id);
3577 let result = self
3578 .inner
3579 .clear_session_snapshot_if_current(runtime_id, expected_current)
3580 .await;
3581 self.note_session_scoped_write(runtime_id);
3582 result
3583 }
3584
3585 async fn persist_input_state(
3586 &self,
3587 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3588 state: &InputStatePersistenceRecord,
3589 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
3590 self.note_session_scoped_write(runtime_id);
3591 let result = self.inner.persist_input_state(runtime_id, state).await;
3592 self.note_session_scoped_write(runtime_id);
3593 result
3594 }
3595
3596 async fn persist_input_states_atomically(
3597 &self,
3598 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3599 states: &[InputStatePersistenceRecord],
3600 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
3601 self.note_session_scoped_write(runtime_id);
3602 let result = self
3603 .inner
3604 .persist_input_states_atomically(runtime_id, states)
3605 .await;
3606 self.note_session_scoped_write(runtime_id);
3607 result
3608 }
3609
3610 async fn compare_and_swap_input_states_atomically(
3611 &self,
3612 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3613 expected: &[StoredInputState],
3614 replacements: &[InputStatePersistenceRecord],
3615 ) -> Result<
3616 meerkat_runtime::store::InputStateBatchCasOutcome,
3617 meerkat_runtime::store::RuntimeStoreError,
3618 > {
3619 self.note_session_scoped_write(runtime_id);
3620 let result = self
3621 .inner
3622 .compare_and_swap_input_states_atomically(runtime_id, expected, replacements)
3623 .await;
3624 self.note_session_scoped_write(runtime_id);
3625 result
3626 }
3627
3628 async fn compare_and_swap_input_states_atomically_with_fence(
3629 &self,
3630 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3631 expected: &[StoredInputState],
3632 replacements: &[InputStatePersistenceRecord],
3633 write_fence: Arc<dyn meerkat_runtime::store::RuntimeStoreWriteFence>,
3634 ) -> Result<
3635 meerkat_runtime::store::FencedInputStateBatchCasOutcome,
3636 meerkat_runtime::store::RuntimeStoreError,
3637 > {
3638 self.note_session_scoped_write(runtime_id);
3639 let result = self
3640 .inner
3641 .compare_and_swap_input_states_atomically_with_fence(
3642 runtime_id,
3643 expected,
3644 replacements,
3645 write_fence,
3646 )
3647 .await;
3648 self.note_session_scoped_write(runtime_id);
3649 result
3650 }
3651
3652 async fn load_input_state(
3653 &self,
3654 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3655 input_id: &meerkat_core::lifecycle::InputId,
3656 ) -> Result<Option<StoredInputState>, meerkat_runtime::store::RuntimeStoreError> {
3657 self.inner.load_input_state(runtime_id, input_id).await
3658 }
3659
3660 async fn observe_machine_lifecycle(
3661 &self,
3662 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3663 ) -> Result<
3664 meerkat_runtime::store::MachineLifecycleObservation,
3665 meerkat_runtime::store::RuntimeStoreError,
3666 > {
3667 self.inner.observe_machine_lifecycle(runtime_id).await
3668 }
3669
3670 async fn compare_and_swap_machine_lifecycle(
3671 &self,
3672 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3673 expected: meerkat_runtime::store::MachineLifecycleExpectedVersion,
3674 replacement: MachineLifecycleCommit,
3675 ) -> Result<
3676 meerkat_runtime::store::MachineLifecycleCasOutcome,
3677 meerkat_runtime::store::RuntimeStoreError,
3678 > {
3679 self.note_session_scoped_write(runtime_id);
3680 let result = self
3681 .inner
3682 .compare_and_swap_machine_lifecycle(runtime_id, expected, replacement)
3683 .await;
3684 self.note_session_scoped_write(runtime_id);
3685 result
3686 }
3687
3688 async fn compare_and_swap_machine_lifecycle_with_fence(
3689 &self,
3690 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3691 expected: meerkat_runtime::store::MachineLifecycleExpectedVersion,
3692 replacement: MachineLifecycleCommit,
3693 write_fence: Arc<dyn meerkat_runtime::store::RuntimeStoreWriteFence>,
3694 ) -> Result<
3695 meerkat_runtime::store::FencedMachineLifecycleCasOutcome,
3696 meerkat_runtime::store::RuntimeStoreError,
3697 > {
3698 self.note_session_scoped_write(runtime_id);
3699 let result = self
3700 .inner
3701 .compare_and_swap_machine_lifecycle_with_fence(
3702 runtime_id,
3703 expected,
3704 replacement,
3705 write_fence,
3706 )
3707 .await;
3708 self.note_session_scoped_write(runtime_id);
3709 result
3710 }
3711
3712 async fn load_machine_lifecycle_record(
3713 &self,
3714 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3715 ) -> Result<Option<Vec<u8>>, meerkat_runtime::store::RuntimeStoreError> {
3716 self.inner.load_machine_lifecycle_record(runtime_id).await
3717 }
3718
3719 async fn commit_machine_lifecycle(
3720 &self,
3721 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3722 commit: MachineLifecycleCommit,
3723 input_states: &[InputStatePersistenceRecord],
3724 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
3725 self.note_session_scoped_write(runtime_id);
3726 let result = self
3727 .inner
3728 .commit_machine_lifecycle(runtime_id, commit, input_states)
3729 .await;
3730 self.note_session_scoped_write(runtime_id);
3731 result
3732 }
3733
3734 async fn commit_unregister_finalization(
3735 &self,
3736 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3737 finalization: meerkat_runtime::store::UnregisterFinalizationCommit,
3738 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
3739 self.note_session_scoped_write(runtime_id);
3740 let result = self
3741 .inner
3742 .commit_unregister_finalization(runtime_id, finalization)
3743 .await;
3744 self.note_session_scoped_write(runtime_id);
3745 result
3746 }
3747
3748 fn supports_compaction_projection_outbox(&self) -> bool {
3753 self.inner.supports_compaction_projection_outbox()
3754 }
3755
3756 async fn load_pending_compaction_projections(
3757 &self,
3758 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3759 ) -> Result<
3760 Vec<meerkat_core::CompactionProjectionIntent>,
3761 meerkat_runtime::store::RuntimeStoreError,
3762 > {
3763 self.inner
3764 .load_pending_compaction_projections(runtime_id)
3765 .await
3766 }
3767
3768 async fn mark_compaction_projection_finalized(
3769 &self,
3770 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3771 projection: &meerkat_core::CompactionProjectionId,
3772 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
3773 self.note_session_scoped_write(runtime_id);
3774 let result = self
3775 .inner
3776 .mark_compaction_projection_finalized(runtime_id, projection)
3777 .await;
3778 self.note_session_scoped_write(runtime_id);
3779 result
3780 }
3781
3782 async fn is_runtime_projection_quarantined(
3783 &self,
3784 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3785 ) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
3786 self.inner
3787 .is_runtime_projection_quarantined(runtime_id)
3788 .await
3789 }
3790
3791 async fn delete_ops_lifecycle(
3792 &self,
3793 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3794 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
3795 self.note_session_scoped_write(runtime_id);
3796 let result = self.inner.delete_ops_lifecycle(runtime_id).await;
3797 self.note_session_scoped_write(runtime_id);
3798 result
3799 }
3800
3801 async fn initialize_ops_lifecycle_if_absent(
3802 &self,
3803 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3804 candidate: &meerkat_runtime::ops_lifecycle::PersistedOpsSnapshot,
3805 ) -> Result<
3806 meerkat_runtime::ops_lifecycle::PersistedOpsSnapshot,
3807 meerkat_runtime::store::RuntimeStoreError,
3808 > {
3809 self.note_session_scoped_write(runtime_id);
3810 let result = self
3811 .inner
3812 .initialize_ops_lifecycle_if_absent(runtime_id, candidate)
3813 .await;
3814 self.note_session_scoped_write(runtime_id);
3815 result
3816 }
3817
3818 async fn persist_ops_lifecycle(
3819 &self,
3820 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3821 snapshot: &meerkat_runtime::ops_lifecycle::PersistedOpsSnapshot,
3822 ) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
3823 self.note_session_scoped_write(runtime_id);
3824 let result = self.inner.persist_ops_lifecycle(runtime_id, snapshot).await;
3825 self.note_session_scoped_write(runtime_id);
3826 result
3827 }
3828
3829 async fn load_ops_lifecycle(
3830 &self,
3831 runtime_id: &meerkat_runtime::LogicalRuntimeId,
3832 ) -> Result<
3833 Option<meerkat_runtime::ops_lifecycle::PersistedOpsSnapshot>,
3834 meerkat_runtime::store::RuntimeStoreError,
3835 > {
3836 self.inner.load_ops_lifecycle(runtime_id).await
3837 }
3838
3839 async fn load_mob_host_binding(
3840 &self,
3841 mob_id: &str,
3842 ) -> Result<Option<Vec<u8>>, meerkat_runtime::store::RuntimeStoreError> {
3843 self.inner.load_mob_host_binding(mob_id).await
3844 }
3845
3846 async fn list_mob_host_bindings(
3847 &self,
3848 ) -> Result<Vec<(String, Vec<u8>)>, meerkat_runtime::store::RuntimeStoreError> {
3849 self.inner.list_mob_host_bindings().await
3850 }
3851
3852 async fn put_mob_host_binding_if_absent(
3853 &self,
3854 mob_id: &str,
3855 record_json: &[u8],
3856 ) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
3857 self.inner
3858 .put_mob_host_binding_if_absent(mob_id, record_json)
3859 .await
3860 }
3861
3862 async fn compare_and_put_mob_host_binding(
3863 &self,
3864 mob_id: &str,
3865 expected_json: &[u8],
3866 next_json: &[u8],
3867 ) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
3868 self.inner
3869 .compare_and_put_mob_host_binding(mob_id, expected_json, next_json)
3870 .await
3871 }
3872
3873 async fn delete_mob_host_binding(
3874 &self,
3875 mob_id: &str,
3876 expected_json: &[u8],
3877 ) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
3878 self.inner
3879 .delete_mob_host_binding(mob_id, expected_json)
3880 .await
3881 }
3882
3883 async fn load_mob_host_revocation(
3884 &self,
3885 mob_id: &str,
3886 ) -> Result<Option<Vec<u8>>, meerkat_runtime::store::RuntimeStoreError> {
3887 self.inner.load_mob_host_revocation(mob_id).await
3888 }
3889
3890 async fn list_mob_host_revocations(
3891 &self,
3892 ) -> Result<Vec<(String, Vec<u8>)>, meerkat_runtime::store::RuntimeStoreError> {
3893 self.inner.list_mob_host_revocations().await
3894 }
3895
3896 async fn revoke_mob_host_binding(
3897 &self,
3898 mob_id: &str,
3899 expected_binding_json: &[u8],
3900 receipt_json: &[u8],
3901 ) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
3902 self.inner
3903 .revoke_mob_host_binding(mob_id, expected_binding_json, receipt_json)
3904 .await
3905 }
3906}
3907
3908#[cfg(test)]
3909static RUNTIME_TURN_TRACES: OnceLock<Mutex<Vec<RuntimeTurnTrace>>> = OnceLock::new();
3910
3911#[cfg(test)]
3912fn runtime_turn_traces() -> &'static Mutex<Vec<RuntimeTurnTrace>> {
3913 RUNTIME_TURN_TRACES.get_or_init(|| Mutex::new(Vec::new()))
3914}
3915
3916#[cfg(test)]
3917#[allow(clippy::expect_used)]
3918fn record_runtime_turn_trace(trace: RuntimeTurnTrace) {
3919 runtime_turn_traces()
3920 .lock()
3921 .expect("runtime turn traces mutex")
3922 .push(trace);
3923}
3924
3925#[cfg(test)]
3926#[allow(dead_code)]
3927#[allow(clippy::expect_used)]
3928pub(crate) fn take_runtime_turn_traces() -> Vec<RuntimeTurnTrace> {
3929 std::mem::take(
3930 &mut *runtime_turn_traces()
3931 .lock()
3932 .expect("runtime turn traces mutex"),
3933 )
3934}
3935
3936#[cfg(not(test))]
3937#[allow(dead_code)]
3938fn record_runtime_turn_trace(_trace: ()) {}
3939
3940fn runtime_turn_diagnostics_enabled() -> bool {
3941 std::env::var("MOBKIT_TRACE_RUNTIME_TURNS").is_ok_and(|value| {
3942 matches!(
3943 value.trim().to_ascii_lowercase().as_str(),
3944 "1" | "true" | "yes" | "on"
3945 )
3946 })
3947}
3948
3949pub fn mob_definition_may_use_image_generation(definition: &MobDefinition) -> bool {
3955 definition.profiles.values().any(|binding| {
3956 binding
3957 .as_inline()
3958 .is_none_or(|profile| profile.tools.image_generation)
3959 })
3960}
3961
3962pub fn mob_definition_may_use_shell(definition: &MobDefinition) -> bool {
3965 definition.profiles.values().any(|binding| {
3966 binding
3967 .as_inline()
3968 .is_none_or(|profile| profile.tools.shell)
3969 })
3970}
3971
3972fn normalize_runtime_turn_request(
3973 mut req: meerkat_core::service::StartTurnRequest,
3974) -> meerkat_core::service::StartTurnRequest {
3975 req.runtime.handling_mode = meerkat_core::types::HandlingMode::Queue;
3981 if let Some(metadata) = req.runtime.turn_metadata.as_mut() {
3984 metadata.render_metadata = None;
3985 }
3986 req
3987}
3988
3989async fn cancel_after_boundary_with_machine_authority_if_live(
3996 service: &dyn MobSessionService,
3997 session_id: &meerkat_core::types::SessionId,
3998 expected_run_id: &meerkat_core::lifecycle::RunId,
3999 authority: meerkat_runtime::MachineSessionControlAuthority,
4000) -> Result<(), SessionError> {
4001 service
4002 .cancel_after_boundary_with_machine_authority(session_id, expected_run_id, authority)
4003 .await
4004 .or_else(|error| match error {
4005 SessionError::NotFound { .. } | SessionError::NotRunning { .. } => Ok(()),
4006 error => Err(error),
4007 })
4008}
4009
4010macro_rules! delegate_mob_session_service {
4013 ($wrapper:ty) => {
4014 #[async_trait]
4015 impl meerkat_core::service::SessionService for $wrapper {
4016 async fn create_session(
4017 &self,
4018 req: CreateSessionRequest,
4019 ) -> Result<meerkat_core::types::RunResult, SessionError> {
4020 let (req, context) = self.prepare_create_request(req).await?;
4021 let result = self.inner.create_session(req).await?;
4022 Ok(self.complete_create(result, context).await)
4023 }
4024 async fn start_turn(
4025 &self,
4026 id: &meerkat_core::types::SessionId,
4027 req: meerkat_core::service::StartTurnRequest,
4028 ) -> Result<meerkat_core::types::RunResult, SessionError> {
4029 self.inner.start_turn(id, req).await
4030 }
4031 async fn reconcile_runtime_compaction_projections(
4032 &self,
4033 id: &meerkat_core::types::SessionId,
4034 intents: Vec<meerkat_core::CompactionProjectionIntent>,
4035 ) -> Result<(), SessionError> {
4036 self.inner
4037 .reconcile_runtime_compaction_projections(id, intents)
4038 .await
4039 }
4040 async fn abort_uncommitted_compaction_projections(
4041 &self,
4042 id: &meerkat_core::types::SessionId,
4043 ) -> Result<(), SessionError> {
4044 self.inner
4045 .abort_uncommitted_compaction_projections(id)
4046 .await
4047 }
4048 async fn abort_rejected_runtime_run_projections(
4049 &self,
4050 id: &meerkat_core::types::SessionId,
4051 ) -> Result<(), SessionError> {
4052 self.inner.abort_rejected_runtime_run_projections(id).await
4053 }
4054 async fn interrupt(
4055 &self,
4056 id: &meerkat_core::types::SessionId,
4057 ) -> Result<(), SessionError> {
4058 self.inner.interrupt(id).await
4059 }
4060 async fn cancel_after_boundary(
4061 &self,
4062 id: &meerkat_core::types::SessionId,
4063 ) -> Result<(), SessionError> {
4064 self.inner.cancel_after_boundary(id).await
4065 }
4066 async fn cancel_after_boundary_for_run(
4067 &self,
4068 id: &meerkat_core::types::SessionId,
4069 expected_run_id: &meerkat_core::lifecycle::RunId,
4070 ) -> Result<(), SessionError> {
4071 self.inner
4072 .cancel_after_boundary_for_run(id, expected_run_id)
4073 .await
4074 }
4075 async fn set_session_client(
4076 &self,
4077 id: &meerkat_core::types::SessionId,
4078 client: Arc<dyn meerkat_core::AgentLlmClient>,
4079 ) -> Result<(), SessionError> {
4080 self.inner
4081 .set_session_client(id, ReplaySanitizingAgentLlmClient::wrap(client))
4082 .await
4083 }
4084 async fn hot_swap_session_llm_identity(
4085 &self,
4086 id: &meerkat_core::types::SessionId,
4087 client: Arc<dyn meerkat_core::AgentLlmClient>,
4088 identity: meerkat_core::session::SessionLlmIdentity,
4089 request_policy: meerkat_core::SessionLlmRequestPolicy,
4090 ) -> Result<(), SessionError> {
4091 self.inner
4092 .hot_swap_session_llm_identity(
4093 id,
4094 ReplaySanitizingAgentLlmClient::wrap(client),
4095 identity,
4096 request_policy,
4097 )
4098 .await
4099 }
4100 async fn update_session_mob_authority_context(
4101 &self,
4102 id: &meerkat_core::types::SessionId,
4103 authority_context: Option<meerkat_core::service::MobToolAuthorityContext>,
4104 ) -> Result<(), SessionError> {
4105 self.inner
4106 .update_session_mob_authority_context(id, authority_context)
4107 .await
4108 }
4109 async fn has_live_session(
4110 &self,
4111 id: &meerkat_core::types::SessionId,
4112 ) -> Result<bool, SessionError> {
4113 self.inner.has_live_session(id).await
4114 }
4115 async fn set_session_tool_visibility_state(
4116 &self,
4117 id: &meerkat_core::types::SessionId,
4118 state: Option<meerkat_core::SessionToolVisibilityState>,
4119 ) -> Result<(), SessionError> {
4120 self.inner
4121 .set_session_tool_visibility_state(id, state)
4122 .await
4123 }
4124 async fn set_session_tool_filter(
4125 &self,
4126 id: &meerkat_core::types::SessionId,
4127 filter: meerkat_core::ToolFilter,
4128 ) -> Result<(), SessionError> {
4129 self.inner.set_session_tool_filter(id, filter).await
4130 }
4131 async fn read(
4132 &self,
4133 id: &meerkat_core::types::SessionId,
4134 ) -> Result<meerkat_core::service::SessionView, SessionError> {
4135 self.inner.read(id).await
4136 }
4137 async fn list(
4138 &self,
4139 query: meerkat_core::service::SessionQuery,
4140 ) -> Result<Vec<meerkat_core::service::SessionSummary>, SessionError> {
4141 self.inner.list(query).await
4142 }
4143 async fn archive(
4144 &self,
4145 id: &meerkat_core::types::SessionId,
4146 ) -> Result<(), SessionError> {
4147 self.inner.archive(id).await
4148 }
4149 async fn subscribe_session_events(
4150 &self,
4151 id: &meerkat_core::types::SessionId,
4152 ) -> Result<meerkat_core::comms::EventStream, meerkat_core::comms::StreamError> {
4153 meerkat_core::service::SessionService::subscribe_session_events(
4154 self.inner.as_ref(),
4155 id,
4156 )
4157 .await
4158 }
4159 async fn record_live_terminal_error(
4160 &self,
4161 id: &meerkat_core::types::SessionId,
4162 cause: meerkat_core::live_adapter::LiveAdapterErrorCode,
4163 ) -> Result<(), SessionError> {
4164 self.inner.record_live_terminal_error(id, cause).await
4165 }
4166 async fn record_live_output_audio_degraded(
4167 &self,
4168 id: &meerkat_core::types::SessionId,
4169 dropped: u64,
4170 ) -> Result<(), SessionError> {
4171 self.inner
4172 .record_live_output_audio_degraded(id, dropped)
4173 .await
4174 }
4175 }
4176
4177 #[async_trait]
4178 impl meerkat_core::service::SessionServiceCommsExt for $wrapper {
4179 async fn comms_runtime(
4180 &self,
4181 id: &meerkat_core::types::SessionId,
4182 ) -> Option<Arc<dyn meerkat_core::agent::CommsRuntime>> {
4183 self.inner.comms_runtime(id).await
4184 }
4185
4186 async fn event_injector(
4187 &self,
4188 id: &meerkat_core::types::SessionId,
4189 ) -> Option<Arc<dyn meerkat_core::EventInjector>> {
4190 self.inner.event_injector(id).await
4191 }
4192
4193 async fn interaction_event_injector(
4194 &self,
4195 id: &meerkat_core::types::SessionId,
4196 ) -> Option<Arc<dyn meerkat_core::event_injector::SubscribableInjector>> {
4197 self.inner.interaction_event_injector(id).await
4198 }
4199 }
4200
4201 #[async_trait]
4202 impl meerkat_core::service::SessionServiceControlExt for $wrapper {
4203 async fn append_system_context(
4204 &self,
4205 id: &meerkat_core::types::SessionId,
4206 req: meerkat_core::service::AppendSystemContextRequest,
4207 ) -> Result<
4208 meerkat_core::service::AppendSystemContextResult,
4209 meerkat_core::service::SessionControlError,
4210 > {
4211 self.inner.append_system_context(id, req).await
4212 }
4213 async fn stage_tool_results(
4214 &self,
4215 id: &meerkat_core::types::SessionId,
4216 req: meerkat_core::service::StageToolResultsRequest,
4217 ) -> Result<meerkat_core::service::StageToolResultsResult, SessionError> {
4218 self.inner.stage_tool_results(id, req).await
4219 }
4220 }
4221
4222 #[async_trait]
4223 impl meerkat_core::service::SessionServiceHistoryExt for $wrapper {
4224 async fn read_history(
4225 &self,
4226 id: &meerkat_core::types::SessionId,
4227 query: meerkat_core::service::SessionHistoryQuery,
4228 ) -> Result<meerkat_core::service::SessionHistoryPage, SessionError> {
4229 self.inner.read_history(id, query).await
4230 }
4231 async fn read_transcript_revision(
4232 &self,
4233 id: &meerkat_core::types::SessionId,
4234 query: meerkat_core::service::SessionTranscriptRevisionQuery,
4235 ) -> Result<meerkat_core::service::SessionTranscriptRevisionPage, SessionError> {
4236 self.inner.read_transcript_revision(id, query).await
4237 }
4238 async fn list_transcript_revisions(
4239 &self,
4240 id: &meerkat_core::types::SessionId,
4241 query: meerkat_core::service::SessionTranscriptRevisionListQuery,
4242 ) -> Result<meerkat_core::service::SessionTranscriptRevisionList, SessionError> {
4243 self.inner.list_transcript_revisions(id, query).await
4244 }
4245 }
4246
4247 #[async_trait]
4248 impl MobSessionService for $wrapper {
4249 async fn load_session_for_resume(
4250 &self,
4251 session_id: &meerkat_core::types::SessionId,
4252 ) -> Result<meerkat_mob::ResumeSessionLoad, SessionError> {
4253 let load = self.inner.load_session_for_resume(session_id).await?;
4254 self.overlay_runtime_archived_terminal(load).await
4255 }
4256 async fn prepare_session_for_resume(
4257 &self,
4258 session_id: &meerkat_core::types::SessionId,
4259 ) -> Result<(), SessionError> {
4260 self.inner.prepare_session_for_resume(session_id).await
4261 }
4262 async fn materialize_session_for_resume(
4263 &self,
4264 session_id: &meerkat_core::types::SessionId,
4265 ) -> Result<meerkat_mob::ResumeSessionLoad, SessionError> {
4266 self.inner.materialize_session_for_resume(session_id).await
4267 }
4268
4269 async fn create_session_under_runtime_turn_boundary(
4270 &self,
4271 req: meerkat_core::service::CreateSessionRequest,
4272 ) -> Result<meerkat_core::RunResult, SessionError> {
4273 let (req, context) = self.prepare_create_request(req).await?;
4274 let result = self
4275 .inner
4276 .create_session_under_runtime_turn_boundary(req)
4277 .await?;
4278 Ok(self.complete_create(result, context).await)
4279 }
4280 async fn create_session_with_actor_witness_under_runtime_turn_boundary(
4281 &self,
4282 req: meerkat_core::service::CreateSessionRequest,
4283 actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
4284 ) -> Result<meerkat_core::RunResult, SessionError> {
4285 let (req, context) = self.prepare_create_request(req).await?;
4286 let result = self
4287 .inner
4288 .create_session_with_actor_witness_under_runtime_turn_boundary(
4289 req,
4290 actor_witness_slot,
4291 )
4292 .await?;
4293 Ok(self.complete_create(result, context).await)
4294 }
4295 async fn create_session_with_machine_archived_resume_authority_under_runtime_turn_boundary(
4296 &self,
4297 req: meerkat_core::service::CreateSessionRequest,
4298 authorization: meerkat_runtime::ArchivedSessionActorMaterializationAuthorization,
4299 ) -> Result<meerkat_core::RunResult, SessionError> {
4300 let (req, context) = self.prepare_create_request(req).await?;
4301 let result = self
4302 .inner
4303 .create_session_with_machine_archived_resume_authority_under_runtime_turn_boundary(
4304 req,
4305 authorization,
4306 )
4307 .await?;
4308 Ok(self.complete_create(result, context).await)
4309 }
4310 async fn create_session_with_machine_archived_resume_authority_and_actor_witness_under_runtime_turn_boundary(
4311 &self,
4312 req: meerkat_core::service::CreateSessionRequest,
4313 authorization: meerkat_runtime::ArchivedSessionActorMaterializationAuthorization,
4314 actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
4315 ) -> Result<meerkat_core::RunResult, SessionError> {
4316 let (req, context) = self.prepare_create_request(req).await?;
4317 let result = self
4318 .inner
4319 .create_session_with_machine_archived_resume_authority_and_actor_witness_under_runtime_turn_boundary(
4320 req,
4321 authorization,
4322 actor_witness_slot,
4323 )
4324 .await?;
4325 Ok(self.complete_create(result, context).await)
4326 }
4327 fn supports_persistent_sessions(&self) -> bool {
4328 self.inner.supports_persistent_sessions()
4329 }
4330 fn supports_runtime_turn_apply(&self) -> bool {
4331 self.inner.supports_runtime_turn_apply()
4332 }
4333 async fn session_known_to_archive_authority(
4339 &self,
4340 session_id: &meerkat_core::types::SessionId,
4341 ) -> Result<bool, SessionError> {
4342 self.inner.session_known_to_archive_authority(session_id).await
4343 }
4344 fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
4345 self.runtime_adapter_override
4346 .clone()
4347 .or_else(|| self.inner.runtime_adapter())
4348 }
4349 async fn interrupt_with_machine_authority(
4350 &self,
4351 session_id: &meerkat_core::types::SessionId,
4352 authority: meerkat_runtime::MachineSessionControlAuthority,
4353 ) -> Result<(), SessionError> {
4354 self.inner
4355 .interrupt_with_machine_authority(session_id, authority)
4356 .await
4357 }
4358 async fn cancel_after_boundary_with_machine_authority(
4359 &self,
4360 session_id: &meerkat_core::types::SessionId,
4361 expected_run_id: &meerkat_core::lifecycle::RunId,
4362 authority: meerkat_runtime::MachineSessionControlAuthority,
4363 ) -> Result<(), SessionError> {
4364 cancel_after_boundary_with_machine_authority_if_live(
4365 self.inner.as_ref(),
4366 session_id,
4367 expected_run_id,
4368 authority,
4369 )
4370 .await
4371 }
4372 async fn cancel_current_after_boundary_with_machine_authority(
4373 &self,
4374 session_id: &meerkat_core::types::SessionId,
4375 authority: meerkat_runtime::MachineSessionControlAuthority,
4376 ) -> Result<(), SessionError> {
4377 self.inner
4378 .cancel_current_after_boundary_with_machine_authority(session_id, authority)
4379 .await
4380 }
4381 async fn live_session_actor_registered(
4382 &self,
4383 session_id: &meerkat_core::types::SessionId,
4384 ) -> Result<bool, SessionError> {
4385 self.inner.live_session_actor_registered(session_id).await
4386 }
4387 async fn session_belongs_to_mob(
4388 &self,
4389 session_id: &meerkat_core::types::SessionId,
4390 mob_id: &meerkat_mob::MobId,
4391 ) -> bool {
4392 self.inner.session_belongs_to_mob(session_id, mob_id).await
4393 }
4394 async fn load_persisted_session(
4395 &self,
4396 session_id: &meerkat_core::types::SessionId,
4397 ) -> Result<Option<meerkat_core::session::Session>, SessionError> {
4398 self.load_persisted_session_absorbed(session_id).await
4402 }
4403 async fn load_revivable_retired_session(
4409 &self,
4410 session_id: &meerkat_core::types::SessionId,
4411 ) -> Result<Option<meerkat_core::session::Session>, SessionError> {
4412 self.inner.load_revivable_retired_session(session_id).await
4413 }
4414 async fn load_persisted_session_metadata(
4415 &self,
4416 session_id: &meerkat_core::types::SessionId,
4417 ) -> Result<Option<meerkat_core::PersistedSessionMetadataView>, SessionError> {
4418 self.inner.load_persisted_session_metadata(session_id).await
4419 }
4420 async fn authorize_revivable_retired_session(
4421 &self,
4422 session_id: &meerkat_core::types::SessionId,
4423 authority: meerkat_runtime::PreparedArchivedResumeCommitLease,
4424 ) -> Result<meerkat_runtime::AuthorizedArchivedResumeCommitLease, SessionError> {
4425 self.inner
4426 .authorize_revivable_retired_session(session_id, authority)
4427 .await
4428 }
4429 async fn create_session_with_machine_archived_resume_authority(
4430 &self,
4431 req: meerkat_core::service::CreateSessionRequest,
4432 authorization: meerkat_runtime::ArchivedSessionActorMaterializationAuthorization,
4433 ) -> Result<meerkat_core::RunResult, SessionError> {
4434 let (req, context) = self.prepare_create_request(req).await?;
4435 let result = self
4436 .inner
4437 .create_session_with_machine_archived_resume_authority(req, authorization)
4438 .await?;
4439 Ok(self.complete_create(result, context).await)
4440 }
4441 async fn subscribe_session_events(
4442 &self,
4443 session_id: &meerkat_core::types::SessionId,
4444 ) -> Result<meerkat_core::comms::EventStream, meerkat_core::comms::StreamError> {
4445 meerkat_mob::MobSessionService::subscribe_session_events(
4446 self.inner.as_ref(),
4447 session_id,
4448 )
4449 .await
4450 }
4451 async fn archive_with_mob_lifecycle_authority(
4452 &self,
4453 session_id: &meerkat_core::types::SessionId,
4454 ) -> Result<(), SessionError> {
4455 match self
4456 .inner
4457 .archive_with_mob_lifecycle_authority(session_id)
4458 .await
4459 {
4460 Err(err)
4461 if is_stopped_session_archive_retire_rejection(&err.to_string()) =>
4462 {
4463 tracing::warn!(
4474 session_id = %session_id,
4475 error = %err,
4476 "archive: tolerating Retire rejection for stopped idle session; archive document committed"
4477 );
4478 Ok(())
4479 }
4480 other => other,
4481 }
4482 }
4483 async fn archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(
4484 &self,
4485 session_id: &meerkat_core::types::SessionId,
4486 ) -> Result<(), SessionError> {
4487 self.inner
4488 .archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(session_id)
4489 .await
4490 }
4491 async fn execution_snapshot(
4492 &self,
4493 session_id: &meerkat_core::types::SessionId,
4494 ) -> Result<Option<meerkat_core::agent::AgentExecutionSnapshot>, SessionError> {
4495 self.inner.execution_snapshot(session_id).await
4496 }
4497 async fn tool_scope_snapshot(
4498 &self,
4499 session_id: &meerkat_core::types::SessionId,
4500 ) -> Result<Option<meerkat_core::ToolScopeSnapshot>, SessionError> {
4501 self.inner.tool_scope_snapshot(session_id).await
4502 }
4503 async fn external_tool_surface_snapshot(
4504 &self,
4505 session_id: &meerkat_core::types::SessionId,
4506 ) -> Result<Option<meerkat_core::ExternalToolSurfaceSnapshot>, SessionError> {
4507 self.inner.external_tool_surface_snapshot(session_id).await
4508 }
4509 async fn peer_ingress_runtime_snapshot(
4510 &self,
4511 session_id: &meerkat_core::types::SessionId,
4512 ) -> Result<Option<meerkat_core::PeerIngressRuntimeSnapshot>, SessionError> {
4513 self.inner.peer_ingress_runtime_snapshot(session_id).await
4514 }
4515 async fn apply_runtime_turn(
4516 &self,
4517 session_id: &meerkat_core::types::SessionId,
4518 run_id: meerkat_core::lifecycle::RunId,
4519 req: meerkat_core::service::StartTurnRequest,
4520 boundary: meerkat_core::lifecycle::run_primitive::RunApplyBoundary,
4521 contributing_input_ids: Vec<meerkat_core::lifecycle::InputId>,
4522 ) -> Result<meerkat_core::lifecycle::core_executor::CoreApplyOutput, SessionError> {
4523 #[cfg(test)]
4524 let boundary_name = format!("{boundary:?}");
4525 #[cfg(test)]
4526 let contributing_count = contributing_input_ids.len();
4527 let run_id_for_log = run_id.to_string();
4528 if runtime_turn_diagnostics_enabled() {
4529 tracing::warn!(
4530 session_id = %session_id,
4531 run_id = %run_id_for_log,
4532 boundary = ?boundary,
4533 contributing_inputs = contributing_input_ids.len(),
4534 "mobkit runtime turn start"
4535 );
4536 }
4537 let result = self
4538 .inner
4539 .apply_runtime_turn(
4540 session_id,
4541 run_id,
4542 normalize_runtime_turn_request(req),
4543 boundary,
4544 contributing_input_ids,
4545 )
4546 .await;
4547 #[cfg(test)]
4548 record_runtime_turn_trace(RuntimeTurnTrace {
4549 session_id: session_id.to_string(),
4550 boundary: boundary_name,
4551 contributing_input_count: contributing_count,
4552 outcome: match &result {
4553 Ok(_) => "ok".to_string(),
4554 Err(error) => format!("err:{error}"),
4555 },
4556 });
4557 if runtime_turn_diagnostics_enabled() {
4558 match &result {
4559 Ok(_) => tracing::warn!(
4560 session_id = %session_id,
4561 run_id = %run_id_for_log,
4562 "mobkit runtime turn ok"
4563 ),
4564 Err(_) => tracing::error!(
4565 session_id = %session_id,
4566 run_id = %run_id_for_log,
4567 "mobkit runtime turn error"
4568 ),
4569 }
4570 }
4571 result
4572 }
4573 async fn prepare_transient_turn_context_for_active_turn(
4574 &self,
4575 session_id: &meerkat_core::types::SessionId,
4576 expected_run_id: &meerkat_core::lifecycle::RunId,
4577 contexts: Vec<meerkat_core::lifecycle::run_primitive::TurnRequestContext>,
4578 ) -> Result<meerkat_core::CoreBoundaryStageOutput, meerkat_core::CoreBoundaryStageError>
4579 {
4580 self.inner
4581 .prepare_transient_turn_context_for_active_turn(
4582 session_id,
4583 expected_run_id,
4584 contexts,
4585 )
4586 .await
4587 }
4588 async fn acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
4589 &self,
4590 session_id: &meerkat_core::types::SessionId,
4591 authority: &meerkat_core::CommittedSessionBoundaryAuthority,
4592 ) -> Result<(), SessionError> {
4593 match self
4594 .inner
4595 .acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
4596 session_id,
4597 authority,
4598 )
4599 .await
4600 {
4601 Err(SessionError::Unsupported(ref detail))
4617 if detail
4618 == "ephemeral session service cannot acknowledge store-owned \
4619 runtime boundaries"
4620 && self.absorbs_unsupported_boundary_acknowledgement() =>
4621 {
4622 Ok(())
4623 }
4624 other => other,
4625 }
4626 }
4627 async fn acquire_runtime_turn_finalization_guard(
4628 &self,
4629 session_id: &meerkat_core::types::SessionId,
4630 ) -> Result<
4631 Box<dyn meerkat_core::lifecycle::CoreExecutorTurnFinalizationGuard>,
4632 SessionError,
4633 > {
4634 self.inner
4635 .acquire_runtime_turn_finalization_guard(session_id)
4636 .await
4637 }
4638 async fn checkpoint_committed_runtime_session_snapshot_under_turn_finalization_boundary(
4639 &self,
4640 session_id: &meerkat_core::types::SessionId,
4641 session_snapshot: Arc<Vec<u8>>,
4642 ) -> Result<(), SessionError> {
4643 self.inner
4644 .checkpoint_committed_runtime_session_snapshot_under_turn_finalization_boundary(
4645 session_id,
4646 session_snapshot,
4647 )
4648 .await
4649 }
4650 async fn discard_live_session_after_runtime_stop_terminalized(
4651 &self,
4652 session_id: &meerkat_core::types::SessionId,
4653 ) -> Result<(), SessionError> {
4654 self.inner
4655 .discard_live_session_after_runtime_stop_terminalized(session_id)
4656 .await
4657 }
4658 async fn discard_live_session_after_runtime_stop_terminalized_under_turn_finalization_boundary(
4659 &self,
4660 session_id: &meerkat_core::types::SessionId,
4661 ) -> Result<(), SessionError> {
4662 self.inner
4663 .discard_live_session_after_runtime_stop_terminalized_under_turn_finalization_boundary(
4664 session_id,
4665 )
4666 .await
4667 }
4668 async fn publish_interaction_terminals(
4669 &self,
4670 session_id: &meerkat_core::types::SessionId,
4671 events: &[meerkat_core::event::AgentEvent],
4672 ) -> Result<
4673 Vec<
4674 meerkat_core::lifecycle::core_executor::CoreInteractionTerminalPublicationReceipt,
4675 >,
4676 SessionError,
4677 > {
4678 self.inner
4679 .publish_interaction_terminals(session_id, events)
4680 .await
4681 }
4682 async fn discard_live_session(
4683 &self,
4684 session_id: &meerkat_core::types::SessionId,
4685 ) -> Result<(), SessionError> {
4686 self.inner.discard_live_session(session_id).await
4687 }
4688 async fn discard_live_session_under_runtime_turn_boundary(
4689 &self,
4690 session_id: &meerkat_core::types::SessionId,
4691 ) -> Result<(), SessionError> {
4692 self.inner
4693 .discard_live_session_under_runtime_turn_boundary(session_id)
4694 .await
4695 }
4696 async fn discard_live_session_actor_under_runtime_turn_boundary(
4697 &self,
4698 witness: &meerkat_session::LiveSessionActorWitness,
4699 ) -> Result<bool, SessionError> {
4700 self.inner
4701 .discard_live_session_actor_under_runtime_turn_boundary(witness)
4702 .await
4703 }
4704 async fn await_event_projection_drain(
4705 &self,
4706 session_id: &meerkat_core::types::SessionId,
4707 ) -> Result<bool, SessionError> {
4708 self.inner.await_event_projection_drain(session_id).await
4709 }
4710 async fn checkpoint_committed_runtime_session_snapshot(
4711 &self,
4712 session_id: &meerkat_core::types::SessionId,
4713 session_snapshot: Arc<Vec<u8>>,
4714 ) -> Result<(), SessionError> {
4715 self.inner
4716 .checkpoint_committed_runtime_session_snapshot(session_id, session_snapshot)
4717 .await
4718 }
4719 async fn cancel_all_checkpointers(&self) {
4720 self.inner.cancel_all_checkpointers().await;
4721 }
4722 async fn rearm_all_checkpointers(&self) {
4723 self.inner.rearm_all_checkpointers().await;
4724 }
4725 }
4726 };
4727}
4728
4729delegate_mob_session_service!(PreBuildMobSessionService);
4730
4731struct AfterCreateMobSessionService {
4736 inner: Arc<dyn MobSessionService>,
4737 after_hook: AfterCreateHook,
4738}
4739
4740impl AfterCreateMobSessionService {
4741 fn prepare_create_request(
4742 &self,
4743 mut req: CreateSessionRequest,
4744 ) -> (CreateSessionRequest, SessionCreatedContext) {
4745 sanitize_create_session_request_llm_override(&mut req);
4746 ensure_shell_tooling_build_substrate(&mut req);
4747 let context = SessionCreatedContext {
4748 model: req.model.clone(),
4749 labels: req.labels.clone().unwrap_or_default(),
4750 system_prompt: req.system_prompt.as_set_prompt().map(ToString::to_string),
4751 };
4752 (req, context)
4753 }
4754
4755 async fn complete_create(
4756 &self,
4757 result: meerkat_core::types::RunResult,
4758 context: SessionCreatedContext,
4759 ) -> meerkat_core::types::RunResult {
4760 (self.after_hook)(result.session_id.clone(), context).await;
4761 result
4762 }
4763}
4764
4765#[async_trait]
4766impl meerkat_core::service::SessionService for AfterCreateMobSessionService {
4767 async fn create_session(
4768 &self,
4769 req: CreateSessionRequest,
4770 ) -> Result<meerkat_core::types::RunResult, SessionError> {
4771 let (req, context) = self.prepare_create_request(req);
4772 let result = self.inner.create_session(req).await?;
4773 Ok(self.complete_create(result, context).await)
4774 }
4775 async fn start_turn(
4776 &self,
4777 id: &meerkat_core::types::SessionId,
4778 req: meerkat_core::service::StartTurnRequest,
4779 ) -> Result<meerkat_core::types::RunResult, SessionError> {
4780 self.inner.start_turn(id, req).await
4781 }
4782 async fn reconcile_runtime_compaction_projections(
4783 &self,
4784 id: &meerkat_core::types::SessionId,
4785 intents: Vec<meerkat_core::CompactionProjectionIntent>,
4786 ) -> Result<(), SessionError> {
4787 self.inner
4788 .reconcile_runtime_compaction_projections(id, intents)
4789 .await
4790 }
4791 async fn abort_uncommitted_compaction_projections(
4792 &self,
4793 id: &meerkat_core::types::SessionId,
4794 ) -> Result<(), SessionError> {
4795 self.inner
4796 .abort_uncommitted_compaction_projections(id)
4797 .await
4798 }
4799 async fn abort_rejected_runtime_run_projections(
4800 &self,
4801 id: &meerkat_core::types::SessionId,
4802 ) -> Result<(), SessionError> {
4803 self.inner.abort_rejected_runtime_run_projections(id).await
4804 }
4805 async fn interrupt(&self, id: &meerkat_core::types::SessionId) -> Result<(), SessionError> {
4806 self.inner.interrupt(id).await
4807 }
4808 async fn cancel_after_boundary(
4809 &self,
4810 id: &meerkat_core::types::SessionId,
4811 ) -> Result<(), SessionError> {
4812 self.inner.cancel_after_boundary(id).await
4813 }
4814 async fn cancel_after_boundary_for_run(
4815 &self,
4816 id: &meerkat_core::types::SessionId,
4817 expected_run_id: &meerkat_core::lifecycle::RunId,
4818 ) -> Result<(), SessionError> {
4819 self.inner
4820 .cancel_after_boundary_for_run(id, expected_run_id)
4821 .await
4822 }
4823 async fn set_session_client(
4824 &self,
4825 id: &meerkat_core::types::SessionId,
4826 client: Arc<dyn meerkat_core::AgentLlmClient>,
4827 ) -> Result<(), SessionError> {
4828 self.inner
4829 .set_session_client(id, ReplaySanitizingAgentLlmClient::wrap(client))
4830 .await
4831 }
4832 async fn hot_swap_session_llm_identity(
4833 &self,
4834 id: &meerkat_core::types::SessionId,
4835 client: Arc<dyn meerkat_core::AgentLlmClient>,
4836 identity: meerkat_core::session::SessionLlmIdentity,
4837 request_policy: meerkat_core::SessionLlmRequestPolicy,
4838 ) -> Result<(), SessionError> {
4839 self.inner
4840 .hot_swap_session_llm_identity(
4841 id,
4842 ReplaySanitizingAgentLlmClient::wrap(client),
4843 identity,
4844 request_policy,
4845 )
4846 .await
4847 }
4848 async fn update_session_mob_authority_context(
4849 &self,
4850 id: &meerkat_core::types::SessionId,
4851 authority_context: Option<meerkat_core::service::MobToolAuthorityContext>,
4852 ) -> Result<(), SessionError> {
4853 self.inner
4854 .update_session_mob_authority_context(id, authority_context)
4855 .await
4856 }
4857 async fn has_live_session(
4858 &self,
4859 id: &meerkat_core::types::SessionId,
4860 ) -> Result<bool, SessionError> {
4861 self.inner.has_live_session(id).await
4862 }
4863 async fn set_session_tool_visibility_state(
4864 &self,
4865 id: &meerkat_core::types::SessionId,
4866 state: Option<meerkat_core::SessionToolVisibilityState>,
4867 ) -> Result<(), SessionError> {
4868 self.inner
4869 .set_session_tool_visibility_state(id, state)
4870 .await
4871 }
4872 async fn set_session_tool_filter(
4873 &self,
4874 id: &meerkat_core::types::SessionId,
4875 filter: meerkat_core::ToolFilter,
4876 ) -> Result<(), SessionError> {
4877 self.inner.set_session_tool_filter(id, filter).await
4878 }
4879 async fn read(
4880 &self,
4881 id: &meerkat_core::types::SessionId,
4882 ) -> Result<meerkat_core::service::SessionView, SessionError> {
4883 self.inner.read(id).await
4884 }
4885 async fn list(
4886 &self,
4887 query: meerkat_core::service::SessionQuery,
4888 ) -> Result<Vec<meerkat_core::service::SessionSummary>, SessionError> {
4889 self.inner.list(query).await
4890 }
4891 async fn archive(&self, id: &meerkat_core::types::SessionId) -> Result<(), SessionError> {
4892 self.inner.archive(id).await
4893 }
4894 async fn subscribe_session_events(
4895 &self,
4896 id: &meerkat_core::types::SessionId,
4897 ) -> Result<meerkat_core::comms::EventStream, meerkat_core::comms::StreamError> {
4898 meerkat_core::service::SessionService::subscribe_session_events(self.inner.as_ref(), id)
4899 .await
4900 }
4901 async fn record_live_terminal_error(
4902 &self,
4903 id: &meerkat_core::types::SessionId,
4904 cause: meerkat_core::live_adapter::LiveAdapterErrorCode,
4905 ) -> Result<(), SessionError> {
4906 self.inner.record_live_terminal_error(id, cause).await
4907 }
4908 async fn record_live_output_audio_degraded(
4909 &self,
4910 id: &meerkat_core::types::SessionId,
4911 dropped: u64,
4912 ) -> Result<(), SessionError> {
4913 self.inner
4914 .record_live_output_audio_degraded(id, dropped)
4915 .await
4916 }
4917}
4918
4919#[async_trait]
4920impl meerkat_core::service::SessionServiceCommsExt for AfterCreateMobSessionService {
4921 async fn comms_runtime(
4922 &self,
4923 id: &meerkat_core::types::SessionId,
4924 ) -> Option<Arc<dyn meerkat_core::agent::CommsRuntime>> {
4925 self.inner.comms_runtime(id).await
4926 }
4927
4928 async fn event_injector(
4929 &self,
4930 id: &meerkat_core::types::SessionId,
4931 ) -> Option<Arc<dyn meerkat_core::EventInjector>> {
4932 self.inner.event_injector(id).await
4933 }
4934
4935 async fn interaction_event_injector(
4936 &self,
4937 id: &meerkat_core::types::SessionId,
4938 ) -> Option<Arc<dyn meerkat_core::event_injector::SubscribableInjector>> {
4939 self.inner.interaction_event_injector(id).await
4940 }
4941}
4942
4943#[async_trait]
4944impl meerkat_core::service::SessionServiceControlExt for AfterCreateMobSessionService {
4945 async fn append_system_context(
4946 &self,
4947 id: &meerkat_core::types::SessionId,
4948 req: meerkat_core::service::AppendSystemContextRequest,
4949 ) -> Result<
4950 meerkat_core::service::AppendSystemContextResult,
4951 meerkat_core::service::SessionControlError,
4952 > {
4953 self.inner.append_system_context(id, req).await
4954 }
4955 async fn stage_tool_results(
4956 &self,
4957 id: &meerkat_core::types::SessionId,
4958 req: meerkat_core::service::StageToolResultsRequest,
4959 ) -> Result<meerkat_core::service::StageToolResultsResult, SessionError> {
4960 self.inner.stage_tool_results(id, req).await
4961 }
4962}
4963
4964#[async_trait]
4965impl meerkat_core::service::SessionServiceHistoryExt for AfterCreateMobSessionService {
4966 async fn read_history(
4967 &self,
4968 id: &meerkat_core::types::SessionId,
4969 query: meerkat_core::service::SessionHistoryQuery,
4970 ) -> Result<meerkat_core::service::SessionHistoryPage, SessionError> {
4971 self.inner.read_history(id, query).await
4972 }
4973 async fn read_transcript_revision(
4974 &self,
4975 id: &meerkat_core::types::SessionId,
4976 query: meerkat_core::service::SessionTranscriptRevisionQuery,
4977 ) -> Result<meerkat_core::service::SessionTranscriptRevisionPage, SessionError> {
4978 self.inner.read_transcript_revision(id, query).await
4979 }
4980 async fn list_transcript_revisions(
4981 &self,
4982 id: &meerkat_core::types::SessionId,
4983 query: meerkat_core::service::SessionTranscriptRevisionListQuery,
4984 ) -> Result<meerkat_core::service::SessionTranscriptRevisionList, SessionError> {
4985 self.inner.list_transcript_revisions(id, query).await
4986 }
4987}
4988
4989#[async_trait]
4990impl MobSessionService for AfterCreateMobSessionService {
4991 async fn load_session_for_resume(
4992 &self,
4993 session_id: &meerkat_core::types::SessionId,
4994 ) -> Result<meerkat_mob::ResumeSessionLoad, SessionError> {
4995 self.inner.load_session_for_resume(session_id).await
4996 }
4997 async fn prepare_session_for_resume(
4998 &self,
4999 session_id: &meerkat_core::types::SessionId,
5000 ) -> Result<(), SessionError> {
5001 self.inner.prepare_session_for_resume(session_id).await
5002 }
5003 async fn materialize_session_for_resume(
5004 &self,
5005 session_id: &meerkat_core::types::SessionId,
5006 ) -> Result<meerkat_mob::ResumeSessionLoad, SessionError> {
5007 self.inner.materialize_session_for_resume(session_id).await
5008 }
5009
5010 async fn create_session_under_runtime_turn_boundary(
5011 &self,
5012 req: meerkat_core::service::CreateSessionRequest,
5013 ) -> Result<meerkat_core::RunResult, SessionError> {
5014 let (req, context) = self.prepare_create_request(req);
5015 let result = self
5016 .inner
5017 .create_session_under_runtime_turn_boundary(req)
5018 .await?;
5019 Ok(self.complete_create(result, context).await)
5020 }
5021
5022 async fn create_session_with_actor_witness_under_runtime_turn_boundary(
5023 &self,
5024 req: meerkat_core::service::CreateSessionRequest,
5025 actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
5026 ) -> Result<meerkat_core::RunResult, SessionError> {
5027 let (req, context) = self.prepare_create_request(req);
5028 let result = self
5029 .inner
5030 .create_session_with_actor_witness_under_runtime_turn_boundary(req, actor_witness_slot)
5031 .await?;
5032 Ok(self.complete_create(result, context).await)
5033 }
5034
5035 async fn create_session_with_machine_archived_resume_authority_under_runtime_turn_boundary(
5036 &self,
5037 req: meerkat_core::service::CreateSessionRequest,
5038 authorization: meerkat_runtime::ArchivedSessionActorMaterializationAuthorization,
5039 ) -> Result<meerkat_core::RunResult, SessionError> {
5040 let (req, context) = self.prepare_create_request(req);
5041 let result = self
5042 .inner
5043 .create_session_with_machine_archived_resume_authority_under_runtime_turn_boundary(
5044 req,
5045 authorization,
5046 )
5047 .await?;
5048 Ok(self.complete_create(result, context).await)
5049 }
5050
5051 async fn create_session_with_machine_archived_resume_authority_and_actor_witness_under_runtime_turn_boundary(
5052 &self,
5053 req: meerkat_core::service::CreateSessionRequest,
5054 authorization: meerkat_runtime::ArchivedSessionActorMaterializationAuthorization,
5055 actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
5056 ) -> Result<meerkat_core::RunResult, SessionError> {
5057 let (req, context) = self.prepare_create_request(req);
5058 let result = self
5059 .inner
5060 .create_session_with_machine_archived_resume_authority_and_actor_witness_under_runtime_turn_boundary(
5061 req,
5062 authorization,
5063 actor_witness_slot,
5064 )
5065 .await?;
5066 Ok(self.complete_create(result, context).await)
5067 }
5068
5069 fn supports_persistent_sessions(&self) -> bool {
5070 self.inner.supports_persistent_sessions()
5071 }
5072
5073 fn supports_runtime_turn_apply(&self) -> bool {
5074 self.inner.supports_runtime_turn_apply()
5075 }
5076
5077 async fn session_known_to_archive_authority(
5080 &self,
5081 session_id: &meerkat_core::types::SessionId,
5082 ) -> Result<bool, SessionError> {
5083 self.inner
5084 .session_known_to_archive_authority(session_id)
5085 .await
5086 }
5087 fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
5088 self.inner.runtime_adapter()
5089 }
5090 async fn interrupt_with_machine_authority(
5091 &self,
5092 session_id: &meerkat_core::types::SessionId,
5093 authority: meerkat_runtime::MachineSessionControlAuthority,
5094 ) -> Result<(), SessionError> {
5095 self.inner
5096 .interrupt_with_machine_authority(session_id, authority)
5097 .await
5098 }
5099 async fn cancel_after_boundary_with_machine_authority(
5100 &self,
5101 session_id: &meerkat_core::types::SessionId,
5102 expected_run_id: &meerkat_core::lifecycle::RunId,
5103 authority: meerkat_runtime::MachineSessionControlAuthority,
5104 ) -> Result<(), SessionError> {
5105 cancel_after_boundary_with_machine_authority_if_live(
5106 self.inner.as_ref(),
5107 session_id,
5108 expected_run_id,
5109 authority,
5110 )
5111 .await
5112 }
5113 async fn cancel_current_after_boundary_with_machine_authority(
5114 &self,
5115 session_id: &meerkat_core::types::SessionId,
5116 authority: meerkat_runtime::MachineSessionControlAuthority,
5117 ) -> Result<(), SessionError> {
5118 self.inner
5119 .cancel_current_after_boundary_with_machine_authority(session_id, authority)
5120 .await
5121 }
5122 async fn live_session_actor_registered(
5123 &self,
5124 session_id: &meerkat_core::types::SessionId,
5125 ) -> Result<bool, SessionError> {
5126 self.inner.live_session_actor_registered(session_id).await
5127 }
5128 async fn session_belongs_to_mob(
5129 &self,
5130 session_id: &meerkat_core::types::SessionId,
5131 mob_id: &meerkat_mob::MobId,
5132 ) -> bool {
5133 self.inner.session_belongs_to_mob(session_id, mob_id).await
5134 }
5135 async fn load_persisted_session(
5136 &self,
5137 session_id: &meerkat_core::types::SessionId,
5138 ) -> Result<Option<meerkat_core::session::Session>, SessionError> {
5139 self.inner.load_persisted_session(session_id).await
5140 }
5141 async fn load_revivable_retired_session(
5145 &self,
5146 session_id: &meerkat_core::types::SessionId,
5147 ) -> Result<Option<meerkat_core::session::Session>, SessionError> {
5148 self.inner.load_revivable_retired_session(session_id).await
5149 }
5150 async fn load_persisted_session_metadata(
5151 &self,
5152 session_id: &meerkat_core::types::SessionId,
5153 ) -> Result<Option<meerkat_core::PersistedSessionMetadataView>, SessionError> {
5154 self.inner.load_persisted_session_metadata(session_id).await
5155 }
5156 async fn authorize_revivable_retired_session(
5157 &self,
5158 session_id: &meerkat_core::types::SessionId,
5159 authority: meerkat_runtime::PreparedArchivedResumeCommitLease,
5160 ) -> Result<meerkat_runtime::AuthorizedArchivedResumeCommitLease, SessionError> {
5161 self.inner
5162 .authorize_revivable_retired_session(session_id, authority)
5163 .await
5164 }
5165 async fn create_session_with_machine_archived_resume_authority(
5166 &self,
5167 req: meerkat_core::service::CreateSessionRequest,
5168 authorization: meerkat_runtime::ArchivedSessionActorMaterializationAuthorization,
5169 ) -> Result<meerkat_core::RunResult, SessionError> {
5170 let (req, context) = self.prepare_create_request(req);
5171 let result = self
5172 .inner
5173 .create_session_with_machine_archived_resume_authority(req, authorization)
5174 .await?;
5175 Ok(self.complete_create(result, context).await)
5176 }
5177 async fn subscribe_session_events(
5178 &self,
5179 session_id: &meerkat_core::types::SessionId,
5180 ) -> Result<meerkat_core::comms::EventStream, meerkat_core::comms::StreamError> {
5181 meerkat_mob::MobSessionService::subscribe_session_events(self.inner.as_ref(), session_id)
5182 .await
5183 }
5184 async fn archive_with_mob_lifecycle_authority(
5185 &self,
5186 session_id: &meerkat_core::types::SessionId,
5187 ) -> Result<(), SessionError> {
5188 self.inner
5189 .archive_with_mob_lifecycle_authority(session_id)
5190 .await
5191 }
5192 async fn archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(
5193 &self,
5194 session_id: &meerkat_core::types::SessionId,
5195 ) -> Result<(), SessionError> {
5196 self.inner
5197 .archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(session_id)
5198 .await
5199 }
5200 async fn execution_snapshot(
5201 &self,
5202 session_id: &meerkat_core::types::SessionId,
5203 ) -> Result<Option<meerkat_core::agent::AgentExecutionSnapshot>, SessionError> {
5204 self.inner.execution_snapshot(session_id).await
5205 }
5206 async fn tool_scope_snapshot(
5207 &self,
5208 session_id: &meerkat_core::types::SessionId,
5209 ) -> Result<Option<meerkat_core::ToolScopeSnapshot>, SessionError> {
5210 self.inner.tool_scope_snapshot(session_id).await
5211 }
5212 async fn external_tool_surface_snapshot(
5213 &self,
5214 session_id: &meerkat_core::types::SessionId,
5215 ) -> Result<Option<meerkat_core::ExternalToolSurfaceSnapshot>, SessionError> {
5216 self.inner.external_tool_surface_snapshot(session_id).await
5217 }
5218 async fn peer_ingress_runtime_snapshot(
5219 &self,
5220 session_id: &meerkat_core::types::SessionId,
5221 ) -> Result<Option<meerkat_core::PeerIngressRuntimeSnapshot>, SessionError> {
5222 self.inner.peer_ingress_runtime_snapshot(session_id).await
5223 }
5224 async fn apply_runtime_turn(
5225 &self,
5226 session_id: &meerkat_core::types::SessionId,
5227 run_id: meerkat_core::lifecycle::RunId,
5228 req: meerkat_core::service::StartTurnRequest,
5229 boundary: meerkat_core::lifecycle::run_primitive::RunApplyBoundary,
5230 contributing_input_ids: Vec<meerkat_core::lifecycle::InputId>,
5231 ) -> Result<meerkat_core::lifecycle::core_executor::CoreApplyOutput, SessionError> {
5232 self.inner
5233 .apply_runtime_turn(session_id, run_id, req, boundary, contributing_input_ids)
5234 .await
5235 }
5236 async fn prepare_transient_turn_context_for_active_turn(
5237 &self,
5238 session_id: &meerkat_core::types::SessionId,
5239 expected_run_id: &meerkat_core::lifecycle::RunId,
5240 contexts: Vec<meerkat_core::lifecycle::run_primitive::TurnRequestContext>,
5241 ) -> Result<meerkat_core::CoreBoundaryStageOutput, meerkat_core::CoreBoundaryStageError> {
5242 self.inner
5243 .prepare_transient_turn_context_for_active_turn(session_id, expected_run_id, contexts)
5244 .await
5245 }
5246 async fn acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
5247 &self,
5248 session_id: &meerkat_core::types::SessionId,
5249 authority: &meerkat_core::CommittedSessionBoundaryAuthority,
5250 ) -> Result<(), SessionError> {
5251 self.inner
5252 .acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
5253 session_id, authority,
5254 )
5255 .await
5256 }
5257 async fn acquire_runtime_turn_finalization_guard(
5258 &self,
5259 session_id: &meerkat_core::types::SessionId,
5260 ) -> Result<Box<dyn meerkat_core::lifecycle::CoreExecutorTurnFinalizationGuard>, SessionError>
5261 {
5262 self.inner
5263 .acquire_runtime_turn_finalization_guard(session_id)
5264 .await
5265 }
5266 async fn checkpoint_committed_runtime_session_snapshot_under_turn_finalization_boundary(
5267 &self,
5268 session_id: &meerkat_core::types::SessionId,
5269 session_snapshot: Arc<Vec<u8>>,
5270 ) -> Result<(), SessionError> {
5271 self.inner
5272 .checkpoint_committed_runtime_session_snapshot_under_turn_finalization_boundary(
5273 session_id,
5274 session_snapshot,
5275 )
5276 .await
5277 }
5278 async fn discard_live_session_after_runtime_stop_terminalized(
5279 &self,
5280 session_id: &meerkat_core::types::SessionId,
5281 ) -> Result<(), SessionError> {
5282 self.inner
5283 .discard_live_session_after_runtime_stop_terminalized(session_id)
5284 .await
5285 }
5286 async fn discard_live_session_after_runtime_stop_terminalized_under_turn_finalization_boundary(
5287 &self,
5288 session_id: &meerkat_core::types::SessionId,
5289 ) -> Result<(), SessionError> {
5290 self.inner
5291 .discard_live_session_after_runtime_stop_terminalized_under_turn_finalization_boundary(
5292 session_id,
5293 )
5294 .await
5295 }
5296 async fn publish_interaction_terminals(
5297 &self,
5298 session_id: &meerkat_core::types::SessionId,
5299 events: &[meerkat_core::event::AgentEvent],
5300 ) -> Result<
5301 Vec<meerkat_core::lifecycle::core_executor::CoreInteractionTerminalPublicationReceipt>,
5302 SessionError,
5303 > {
5304 self.inner
5305 .publish_interaction_terminals(session_id, events)
5306 .await
5307 }
5308 async fn discard_live_session(
5309 &self,
5310 session_id: &meerkat_core::types::SessionId,
5311 ) -> Result<(), SessionError> {
5312 self.inner.discard_live_session(session_id).await
5313 }
5314 async fn discard_live_session_under_runtime_turn_boundary(
5315 &self,
5316 session_id: &meerkat_core::types::SessionId,
5317 ) -> Result<(), SessionError> {
5318 self.inner
5319 .discard_live_session_under_runtime_turn_boundary(session_id)
5320 .await
5321 }
5322 async fn discard_live_session_actor_under_runtime_turn_boundary(
5323 &self,
5324 witness: &meerkat_session::LiveSessionActorWitness,
5325 ) -> Result<bool, SessionError> {
5326 self.inner
5327 .discard_live_session_actor_under_runtime_turn_boundary(witness)
5328 .await
5329 }
5330 async fn await_event_projection_drain(
5331 &self,
5332 session_id: &meerkat_core::types::SessionId,
5333 ) -> Result<bool, SessionError> {
5334 self.inner.await_event_projection_drain(session_id).await
5335 }
5336 async fn checkpoint_committed_runtime_session_snapshot(
5337 &self,
5338 session_id: &meerkat_core::types::SessionId,
5339 session_snapshot: Arc<Vec<u8>>,
5340 ) -> Result<(), SessionError> {
5341 self.inner
5342 .checkpoint_committed_runtime_session_snapshot(session_id, session_snapshot)
5343 .await
5344 }
5345 async fn cancel_all_checkpointers(&self) {
5346 self.inner.cancel_all_checkpointers().await;
5347 }
5348 async fn rearm_all_checkpointers(&self) {
5349 self.inner.rearm_all_checkpointers().await;
5350 }
5351}
5352
5353pub struct MobBootstrapSpec {
5355 pub definition: MobDefinition,
5356 pub storage: MobStorage,
5357 pub session_service: Arc<dyn MobSessionService>,
5358 pub binary_blob_store: Option<Arc<dyn BinaryBlobStore>>,
5359 pub(crate) agent_mob_mcp_state: Option<Arc<meerkat_mob_mcp::MobMcpState>>,
5360 pub(crate) implicit_delegate_retirement_overrides: Option<ImplicitDelegateRetirementOverrides>,
5361 pub(crate) agent_mob_default_llm_client_slot: Option<SharedDefaultLlmClientSlot>,
5362 pub(crate) console_spawn_sink_slot: Option<SharedConsoleSpawnSinkSlot>,
5363 pub(crate) identity_runtime_slot: Option<SharedIdentityRuntimeSlot>,
5364 pub options: MobBootstrapOptions,
5365 pub runtime_adapter: Option<Arc<meerkat_runtime::MeerkatMachine>>,
5372 pub(crate) spawn_member_customizer: Option<Arc<dyn meerkat_mob::SpawnMemberCustomizer>>,
5376 pub(crate) default_external_tools_provider: Option<meerkat_mob::ExternalToolsProvider>,
5385 pub(crate) workgraph_service: Option<meerkat::WorkGraphService>,
5391 pub(crate) workgraph_admission_slots: Vec<crate::workgraph_admission::WorkGraphAdmissionSlot>,
5397 pub(crate) workgraph_admission_sidecar: Option<PathBuf>,
5402 pub resolved_storage: Option<ResolvedStorageSummary>,
5408 pub(crate) session_write_epochs: Option<Arc<SessionSnapshotWriteEpochs>>,
5413 pub committed_boundary_recoverer:
5419 Option<Arc<dyn crate::identity_first::bridge::CommittedBoundaryRecoverer>>,
5420 pub(crate) dispatch_taint_slot: crate::memory::dispatch_taint::DispatchTaintSlot,
5424 pub(crate) _ephemeral_dir: Option<Arc<tempfile::TempDir>>,
5427}
5428
5429impl MobBootstrapSpec {
5430 pub fn new(
5431 definition: MobDefinition,
5432 storage: MobStorage,
5433 session_service: Arc<dyn MobSessionService>,
5434 ) -> Self {
5435 let dispatch_taint_slot = crate::memory::dispatch_taint::DispatchTaintSlot::default();
5441 let session_service = Arc::new(PreBuildMobSessionService {
5442 inner: session_service,
5443 hook: no_op_pre_build_hook(),
5444 dispatch_taint: Some(dispatch_taint_slot.clone()),
5445 after_create_hook: None,
5446 runtime_adapter_override: None,
5447 session_read_absorber: None,
5448 archived_terminal_authority: None,
5449 }) as Arc<dyn MobSessionService>;
5450 Self {
5451 definition,
5452 storage,
5453 session_service,
5454 binary_blob_store: None,
5455 agent_mob_mcp_state: None,
5456 implicit_delegate_retirement_overrides: None,
5457 agent_mob_default_llm_client_slot: None,
5458 console_spawn_sink_slot: None,
5459 identity_runtime_slot: None,
5460 options: MobBootstrapOptions {
5461 allow_ephemeral_sessions: true,
5462 notify_orchestrator_on_resume: true,
5463 default_llm_client: None,
5464 },
5465 runtime_adapter: None,
5466 spawn_member_customizer: None,
5467 default_external_tools_provider: None,
5468 workgraph_service: None,
5469 workgraph_admission_slots: Vec::new(),
5470 workgraph_admission_sidecar: None,
5471 resolved_storage: None,
5472 session_write_epochs: None,
5473 committed_boundary_recoverer: None,
5474 dispatch_taint_slot,
5475 _ephemeral_dir: None,
5476 }
5477 }
5478
5479 pub fn dispatch_taint_slot(&self) -> crate::memory::dispatch_taint::DispatchTaintSlot {
5485 self.dispatch_taint_slot.clone()
5486 }
5487
5488 #[must_use]
5492 pub fn with_resolved_storage(mut self, summary: ResolvedStorageSummary) -> Self {
5493 self.resolved_storage = Some(summary);
5494 self
5495 }
5496
5497 pub fn with_options(mut self, options: MobBootstrapOptions) -> Self {
5498 self.options = options;
5499 self
5500 }
5501
5502 pub fn with_agent_mob_tools(
5522 mut self,
5523 mob_tools_slot: Arc<
5524 std::sync::RwLock<Option<Arc<dyn meerkat_core::service::MobToolsFactory>>>,
5525 >,
5526 ) -> Self {
5527 let (
5528 agent_mob_mcp_state,
5529 implicit_delegate_retirement_overrides,
5530 agent_mob_default_llm_client_slot,
5531 console_spawn_sink_slot,
5532 identity_runtime_slot,
5533 ) = install_agent_mob_tools(
5534 &self.definition,
5535 mob_tools_slot,
5536 Arc::clone(&self.session_service),
5537 self.workgraph_service.clone(),
5538 None,
5539 );
5540 self.agent_mob_mcp_state = Some(agent_mob_mcp_state);
5541 self.implicit_delegate_retirement_overrides = Some(implicit_delegate_retirement_overrides);
5542 self.agent_mob_default_llm_client_slot = Some(agent_mob_default_llm_client_slot);
5543 self.console_spawn_sink_slot = Some(console_spawn_sink_slot);
5544 self.identity_runtime_slot = Some(identity_runtime_slot);
5545 self
5546 }
5547
5548 #[must_use]
5557 pub fn with_workgraph_service(mut self, service: Option<meerkat::WorkGraphService>) -> Self {
5558 self.workgraph_service = service;
5559 self
5560 }
5561
5562 #[must_use]
5570 pub fn with_workgraph_admission_slot(
5571 mut self,
5572 slot: crate::workgraph_admission::WorkGraphAdmissionSlot,
5573 ) -> Self {
5574 self.workgraph_admission_slots.push(slot);
5575 self
5576 }
5577
5578 #[must_use]
5586 pub fn with_workgraph_admission_sidecar(mut self, state_dir: &Path) -> Self {
5587 self.workgraph_admission_sidecar =
5588 Some(crate::workgraph_admission::workgraph_admission_sidecar_path(state_dir));
5589 self
5590 }
5591
5592 pub fn with_default_external_tools_provider(
5599 mut self,
5600 provider: meerkat_mob::ExternalToolsProvider,
5601 ) -> Self {
5602 self.default_external_tools_provider = Some(provider);
5603 self
5604 }
5605
5606 pub fn with_session_runtime_adapter(
5614 mut self,
5615 adapter: Arc<meerkat_runtime::MeerkatMachine>,
5616 ) -> Self {
5617 self.session_service = Arc::new(PreBuildMobSessionService {
5618 inner: self.session_service,
5619 hook: no_op_pre_build_hook(),
5620 dispatch_taint: None,
5621 after_create_hook: None,
5622 runtime_adapter_override: Some(adapter),
5623 session_read_absorber: None,
5624 archived_terminal_authority: None,
5625 });
5626 self
5627 }
5628
5629 pub fn with_after_create_hook(mut self, hook: AfterCreateHook) -> Self {
5635 self.session_service = Arc::new(AfterCreateMobSessionService {
5636 inner: self.session_service,
5637 after_hook: hook,
5638 });
5639 self
5640 }
5641
5642 pub fn with_session_write_epochs(mut self, epochs: &SessionWriteEpochsHandle) -> Self {
5659 self.session_write_epochs = Some(Arc::clone(&epochs.epochs));
5660 self.session_service = Arc::new(PreBuildMobSessionService {
5661 inner: self.session_service,
5662 hook: no_op_pre_build_hook(),
5663 dispatch_taint: None,
5664 after_create_hook: None,
5665 runtime_adapter_override: None,
5666 session_read_absorber: Some(Arc::new(SessionDocumentReadAbsorber::new(Arc::clone(
5667 &epochs.epochs,
5668 )))),
5669 archived_terminal_authority: None,
5670 });
5671 self
5672 }
5673
5674 #[must_use]
5687 pub fn with_runtime_archived_terminal_authority(
5688 mut self,
5689 runtime_store: Arc<dyn meerkat_runtime::RuntimeStore>,
5690 ) -> Self {
5691 self.session_service = Arc::new(PreBuildMobSessionService {
5692 inner: self.session_service,
5693 hook: no_op_pre_build_hook(),
5694 dispatch_taint: None,
5695 after_create_hook: None,
5696 runtime_adapter_override: None,
5697 session_read_absorber: None,
5698 archived_terminal_authority: Some(runtime_store),
5699 });
5700 self
5701 }
5702
5703 pub fn ephemeral(
5708 definition: MobDefinition,
5709 storage: MobStorage,
5710 store_path: PathBuf,
5711 max_sessions: usize,
5712 session_store: Option<Arc<dyn AgentSessionStore>>,
5713 ) -> Self {
5714 Self::ephemeral_inner(
5715 definition,
5716 storage,
5717 store_path,
5718 max_sessions,
5719 session_store,
5720 None,
5721 CapabilityFlags::default(),
5722 None,
5723 None,
5724 )
5725 }
5726
5727 pub fn ephemeral_with_hook(
5731 definition: MobDefinition,
5732 storage: MobStorage,
5733 store_path: PathBuf,
5734 max_sessions: usize,
5735 session_store: Option<Arc<dyn AgentSessionStore>>,
5736 hook: impl Fn(
5737 &mut CreateSessionRequest,
5738 ) -> std::pin::Pin<
5739 Box<dyn std::future::Future<Output = Result<(), SessionError>> + Send + '_>,
5740 > + Send
5741 + Sync
5742 + 'static,
5743 ) -> Self {
5744 Self::ephemeral_inner(
5745 definition,
5746 storage,
5747 store_path,
5748 max_sessions,
5749 session_store,
5750 Some(Arc::new(hook)),
5751 CapabilityFlags::default(),
5752 None,
5753 None,
5754 )
5755 }
5756
5757 #[allow(clippy::too_many_arguments)]
5758 pub(crate) fn ephemeral_inner(
5759 definition: MobDefinition,
5760 storage: MobStorage,
5761 store_path: PathBuf,
5762 max_sessions: usize,
5763 session_store: Option<Arc<dyn AgentSessionStore>>,
5764 hook: Option<PreBuildHook>,
5765 mut caps: CapabilityFlags,
5766 after_create_hook: Option<AfterCreateHook>,
5767 agent_config: Option<Config>,
5768 ) -> Self {
5769 caps.image_generation |= mob_definition_may_use_image_generation(&definition);
5770 let binary_blob_store: Arc<dyn BinaryBlobStore> = Arc::new(ObjectStoreBlobStore::memory());
5771 let blob_store: Arc<dyn meerkat_core::BlobStore> =
5772 Arc::new(Base64BlobStoreAdapter::new(binary_blob_store.clone()));
5773 let runtime_adapter = if caps.image_generation {
5774 let runtime_store: Arc<dyn meerkat_runtime::RuntimeStore> =
5775 Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
5776 Some(Arc::new(meerkat_runtime::MeerkatMachine::persistent(
5777 runtime_store,
5778 Arc::clone(&blob_store),
5779 )))
5780 } else {
5781 None
5782 };
5783 let mut factory = AgentFactory::new(&store_path)
5784 .builtins(caps.builtins)
5785 .shell(caps.shell)
5786 .mob(caps.mob)
5787 .comms(caps.comms)
5788 .memory(caps.memory);
5789 if let Some(machine) = runtime_adapter.clone() {
5790 factory = factory.with_image_generation_machine(machine);
5791 }
5792 let config = agent_config.unwrap_or_default();
5793 let mut builder = FactoryAgentBuilder::new(factory, config);
5794 builder.default_blob_store = Some(blob_store);
5795 if let Some(store) = session_store {
5796 builder.default_session_store = Some(store);
5797 }
5798 let (session_llm_reconfigure_blueprint, session_llm_default_client_slot) =
5799 session_llm_reconfigure_blueprint(&builder, &store_path);
5800 let mob_tools_slot = Arc::clone(&builder.default_mob_tools);
5801 let (workgraph_service, workgraph_admission_slot) =
5805 crate::workgraph_wiring::attach_workgraph_tools_ephemeral(
5806 &builder,
5807 definition.id.as_str(),
5808 );
5809 let concrete_session_service = Arc::new(meerkat_session::EphemeralSessionService::new(
5810 builder,
5811 max_sessions,
5812 ));
5813 let effective_runtime_adapter = runtime_adapter
5814 .clone()
5815 .or_else(|| MobSessionService::runtime_adapter(concrete_session_service.as_ref()));
5816 let reconfigure_service: Arc<
5817 dyn meerkat::session_runtime::llm_reconfigure::SessionRuntimeLlmReconfigureService,
5818 > = concrete_session_service.clone();
5819 if let Some(effective_runtime_adapter) = effective_runtime_adapter.as_ref() {
5820 session_llm_reconfigure_blueprint
5821 .install(effective_runtime_adapter, reconfigure_service);
5822 } else {
5823 tracing::error!(
5824 "ephemeral session service has no runtime adapter; runtime LLM reconfiguration is unavailable"
5825 );
5826 }
5827 let session_service: Arc<dyn MobSessionService> = concrete_session_service;
5828 let hook = hook.unwrap_or_else(no_op_pre_build_hook);
5829 let after_create_hook = if let Some(runtime_adapter) = runtime_adapter {
5830 let user_after_create_hook = after_create_hook.clone();
5831 Some(Arc::new(
5832 move |session_id: meerkat_core::types::SessionId, ctx: SessionCreatedContext| {
5833 let runtime_adapter = runtime_adapter.clone();
5834 let user_after_create_hook = user_after_create_hook.clone();
5835 Box::pin(async move {
5836 if let Err(error) =
5840 runtime_adapter.register_session(session_id.clone()).await
5841 {
5842 tracing::error!(
5843 session_id = %session_id,
5844 error = %error,
5845 "post-create session runtime registration failed"
5846 );
5847 }
5848 if let Some(user_after_create_hook) = user_after_create_hook {
5849 user_after_create_hook(session_id, ctx).await;
5850 }
5851 })
5852 as std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>>
5853 },
5854 ) as AfterCreateHook)
5855 } else {
5856 after_create_hook
5857 };
5858 let session_service = Arc::new(PreBuildMobSessionService {
5859 inner: session_service,
5860 hook,
5861 dispatch_taint: None,
5862 after_create_hook,
5863 runtime_adapter_override: effective_runtime_adapter.clone(),
5864 session_read_absorber: None,
5865 archived_terminal_authority: None,
5866 }) as Arc<dyn MobSessionService>;
5867 let (
5868 agent_mob_mcp_state,
5869 implicit_delegate_retirement_overrides,
5870 agent_mob_default_llm_client_slot,
5871 console_spawn_sink_slot,
5872 identity_runtime_slot,
5873 ) = install_agent_mob_tools(
5874 &definition,
5875 mob_tools_slot,
5876 Arc::clone(&session_service),
5877 Some(workgraph_service.clone()),
5878 Some(session_llm_default_client_slot),
5879 );
5880 let mut spec = Self::new(definition, storage, session_service);
5881 spec.agent_mob_mcp_state = Some(agent_mob_mcp_state);
5882 spec.implicit_delegate_retirement_overrides = Some(implicit_delegate_retirement_overrides);
5883 spec.agent_mob_default_llm_client_slot = Some(agent_mob_default_llm_client_slot);
5884 spec.console_spawn_sink_slot = Some(console_spawn_sink_slot);
5885 spec.identity_runtime_slot = Some(identity_runtime_slot);
5886 spec.runtime_adapter = effective_runtime_adapter;
5887 spec.binary_blob_store = Some(binary_blob_store);
5888 spec.workgraph_service = Some(workgraph_service);
5889 spec.workgraph_admission_slots
5890 .push(workgraph_admission_slot);
5891 let mut slots = vec![
5895 StorageSlotSummary::declared_ephemeral(
5896 "sessions",
5897 "EphemeralSessionService",
5898 "declared by the ephemeral launch mode",
5899 ),
5900 blob_slot_summary(BlobDurability::DeclaredEphemeral),
5901 ];
5902 slots.extend(scratch_ring_buffer_slots());
5903 spec.resolved_storage = Some(
5904 ResolvedStorageSummary::new(BlobDurability::DeclaredEphemeral, None).with_slots(slots),
5905 );
5906 spec
5907 }
5908
5909 pub fn persistent(
5922 definition: MobDefinition,
5923 storage: MobStorage,
5924 store_path: PathBuf,
5925 max_sessions: usize,
5926 session_store: Arc<dyn SessionStore>,
5927 ) -> Result<Self, StorageResolutionError> {
5928 Self::persistent_inner(
5929 definition,
5930 storage,
5931 store_path,
5932 max_sessions,
5933 session_store,
5934 "caller-supplied session store",
5935 None,
5936 false,
5937 false,
5938 None,
5939 None,
5940 CapabilityFlags::default(),
5941 None,
5942 None,
5943 )
5944 }
5945
5946 pub fn persistent_with_hook(
5956 definition: MobDefinition,
5957 storage: MobStorage,
5958 store_path: PathBuf,
5959 max_sessions: usize,
5960 session_store: Arc<dyn SessionStore>,
5961 hook: impl Fn(
5962 &mut CreateSessionRequest,
5963 ) -> std::pin::Pin<
5964 Box<dyn std::future::Future<Output = Result<(), SessionError>> + Send + '_>,
5965 > + Send
5966 + Sync
5967 + 'static,
5968 ) -> Result<Self, StorageResolutionError> {
5969 Self::persistent_inner(
5970 definition,
5971 storage,
5972 store_path,
5973 max_sessions,
5974 session_store,
5975 "caller-supplied session store",
5976 None,
5977 false,
5978 false,
5979 None,
5980 Some(Arc::new(hook)),
5981 CapabilityFlags::default(),
5982 None,
5983 None,
5984 )
5985 }
5986
5987 #[allow(clippy::too_many_arguments)]
5988 pub(crate) fn persistent_inner(
5989 definition: MobDefinition,
5990 storage: MobStorage,
5991 store_path: PathBuf,
5992 max_sessions: usize,
5993 session_store: Arc<dyn SessionStore>,
5994 session_store_kind: &str,
5995 custom_blob_store: Option<BlobStoreInjection>,
5996 ephemeral_blobs: bool,
5997 ephemeral_runtime_store: bool,
5998 schedule_store: Option<Arc<dyn meerkat::ScheduleStore>>,
5999 hook: Option<PreBuildHook>,
6000 caps: CapabilityFlags,
6001 after_create_hook: Option<AfterCreateHook>,
6002 agent_config: Option<Config>,
6003 ) -> Result<Self, StorageResolutionError> {
6004 Self::persistent_inner_with_provider_stores(
6005 definition,
6006 storage,
6007 store_path,
6008 max_sessions,
6009 session_store,
6010 session_store_kind,
6011 custom_blob_store,
6012 ephemeral_blobs,
6013 ephemeral_runtime_store,
6014 schedule_store,
6015 hook,
6016 caps,
6017 after_create_hook,
6018 agent_config,
6019 None,
6020 )
6021 }
6022
6023 #[allow(clippy::too_many_arguments)]
6029 pub(crate) fn persistent_inner_with_provider_stores(
6030 definition: MobDefinition,
6031 storage: MobStorage,
6032 store_path: PathBuf,
6033 max_sessions: usize,
6034 session_store: Arc<dyn SessionStore>,
6035 session_store_kind: &str,
6036 custom_blob_store: Option<BlobStoreInjection>,
6037 ephemeral_blobs: bool,
6038 ephemeral_runtime_store: bool,
6039 schedule_store: Option<Arc<dyn meerkat::ScheduleStore>>,
6040 hook: Option<PreBuildHook>,
6041 mut caps: CapabilityFlags,
6042 after_create_hook: Option<AfterCreateHook>,
6043 agent_config: Option<Config>,
6044 provider_meerkat_stores: Option<crate::storage_provider::ProviderMeerkatStores>,
6045 ) -> Result<Self, StorageResolutionError> {
6046 caps.image_generation |= mob_definition_may_use_image_generation(&definition);
6047 let (binary_blob_store, blob_store, blob_durability): (
6053 Arc<dyn BinaryBlobStore>,
6054 Arc<dyn meerkat_core::BlobStore>,
6055 BlobDurability,
6056 ) = if let Some(injection) = custom_blob_store {
6057 let (binary_blob_store, blob_store) = injection.into_pair();
6058 let persistent = binary_blob_store.is_persistent();
6059 (
6060 binary_blob_store,
6061 blob_store,
6062 BlobDurability::Custom { persistent },
6063 )
6064 } else if ephemeral_blobs {
6065 let binary_blob_store: Arc<dyn BinaryBlobStore> =
6066 Arc::new(ObjectStoreBlobStore::memory());
6067 let blob_store: Arc<dyn meerkat_core::BlobStore> =
6068 Arc::new(Base64BlobStoreAdapter::new(binary_blob_store.clone()));
6069 (
6070 binary_blob_store,
6071 blob_store,
6072 BlobDurability::DeclaredEphemeral,
6073 )
6074 } else {
6075 let blob_path = store_path.join(crate::storage_layout::BLOB_ROOT_DIR_NAME);
6076 let binary_blob_store: Arc<dyn BinaryBlobStore> =
6077 match ObjectStoreBlobStore::local(blob_path.clone()) {
6078 Ok(store) => Arc::new(store),
6079 Err(err) => {
6080 return Err(BlobStoreResolutionError::OpenFailed {
6081 path: blob_path,
6082 message: err.to_string(),
6083 }
6084 .into());
6085 }
6086 };
6087 let blob_store: Arc<dyn meerkat_core::BlobStore> =
6088 Arc::new(Base64BlobStoreAdapter::new(binary_blob_store.clone()));
6089 (
6090 binary_blob_store,
6091 blob_store,
6092 BlobDurability::PersistentDisk,
6093 )
6094 };
6095 if !binary_blob_store.is_persistent() && !ephemeral_blobs {
6099 return Err(BlobStoreResolutionError::NonPersistentUndeclared.into());
6100 }
6101 let session_store_incremental = Some(probe_session_store_incremental(
6105 &session_store,
6106 session_store_kind,
6107 ));
6108 let (runtime_store, runtime_store_slot): (
6122 Arc<dyn meerkat_runtime::RuntimeStore>,
6123 StorageSlotSummary,
6124 ) = if ephemeral_runtime_store {
6125 (
6126 Arc::new(meerkat_runtime::InMemoryRuntimeStore::new()),
6127 StorageSlotSummary::declared_ephemeral(
6128 "runtime",
6129 "InMemoryRuntimeStore",
6130 "explicitly declared: sessions do not survive process restart",
6131 ),
6132 )
6133 } else if let Some(provider) = provider_meerkat_stores.as_ref() {
6134 (
6138 Arc::clone(&provider.runtime_store),
6139 provider.runtime_slot_summary(),
6140 )
6141 } else {
6142 (
6143 build_persistent_runtime_store(&store_path)?,
6144 StorageSlotSummary::persistent("runtime", "SqliteRuntimeStore"),
6145 )
6146 };
6147 let session_read_epochs = Arc::new(SessionSnapshotWriteEpochs::default());
6157 let runtime_store: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
6158 SessionStoreBackedRuntimeStore::with_write_epochs_and_durable_projection(
6159 runtime_store,
6160 Arc::clone(&session_read_epochs),
6161 session_store.clone(),
6162 ),
6163 );
6164 let runtime_adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent(
6165 Arc::clone(&runtime_store),
6166 Arc::clone(&blob_store),
6167 ));
6168 let mut factory = AgentFactory::new(&store_path)
6169 .builtins(caps.builtins)
6170 .shell(caps.shell)
6171 .mob(caps.mob)
6172 .comms(caps.comms)
6173 .memory(caps.memory);
6174 if caps.image_generation {
6175 factory = factory.with_image_generation_machine(runtime_adapter.clone());
6176 }
6177 let config = agent_config.unwrap_or_default();
6178 let mut builder = FactoryAgentBuilder::new(factory, config);
6179 builder.default_session_store = Some(Arc::new(StoreAdapter::new(session_store.clone())));
6180 builder.default_blob_store = Some(blob_store.clone());
6181 let (job_store, job_store_slot): (Arc<dyn meerkat::DetachedJobStore>, StorageSlotSummary) =
6182 if let Some(provider) = provider_meerkat_stores.as_ref() {
6183 (Arc::clone(&provider.job_store), provider.job_slot_summary())
6184 } else {
6185 let path = meerkat_store::realm_paths_in(
6186 &store_path,
6187 crate::storage_provider::MEERKAT_LEVEL_REALM_ID,
6188 )
6189 .jobs_sqlite_path;
6190 let store =
6191 meerkat::SqliteDetachedJobStore::open(path.clone()).map_err(|error| {
6192 crate::storage_health::JobStoreResolutionError {
6193 path,
6194 message: error.to_string(),
6195 }
6196 })?;
6197 (
6198 Arc::new(store),
6199 StorageSlotSummary::persistent("jobs", "SqliteDetachedJobStore"),
6200 )
6201 };
6202 builder.default_detached_job_store = Some(job_store);
6203 let (session_llm_reconfigure_blueprint, session_llm_default_client_slot) =
6204 session_llm_reconfigure_blueprint(&builder, &store_path);
6205 let mob_tools_slot = Arc::clone(&builder.default_mob_tools);
6206 let schedule_slot = schedule_store.map(|store| {
6211 let _attached =
6212 crate::schedule_wiring::attach_schedule_tools_with_store(&builder, store);
6213 StorageSlotSummary::persistent("schedule", "custom schedule store").with_detail(
6214 "caller-injected store; durability rides with the injector. Library mode \
6215 attaches schedule tools without a firing host",
6216 )
6217 });
6218 let (workgraph_service, workgraph_admission_slot, workgraph_slot) =
6223 if let Some(provider) = provider_meerkat_stores.as_ref() {
6224 let service = meerkat::WorkGraphService::with_scope(
6225 Arc::clone(&provider.workgraph_store),
6226 definition.id.as_str(),
6227 meerkat::WorkNamespace::default(),
6228 );
6229 let slot = crate::workgraph_wiring::install_workgraph_tools(&builder, &service);
6230 (Some(service), Some(slot), provider.workgraph_slot_summary())
6231 } else {
6232 match crate::workgraph_wiring::attach_workgraph_tools_reporting(
6233 &builder,
6234 &store_path,
6235 definition.id.as_str(),
6236 ) {
6237 Ok((service, slot)) => (
6238 Some(service),
6239 Some(slot),
6240 StorageSlotSummary::persistent("workgraph", "SqliteWorkGraphStore"),
6241 ),
6242 Err(error) => (
6243 None,
6244 None,
6245 StorageSlotSummary::degraded(
6246 "workgraph",
6247 format!("workgraph store failed to open; workgraph disabled: {error}"),
6248 ),
6249 ),
6250 }
6251 };
6252 let archived_terminal_authority = Arc::clone(&runtime_store);
6253 let concrete_session_service = Arc::new(meerkat_session::PersistentSessionService::new(
6254 builder,
6255 max_sessions,
6256 session_store,
6257 runtime_store,
6258 blob_store,
6259 ));
6260 let reconfigure_service: Arc<
6261 dyn meerkat::session_runtime::llm_reconfigure::SessionRuntimeLlmReconfigureService,
6262 > = concrete_session_service.clone();
6263 session_llm_reconfigure_blueprint.install(&runtime_adapter, reconfigure_service);
6264 let committed_boundary_recoverer: Arc<
6268 dyn crate::identity_first::bridge::CommittedBoundaryRecoverer,
6269 > = concrete_session_service.clone();
6270 let session_service: Arc<dyn MobSessionService> = concrete_session_service;
6271 let hook = hook.unwrap_or_else(no_op_pre_build_hook);
6272 let session_service = Arc::new(PreBuildMobSessionService {
6273 inner: session_service,
6274 hook,
6275 dispatch_taint: None,
6276 after_create_hook,
6277 runtime_adapter_override: None,
6278 session_read_absorber: Some(Arc::new(SessionDocumentReadAbsorber::new(Arc::clone(
6279 &session_read_epochs,
6280 )))),
6281 archived_terminal_authority: Some(archived_terminal_authority),
6282 }) as Arc<dyn MobSessionService>;
6283 let (
6284 agent_mob_mcp_state,
6285 implicit_delegate_retirement_overrides,
6286 agent_mob_default_llm_client_slot,
6287 console_spawn_sink_slot,
6288 identity_runtime_slot,
6289 ) = install_agent_mob_tools(
6290 &definition,
6291 mob_tools_slot,
6292 Arc::clone(&session_service),
6293 workgraph_service.clone(),
6294 Some(session_llm_default_client_slot),
6295 );
6296 let mut spec = Self::new(definition, storage, session_service);
6297 spec.committed_boundary_recoverer = Some(committed_boundary_recoverer);
6298 spec.session_write_epochs = Some(session_read_epochs);
6299 spec.agent_mob_mcp_state = Some(agent_mob_mcp_state);
6300 spec.implicit_delegate_retirement_overrides = Some(implicit_delegate_retirement_overrides);
6301 spec.agent_mob_default_llm_client_slot = Some(agent_mob_default_llm_client_slot);
6302 spec.console_spawn_sink_slot = Some(console_spawn_sink_slot);
6303 spec.identity_runtime_slot = Some(identity_runtime_slot);
6304 spec.runtime_adapter = Some(runtime_adapter);
6305 spec.binary_blob_store = Some(binary_blob_store);
6306 spec.workgraph_service = workgraph_service;
6307 if let Some(slot) = workgraph_admission_slot {
6308 spec.workgraph_admission_slots.push(slot);
6312 spec.workgraph_admission_sidecar =
6313 Some(crate::workgraph_admission::workgraph_admission_sidecar_path(&store_path));
6314 }
6315 let mut slots = vec![
6316 StorageSlotSummary::persistent("sessions", session_store_kind).with_detail(
6317 if session_store_kind == "SqliteSessionStore" {
6318 "builder-opened SQLite store under the state directory"
6319 } else {
6320 "caller-injected store; durability rides with the injector"
6321 },
6322 ),
6323 runtime_store_slot,
6324 blob_slot_summary(blob_durability),
6325 workgraph_slot,
6326 job_store_slot,
6327 ];
6328 if let Some(slot) = schedule_slot {
6329 slots.push(slot);
6330 }
6331 slots.extend(scratch_ring_buffer_slots());
6332 spec.resolved_storage = Some(
6333 ResolvedStorageSummary::new(blob_durability, session_store_incremental)
6334 .with_slots(slots),
6335 );
6336 Ok(spec)
6337 }
6338
6339 #[allow(clippy::too_many_arguments)]
6340 pub(crate) fn ephemeral_runtime_backed_inner(
6341 definition: MobDefinition,
6342 storage: MobStorage,
6343 store_path: PathBuf,
6344 max_sessions: usize,
6345 custom_session_store: Option<Arc<dyn SessionStore>>,
6346 session_store_kind: &str,
6347 custom_blob_store: Option<BlobStoreInjection>,
6348 schedule_store: Option<Arc<dyn meerkat::ScheduleStore>>,
6349 hook: Option<PreBuildHook>,
6350 caps: CapabilityFlags,
6351 after_create_hook: Option<AfterCreateHook>,
6352 agent_config: Option<Config>,
6353 ) -> Self {
6354 Self::ephemeral_runtime_backed_with_provider_stores(
6355 definition,
6356 storage,
6357 store_path,
6358 max_sessions,
6359 custom_session_store,
6360 session_store_kind,
6361 custom_blob_store,
6362 schedule_store,
6363 hook,
6364 caps,
6365 after_create_hook,
6366 agent_config,
6367 None,
6368 )
6369 }
6370
6371 #[allow(clippy::too_many_arguments)]
6376 pub(crate) fn ephemeral_runtime_backed_with_provider_stores(
6377 definition: MobDefinition,
6378 storage: MobStorage,
6379 store_path: PathBuf,
6380 max_sessions: usize,
6381 custom_session_store: Option<Arc<dyn SessionStore>>,
6382 session_store_kind: &str,
6383 custom_blob_store: Option<BlobStoreInjection>,
6384 schedule_store: Option<Arc<dyn meerkat::ScheduleStore>>,
6385 hook: Option<PreBuildHook>,
6386 mut caps: CapabilityFlags,
6387 after_create_hook: Option<AfterCreateHook>,
6388 agent_config: Option<Config>,
6389 provider_meerkat_stores: Option<crate::storage_provider::ProviderMeerkatStores>,
6390 ) -> Self {
6391 caps.image_generation |= mob_definition_may_use_image_generation(&definition);
6392 let config = agent_config.unwrap_or_default();
6393 let has_custom_session_store = custom_session_store.is_some();
6394 let session_store: Arc<dyn SessionStore> = custom_session_store
6395 .clone()
6396 .unwrap_or_else(|| Arc::new(meerkat_store::MemoryStore::new()));
6397 let (binary_blob_store, blob_store, blob_durability): (
6404 Arc<dyn BinaryBlobStore>,
6405 Arc<dyn meerkat_core::BlobStore>,
6406 BlobDurability,
6407 ) = if let Some(injection) = custom_blob_store {
6408 let (binary_blob_store, blob_store) = injection.into_pair();
6409 let persistent = binary_blob_store.is_persistent();
6410 (
6411 binary_blob_store,
6412 blob_store,
6413 BlobDurability::Custom { persistent },
6414 )
6415 } else {
6416 let binary_blob_store: Arc<dyn BinaryBlobStore> =
6417 Arc::new(ObjectStoreBlobStore::memory());
6418 let blob_store: Arc<dyn meerkat_core::BlobStore> =
6419 Arc::new(Base64BlobStoreAdapter::new(binary_blob_store.clone()));
6420 (
6421 binary_blob_store,
6422 blob_store,
6423 BlobDurability::DeclaredEphemeral,
6424 )
6425 };
6426 let session_store_incremental = custom_session_store
6429 .as_ref()
6430 .map(|store| probe_session_store_incremental(store, session_store_kind));
6431 let runtime_store: Arc<dyn meerkat_runtime::RuntimeStore> =
6439 if let Some(provider) = provider_meerkat_stores.as_ref() {
6440 if let Some(custom_session_store) = custom_session_store.clone() {
6447 Arc::new(SessionStoreBackedRuntimeStore::new(
6448 Arc::clone(&provider.runtime_store),
6449 custom_session_store,
6450 ))
6451 } else {
6452 Arc::clone(&provider.runtime_store)
6453 }
6454 } else if let Some(custom_session_store) = custom_session_store.clone() {
6455 Arc::new(SessionStoreBackedRuntimeStore::new(
6456 Arc::new(meerkat_runtime::InMemoryRuntimeStore::new()),
6457 custom_session_store,
6458 ))
6459 } else {
6460 Arc::new(meerkat_runtime::InMemoryRuntimeStore::new())
6461 };
6462 let runtime_adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent(
6463 Arc::clone(&runtime_store),
6464 Arc::clone(&blob_store),
6465 ));
6466 let mut factory = AgentFactory::new(&store_path)
6467 .builtins(caps.builtins)
6468 .shell(caps.shell)
6469 .mob(caps.mob)
6470 .comms(caps.comms)
6471 .memory(caps.memory);
6472 if caps.image_generation {
6473 factory = factory.with_image_generation_machine(runtime_adapter.clone());
6474 }
6475 let mut builder = FactoryAgentBuilder::new(factory, config);
6476 builder.default_session_store = Some(Arc::new(StoreAdapter::new(session_store.clone())));
6477 builder.default_blob_store = Some(blob_store.clone());
6478 if let Some(provider) = provider_meerkat_stores.as_ref() {
6479 builder.default_detached_job_store = Some(Arc::clone(&provider.job_store));
6480 }
6481 let (session_llm_reconfigure_blueprint, session_llm_default_client_slot) =
6482 session_llm_reconfigure_blueprint(&builder, &store_path);
6483 let mob_tools_slot = Arc::clone(&builder.default_mob_tools);
6484 let schedule_slot = schedule_store.map(|store| {
6489 let _attached =
6490 crate::schedule_wiring::attach_schedule_tools_with_store(&builder, store);
6491 StorageSlotSummary::persistent("schedule", "custom schedule store").with_detail(
6492 "caller-injected store; durability rides with the injector. Library mode \
6493 attaches schedule tools without a firing host",
6494 )
6495 });
6496 let (workgraph_service, workgraph_admission_slot) =
6497 if let Some(provider) = provider_meerkat_stores.as_ref() {
6498 let service = meerkat::WorkGraphService::with_scope(
6502 Arc::clone(&provider.workgraph_store),
6503 definition.id.as_str(),
6504 meerkat::WorkNamespace::default(),
6505 );
6506 let slot = crate::workgraph_wiring::install_workgraph_tools(&builder, &service);
6507 (service, slot)
6508 } else {
6509 crate::workgraph_wiring::attach_workgraph_tools_ephemeral(
6510 &builder,
6511 definition.id.as_str(),
6512 )
6513 };
6514 let session_service: Arc<dyn MobSessionService> =
6515 if let Some(custom_session_store) = custom_session_store {
6516 let concrete_session_service =
6517 Arc::new(meerkat_session::PersistentSessionService::new(
6518 builder,
6519 max_sessions,
6520 custom_session_store,
6521 runtime_store.clone(),
6522 blob_store,
6523 ));
6524 let reconfigure_service: Arc<
6525 dyn meerkat::session_runtime::llm_reconfigure::SessionRuntimeLlmReconfigureService,
6526 > = concrete_session_service.clone();
6527 session_llm_reconfigure_blueprint.install(&runtime_adapter, reconfigure_service);
6528 concrete_session_service
6529 } else {
6530 let concrete_session_service = Arc::new(
6531 meerkat_session::EphemeralSessionService::new(builder, max_sessions),
6532 );
6533 let reconfigure_service: Arc<
6534 dyn meerkat::session_runtime::llm_reconfigure::SessionRuntimeLlmReconfigureService,
6535 > = concrete_session_service.clone();
6536 session_llm_reconfigure_blueprint.install(&runtime_adapter, reconfigure_service);
6537 concrete_session_service
6538 };
6539 let hook = hook.unwrap_or_else(no_op_pre_build_hook);
6540 let runtime_adapter_for_after_create = runtime_adapter.clone();
6541 let combined_after_create_hook: AfterCreateHook = Arc::new(move |session_id, ctx| {
6542 let runtime_adapter = runtime_adapter_for_after_create.clone();
6543 let after_create_hook = after_create_hook.clone();
6544 Box::pin(async move {
6545 if let Err(error) = runtime_adapter.register_session(session_id.clone()).await {
6549 tracing::error!(
6550 session_id = %session_id,
6551 error = %error,
6552 "post-create session runtime registration failed"
6553 );
6554 }
6555 if let Some(after_create_hook) = after_create_hook {
6556 after_create_hook(session_id, ctx).await;
6557 }
6558 })
6559 });
6560 let session_service = Arc::new(PreBuildMobSessionService {
6561 inner: session_service,
6562 hook,
6563 dispatch_taint: None,
6564 after_create_hook: Some(combined_after_create_hook),
6565 runtime_adapter_override: Some(runtime_adapter.clone()),
6566 session_read_absorber: None,
6567 archived_terminal_authority: None,
6568 }) as Arc<dyn MobSessionService>;
6569 let (
6570 agent_mob_mcp_state,
6571 implicit_delegate_retirement_overrides,
6572 agent_mob_default_llm_client_slot,
6573 console_spawn_sink_slot,
6574 identity_runtime_slot,
6575 ) = install_agent_mob_tools(
6576 &definition,
6577 mob_tools_slot,
6578 Arc::clone(&session_service),
6579 Some(workgraph_service.clone()),
6580 Some(session_llm_default_client_slot),
6581 );
6582 let mut spec = Self::new(definition, storage, session_service);
6583 spec.agent_mob_mcp_state = Some(agent_mob_mcp_state);
6584 spec.implicit_delegate_retirement_overrides = Some(implicit_delegate_retirement_overrides);
6585 spec.agent_mob_default_llm_client_slot = Some(agent_mob_default_llm_client_slot);
6586 spec.console_spawn_sink_slot = Some(console_spawn_sink_slot);
6587 spec.identity_runtime_slot = Some(identity_runtime_slot);
6588 spec.runtime_adapter = Some(runtime_adapter);
6589 spec.binary_blob_store = Some(binary_blob_store);
6590 spec.workgraph_service = Some(workgraph_service);
6591 spec.workgraph_admission_slots
6592 .push(workgraph_admission_slot);
6593 let mut slots = vec![
6594 if has_custom_session_store {
6595 StorageSlotSummary::persistent("sessions", session_store_kind)
6596 .with_detail("caller-injected store; durability rides with the injector")
6597 } else {
6598 StorageSlotSummary::declared_ephemeral(
6599 "sessions",
6600 "MemoryStore",
6601 "declared by the ephemeral launch mode",
6602 )
6603 },
6604 if let Some(provider) = provider_meerkat_stores.as_ref() {
6605 provider.runtime_slot_summary()
6606 } else {
6607 StorageSlotSummary::declared_ephemeral(
6608 "runtime",
6609 "InMemoryRuntimeStore",
6610 "declared by the ephemeral launch mode",
6611 )
6612 },
6613 blob_slot_summary(blob_durability),
6614 if let Some(provider) = provider_meerkat_stores.as_ref() {
6615 provider.workgraph_slot_summary()
6616 } else {
6617 StorageSlotSummary::declared_ephemeral(
6618 "workgraph",
6619 "MemoryWorkGraphStore",
6620 "declared by the ephemeral launch mode",
6621 )
6622 },
6623 if let Some(provider) = provider_meerkat_stores.as_ref() {
6624 provider.job_slot_summary()
6625 } else {
6626 StorageSlotSummary::declared_ephemeral(
6627 "jobs",
6628 "disabled",
6629 "semantic detached admission is unavailable in ephemeral launch mode",
6630 )
6631 },
6632 ];
6633 if let Some(slot) = schedule_slot {
6634 slots.push(slot);
6635 }
6636 slots.extend(scratch_ring_buffer_slots());
6637 spec.resolved_storage = Some(
6638 ResolvedStorageSummary::new(blob_durability, session_store_incremental)
6639 .with_slots(slots),
6640 );
6641 spec
6642 }
6643}
6644
6645#[derive(Debug)]
6647pub enum MobRuntimeError {
6648 Mob(MobError),
6649 InvalidInput(&'static str),
6650 InvalidConfig(String),
6651}
6652
6653impl std::fmt::Display for MobRuntimeError {
6654 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
6655 match self {
6656 Self::Mob(err) => write!(f, "{err}"),
6657 Self::InvalidInput(message) => write!(f, "{message}"),
6658 Self::InvalidConfig(message) => write!(f, "{message}"),
6659 }
6660 }
6661}
6662
6663impl std::error::Error for MobRuntimeError {}
6664
6665impl From<MobError> for MobRuntimeError {
6666 fn from(value: MobError) -> Self {
6667 Self::Mob(value)
6668 }
6669}
6670
6671#[derive(Clone, Debug)]
6680pub struct SessionCreatedContext {
6681 pub model: String,
6682 pub labels: std::collections::BTreeMap<String, String>,
6683 pub system_prompt: Option<String>,
6684}
6685
6686#[async_trait]
6693pub trait SessionHook: Send + Sync {
6694 async fn before_create(&self, _req: &mut CreateSessionRequest) -> Result<(), SessionError> {
6698 Ok(())
6699 }
6700
6701 async fn after_create(
6704 &self,
6705 _session_id: &meerkat_core::types::SessionId,
6706 _ctx: &SessionCreatedContext,
6707 ) {
6708 }
6709}
6710
6711#[derive(Clone, Copy, Debug)]
6713pub struct CapabilityFlags {
6714 pub builtins: bool,
6715 pub shell: bool,
6716 pub mob: bool,
6717 pub comms: bool,
6718 pub memory: bool,
6719 pub image_generation: bool,
6720}
6721
6722impl Default for CapabilityFlags {
6723 fn default() -> Self {
6724 Self {
6725 builtins: true,
6726 shell: true,
6727 mob: true,
6728 comms: true,
6729 memory: true,
6730 image_generation: false,
6731 }
6732 }
6733}
6734
6735pub type RealMobRuntime = MobRuntime;
6737
6738pub fn auto_mark_declared_resume_overrides(definition: &mut MobDefinition) {
6773 let MobDefinition {
6774 profiles, models, ..
6775 } = definition;
6776 for binding in profiles.values_mut() {
6777 let Some(profile) = binding.as_inline_mut() else {
6778 continue;
6779 };
6780 let mut declared = Vec::new();
6781 let coherent_provider = profile
6782 .provider
6783 .or_else(|| {
6787 profile
6788 .self_hosted_server_id
6789 .as_ref()
6790 .map(|_| Provider::SelfHosted)
6791 })
6792 .or_else(|| {
6793 meerkat_models::canonical()
6794 .infer_provider(&profile.model)
6795 .or_else(|| models.get(&profile.model).map(|entry| entry.provider))
6796 .filter(|provider| !matches!(provider, Provider::SelfHosted | Provider::Other))
6802 });
6803 if let Some(provider) = coherent_provider {
6804 profile.provider = Some(provider);
6805 declared.push(meerkat_mob::ResumeOverrideField::Model);
6806 declared.push(meerkat_mob::ResumeOverrideField::Provider);
6807 }
6808 if profile.provider_params.is_some() {
6809 declared.push(meerkat_mob::ResumeOverrideField::ProviderParams);
6810 }
6811 for field in declared {
6812 if !profile.resume_overrides.contains(&field) {
6813 profile.resume_overrides.push(field);
6814 }
6815 }
6816 }
6817}
6818
6819#[derive(Clone)]
6821pub struct MobRuntime {
6822 handle: MobHandle,
6823 session_service: Option<Arc<dyn MobSessionService>>,
6824 agent_mob_mcp_state: Option<Arc<meerkat_mob_mcp::MobMcpState>>,
6825 implicit_delegate_retirement_overrides: Option<ImplicitDelegateRetirementOverrides>,
6826 binary_blob_store: Option<Arc<dyn BinaryBlobStore>>,
6827 baseline_member_specs: Arc<tokio::sync::RwLock<Vec<SpawnMemberSpec>>>,
6828 console_spawn_sink_slot: Option<SharedConsoleSpawnSinkSlot>,
6831 identity_runtime_slot: Option<SharedIdentityRuntimeSlot>,
6832 workgraph_service: Option<meerkat::WorkGraphService>,
6835 workgraph_admission: Arc<crate::workgraph_admission::WorkGraphAdmission>,
6844 resolved_storage: Option<ResolvedStorageSummary>,
6847 session_write_epochs: Option<Arc<SessionSnapshotWriteEpochs>>,
6851 committed_boundary_recoverer:
6854 Option<Arc<dyn crate::identity_first::bridge::CommittedBoundaryRecoverer>>,
6855 _ephemeral_dir: Option<Arc<tempfile::TempDir>>,
6858}
6859
6860impl MobRuntime {
6861 pub async fn bootstrap(mut spec: MobBootstrapSpec) -> Result<Self, MobRuntimeError> {
6862 auto_mark_declared_resume_overrides(&mut spec.definition);
6866 let ephemeral_dir = spec._ephemeral_dir.clone();
6867 let session_service = spec.session_service.clone();
6868 let binary_blob_store = spec.binary_blob_store.clone();
6869 let mob_id = spec.definition.id.clone();
6870 let agent_mob_mcp_state = spec.agent_mob_mcp_state.clone();
6871 let implicit_delegate_retirement_overrides =
6872 spec.implicit_delegate_retirement_overrides.clone();
6873 let console_spawn_sink_slot = spec.console_spawn_sink_slot.clone();
6874 let identity_runtime_slot = spec.identity_runtime_slot.clone();
6875 let default_llm_client = spec
6876 .options
6877 .default_llm_client
6878 .clone()
6879 .map(ReplaySanitizingLlmClient::wrap)
6880 .map(ProviderAgnosticLlmClient::wrap);
6883 if let Some(slot) = spec.agent_mob_default_llm_client_slot.as_ref() {
6884 *slot
6885 .write()
6886 .unwrap_or_else(std::sync::PoisonError::into_inner) = default_llm_client.clone();
6887 }
6888 let effective_runtime_adapter = spec
6889 .runtime_adapter
6890 .clone()
6891 .or_else(|| session_service.runtime_adapter());
6892
6893 let mut builder = MobBuilder::new(spec.definition, spec.storage);
6894
6895 if let Some(adapter) = effective_runtime_adapter {
6902 builder = builder.with_runtime_adapter(adapter);
6903 }
6904
6905 builder = builder
6906 .with_session_service(session_service.clone())
6907 .allow_ephemeral_sessions(spec.options.allow_ephemeral_sessions)
6908 .notify_orchestrator_on_resume(spec.options.notify_orchestrator_on_resume);
6909
6910 if let Some(customizer) = spec.spawn_member_customizer.clone() {
6911 builder = builder.with_spawn_member_customizer(customizer);
6912 }
6913
6914 if let Some(provider) = spec.default_external_tools_provider.clone() {
6915 builder = builder.with_default_external_tools_provider(Some(provider));
6916 }
6917
6918 builder = builder.with_workgraph_service(spec.workgraph_service.clone());
6923
6924 if let Some(client) = default_llm_client {
6925 builder = builder.with_default_llm_client(client);
6926 }
6927
6928 let handle = builder.create().await?;
6929 if let Some(state) = agent_mob_mcp_state.as_ref() {
6930 state.mob_insert_handle(mob_id, handle.clone()).await;
6931 }
6932 let workgraph_admission = Arc::new(crate::workgraph_admission::WorkGraphAdmission::new(
6940 handle.clone(),
6941 Some(Arc::clone(&session_service)),
6942 spec.workgraph_admission_sidecar,
6943 ));
6944 for slot in &spec.workgraph_admission_slots {
6945 *slot
6946 .write()
6947 .unwrap_or_else(std::sync::PoisonError::into_inner) =
6948 Some(Arc::clone(&workgraph_admission));
6949 }
6950 Ok(Self {
6951 handle,
6952 session_service: Some(session_service),
6953 agent_mob_mcp_state,
6954 implicit_delegate_retirement_overrides,
6955 binary_blob_store,
6956 baseline_member_specs: Arc::new(tokio::sync::RwLock::new(Vec::new())),
6957 console_spawn_sink_slot,
6958 identity_runtime_slot,
6959 workgraph_service: spec.workgraph_service,
6960 workgraph_admission,
6961 resolved_storage: spec.resolved_storage,
6962 session_write_epochs: spec.session_write_epochs,
6963 committed_boundary_recoverer: spec.committed_boundary_recoverer,
6964 _ephemeral_dir: ephemeral_dir,
6965 })
6966 }
6967
6968 pub fn from_handle(handle: MobHandle) -> Self {
6969 let workgraph_admission = Arc::new(crate::workgraph_admission::WorkGraphAdmission::new(
6970 handle.clone(),
6971 None,
6972 None,
6973 ));
6974 Self {
6975 handle,
6976 session_service: None,
6977 agent_mob_mcp_state: None,
6978 implicit_delegate_retirement_overrides: None,
6979 binary_blob_store: None,
6980 baseline_member_specs: Arc::new(tokio::sync::RwLock::new(Vec::new())),
6981 console_spawn_sink_slot: None,
6982 identity_runtime_slot: None,
6983 workgraph_service: None,
6984 workgraph_admission,
6985 resolved_storage: None,
6986 session_write_epochs: None,
6987 committed_boundary_recoverer: None,
6988 _ephemeral_dir: None,
6989 }
6990 }
6991
6992 pub fn handle(&self) -> MobHandle {
6993 self.handle.clone()
6994 }
6995
6996 pub fn agent_mob_mcp_state(&self) -> Option<Arc<meerkat_mob_mcp::MobMcpState>> {
6997 self.agent_mob_mcp_state.clone()
6998 }
6999
7000 pub fn workgraph_service(&self) -> Option<meerkat::WorkGraphService> {
7002 self.workgraph_service.clone()
7003 }
7004
7005 pub(crate) fn workgraph_admission(
7011 &self,
7012 ) -> Arc<crate::workgraph_admission::WorkGraphAdmission> {
7013 Arc::clone(&self.workgraph_admission)
7014 }
7015
7016 pub(crate) fn install_console_spawn_sink(&self, sink: ConsoleSpawnSink) {
7019 if let Some(slot) = self.console_spawn_sink_slot.as_ref() {
7020 *slot
7021 .write()
7022 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(sink);
7023 }
7024 }
7025
7026 pub(crate) fn console_spawn_sink(&self) -> Option<ConsoleSpawnSink> {
7028 self.console_spawn_sink_slot.as_ref().and_then(|slot| {
7029 slot.read()
7030 .unwrap_or_else(std::sync::PoisonError::into_inner)
7031 .clone()
7032 })
7033 }
7034
7035 pub(crate) fn install_identity_runtime_authority(
7036 &self,
7037 identity_runtime: Arc<crate::identity_first::IdentityRuntime>,
7038 ) {
7039 if let Some(slot) = self.identity_runtime_slot.as_ref() {
7040 *slot
7041 .write()
7042 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(identity_runtime);
7043 }
7044 }
7045
7046 pub(crate) fn clear_identity_runtime_authority(&self) {
7051 if let Some(slot) = self.identity_runtime_slot.as_ref() {
7052 *slot
7053 .write()
7054 .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
7055 }
7056 }
7057
7058 pub(crate) async fn console_identity_labels(
7061 &self,
7062 ) -> BTreeMap<String, BTreeMap<String, String>> {
7063 match self.console_spawn_sink() {
7064 Some(sink) => sink.identity_labels_snapshot().await,
7065 None => BTreeMap::new(),
7066 }
7067 }
7068
7069 pub(crate) fn implicit_delegate_retirement_overrides(
7070 &self,
7071 ) -> Option<ImplicitDelegateRetirementOverrides> {
7072 self.implicit_delegate_retirement_overrides.clone()
7073 }
7074
7075 pub async fn set_baseline_member_specs(&self, specs: Vec<SpawnMemberSpec>) {
7076 *self.baseline_member_specs.write().await = specs;
7077 }
7078
7079 pub async fn baseline_member_specs(&self) -> Vec<SpawnMemberSpec> {
7080 self.baseline_member_specs.read().await.clone()
7081 }
7082
7083 pub(crate) fn session_document_write_epoch(&self, session_id_str: &str) -> Option<u64> {
7091 let epochs = self.session_write_epochs.as_ref()?;
7092 let session_id = meerkat_core::types::SessionId::parse(session_id_str).ok()?;
7093 Some(epochs.observe(&session_id))
7094 }
7095
7096 pub async fn read_session_history(
7097 &self,
7098 session_id_str: &str,
7099 offset: usize,
7100 limit: Option<usize>,
7101 ) -> Result<SessionHistoryPage, MobRuntimeError> {
7102 if session_id_str.trim().is_empty() {
7103 return Err(MobRuntimeError::InvalidInput(
7104 "session_id must not be empty",
7105 ));
7106 }
7107 let Some(session_service) = self.session_service.as_ref() else {
7108 return Err(MobRuntimeError::InvalidInput(
7109 "session history unavailable for this runtime",
7110 ));
7111 };
7112 let session_id = meerkat_core::types::SessionId::parse(session_id_str)
7113 .map_err(|_| MobRuntimeError::InvalidInput("invalid session_id format"))?;
7114 SessionServiceHistoryExt::read_history(
7115 session_service.as_ref(),
7116 &session_id,
7117 SessionHistoryQuery { offset, limit },
7118 )
7119 .await
7120 .map_err(|err| MobRuntimeError::Mob(MobError::Internal(err.to_string())))
7121 }
7122
7123 #[allow(dead_code)]
7124 pub(crate) async fn runtime_state_for_session(
7125 &self,
7126 session_id_str: &str,
7127 ) -> Result<Option<meerkat_runtime::RuntimeState>, MobRuntimeError> {
7128 if session_id_str.trim().is_empty() {
7129 return Err(MobRuntimeError::InvalidInput(
7130 "session_id must not be empty",
7131 ));
7132 }
7133 let Some(session_service) = self.session_service.as_ref() else {
7134 return Ok(None);
7135 };
7136 let Some(runtime_adapter) = session_service.runtime_adapter() else {
7137 return Ok(None);
7138 };
7139 let session_id = meerkat_core::types::SessionId::parse(session_id_str)
7140 .map_err(|_| MobRuntimeError::InvalidInput("invalid session_id format"))?;
7141 let state = meerkat_runtime::service_ext::SessionServiceRuntimeExt::runtime_state(
7142 runtime_adapter.as_ref(),
7143 &session_id,
7144 )
7145 .await
7146 .map_err(|err| MobRuntimeError::Mob(MobError::Internal(err.to_string())))?;
7147 Ok(Some(state))
7148 }
7149
7150 #[allow(dead_code)]
7151 pub(crate) async fn comms_runtime_for_session(
7152 &self,
7153 session_id_str: &str,
7154 ) -> Result<Option<Arc<dyn CommsRuntime>>, MobRuntimeError> {
7155 if session_id_str.trim().is_empty() {
7156 return Err(MobRuntimeError::InvalidInput(
7157 "session_id must not be empty",
7158 ));
7159 }
7160 let Some(session_service) = self.session_service.as_ref() else {
7161 return Ok(None);
7162 };
7163 let session_id = meerkat_core::types::SessionId::parse(session_id_str)
7164 .map_err(|_| MobRuntimeError::InvalidInput("invalid session_id format"))?;
7165 Ok(
7166 meerkat_core::service::SessionServiceCommsExt::comms_runtime(
7167 session_service.as_ref(),
7168 &session_id,
7169 )
7170 .await,
7171 )
7172 }
7173
7174 #[allow(dead_code)]
7175 pub(crate) async fn active_input_ids_for_session(
7176 &self,
7177 session_id_str: &str,
7178 ) -> Result<Option<Vec<String>>, MobRuntimeError> {
7179 if session_id_str.trim().is_empty() {
7180 return Err(MobRuntimeError::InvalidInput(
7181 "session_id must not be empty",
7182 ));
7183 }
7184 let Some(session_service) = self.session_service.as_ref() else {
7185 return Ok(None);
7186 };
7187 let Some(runtime_adapter) = session_service.runtime_adapter() else {
7188 return Ok(None);
7189 };
7190 let session_id = meerkat_core::types::SessionId::parse(session_id_str)
7191 .map_err(|_| MobRuntimeError::InvalidInput("invalid session_id format"))?;
7192 let input_ids = meerkat_runtime::service_ext::SessionServiceRuntimeExt::list_active_inputs(
7193 runtime_adapter.as_ref(),
7194 &session_id,
7195 )
7196 .await
7197 .map_err(|err| MobRuntimeError::Mob(MobError::Internal(err.to_string())))?;
7198 Ok(Some(
7199 input_ids.into_iter().map(|id| id.to_string()).collect(),
7200 ))
7201 }
7202
7203 #[allow(dead_code)]
7204 pub(crate) async fn ensure_comms_drain_for_session(
7205 &self,
7206 session_id_str: &str,
7207 ) -> Result<Option<bool>, MobRuntimeError> {
7208 if session_id_str.trim().is_empty() {
7209 return Err(MobRuntimeError::InvalidInput(
7210 "session_id must not be empty",
7211 ));
7212 }
7213 let Some(session_service) = self.session_service.as_ref() else {
7214 return Ok(None);
7215 };
7216 let Some(runtime_adapter) = session_service.runtime_adapter() else {
7217 return Ok(None);
7218 };
7219 let session_id = meerkat_core::types::SessionId::parse(session_id_str)
7220 .map_err(|_| MobRuntimeError::InvalidInput("invalid session_id format"))?;
7221 let comms_runtime = meerkat_core::service::SessionServiceCommsExt::comms_runtime(
7222 session_service.as_ref(),
7223 &session_id,
7224 )
7225 .await;
7226 if let Some(comms) = comms_runtime {
7227 let _handle = meerkat_runtime::comms_drain::spawn_comms_drain(
7228 runtime_adapter.clone(),
7229 session_id,
7230 comms,
7231 None,
7232 );
7233 Ok(Some(true))
7234 } else {
7235 Ok(Some(false))
7236 }
7237 }
7238
7239 pub fn session_service(&self) -> Option<&Arc<dyn MobSessionService>> {
7245 self.session_service.as_ref()
7246 }
7247
7248 pub fn committed_boundary_recoverer(
7252 &self,
7253 ) -> Option<Arc<dyn crate::identity_first::bridge::CommittedBoundaryRecoverer>> {
7254 self.committed_boundary_recoverer.clone()
7255 }
7256
7257 pub fn binary_blob_store(&self) -> Option<Arc<dyn BinaryBlobStore>> {
7258 self.binary_blob_store.clone()
7259 }
7260
7261 pub fn resolved_storage(&self) -> Option<ResolvedStorageSummary> {
7265 self.resolved_storage.clone()
7266 }
7267}
7268
7269pub fn member_entry_to_json(entry: &meerkat_mob::runtime::MobMemberListEntry) -> serde_json::Value {
7276 let mut value = serde_json::to_value(entry).unwrap_or(serde_json::Value::Null);
7277 if let Some(object) = value.as_object_mut() {
7281 if let Some(serde_json::Value::String(id)) = object.get_mut("agent_identity") {
7282 *id = crate::member_comms_id::runtime_alias_str(id).into_owned();
7283 }
7284 if let Some(serde_json::Value::Array(peers)) = object.get_mut("wired_to") {
7285 for peer in peers {
7286 if let serde_json::Value::String(peer_id) = peer {
7287 *peer_id = crate::member_comms_id::runtime_alias_str(peer_id).into_owned();
7288 }
7289 }
7290 }
7291 object.insert(
7297 "state".to_string(),
7298 serde_json::Value::String(member_status_state_string(entry.status)),
7299 );
7300 }
7301 value
7302}
7303
7304#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
7305pub struct ResolvedToolsSnapshot {
7306 pub identity: String,
7307 pub session_id: String,
7308 pub tools: Vec<String>,
7309}
7310
7311pub async fn resolved_tools_for_session(
7312 session_service: Option<&Arc<dyn MobSessionService>>,
7313 identity: &str,
7314 session_id: meerkat_core::types::SessionId,
7315) -> Result<ResolvedToolsSnapshot, MobRuntimeError> {
7316 let Some(session_service) = session_service else {
7317 return Err(MobRuntimeError::InvalidInput(
7318 "resolved tools unavailable for this runtime",
7319 ));
7320 };
7321 let scope = session_service
7322 .tool_scope_snapshot(&session_id)
7323 .await
7324 .map_err(|err| MobRuntimeError::Mob(MobError::Internal(err.to_string())))?
7325 .ok_or(MobRuntimeError::InvalidInput(
7326 "identity tool scope is unavailable",
7327 ))?;
7328 let mut tools = scope
7329 .visible_names
7330 .into_iter()
7331 .map(meerkat_core::types::ToolName::into_string)
7332 .collect::<Vec<_>>();
7333 tools.sort();
7334 Ok(ResolvedToolsSnapshot {
7335 identity: identity.to_string(),
7336 session_id: session_id.to_string(),
7337 tools,
7338 })
7339}
7340
7341pub async fn resolved_tools_for_member(
7342 handle: &MobHandle,
7343 session_service: Option<&Arc<dyn MobSessionService>>,
7344 member_id: &str,
7345) -> Result<ResolvedToolsSnapshot, MobRuntimeError> {
7346 if member_id.trim().is_empty() {
7347 return Err(MobRuntimeError::InvalidInput("identity must not be empty"));
7348 }
7349 let mid = crate::member_comms_id::mob_member_id(member_id);
7350 let status = handle.member_status(&mid).await?;
7351 let Some(session_id) = status.current_session_id else {
7352 return Err(MobRuntimeError::InvalidInput(
7353 "identity has no current session",
7354 ));
7355 };
7356 resolved_tools_for_session(session_service, member_id, session_id).await
7357}
7358
7359pub fn console_agent_event_payload(event: &meerkat_core::AgentEvent) -> Value {
7372 use meerkat_core::AgentEvent;
7373 use meerkat_core::event::agent_event_type;
7374
7375 let mut payload = serde_json::to_value(event).unwrap_or_else(|_| serde_json::json!({}));
7376 let record = match payload.as_object_mut() {
7377 Some(record) => record,
7378 None => return payload,
7379 };
7380 let is_tool_event = matches!(
7381 agent_event_type(event),
7382 "tool_call_requested"
7383 | "tool_result_received"
7384 | "tool_execution_started"
7385 | "tool_execution_completed"
7386 | "tool_execution_timed_out"
7387 );
7388 if is_tool_event
7389 && !record.contains_key("tool_call_id")
7390 && let Some(id) = record.get("id").cloned()
7391 {
7392 record.insert("tool_call_id".to_string(), id);
7393 }
7394 if let AgentEvent::ToolExecutionCompleted { content, .. } = event
7395 && !record.contains_key("result")
7396 {
7397 record.insert(
7398 "result".to_string(),
7399 Value::String(meerkat_core::types::text_content(content)),
7400 );
7401 }
7402 payload
7403}
7404
7405pub fn content_input_has_images(content: &meerkat_core::ContentInput) -> bool {
7406 match content {
7407 meerkat_core::ContentInput::Text(_) => false,
7408 meerkat_core::ContentInput::Blocks(blocks) => blocks
7409 .iter()
7410 .any(|block| matches!(block, meerkat_core::ContentBlock::Image { .. })),
7411 }
7412}
7413
7414pub fn model_capabilities_for_model(
7415 provider: Provider,
7416 model: &str,
7417) -> crate::runtime::ConsoleModelCapabilities {
7418 let image_input = meerkat_models::profile_for(provider, model)
7419 .map(|profile| profile.vision)
7420 .unwrap_or(false);
7421 crate::runtime::ConsoleModelCapabilities { image_input }
7422}
7423
7424pub fn model_capabilities_for_profile(
7425 profile: &Profile,
7426) -> crate::runtime::ConsoleModelCapabilities {
7427 let image_input = meerkat_models::infer_provider(&profile.model)
7428 .and_then(|provider| meerkat_models::profile_for(provider, &profile.model))
7429 .map(|profile| profile.vision)
7430 .unwrap_or(false);
7431 crate::runtime::ConsoleModelCapabilities { image_input }
7432}
7433
7434pub fn model_capabilities_for_role(
7435 definition: &MobDefinition,
7436 role: &str,
7437) -> crate::runtime::ConsoleModelCapabilities {
7438 let profile_name = ProfileName::from(role);
7439 definition
7440 .resolve_inline_profile(&profile_name)
7441 .map(model_capabilities_for_profile)
7442 .unwrap_or(crate::runtime::ConsoleModelCapabilities { image_input: false })
7443}
7444
7445pub fn model_capabilities_for_member_entry(
7446 definition: &MobDefinition,
7447 entry: &meerkat_mob::runtime::MobMemberListEntry,
7448) -> crate::runtime::ConsoleModelCapabilities {
7449 model_capabilities_for_role(definition, entry.role.as_str())
7450}
7451
7452pub async fn model_capabilities_for_member(
7453 handle: &MobHandle,
7454 session_service: Option<&Arc<dyn MobSessionService>>,
7455 member_id: &meerkat_mob::ids::AgentIdentity,
7456) -> crate::runtime::ConsoleModelCapabilities {
7457 if let Some(service) = session_service
7458 && let Some(session_id) = handle.resolve_bridge_session_id(member_id).await
7459 && let Ok(view) = service.read(&session_id).await
7460 {
7461 return model_capabilities_for_model(view.state.provider, &view.state.model);
7462 }
7463
7464 handle
7467 .get_member(member_id)
7468 .await
7469 .ok()
7470 .flatten()
7471 .map(|member| model_capabilities_for_role(handle.definition(), member.role.as_str()))
7472 .unwrap_or(crate::runtime::ConsoleModelCapabilities { image_input: false })
7473}
7474
7475pub async fn assert_member_accepts_images(
7476 handle: &MobHandle,
7477 session_service: Option<&Arc<dyn MobSessionService>>,
7478 member_id: &str,
7479 content: &meerkat_core::ContentInput,
7480) -> Result<(), MobRuntimeError> {
7481 if !content_input_has_images(content) {
7482 return Ok(());
7483 }
7484 let mid = crate::member_comms_id::mob_member_id(member_id);
7487 let Some(member) = handle
7488 .get_member(&mid)
7489 .await
7490 .map_err(|_| MobRuntimeError::InvalidInput("member lookup failed"))?
7491 else {
7492 return Err(MobRuntimeError::InvalidInput("member not found"));
7493 };
7494 let caps = model_capabilities_for_member(handle, session_service, &member.agent_identity).await;
7495 if caps.image_input {
7496 Ok(())
7497 } else {
7498 Err(MobRuntimeError::InvalidInput(
7499 "target member model cannot accept image input",
7500 ))
7501 }
7502}
7503
7504pub async fn send_message_on_mob(
7514 handle: &MobHandle,
7515 member_id: &str,
7516 content: impl Into<meerkat_core::ContentInput>,
7517) -> Result<String, MobRuntimeError> {
7518 send_message_on_mob_with_mode(
7519 handle,
7520 member_id,
7521 content,
7522 meerkat_core::types::HandlingMode::Queue,
7523 )
7524 .await
7525}
7526
7527pub async fn send_message_on_mob_with_mode(
7530 handle: &MobHandle,
7531 member_id: &str,
7532 content: impl Into<meerkat_core::ContentInput>,
7533 handling_mode: meerkat_core::types::HandlingMode,
7534) -> Result<String, MobRuntimeError> {
7535 if member_id.trim().is_empty() {
7536 return Err(MobRuntimeError::InvalidInput("member_id must not be empty"));
7537 }
7538 let content = content.into();
7539 let is_empty = match &content {
7540 meerkat_core::ContentInput::Text(s) => s.trim().is_empty(),
7541 meerkat_core::ContentInput::Blocks(blocks) => blocks.is_empty(),
7542 };
7543 if is_empty {
7544 return Err(MobRuntimeError::InvalidInput("content must not be empty"));
7545 }
7546 let mid = crate::member_comms_id::mob_member_id(member_id);
7549 let _receipt = handle
7550 .member(&mid)
7551 .await?
7552 .send(content, handling_mode)
7553 .await?;
7554 if let Some(session_id) = handle.resolve_bridge_session_id(&mid).await {
7555 return Ok(session_id.to_string());
7556 }
7557
7558 let status = handle.member_status(&mid).await?;
7559 if status.external_member.is_some() {
7560 return Ok(String::new());
7561 }
7562
7563 Err(MobRuntimeError::Mob(MobError::Internal(
7564 "member has no bridge session after send".to_string(),
7565 )))
7566}
7567
7568#[cfg(test)]
7569#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
7570mod tests {
7571 use super::*;
7572
7573 struct EmptyDispatcher;
7574
7575 #[async_trait::async_trait]
7576 impl meerkat_core::AgentToolDispatcher for EmptyDispatcher {
7577 fn tools(&self) -> Arc<[Arc<meerkat_core::types::ToolDef>]> {
7578 Vec::<Arc<meerkat_core::types::ToolDef>>::new().into()
7579 }
7580
7581 async fn dispatch(
7582 &self,
7583 call: meerkat_core::types::ToolCallView<'_>,
7584 ) -> Result<meerkat_core::ToolDispatchOutcome, meerkat_core::ToolError> {
7585 Err(meerkat_core::ToolError::not_found(call.name))
7586 }
7587
7588 fn capabilities(&self) -> meerkat_core::agent::DispatcherCapabilities {
7589 meerkat_core::agent::DispatcherCapabilities::default()
7590 }
7591 }
7592
7593 fn wrapper_with_overrides(
7594 overrides: ImplicitDelegateRetirementOverrides,
7595 ) -> AutoWireParentMobToolDispatcher {
7596 AutoWireParentMobToolDispatcher {
7597 inner: Arc::new(EmptyDispatcher),
7598 implicit_delegate_retirement_overrides: overrides,
7599 console_spawn_sink: new_console_spawn_sink_slot(),
7600 identity_runtime: Arc::new(std::sync::RwLock::new(None)),
7601 protected_mob_id: "test-mob".to_string(),
7602 spawner_comms_name: None,
7603 }
7604 }
7605
7606 #[test]
7607 fn raw_mob_tools_detect_public_and_encoded_generated_aliases() {
7608 let encoded = crate::member_comms_id::mob_member_id_str("rt:worker:0").into_owned();
7609 assert_eq!(
7610 reserved_raw_member_tool_argument(
7611 "mob_spawn_member",
7612 &serde_json::json!({"member_id": "rt:worker:0"}),
7613 ),
7614 Some(("member_id", "rt:worker:0".to_string()))
7615 );
7616 assert_eq!(
7617 reserved_raw_member_tool_argument(
7618 "mob_wire",
7619 &serde_json::json!({"member_id": "classic", "peer": {"local": encoded}}),
7620 ),
7621 Some(("peer.local", "rt:worker:0".to_string()))
7622 );
7623 assert_eq!(
7624 reserved_raw_member_tool_argument(
7625 "delegate",
7626 &serde_json::json!({"member_id": "mk--victim"}),
7627 ),
7628 Some(("member_id", "victim".to_string()))
7629 );
7630 assert_eq!(
7631 reserved_raw_member_tool_argument(
7632 "spawn_many_members",
7633 &serde_json::json!({
7634 "specs": [
7635 {"profile": "worker", "member_id": "classic"},
7636 {"profile": "worker", "member_id": encoded},
7637 ]
7638 }),
7639 ),
7640 Some(("specs[].member_id", "rt:worker:0".to_string()))
7641 );
7642 }
7643
7644 #[tokio::test]
7645 async fn raw_mob_tool_dispatch_fails_closed_before_lower_plane() {
7646 use meerkat_core::AgentToolDispatcher;
7647
7648 let dispatcher = wrapper_with_overrides(ImplicitDelegateRetirementOverrides::default());
7649 let args = serde_json::value::RawValue::from_string(
7650 serde_json::json!({"mob_id": "m", "member_id": "rt:worker:0"}).to_string(),
7651 )
7652 .expect("raw args");
7653 let error = dispatcher
7654 .dispatch(meerkat_core::types::ToolCallView {
7655 id: "call-reserved",
7656 name: "mob_retire_member",
7657 args: &args,
7658 })
7659 .await
7660 .expect_err("reserved alias must not reach the raw dispatcher");
7661
7662 assert!(error.to_string().contains("reserved rt:* / mk--"));
7663
7664 let batch_args = serde_json::value::RawValue::from_string(
7665 serde_json::json!({
7666 "specs": [{"profile": "worker", "member_id": "rt:worker:0"}]
7667 })
7668 .to_string(),
7669 )
7670 .expect("raw args");
7671 let batch_error = dispatcher
7672 .dispatch(meerkat_core::types::ToolCallView {
7673 id: "call-reserved-batch",
7674 name: "spawn_many_members",
7675 args: &batch_args,
7676 })
7677 .await
7678 .expect_err("reserved batch member must not reach the raw dispatcher");
7679 assert!(batch_error.to_string().contains("specs[].member_id"));
7680 }
7681
7682 #[tokio::test]
7683 async fn raw_operator_tool_rejects_registered_plain_durable_identity() {
7684 use meerkat_core::AgentToolDispatcher;
7685
7686 let dispatcher = wrapper_with_overrides(ImplicitDelegateRetirementOverrides::default());
7687 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
7688 crate::identity_first::IdentityRuntimeConfig {
7689 continuity_store: Arc::new(
7690 crate::identity_first::LocalContinuityStore::in_memory()
7691 .expect("continuity store"),
7692 ),
7693 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7694 runtime_instance_id: "raw-operator-test".to_string(),
7695 has_runtime_store: true,
7696 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7697 bridge: None,
7698 default_timeout: None,
7699 },
7700 ));
7701 let identity =
7702 crate::identity_first::AgentIdentity::parse("lead").expect("durable identity");
7703 identity_runtime
7704 .register(
7705 crate::identity_first::DurableAgentSpec {
7706 identity,
7707 profile: meerkat_mob::ProfileName::from("worker"),
7708 addressability: crate::identity_first::AgentAddressability::Addressable,
7709 display_name: None,
7710 labels: BTreeMap::new(),
7711 context: None,
7712 additional_instructions: Vec::new(),
7713 initial_message: None,
7714 runtime_mode_override: None,
7715 backend: None,
7716 binding: None,
7717 },
7718 crate::identity_first::IdentityLifecycleState::Dormant,
7719 None,
7720 None,
7721 )
7722 .await;
7723 *dispatcher
7724 .identity_runtime
7725 .write()
7726 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(identity_runtime);
7727
7728 let args = serde_json::value::RawValue::from_string(
7729 serde_json::json!({"member_id": "lead"}).to_string(),
7730 )
7731 .expect("raw args");
7732 let error = dispatcher
7733 .dispatch(meerkat_core::types::ToolCallView {
7734 id: "call-owned",
7735 name: "force_cancel_member",
7736 args: &args,
7737 })
7738 .await
7739 .expect_err("registered durable identity must not reach raw force cancel");
7740 assert!(
7741 error
7742 .to_string()
7743 .contains("owned by the attached IdentityRuntime")
7744 );
7745 }
7746
7747 #[test]
7748 fn delegate_tool_schema_exposes_idle_retire_secs() {
7749 let tool = meerkat_core::types::ToolDef::new(
7750 "delegate",
7751 "Delegate work",
7752 serde_json::json!({
7753 "type": "object",
7754 "properties": {
7755 "task": {"type": "string"}
7756 },
7757 "required": ["task"]
7758 }),
7759 );
7760
7761 let patched = delegate_tool_def_with_idle_retire_secs(&tool);
7762 let idle_retire_secs = &patched.input_schema["properties"]["idle_retire_secs"];
7763
7764 assert!(patched.description.contains("IDLE RETIREMENT:"));
7765 assert_eq!(idle_retire_secs["anyOf"][0]["type"], "integer");
7766 assert_eq!(idle_retire_secs["anyOf"][0]["minimum"], 0);
7767 assert_eq!(idle_retire_secs["anyOf"][1]["type"], "null");
7768 }
7769
7770 #[test]
7771 fn mob_spawn_tool_schema_exposes_opt_in_idle_retire_secs() {
7772 let tool = meerkat_core::types::ToolDef::new(
7773 "mob_spawn_member",
7774 "Spawn member",
7775 serde_json::json!({
7776 "type": "object",
7777 "properties": {
7778 "profile": {"type": "string"},
7779 "member_id": {"type": "string"}
7780 },
7781 "required": ["profile", "member_id"]
7782 }),
7783 );
7784
7785 let patched = mob_spawn_tool_def_with_idle_retire_secs(&tool);
7786 let idle_retire_secs = &patched.input_schema["properties"]["idle_retire_secs"];
7787
7788 assert!(
7789 patched
7790 .description
7791 .contains("Omit idle_retire_secs to leave this spawned member out")
7792 );
7793 assert_eq!(idle_retire_secs["anyOf"][0]["type"], "integer");
7794 assert_eq!(idle_retire_secs["anyOf"][0]["minimum"], 0);
7795 assert_eq!(idle_retire_secs["anyOf"][1]["type"], "null");
7796 }
7797
7798 #[tokio::test]
7799 async fn auto_wire_wrapper_preserves_ops_lifecycle_binding() {
7800 use meerkat_core::AgentToolDispatcher;
7801 use std::sync::atomic::{AtomicBool, Ordering};
7802
7803 struct BindAwareDispatcher {
7804 bound: Arc<AtomicBool>,
7805 }
7806
7807 #[async_trait::async_trait]
7808 impl meerkat_core::AgentToolDispatcher for BindAwareDispatcher {
7809 fn tools(&self) -> Arc<[Arc<meerkat_core::types::ToolDef>]> {
7810 Vec::<Arc<meerkat_core::types::ToolDef>>::new().into()
7811 }
7812
7813 async fn dispatch(
7814 &self,
7815 call: meerkat_core::types::ToolCallView<'_>,
7816 ) -> Result<meerkat_core::ToolDispatchOutcome, meerkat_core::ToolError> {
7817 Err(meerkat_core::ToolError::not_found(call.name))
7818 }
7819
7820 fn capabilities(&self) -> meerkat_core::agent::DispatcherCapabilities {
7821 meerkat_core::agent::DispatcherCapabilities {
7822 ops_lifecycle: true,
7823 }
7824 }
7825
7826 fn bind_ops_lifecycle(
7827 self: Arc<Self>,
7828 _registry: Arc<dyn meerkat_core::ops_lifecycle::OpsLifecycleRegistry>,
7829 _owner_bridge_session_id: meerkat_core::types::SessionId,
7830 ) -> Result<meerkat_core::agent::BindOutcome, meerkat_core::agent::OpsLifecycleBindError>
7831 {
7832 self.bound.store(true, Ordering::SeqCst);
7833 Ok(meerkat_core::agent::BindOutcome::Bound(self))
7834 }
7835 }
7836
7837 let bound = Arc::new(AtomicBool::new(false));
7838 let dispatcher = Arc::new(AutoWireParentMobToolDispatcher {
7839 inner: Arc::new(BindAwareDispatcher {
7840 bound: Arc::clone(&bound),
7841 }),
7842 implicit_delegate_retirement_overrides: ImplicitDelegateRetirementOverrides::default(),
7843 console_spawn_sink: new_console_spawn_sink_slot(),
7844 identity_runtime: Arc::new(std::sync::RwLock::new(None)),
7845 protected_mob_id: "test-mob".to_string(),
7846 spawner_comms_name: None,
7847 });
7848
7849 assert!(dispatcher.capabilities().ops_lifecycle);
7850 let outcome = dispatcher
7851 .bind_ops_lifecycle(
7852 Arc::new(meerkat_runtime::ops_lifecycle::RuntimeOpsLifecycleRegistry::new()),
7853 meerkat_core::types::SessionId::new(),
7854 )
7855 .expect("wrapper should delegate ops lifecycle binding");
7856
7857 assert!(outcome.was_bound());
7858 assert!(bound.load(Ordering::SeqCst));
7859 assert!(outcome.into_dispatcher().capabilities().ops_lifecycle);
7860 }
7861
7862 #[tokio::test]
7863 async fn auto_wire_wrapper_preserves_objective_dispatch_context() {
7864 struct ContextAwareDispatcher {
7865 observed_objective: Arc<std::sync::Mutex<Option<String>>>,
7866 }
7867
7868 #[async_trait::async_trait]
7869 impl meerkat_core::AgentToolDispatcher for ContextAwareDispatcher {
7870 fn tools(&self) -> Arc<[Arc<meerkat_core::types::ToolDef>]> {
7871 Vec::<Arc<meerkat_core::types::ToolDef>>::new().into()
7872 }
7873
7874 async fn dispatch(
7875 &self,
7876 call: meerkat_core::types::ToolCallView<'_>,
7877 ) -> Result<meerkat_core::ToolDispatchOutcome, meerkat_core::ToolError> {
7878 Err(meerkat_core::ToolError::execution_failed(format!(
7879 "plain dispatch unexpectedly used for {}",
7880 call.name
7881 )))
7882 }
7883
7884 async fn dispatch_with_context(
7885 &self,
7886 call: meerkat_core::types::ToolCallView<'_>,
7887 context: &meerkat_core::ToolDispatchContext,
7888 ) -> Result<meerkat_core::ToolDispatchOutcome, meerkat_core::ToolError> {
7889 *self
7890 .observed_objective
7891 .lock()
7892 .unwrap_or_else(std::sync::PoisonError::into_inner) = context
7893 .turn_metadata(meerkat_core::agent::TOOL_DISPATCH_OBJECTIVE_ID_KEY)
7894 .and_then(Value::as_str)
7895 .map(ToString::to_string);
7896 Ok(meerkat_core::ToolDispatchOutcome::sync_result(
7897 meerkat_core::types::ToolResult::new(
7898 call.id.to_string(),
7899 "{}".to_string(),
7900 false,
7901 ),
7902 ))
7903 }
7904 }
7905
7906 let observed_objective = Arc::new(std::sync::Mutex::new(None));
7907 let dispatcher = AutoWireParentMobToolDispatcher {
7908 inner: Arc::new(ContextAwareDispatcher {
7909 observed_objective: Arc::clone(&observed_objective),
7910 }),
7911 implicit_delegate_retirement_overrides: ImplicitDelegateRetirementOverrides::default(),
7912 console_spawn_sink: new_console_spawn_sink_slot(),
7913 identity_runtime: Arc::new(std::sync::RwLock::new(None)),
7914 protected_mob_id: "test-mob".to_string(),
7915 spawner_comms_name: None,
7916 };
7917 let objective_id = uuid::Uuid::new_v4().to_string();
7918 let context =
7919 meerkat_core::ToolDispatchContext::default().with_turn_metadata(BTreeMap::from([(
7920 meerkat_core::agent::TOOL_DISPATCH_OBJECTIVE_ID_KEY.to_string(),
7921 Value::String(objective_id.clone()),
7922 )]));
7923 let args = serde_json::value::RawValue::from_string(
7924 serde_json::json!({"task": "review"}).to_string(),
7925 )
7926 .expect("raw delegate args");
7927
7928 meerkat_core::AgentToolDispatcher::dispatch_with_context(
7929 &dispatcher,
7930 meerkat_core::types::ToolCallView {
7931 id: "call-objective",
7932 name: "delegate",
7933 args: &args,
7934 },
7935 &context,
7936 )
7937 .await
7938 .expect("context-aware delegate dispatch");
7939
7940 assert_eq!(
7941 observed_objective
7942 .lock()
7943 .unwrap_or_else(std::sync::PoisonError::into_inner)
7944 .as_deref(),
7945 Some(objective_id.as_str())
7946 );
7947 }
7948
7949 #[test]
7950 fn delegate_idle_retire_secs_arg_is_stripped_and_parsed() {
7951 let mut args = serde_json::json!({
7952 "task": "inspect",
7953 "idle_retire_secs": 42
7954 });
7955
7956 let parsed = delegate_idle_retire_override_from_args("delegate", &mut args)
7957 .expect("valid idle retire arg");
7958
7959 assert_eq!(parsed, Some(DelegateIdleRetireOverride::Seconds(42)));
7960 assert!(args.get("idle_retire_secs").is_none());
7961 }
7962
7963 #[test]
7964 fn delegate_idle_retire_secs_null_disables_member_retirement() {
7965 let mut args = serde_json::json!({
7966 "task": "inspect",
7967 "idle_retire_secs": null
7968 });
7969
7970 let parsed = delegate_idle_retire_override_from_args("delegate", &mut args)
7971 .expect("valid idle retire arg");
7972
7973 assert_eq!(parsed, Some(DelegateIdleRetireOverride::Disabled));
7974 assert!(args.get("idle_retire_secs").is_none());
7975 }
7976
7977 #[test]
7978 fn delegate_idle_retire_secs_omitted_inherits_runtime_default() {
7979 let mut args = serde_json::json!({"task": "inspect"});
7980
7981 let parsed = delegate_idle_retire_override_from_args("delegate", &mut args)
7982 .expect("omitted idle retire arg");
7983
7984 assert_eq!(parsed, None);
7985 assert_eq!(args, serde_json::json!({"task": "inspect"}));
7986 }
7987
7988 #[test]
7989 fn delegate_idle_retire_secs_rejects_negative_or_fractional_values() {
7990 let mut negative = serde_json::json!({"task": "inspect", "idle_retire_secs": -1});
7991 let mut fractional = serde_json::json!({"task": "inspect", "idle_retire_secs": 1.5});
7992
7993 assert!(delegate_idle_retire_override_from_args("delegate", &mut negative).is_err());
7994 assert!(delegate_idle_retire_override_from_args("delegate", &mut fractional).is_err());
7995 }
7996
7997 #[test]
7998 fn mob_spawn_idle_retire_targets_use_args_when_result_omits_mob_id() {
7999 let args = serde_json::json!({
8000 "mob_id": "ob3",
8001 "profile": "review-worker",
8002 "member_id": "review-worker-vibe-forward",
8003 });
8004 let fallback_targets = idle_retire_targets_from_spawn_args(&args);
8005
8006 assert_eq!(
8007 fallback_targets,
8008 vec![IdleRetireTarget {
8009 mob_id: "ob3".to_string(),
8010 member_id: "review-worker-vibe-forward".to_string(),
8011 }]
8012 );
8013 assert_eq!(
8014 idle_retire_targets_from_outcome_text(
8015 r#"{"agent_identity":"review-worker-vibe-forward","member_ref":"opaque"}"#,
8016 &fallback_targets,
8017 ),
8018 fallback_targets
8019 );
8020 }
8021
8022 #[test]
8023 fn mob_spawn_idle_retire_targets_support_canonical_specs_shape() {
8024 let args = serde_json::json!({
8025 "mob_id": "ob3",
8026 "specs": [
8027 {"profile": "person-worker", "agent_identity": "person-worker-a"},
8028 {"profile": "person-worker", "member_id": "person-worker-b", "mob_id": "other"}
8029 ]
8030 });
8031 let fallback_targets = idle_retire_targets_from_spawn_args(&args);
8032
8033 assert_eq!(
8034 fallback_targets,
8035 vec![
8036 IdleRetireTarget {
8037 mob_id: "ob3".to_string(),
8038 member_id: "person-worker-a".to_string(),
8039 },
8040 IdleRetireTarget {
8041 mob_id: "other".to_string(),
8042 member_id: "person-worker-b".to_string(),
8043 },
8044 ]
8045 );
8046 assert_eq!(
8047 idle_retire_targets_from_outcome_text(
8048 r#"{"members":[{"agent_identity":"person-worker-a"},{"agent_identity":"person-worker-b","mob_id":"other"}]}"#,
8049 &fallback_targets,
8050 ),
8051 fallback_targets
8052 );
8053 }
8054
8055 #[tokio::test]
8056 async fn implicit_delegate_retirement_overrides_round_trip_per_member() {
8057 let overrides = ImplicitDelegateRetirementOverrides::default();
8058
8059 overrides
8060 .set("mob-a", "worker-1", DelegateIdleRetireOverride::Seconds(12))
8061 .await;
8062 overrides
8063 .set("mob-a", "worker-2", DelegateIdleRetireOverride::Disabled)
8064 .await;
8065
8066 assert_eq!(
8067 overrides.get("mob-a", "worker-1").await,
8068 Some(DelegateIdleRetireOverride::Seconds(12))
8069 );
8070 assert_eq!(
8071 overrides.get("mob-a", "worker-2").await,
8072 Some(DelegateIdleRetireOverride::Disabled)
8073 );
8074 assert_eq!(overrides.get("mob-a", "worker-3").await, None);
8075 }
8076
8077 #[tokio::test]
8078 async fn mob_spawn_idle_retire_registration_uses_spawn_args_when_result_omits_mob_id() {
8079 let overrides = ImplicitDelegateRetirementOverrides::default();
8080 let dispatcher = wrapper_with_overrides(overrides.clone());
8081 let fallback_targets = idle_retire_targets_from_spawn_args(&serde_json::json!({
8082 "mob_id": "ob3",
8083 "member_id": "review-worker-vibe-forward",
8084 }));
8085 let outcome =
8086 meerkat_core::ToolDispatchOutcome::sync_result(meerkat_core::types::ToolResult::new(
8087 "spawn-1".to_string(),
8088 r#"{"agent_identity":"review-worker-vibe-forward","member_ref":"opaque"}"#
8089 .to_string(),
8090 false,
8091 ));
8092
8093 dispatcher
8094 .register_idle_retire_override_from_outcome(
8095 &outcome,
8096 Some(DelegateIdleRetireOverride::Seconds(900)),
8097 &fallback_targets,
8098 )
8099 .await;
8100
8101 assert_eq!(
8102 overrides.get("ob3", "review-worker-vibe-forward").await,
8103 Some(DelegateIdleRetireOverride::Seconds(900))
8104 );
8105 }
8106
8107 #[test]
8108 fn image_generation_substrate_defaults_off_for_inline_profiles() {
8109 let definition = meerkat_mob::MobDefinition::from_toml(
8110 r#"
8111[mob]
8112id = "test"
8113
8114[profiles.worker]
8115model = "gpt-5.5"
8116
8117[profiles.worker.tools]
8118builtins = true
8119"#,
8120 )
8121 .unwrap_or_else(|e| panic!("{e}"));
8122
8123 assert!(
8124 !mob_definition_may_use_image_generation(&definition),
8125 "inline profiles should not wire the image substrate unless a profile opts in"
8126 );
8127 }
8128
8129 #[test]
8130 fn image_generation_substrate_follows_profile_tool_config() {
8131 let definition = meerkat_mob::MobDefinition::from_toml(
8132 r#"
8133[mob]
8134id = "test"
8135
8136[profiles.commander]
8137model = "gpt-5.5"
8138
8139[profiles.commander.tools]
8140builtins = true
8141image_generation = true
8142
8143[profiles.investigator]
8144model = "gpt-5.5"
8145
8146[profiles.investigator.tools]
8147builtins = true
8148image_generation = false
8149"#,
8150 )
8151 .unwrap_or_else(|e| panic!("{e}"));
8152
8153 let commander = definition.profiles["commander"].as_inline().unwrap();
8154 let investigator = definition.profiles["investigator"].as_inline().unwrap();
8155 assert!(commander.tools.image_generation);
8156 assert!(!investigator.tools.image_generation);
8157 assert!(
8158 mob_definition_may_use_image_generation(&definition),
8159 "one opt-in profile is enough to wire substrate; Meerkat gates visibility per profile"
8160 );
8161 }
8162
8163 #[test]
8164 fn shell_substrate_defaults_off_for_inline_profiles() {
8165 let definition = meerkat_mob::MobDefinition::from_toml(
8166 r#"
8167[mob]
8168id = "test"
8169
8170[profiles.worker]
8171model = "gpt-5.5"
8172
8173[profiles.worker.tools]
8174builtins = true
8175"#,
8176 )
8177 .unwrap_or_else(|e| panic!("{e}"));
8178
8179 assert!(
8180 !mob_definition_may_use_shell(&definition),
8181 "inline profiles should not wire the shell substrate unless a profile opts in"
8182 );
8183 }
8184
8185 #[test]
8186 fn shell_substrate_follows_profile_tool_config() {
8187 let definition = meerkat_mob::MobDefinition::from_toml(
8188 r#"
8189[mob]
8190id = "test"
8191
8192[profiles.domain]
8193model = "gpt-5.5"
8194
8195[profiles.domain.tools]
8196builtins = true
8197shell = false
8198
8199[profiles.security]
8200model = "gpt-5.5"
8201
8202[profiles.security.tools]
8203builtins = true
8204shell = true
8205"#,
8206 )
8207 .unwrap_or_else(|e| panic!("{e}"));
8208
8209 let domain = definition.profiles["domain"].as_inline().unwrap();
8210 let security = definition.profiles["security"].as_inline().unwrap();
8211 assert!(!domain.tools.shell);
8212 assert!(security.tools.shell);
8213 assert!(
8214 mob_definition_may_use_shell(&definition),
8215 "one opt-in profile is enough to wire substrate; Meerkat gates visibility per profile"
8216 );
8217 }
8218
8219 #[test]
8220 fn shell_tooling_forces_builtin_substrate_without_exposing_broad_builtins() {
8221 let mut req = CreateSessionRequest {
8222 model: "gpt-5.5".to_string(),
8223 prompt: meerkat_core::ContentInput::Text("test".to_string()),
8224 system_prompt: meerkat_core::config::SystemPromptOverride::Inherit,
8225 max_tokens: None,
8226 event_tx: None,
8227 initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
8228 build: Some(meerkat_core::service::SessionBuildOptions {
8229 override_builtins: meerkat_core::ToolCategoryOverride::Disable,
8230 override_shell: meerkat_core::ToolCategoryOverride::Enable,
8231 ..Default::default()
8232 }),
8233 labels: None,
8234 deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::default(),
8235 injected_context: Vec::new(),
8236 };
8237
8238 ensure_shell_tooling_build_substrate(&mut req);
8239
8240 let build = req.build.expect("build options");
8241 assert_eq!(
8242 build.override_shell,
8243 meerkat_core::ToolCategoryOverride::Enable
8244 );
8245 assert_eq!(
8246 build.override_builtins,
8247 meerkat_core::ToolCategoryOverride::Enable,
8248 "shell-only profiles must still enable Meerkat's builtin substrate"
8249 );
8250 let allow = match build.initial_tool_filter.expect("shell visibility filter") {
8251 meerkat_core::ToolFilter::Allow(allow) => allow,
8252 other => panic!("expected shell/comms allow filter, got {other:?}"),
8253 };
8254 for tool in SHELL_BUILTIN_TOOL_NAMES
8255 .iter()
8256 .chain(COMMS_TOOL_NAMES.iter())
8257 {
8258 assert!(allow.contains(tool), "missing expected tool {tool}");
8259 }
8260 for broad_builtin in ["task_list", "task_create", "apply_patch", "browse_skills"] {
8261 assert!(
8262 !allow.contains(broad_builtin),
8263 "shell-only filter must not expose broad builtin {broad_builtin}",
8264 );
8265 }
8266 }
8267
8268 #[test]
8269 fn non_shell_profiles_keep_builtin_override_unchanged() {
8270 let mut req = CreateSessionRequest {
8271 model: "gpt-5.5".to_string(),
8272 prompt: meerkat_core::ContentInput::Text("test".to_string()),
8273 system_prompt: meerkat_core::config::SystemPromptOverride::Inherit,
8274 max_tokens: None,
8275 event_tx: None,
8276 initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
8277 build: Some(meerkat_core::service::SessionBuildOptions {
8278 override_builtins: meerkat_core::ToolCategoryOverride::Disable,
8279 override_shell: meerkat_core::ToolCategoryOverride::Disable,
8280 ..Default::default()
8281 }),
8282 labels: None,
8283 deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::default(),
8284 injected_context: Vec::new(),
8285 };
8286
8287 ensure_shell_tooling_build_substrate(&mut req);
8288
8289 let build = req.build.expect("build options");
8290 assert_eq!(
8291 build.override_builtins,
8292 meerkat_core::ToolCategoryOverride::Disable
8293 );
8294 assert_eq!(
8295 build.override_shell,
8296 meerkat_core::ToolCategoryOverride::Disable
8297 );
8298 }
8299
8300 #[test]
8301 fn image_generation_profiles_can_disable_builtins_with_meerkat_062() {
8302 let definition = meerkat_mob::MobDefinition::from_toml(
8303 r#"
8304[mob]
8305id = "test"
8306
8307[profiles.commander]
8308model = "gpt-5.5"
8309
8310[profiles.commander.tools]
8311builtins = false
8312image_generation = true
8313"#,
8314 )
8315 .unwrap_or_else(|e| panic!("{e}"));
8316
8317 let commander = definition.profiles["commander"].as_inline().unwrap();
8318 assert!(!commander.tools.builtins);
8319 assert!(commander.tools.image_generation);
8320 assert!(
8321 mob_definition_may_use_image_generation(&definition),
8322 "image generation now has its own Meerkat tool gate"
8323 );
8324 }
8325
8326 #[test]
8327 fn image_generation_substrate_is_conservative_for_realm_profile_refs() {
8328 let definition = meerkat_mob::MobDefinition::from_toml(
8329 r#"
8330[mob]
8331id = "test"
8332
8333[profiles.worker]
8334realm_profile = "worker-v2"
8335"#,
8336 )
8337 .unwrap_or_else(|e| panic!("{e}"));
8338
8339 assert!(
8340 mob_definition_may_use_image_generation(&definition),
8341 "realm profiles resolve at spawn time, so MobKit wires substrate and lets Meerkat enforce profile policy"
8342 );
8343 }
8344
8345 #[test]
8346 fn sanitize_llm_request_drops_replay_unsafe_server_tool_blocks() {
8347 let request = meerkat_client::LlmRequest::new(
8348 "gpt-5.5",
8349 vec![meerkat_core::Message::BlockAssistant(
8350 meerkat_core::BlockAssistantMessage::new(
8351 vec![
8352 meerkat_core::AssistantBlock::Text {
8353 text: "done".to_string(),
8354 meta: None,
8355 },
8356 meerkat_core::AssistantBlock::ServerToolContent {
8357 id: Some("ws-stream".to_string()),
8358 kind: meerkat_core::ServerToolKind::WebSearch,
8359 content: serde_json::json!({
8360 "type": "response.web_search_call.searching",
8361 "item_id": "ws_123"
8362 }),
8363 meta: None,
8364 },
8365 meerkat_core::AssistantBlock::ServerToolContent {
8366 id: Some("ws_123".to_string()),
8367 kind: meerkat_core::ServerToolKind::ProviderNative {
8368 name: "web_search_call".to_string(),
8369 },
8370 content: serde_json::json!({
8371 "type": "web_search_call",
8372 "id": "ws_123",
8373 "status": "completed"
8374 }),
8375 meta: None,
8376 },
8377 meerkat_core::AssistantBlock::ServerToolContent {
8378 id: None,
8379 kind: meerkat_core::ServerToolKind::ProviderNative {
8380 name: "web_search_annotations".to_string(),
8381 },
8382 content: serde_json::json!({
8383 "type": "message_annotations",
8384 "annotations": []
8385 }),
8386 meta: None,
8387 },
8388 ],
8389 meerkat_core::StopReason::EndTurn,
8390 ),
8391 )],
8392 );
8393
8394 let sanitized = sanitize_llm_request_for_stateless_replay(&request);
8395 let meerkat_core::Message::BlockAssistant(assistant) = &sanitized.messages[0] else {
8396 panic!("expected block assistant");
8397 };
8398
8399 assert_eq!(assistant.blocks.len(), 2);
8400 assert!(matches!(
8401 assistant.blocks[0],
8402 meerkat_core::AssistantBlock::Text { .. }
8403 ));
8404 assert!(matches!(
8405 assistant.blocks[1],
8406 meerkat_core::AssistantBlock::ServerToolContent { ref kind, .. }
8407 if kind.provider_name() == "web_search_call"
8408 ));
8409 }
8410
8411 #[test]
8412 fn sanitize_llm_request_preserves_generated_images_for_meerkat_062() {
8413 let request = meerkat_client::LlmRequest::new(
8414 "gpt-5.5",
8415 vec![meerkat_core::Message::BlockAssistant(
8416 meerkat_core::BlockAssistantMessage::new(
8417 vec![
8418 meerkat_core::AssistantBlock::Text {
8419 text: "visible".to_string(),
8420 meta: None,
8421 },
8422 generated_image_block_for_test(),
8423 ],
8424 meerkat_core::StopReason::EndTurn,
8425 ),
8426 )],
8427 );
8428
8429 let sanitized = sanitize_llm_request_for_stateless_replay(&request);
8430
8431 let meerkat_core::Message::BlockAssistant(original_assistant) = &request.messages[0] else {
8432 panic!("expected original block assistant");
8433 };
8434 assert!(
8435 original_assistant
8436 .blocks
8437 .iter()
8438 .any(|block| matches!(block, meerkat_core::AssistantBlock::Image { .. })),
8439 "request-view sanitization must not rewrite canonical caller-owned messages"
8440 );
8441
8442 let meerkat_core::Message::BlockAssistant(sanitized_assistant) = &sanitized.messages[0]
8443 else {
8444 panic!("expected sanitized block assistant");
8445 };
8446 assert!(
8447 sanitized_assistant
8448 .blocks
8449 .iter()
8450 .any(|block| matches!(block, meerkat_core::AssistantBlock::Image { .. })),
8451 "Meerkat 0.6.2 owns provider replay projection for generated images"
8452 );
8453 }
8454
8455 #[derive(Default)]
8456 struct CapturingLlmClient {
8457 projected_messages: std::sync::Mutex<Vec<meerkat_core::Message>>,
8458 }
8459
8460 #[async_trait]
8461 impl LlmClient for CapturingLlmClient {
8462 fn project_replay_messages(
8463 &self,
8464 messages: &[meerkat_core::Message],
8465 ) -> Result<Vec<meerkat_core::Message>, meerkat_client::LlmError> {
8466 *self
8467 .projected_messages
8468 .lock()
8469 .unwrap_or_else(std::sync::PoisonError::into_inner) = messages.to_vec();
8470 Ok(messages.to_vec())
8471 }
8472
8473 fn stream<'a>(&'a self, _request: &'a LlmRequest) -> LlmStream<'a> {
8474 Box::pin(futures::stream::iter([Ok(
8475 meerkat_client::LlmEvent::Done {
8476 outcome: meerkat_client::LlmDoneOutcome::Success {
8477 stop_reason: meerkat_core::StopReason::EndTurn,
8478 },
8479 },
8480 )]))
8481 }
8482
8483 fn provider(&self) -> meerkat_core::Provider {
8484 meerkat_core::Provider::OpenAI
8485 }
8486
8487 async fn health_check(&self) -> Result<(), meerkat_client::LlmError> {
8488 Ok(())
8489 }
8490 }
8491
8492 #[test]
8493 fn replay_sanitizing_llm_client_delegates_provider_projection() {
8494 let capture = Arc::new(CapturingLlmClient::default());
8495 let inner: Arc<dyn LlmClient> = capture.clone();
8496 let wrapped = ReplaySanitizingLlmClient::new(inner);
8497 let messages = vec![meerkat_core::Message::BlockAssistant(
8498 meerkat_core::BlockAssistantMessage::new(
8499 vec![
8500 meerkat_core::AssistantBlock::Text {
8501 text: "visible".to_string(),
8502 meta: None,
8503 },
8504 meerkat_core::AssistantBlock::ServerToolContent {
8505 id: Some("ws-stream".to_string()),
8506 kind: meerkat_core::ServerToolKind::WebSearch,
8507 content: serde_json::json!({
8508 "type": "response.web_search_call.searching",
8509 "item_id": "ws_123"
8510 }),
8511 meta: None,
8512 },
8513 ],
8514 meerkat_core::StopReason::EndTurn,
8515 ),
8516 )];
8517
8518 let projected = wrapped
8519 .project_replay_messages(&messages)
8520 .expect("wrapped client should delegate provider projection");
8521
8522 let seen = capture
8523 .projected_messages
8524 .lock()
8525 .unwrap_or_else(std::sync::PoisonError::into_inner)
8526 .clone();
8527 let meerkat_core::Message::BlockAssistant(assistant) = &seen[0] else {
8528 panic!("expected block assistant");
8529 };
8530 assert_eq!(
8531 assistant.blocks.len(),
8532 1,
8533 "MobKit sanitization must happen before Meerkat provider projection"
8534 );
8535 assert!(matches!(
8536 assistant.blocks[0],
8537 meerkat_core::AssistantBlock::Text { .. }
8538 ));
8539 assert_eq!(
8540 serde_json::to_value(&projected).expect("projected messages serialize"),
8541 serde_json::to_value(&seen).expect("seen messages serialize")
8542 );
8543 }
8544
8545 #[derive(Default)]
8546 struct CapturingAgentLlmClient {
8547 seen_messages: std::sync::Mutex<Vec<meerkat_core::Message>>,
8548 fallback_prepare_calls: std::sync::atomic::AtomicUsize,
8549 fallback_commit_calls: std::sync::atomic::AtomicUsize,
8550 fallback_schema_calls: std::sync::atomic::AtomicUsize,
8551 stream_observation_starts: std::sync::atomic::AtomicUsize,
8552 }
8553
8554 #[async_trait]
8555 impl meerkat_core::AgentLlmClient for CapturingAgentLlmClient {
8556 async fn stream_response(
8557 &self,
8558 messages: &[meerkat_core::Message],
8559 _tools: &[Arc<meerkat_core::ToolDef>],
8560 _max_tokens: u32,
8561 _temperature: Option<f32>,
8562 _provider_params: Option<
8563 &meerkat_core::lifecycle::run_primitive::ProviderParamsOverride,
8564 >,
8565 ) -> Result<meerkat_core::agent::LlmStreamResult, meerkat_core::AgentError> {
8566 *self
8567 .seen_messages
8568 .lock()
8569 .unwrap_or_else(std::sync::PoisonError::into_inner) = messages.to_vec();
8570 Ok(meerkat_core::agent::LlmStreamResult::new(
8571 Vec::new(),
8572 meerkat_core::StopReason::EndTurn,
8573 meerkat_core::Usage::default(),
8574 ))
8575 }
8576
8577 fn provider(&self) -> meerkat_core::Provider {
8578 meerkat_core::Provider::OpenAI
8579 }
8580
8581 fn model(&self) -> &'static str {
8582 "gpt-5.5"
8583 }
8584
8585 fn prepare_model_fallback(
8586 &self,
8587 _failure: &meerkat_core::AgentError,
8588 ) -> Option<meerkat_core::agent::AgentLlmFallbackSwitch> {
8589 self.fallback_prepare_calls
8590 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8591 None
8592 }
8593
8594 fn commit_model_fallback(
8595 &self,
8596 _previous_identity: &meerkat_core::SessionLlmIdentity,
8597 _target_identity: &meerkat_core::SessionLlmIdentity,
8598 ) -> Result<(), meerkat_core::AgentError> {
8599 self.fallback_commit_calls
8600 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8601 Ok(())
8602 }
8603
8604 fn active_model_fallback_identity(&self) -> Option<meerkat_core::SessionLlmIdentity> {
8605 Some(test_session_llm_identity("fallback-model"))
8606 }
8607
8608 fn compile_model_fallback_schema(
8609 &self,
8610 _target_identity: &meerkat_core::SessionLlmIdentity,
8611 _output_schema: &meerkat_core::OutputSchema,
8612 ) -> Result<meerkat_core::schema::CompiledSchema, meerkat_core::AgentError> {
8613 self.fallback_schema_calls
8614 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8615 Err(meerkat_core::AgentError::ConfigError(
8616 "fallback schema probe".to_string(),
8617 ))
8618 }
8619
8620 fn begin_stream_output_observation(&self) {
8621 self.stream_observation_starts
8622 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8623 }
8624
8625 fn stream_output_observed(&self) -> bool {
8626 true
8627 }
8628 }
8629
8630 fn test_session_llm_identity(model: &str) -> meerkat_core::SessionLlmIdentity {
8631 meerkat_core::SessionLlmIdentity {
8632 model: model.to_string(),
8633 provider: meerkat_core::Provider::OpenAI,
8634 self_hosted_server_id: None,
8635 provider_params: None,
8636 auth_binding: None,
8637 }
8638 }
8639
8640 #[test]
8641 fn sanitize_agent_llm_client_forwards_fallback_and_stream_observation_state() {
8642 let capture = Arc::new(CapturingAgentLlmClient::default());
8643 let inner: Arc<dyn meerkat_core::AgentLlmClient> = capture.clone();
8644 let wrapped = ReplaySanitizingAgentLlmClient::new(inner);
8645 let previous = test_session_llm_identity("primary-model");
8646 let target = test_session_llm_identity("fallback-model");
8647
8648 assert!(
8649 meerkat_core::AgentLlmClient::prepare_model_fallback(
8650 &wrapped,
8651 &meerkat_core::AgentError::ConfigError("probe".to_string()),
8652 )
8653 .is_none()
8654 );
8655 meerkat_core::AgentLlmClient::commit_model_fallback(&wrapped, &previous, &target)
8656 .expect("fallback activation should forward");
8657 assert_eq!(
8658 meerkat_core::AgentLlmClient::active_model_fallback_identity(&wrapped)
8659 .expect("active fallback identity should forward")
8660 .model,
8661 "fallback-model"
8662 );
8663 let schema = meerkat_core::OutputSchema::new(serde_json::json!({"type": "object"}))
8664 .expect("valid test schema");
8665 let error =
8666 meerkat_core::AgentLlmClient::compile_model_fallback_schema(&wrapped, &target, &schema)
8667 .expect_err("fallback schema probe error should forward");
8668 assert!(error.to_string().contains("fallback schema probe"));
8669 meerkat_core::AgentLlmClient::begin_stream_output_observation(&wrapped);
8670 assert!(meerkat_core::AgentLlmClient::stream_output_observed(
8671 &wrapped
8672 ));
8673
8674 assert_eq!(
8675 capture
8676 .fallback_prepare_calls
8677 .load(std::sync::atomic::Ordering::SeqCst),
8678 1
8679 );
8680 assert_eq!(
8681 capture
8682 .fallback_commit_calls
8683 .load(std::sync::atomic::Ordering::SeqCst),
8684 1
8685 );
8686 assert_eq!(
8687 capture
8688 .fallback_schema_calls
8689 .load(std::sync::atomic::Ordering::SeqCst),
8690 1
8691 );
8692 assert_eq!(
8693 capture
8694 .stream_observation_starts
8695 .load(std::sync::atomic::Ordering::SeqCst),
8696 1
8697 );
8698 }
8699
8700 #[tokio::test]
8701 async fn sanitize_agent_llm_client_drops_replay_unsafe_server_tool_blocks() {
8702 let capture = Arc::new(CapturingAgentLlmClient::default());
8703 let inner: Arc<dyn meerkat_core::AgentLlmClient> = capture.clone();
8704 let wrapped = ReplaySanitizingAgentLlmClient::wrap(inner);
8705 let messages = vec![meerkat_core::Message::BlockAssistant(
8706 meerkat_core::BlockAssistantMessage::new(
8707 vec![
8708 meerkat_core::AssistantBlock::Text {
8709 text: "visible".to_string(),
8710 meta: None,
8711 },
8712 meerkat_core::AssistantBlock::ServerToolContent {
8713 id: Some("ws-stream".to_string()),
8714 kind: meerkat_core::ServerToolKind::WebSearch,
8715 content: serde_json::json!({
8716 "type": "response.web_search_call.searching",
8717 "item_id": "ws_123"
8718 }),
8719 meta: None,
8720 },
8721 meerkat_core::AssistantBlock::ServerToolContent {
8722 id: Some("ok".to_string()),
8723 kind: meerkat_core::ServerToolKind::ProviderNative {
8724 name: "web_search_call".to_string(),
8725 },
8726 content: serde_json::json!({
8727 "type": "web_search_call",
8728 "id": "ws_123",
8729 "status": "completed"
8730 }),
8731 meta: None,
8732 },
8733 ],
8734 meerkat_core::StopReason::EndTurn,
8735 ),
8736 )];
8737 let tools: Vec<Arc<meerkat_core::ToolDef>> = Vec::new();
8738
8739 wrapped
8740 .stream_response(&messages, &tools, 512, None, None)
8741 .await
8742 .expect("wrapped client should delegate");
8743
8744 let seen = capture
8745 .seen_messages
8746 .lock()
8747 .unwrap_or_else(std::sync::PoisonError::into_inner)
8748 .clone();
8749 let meerkat_core::Message::BlockAssistant(assistant) = &seen[0] else {
8750 panic!("expected block assistant");
8751 };
8752 assert_eq!(assistant.blocks.len(), 2);
8753 assert!(matches!(
8754 assistant.blocks[0],
8755 meerkat_core::AssistantBlock::Text { .. }
8756 ));
8757 assert!(matches!(
8758 assistant.blocks[1],
8759 meerkat_core::AssistantBlock::ServerToolContent { ref kind, .. }
8760 if kind.provider_name() == "web_search_call"
8761 ));
8762 }
8763
8764 fn generated_image_block_for_test() -> meerkat_core::AssistantBlock {
8765 serde_json::from_value(serde_json::json!({
8766 "block_type": "image",
8767 "data": {
8768 "image_id": "00000000-0000-0000-0000-000000000051",
8769 "blob_ref": {
8770 "blob_id": "sha256:test-generated-image",
8771 "media_type": "image/png"
8772 },
8773 "media_type": "image/png",
8774 "width": 1024,
8775 "height": 1024,
8776 "revised_prompt": { "disposition": "not_requested" },
8777 "meta": { "provider": "not_emitted" }
8778 }
8779 }))
8780 .expect("test image block should deserialize")
8781 }
8782
8783 #[test]
8784 fn sanitize_message_preserves_assistant_image_blocks() {
8785 let message =
8786 meerkat_core::Message::BlockAssistant(meerkat_core::BlockAssistantMessage::new(
8787 vec![
8788 meerkat_core::AssistantBlock::Text {
8789 text: "Here is the image.".to_string(),
8790 meta: None,
8791 },
8792 generated_image_block_for_test(),
8793 ],
8794 meerkat_core::StopReason::EndTurn,
8795 ));
8796
8797 let sanitized = sanitize_message_for_stateless_replay(message);
8798 let meerkat_core::Message::BlockAssistant(assistant) = sanitized else {
8799 panic!("expected block assistant");
8800 };
8801
8802 assert_eq!(assistant.blocks.len(), 2);
8803 assert!(matches!(
8804 assistant.blocks[0],
8805 meerkat_core::AssistantBlock::Text { .. }
8806 ));
8807 assert!(
8808 matches!(
8809 assistant.blocks[1],
8810 meerkat_core::AssistantBlock::Image { .. }
8811 ),
8812 "generated image blocks should reach Meerkat's provider projection"
8813 );
8814 }
8815
8816 #[test]
8819 fn persistent_with_hook_wraps_session_service() {
8820 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
8821 let store_path = dir.path().to_path_buf();
8822 let Ok(sqlite) = meerkat_store::SqliteSessionStore::open(store_path.join("sessions.db"))
8823 else {
8824 panic!("failed to open sqlite session store");
8825 };
8826 let session_store: Arc<dyn SessionStore> = Arc::new(sqlite);
8827 let Ok(definition) = meerkat_mob::MobDefinition::from_toml("[mob]\nid = \"test\"\n") else {
8828 panic!("failed to parse minimal mob definition");
8829 };
8830
8831 let hook_called = Arc::new(std::sync::atomic::AtomicBool::new(false));
8832 let hook_called_clone = hook_called.clone();
8833
8834 let spec = MobBootstrapSpec::persistent_with_hook(
8835 definition,
8836 meerkat_mob::MobStorage::in_memory(),
8837 store_path.clone(),
8838 4,
8839 session_store,
8840 move |_req: &mut CreateSessionRequest| {
8841 hook_called_clone.store(true, std::sync::atomic::Ordering::Relaxed);
8842 Box::pin(async { Ok(()) })
8843 },
8844 )
8845 .unwrap_or_else(|e| panic!("{e}"));
8846
8847 assert!(
8855 spec.runtime_adapter.is_some(),
8856 "persistent_with_hook must provide a runtime adapter via spec.runtime_adapter"
8857 );
8858 assert!(
8859 spec.session_service.runtime_adapter().is_some(),
8860 "session service must own a runtime_store so archive/retire don't \
8861 hit the store-only-projection rejection in meerkat-session"
8862 );
8863 assert!(
8864 store_path.join("runtime.sqlite").exists(),
8865 "persistent_inner must open a SqliteRuntimeStore at <store_path>/runtime.sqlite"
8866 );
8867
8868 assert!(
8873 !hook_called.load(std::sync::atomic::Ordering::Relaxed),
8874 "hook must not be called before create_session"
8875 );
8876 }
8877
8878 #[test]
8880 fn ephemeral_with_hook_creates_spec() {
8881 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
8882 let store_path = dir.path().to_path_buf();
8883 let Ok(definition) = meerkat_mob::MobDefinition::from_toml("[mob]\nid = \"test\"\n") else {
8884 panic!("failed to parse minimal mob definition");
8885 };
8886
8887 let hook_called = Arc::new(std::sync::atomic::AtomicBool::new(false));
8888 let hook_called_clone = hook_called.clone();
8889
8890 let spec = MobBootstrapSpec::ephemeral_with_hook(
8891 definition,
8892 meerkat_mob::MobStorage::in_memory(),
8893 store_path,
8894 4,
8895 None,
8896 move |_req: &mut CreateSessionRequest| {
8897 hook_called_clone.store(true, std::sync::atomic::Ordering::Relaxed);
8898 Box::pin(async { Ok(()) })
8899 },
8900 );
8901
8902 let runtime_adapter = spec
8903 .runtime_adapter
8904 .as_ref()
8905 .unwrap_or_else(|| panic!("ephemeral_with_hook must retain its runtime adapter"));
8906 assert!(
8907 runtime_adapter.has_session_llm_reconfigure_host(),
8908 "ephemeral_with_hook must install the live LLM reconfiguration host"
8909 );
8910
8911 assert!(
8913 !hook_called.load(std::sync::atomic::Ordering::Relaxed),
8914 "hook must not be called before create_session"
8915 );
8916 }
8917
8918 #[tokio::test]
8922 async fn pre_build_hook_mutates_create_session_request() {
8923 use std::sync::Mutex;
8924
8925 let captured = Arc::new(Mutex::new(None::<(String, Option<String>)>));
8926 let captured_clone = captured.clone();
8927
8928 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
8930 let factory = AgentFactory::new(dir.path()).builtins(true);
8931 let config = Config::default();
8932 let builder = FactoryAgentBuilder::new(factory, config);
8933 let inner: Arc<dyn MobSessionService> =
8934 Arc::new(meerkat_session::EphemeralSessionService::new(builder, 4));
8935
8936 let hook: PreBuildHook = Arc::new(move |req: &mut CreateSessionRequest| {
8938 req.model = "hooked-model".to_string();
8939 req.system_prompt =
8940 meerkat_core::config::SystemPromptOverride::Set("injected-prompt".to_string());
8941 let labels = req.labels.get_or_insert_with(Default::default);
8942 labels.insert("hook_label".to_string(), "hook_value".to_string());
8943 let mut lock = captured_clone
8945 .lock()
8946 .unwrap_or_else(std::sync::PoisonError::into_inner);
8947 *lock = Some((
8948 req.model.clone(),
8949 req.system_prompt.as_set_prompt().map(ToString::to_string),
8950 ));
8951 Box::pin(async { Ok(()) })
8952 });
8953 let wrapped = PreBuildMobSessionService {
8954 inner,
8955 hook,
8956 dispatch_taint: None,
8957 after_create_hook: None,
8958 runtime_adapter_override: None,
8959 session_read_absorber: None,
8960 archived_terminal_authority: None,
8961 };
8962
8963 let req = CreateSessionRequest {
8964 model: "original-model".to_string(),
8965 prompt: meerkat_core::ContentInput::Text("test".to_string()),
8966 system_prompt: meerkat_core::config::SystemPromptOverride::Inherit,
8967 max_tokens: None,
8968 event_tx: None,
8969 initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
8970 build: None,
8971 labels: None,
8972 deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::default(),
8973 injected_context: Vec::new(),
8974 };
8975
8976 let _ = meerkat_core::service::SessionService::create_session(&wrapped, req).await;
8978
8979 let (model, prompt) = captured
8980 .lock()
8981 .unwrap_or_else(std::sync::PoisonError::into_inner)
8982 .clone()
8983 .expect("hook must have been called");
8984 assert_eq!(model, "hooked-model", "hook must mutate the model");
8985 assert_eq!(
8986 prompt.as_deref(),
8987 Some("injected-prompt"),
8988 "hook must set the system prompt"
8989 );
8990 }
8991
8992 #[tokio::test]
9003 async fn config_retry_reaches_agent_effective_retry_policy() {
9004 use meerkat_session::SessionAgentBuilder as _;
9005
9006 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
9007 let factory = AgentFactory::new(dir.path()).builtins(true);
9008
9009 let mut config = Config::default();
9011 assert_ne!(
9012 config.retry.max_retries, 11,
9013 "test sentinel must differ from default"
9014 );
9015 config.retry.max_retries = 11;
9016 config.retry.initial_delay = std::time::Duration::from_millis(125);
9017 config.retry.max_delay = std::time::Duration::from_secs(7);
9018 config.retry.multiplier = 3.5;
9019
9020 let mut builder = FactoryAgentBuilder::new(factory, config);
9021 builder.default_llm_client = Some(Arc::new(CapturingLlmClient::default()));
9023
9024 let req = CreateSessionRequest {
9025 model: "mock-model".to_string(),
9026 prompt: meerkat_core::ContentInput::Text("noop".to_string()),
9027 system_prompt: meerkat_core::config::SystemPromptOverride::Inherit,
9028 max_tokens: None,
9029 event_tx: None,
9030 initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
9031 build: None,
9032 labels: None,
9033 deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
9034 injected_context: Vec::new(),
9035 };
9036
9037 let (event_tx, _event_rx) = tokio::sync::mpsc::channel(8);
9038 let factory_agent = builder
9039 .build_agent(&req, event_tx)
9040 .await
9041 .unwrap_or_else(|e| panic!("build_agent should succeed offline: {e}"));
9042
9043 let effective = factory_agent.agent().retry_policy();
9044 assert_eq!(
9045 effective.max_retries, 11,
9046 "Config.retry.max_retries must be plumbed into the agent's effective \
9047 RetryPolicy, not the default 3"
9048 );
9049 assert_eq!(
9050 effective.initial_delay,
9051 std::time::Duration::from_millis(125),
9052 "Config.retry.initial_delay must be plumbed, not the 500ms default"
9053 );
9054 assert_eq!(
9055 effective.max_delay,
9056 std::time::Duration::from_secs(7),
9057 "Config.retry.max_delay must be plumbed, not the 30s default"
9058 );
9059 assert!(
9060 (effective.multiplier - 3.5).abs() < f64::EPSILON,
9061 "Config.retry.multiplier must be plumbed, not the 2.0 default"
9062 );
9063 }
9064
9065 struct AbsorberInnerProbe {
9068 session: meerkat_core::session::Session,
9069 loads: std::sync::atomic::AtomicU64,
9070 }
9071
9072 #[async_trait]
9073 impl meerkat_core::service::SessionService for AbsorberInnerProbe {
9074 async fn create_session(
9075 &self,
9076 _req: CreateSessionRequest,
9077 ) -> Result<meerkat_core::types::RunResult, SessionError> {
9078 Err(SessionError::Unsupported("create_session".to_string()))
9079 }
9080
9081 async fn start_turn(
9082 &self,
9083 _id: &meerkat_core::types::SessionId,
9084 _req: meerkat_core::service::StartTurnRequest,
9085 ) -> Result<meerkat_core::types::RunResult, SessionError> {
9086 Err(SessionError::Unsupported("start_turn".to_string()))
9087 }
9088
9089 async fn interrupt(
9090 &self,
9091 _id: &meerkat_core::types::SessionId,
9092 ) -> Result<(), SessionError> {
9093 Ok(())
9094 }
9095
9096 async fn read(
9097 &self,
9098 id: &meerkat_core::types::SessionId,
9099 ) -> Result<meerkat_core::service::SessionView, SessionError> {
9100 Err(SessionError::NotFound { id: id.clone() })
9101 }
9102
9103 async fn list(
9104 &self,
9105 _query: meerkat_core::service::SessionQuery,
9106 ) -> Result<Vec<meerkat_core::service::SessionSummary>, SessionError> {
9107 Ok(Vec::new())
9108 }
9109
9110 async fn archive(&self, _id: &meerkat_core::types::SessionId) -> Result<(), SessionError> {
9111 Ok(())
9112 }
9113 }
9114
9115 #[async_trait]
9116 impl meerkat_core::service::SessionServiceCommsExt for AbsorberInnerProbe {}
9117
9118 #[async_trait]
9119 impl meerkat_core::service::SessionServiceControlExt for AbsorberInnerProbe {
9120 async fn append_system_context(
9121 &self,
9122 _id: &meerkat_core::types::SessionId,
9123 _req: meerkat_core::service::AppendSystemContextRequest,
9124 ) -> Result<
9125 meerkat_core::service::AppendSystemContextResult,
9126 meerkat_core::service::SessionControlError,
9127 > {
9128 Err(SessionError::Unsupported("append_system_context".to_string()).into())
9129 }
9130
9131 async fn stage_tool_results(
9132 &self,
9133 _id: &meerkat_core::types::SessionId,
9134 _req: meerkat_core::service::StageToolResultsRequest,
9135 ) -> Result<meerkat_core::service::StageToolResultsResult, SessionError> {
9136 Err(SessionError::Unsupported("stage_tool_results".to_string()))
9137 }
9138 }
9139
9140 #[async_trait]
9141 impl meerkat_core::service::SessionServiceHistoryExt for AbsorberInnerProbe {
9142 async fn read_history(
9143 &self,
9144 id: &meerkat_core::types::SessionId,
9145 _query: meerkat_core::service::SessionHistoryQuery,
9146 ) -> Result<meerkat_core::service::SessionHistoryPage, SessionError> {
9147 Err(SessionError::NotFound { id: id.clone() })
9148 }
9149 }
9150
9151 #[async_trait]
9152 impl MobSessionService for AbsorberInnerProbe {
9153 async fn prepare_session_for_resume(
9154 &self,
9155 _session_id: &meerkat_core::types::SessionId,
9156 ) -> Result<(), SessionError> {
9157 Ok(())
9158 }
9159 async fn acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
9160 &self,
9161 _session_id: &meerkat_core::types::SessionId,
9162 _authority: &meerkat_core::CommittedSessionBoundaryAuthority,
9163 ) -> Result<(), SessionError> {
9164 Err(SessionError::Unsupported(
9165 "test double does not acknowledge store-owned runtime boundaries".to_string(),
9166 ))
9167 }
9168 async fn load_session_for_resume(
9169 &self,
9170 session_id: &meerkat_core::types::SessionId,
9171 ) -> Result<meerkat_mob::ResumeSessionLoad, SessionError> {
9172 if let Some(session) = self.load_persisted_session(session_id).await? {
9175 return Ok(meerkat_mob::ResumeSessionLoad::Active(Box::new(session)));
9176 }
9177 if let Some(session) = self.load_revivable_retired_session(session_id).await? {
9178 return Ok(meerkat_mob::ResumeSessionLoad::Revivable(Box::new(session)));
9179 }
9180 Ok(meerkat_mob::ResumeSessionLoad::Absent)
9181 }
9182
9183 async fn create_session_under_runtime_turn_boundary(
9184 &self,
9185 req: CreateSessionRequest,
9186 ) -> Result<meerkat_core::types::RunResult, SessionError> {
9187 meerkat_core::SessionService::create_session(self, req).await
9188 }
9189
9190 async fn archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(
9191 &self,
9192 session_id: &meerkat_core::types::SessionId,
9193 ) -> Result<(), SessionError> {
9194 meerkat_core::SessionService::archive(self, session_id).await
9195 }
9196
9197 async fn discard_live_session_under_runtime_turn_boundary(
9198 &self,
9199 _session_id: &meerkat_core::types::SessionId,
9200 ) -> Result<(), SessionError> {
9201 Ok(())
9202 }
9203
9204 async fn load_persisted_session(
9205 &self,
9206 session_id: &meerkat_core::types::SessionId,
9207 ) -> Result<Option<meerkat_core::session::Session>, SessionError> {
9208 self.loads.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
9209 if session_id == self.session.id() {
9210 Ok(Some(self.session.clone()))
9211 } else {
9212 Ok(None)
9213 }
9214 }
9215 }
9216
9217 #[tokio::test]
9223 async fn session_read_absorber_serves_unchanged_documents_until_a_write_epoch_advances() {
9224 let session =
9225 meerkat_core::session::Session::with_id(meerkat_core::types::SessionId::new());
9226 let session_id = session.id().clone();
9227 let probe = Arc::new(AbsorberInnerProbe {
9228 session,
9229 loads: std::sync::atomic::AtomicU64::new(0),
9230 });
9231 let epochs = Arc::new(SessionSnapshotWriteEpochs::default());
9232 let wrapped = PreBuildMobSessionService {
9233 inner: probe.clone(),
9234 hook: no_op_pre_build_hook(),
9235 dispatch_taint: None,
9236 after_create_hook: None,
9237 runtime_adapter_override: None,
9238 session_read_absorber: Some(Arc::new(SessionDocumentReadAbsorber::new(Arc::clone(
9239 &epochs,
9240 )))),
9241 archived_terminal_authority: None,
9242 };
9243
9244 for _ in 0..5 {
9247 let loaded = MobSessionService::load_persisted_session(&wrapped, &session_id)
9248 .await
9249 .expect("absorbed load");
9250 assert_eq!(loaded.expect("absorbed document present").id(), &session_id);
9251 }
9252 assert_eq!(
9253 probe.loads.load(std::sync::atomic::Ordering::SeqCst),
9254 1,
9255 "unchanged-session reads must be absorbed after the first inner load"
9256 );
9257
9258 let facade = SessionStoreBackedRuntimeStore::with_write_epochs(
9261 Arc::new(meerkat_runtime::InMemoryRuntimeStore::new()),
9262 Arc::clone(&epochs),
9263 );
9264 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(&session_id);
9265 meerkat_runtime::RuntimeStore::clear_session_snapshot(&facade, &runtime_id)
9266 .await
9267 .expect("facade-observed session-scoped write");
9268
9269 let reloaded = MobSessionService::load_persisted_session(&wrapped, &session_id)
9270 .await
9271 .expect("post-write load");
9272 assert!(reloaded.is_some());
9273 assert_eq!(
9274 probe.loads.load(std::sync::atomic::Ordering::SeqCst),
9275 2,
9276 "a session-scoped write must force the next read back to the inner service"
9277 );
9278
9279 let other_id = meerkat_core::types::SessionId::new();
9281 for _ in 0..2 {
9282 assert!(
9283 MobSessionService::load_persisted_session(&wrapped, &other_id)
9284 .await
9285 .expect("absent load")
9286 .is_none()
9287 );
9288 }
9289 assert_eq!(
9290 probe.loads.load(std::sync::atomic::Ordering::SeqCst),
9291 4,
9292 "absent documents are not cached"
9293 );
9294 }
9295
9296 #[derive(Default)]
9297 struct ForwardingProbe {
9298 calls: Mutex<Vec<&'static str>>,
9299 cancel_outcome: std::sync::atomic::AtomicU8,
9300 }
9301
9302 impl ForwardingProbe {
9303 fn record(&self, call: &'static str) {
9304 self.calls
9305 .lock()
9306 .unwrap_or_else(std::sync::PoisonError::into_inner)
9307 .push(call);
9308 }
9309
9310 fn calls(&self) -> Vec<&'static str> {
9311 self.calls
9312 .lock()
9313 .unwrap_or_else(std::sync::PoisonError::into_inner)
9314 .clone()
9315 }
9316
9317 fn set_cancel_outcome(&self, outcome: u8) {
9318 self.cancel_outcome
9319 .store(outcome, std::sync::atomic::Ordering::Relaxed);
9320 }
9321 }
9322
9323 #[async_trait]
9324 impl meerkat_core::service::SessionService for ForwardingProbe {
9325 async fn create_session(
9326 &self,
9327 _req: CreateSessionRequest,
9328 ) -> Result<meerkat_core::types::RunResult, SessionError> {
9329 Err(SessionError::Unsupported("create_session".to_string()))
9330 }
9331
9332 async fn start_turn(
9333 &self,
9334 _id: &meerkat_core::types::SessionId,
9335 _req: meerkat_core::service::StartTurnRequest,
9336 ) -> Result<meerkat_core::types::RunResult, SessionError> {
9337 Err(SessionError::Unsupported("start_turn".to_string()))
9338 }
9339
9340 async fn reconcile_runtime_compaction_projections(
9341 &self,
9342 _id: &meerkat_core::types::SessionId,
9343 _intents: Vec<meerkat_core::CompactionProjectionIntent>,
9344 ) -> Result<(), SessionError> {
9345 self.record("reconcile_runtime_compaction_projections");
9346 Ok(())
9347 }
9348
9349 async fn abort_uncommitted_compaction_projections(
9350 &self,
9351 _id: &meerkat_core::types::SessionId,
9352 ) -> Result<(), SessionError> {
9353 self.record("abort_uncommitted_compaction_projections");
9354 Ok(())
9355 }
9356
9357 async fn abort_rejected_runtime_run_projections(
9358 &self,
9359 _id: &meerkat_core::types::SessionId,
9360 ) -> Result<(), SessionError> {
9361 self.record("abort_rejected_runtime_run_projections");
9362 Ok(())
9363 }
9364
9365 async fn interrupt(
9366 &self,
9367 _id: &meerkat_core::types::SessionId,
9368 ) -> Result<(), SessionError> {
9369 self.record("interrupt");
9370 Ok(())
9371 }
9372
9373 async fn read(
9374 &self,
9375 id: &meerkat_core::types::SessionId,
9376 ) -> Result<meerkat_core::service::SessionView, SessionError> {
9377 Err(SessionError::NotFound { id: id.clone() })
9378 }
9379
9380 async fn list(
9381 &self,
9382 _query: meerkat_core::service::SessionQuery,
9383 ) -> Result<Vec<meerkat_core::service::SessionSummary>, SessionError> {
9384 Ok(Vec::new())
9385 }
9386
9387 async fn archive(&self, _id: &meerkat_core::types::SessionId) -> Result<(), SessionError> {
9388 self.record("archive");
9389 Ok(())
9390 }
9391
9392 async fn record_live_terminal_error(
9393 &self,
9394 _id: &meerkat_core::types::SessionId,
9395 _cause: meerkat_core::live_adapter::LiveAdapterErrorCode,
9396 ) -> Result<(), SessionError> {
9397 self.record("record_live_terminal_error");
9398 Ok(())
9399 }
9400
9401 async fn record_live_output_audio_degraded(
9402 &self,
9403 _id: &meerkat_core::types::SessionId,
9404 _dropped: u64,
9405 ) -> Result<(), SessionError> {
9406 self.record("record_live_output_audio_degraded");
9407 Ok(())
9408 }
9409 }
9410
9411 #[async_trait]
9412 impl meerkat_core::service::SessionServiceCommsExt for ForwardingProbe {}
9413
9414 #[async_trait]
9415 impl meerkat_core::service::SessionServiceControlExt for ForwardingProbe {
9416 async fn append_system_context(
9417 &self,
9418 _id: &meerkat_core::types::SessionId,
9419 _req: meerkat_core::service::AppendSystemContextRequest,
9420 ) -> Result<
9421 meerkat_core::service::AppendSystemContextResult,
9422 meerkat_core::service::SessionControlError,
9423 > {
9424 self.record("append_system_context");
9425 Ok(meerkat_core::service::AppendSystemContextResult {
9426 status: meerkat_core::service::AppendSystemContextStatus::Applied,
9427 })
9428 }
9429
9430 async fn stage_tool_results(
9431 &self,
9432 _id: &meerkat_core::types::SessionId,
9433 _req: meerkat_core::service::StageToolResultsRequest,
9434 ) -> Result<meerkat_core::service::StageToolResultsResult, SessionError> {
9435 self.record("stage_tool_results");
9436 Ok(meerkat_core::service::StageToolResultsResult {
9437 accepted_result_count: 7,
9438 disposition: meerkat_core::service::StageToolResultsDisposition::Staged,
9442 })
9443 }
9444 }
9445
9446 #[async_trait]
9447 impl meerkat_core::service::SessionServiceHistoryExt for ForwardingProbe {
9448 async fn read_history(
9449 &self,
9450 id: &meerkat_core::types::SessionId,
9451 _query: meerkat_core::service::SessionHistoryQuery,
9452 ) -> Result<meerkat_core::service::SessionHistoryPage, SessionError> {
9453 Err(SessionError::NotFound { id: id.clone() })
9454 }
9455
9456 async fn read_transcript_revision(
9457 &self,
9458 id: &meerkat_core::types::SessionId,
9459 _query: meerkat_core::service::SessionTranscriptRevisionQuery,
9460 ) -> Result<meerkat_core::service::SessionTranscriptRevisionPage, SessionError> {
9461 self.record("read_transcript_revision");
9462 Err(SessionError::NotFound { id: id.clone() })
9463 }
9464
9465 async fn list_transcript_revisions(
9466 &self,
9467 id: &meerkat_core::types::SessionId,
9468 _query: meerkat_core::service::SessionTranscriptRevisionListQuery,
9469 ) -> Result<meerkat_core::service::SessionTranscriptRevisionList, SessionError> {
9470 self.record("list_transcript_revisions");
9471 Err(SessionError::NotFound { id: id.clone() })
9472 }
9473 }
9474
9475 #[async_trait]
9476 impl MobSessionService for ForwardingProbe {
9477 async fn prepare_session_for_resume(
9478 &self,
9479 _session_id: &meerkat_core::types::SessionId,
9480 ) -> Result<(), SessionError> {
9481 Ok(())
9482 }
9483 async fn acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
9484 &self,
9485 _session_id: &meerkat_core::types::SessionId,
9486 _authority: &meerkat_core::CommittedSessionBoundaryAuthority,
9487 ) -> Result<(), SessionError> {
9488 Err(SessionError::Unsupported(
9489 "test double does not acknowledge store-owned runtime boundaries".to_string(),
9490 ))
9491 }
9492 async fn load_session_for_resume(
9493 &self,
9494 session_id: &meerkat_core::types::SessionId,
9495 ) -> Result<meerkat_mob::ResumeSessionLoad, SessionError> {
9496 if let Some(session) = self.load_persisted_session(session_id).await? {
9499 return Ok(meerkat_mob::ResumeSessionLoad::Active(Box::new(session)));
9500 }
9501 if let Some(session) = self.load_revivable_retired_session(session_id).await? {
9502 return Ok(meerkat_mob::ResumeSessionLoad::Revivable(Box::new(session)));
9503 }
9504 Ok(meerkat_mob::ResumeSessionLoad::Absent)
9505 }
9506
9507 async fn create_session_under_runtime_turn_boundary(
9508 &self,
9509 req: CreateSessionRequest,
9510 ) -> Result<meerkat_core::types::RunResult, SessionError> {
9511 meerkat_core::SessionService::create_session(self, req).await
9512 }
9513
9514 fn supports_persistent_sessions(&self) -> bool {
9515 true
9516 }
9517
9518 fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
9519 Some(Arc::new(meerkat_runtime::MeerkatMachine::ephemeral()))
9520 }
9521
9522 async fn cancel_after_boundary_with_machine_authority(
9523 &self,
9524 session_id: &meerkat_core::types::SessionId,
9525 _expected_run_id: &meerkat_core::lifecycle::RunId,
9526 _authority: meerkat_runtime::MachineSessionControlAuthority,
9527 ) -> Result<(), SessionError> {
9528 self.record("cancel_after_boundary_with_machine_authority");
9529 match self
9530 .cancel_outcome
9531 .load(std::sync::atomic::Ordering::Relaxed)
9532 {
9533 0 => Err(SessionError::NotFound {
9534 id: session_id.clone(),
9535 }),
9536 1 => Err(SessionError::NotRunning {
9537 id: session_id.clone(),
9538 }),
9539 2 => Err(SessionError::Unsupported(
9540 "synthetic cancel rejection".to_string(),
9541 )),
9542 _ => Ok(()),
9543 }
9544 }
9545
9546 async fn archive_with_mob_lifecycle_authority(
9547 &self,
9548 _session_id: &meerkat_core::types::SessionId,
9549 ) -> Result<(), SessionError> {
9550 self.record("archive_with_mob_lifecycle_authority");
9551 Ok(())
9552 }
9553
9554 async fn archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(
9555 &self,
9556 _session_id: &meerkat_core::types::SessionId,
9557 ) -> Result<(), SessionError> {
9558 self.record("archive_with_mob_lifecycle_authority_under_runtime_turn_boundary");
9559 Ok(())
9560 }
9561
9562 async fn discard_live_session_under_runtime_turn_boundary(
9563 &self,
9564 _session_id: &meerkat_core::types::SessionId,
9565 ) -> Result<(), SessionError> {
9566 self.record("discard_live_session_under_runtime_turn_boundary");
9567 Ok(())
9568 }
9569
9570 async fn session_known_to_archive_authority(
9571 &self,
9572 _session_id: &meerkat_core::types::SessionId,
9573 ) -> Result<bool, SessionError> {
9574 self.record("session_known_to_archive_authority");
9575 Ok(true)
9576 }
9577
9578 async fn prepare_transient_turn_context_for_active_turn(
9579 &self,
9580 _session_id: &meerkat_core::types::SessionId,
9581 _expected_run_id: &meerkat_core::lifecycle::RunId,
9582 _contexts: Vec<meerkat_core::lifecycle::run_primitive::TurnRequestContext>,
9583 ) -> Result<meerkat_core::CoreBoundaryStageOutput, meerkat_core::CoreBoundaryStageError>
9584 {
9585 self.record("prepare_transient_turn_context_for_active_turn");
9586 Err(meerkat_core::CoreBoundaryStageError::unavailable(
9587 "probe has no boundary authority",
9588 ))
9589 }
9590 }
9591
9592 #[tokio::test]
9593 async fn pre_build_wrapper_forwards_mob_authority_and_control_extensions() {
9594 let probe = Arc::new(ForwardingProbe::default());
9595 let inner: Arc<dyn MobSessionService> = probe.clone();
9596 let wrapped = PreBuildMobSessionService {
9597 inner,
9598 hook: no_op_pre_build_hook(),
9599 dispatch_taint: None,
9600 after_create_hook: None,
9601 runtime_adapter_override: Some(Arc::new(meerkat_runtime::MeerkatMachine::ephemeral())),
9602 session_read_absorber: None,
9603 archived_terminal_authority: None,
9604 };
9605 let session_id = meerkat_core::types::SessionId::new();
9606 let run_id = meerkat_core::lifecycle::RunId::new();
9607
9608 MobSessionService::cancel_after_boundary_with_machine_authority(
9609 &wrapped,
9610 &session_id,
9611 &run_id,
9612 wrapped
9613 .runtime_adapter()
9614 .expect("wrapper should expose runtime adapter")
9615 .session_control_authority(),
9616 )
9617 .await
9618 .expect("machine-authorized cancel should treat a missing live session as quiesced");
9619
9620 MobSessionService::archive_with_mob_lifecycle_authority(&wrapped, &session_id)
9621 .await
9622 .expect("archive_with_mob_lifecycle_authority should forward to inner service");
9623 meerkat_core::service::SessionService::reconcile_runtime_compaction_projections(
9624 &wrapped,
9625 &session_id,
9626 Vec::new(),
9627 )
9628 .await
9629 .expect("runtime compaction reconciliation should forward to inner service");
9630 meerkat_core::service::SessionService::abort_uncommitted_compaction_projections(
9631 &wrapped,
9632 &session_id,
9633 )
9634 .await
9635 .expect("runtime compaction abort should forward to inner service");
9636 meerkat_core::service::SessionService::abort_rejected_runtime_run_projections(
9637 &wrapped,
9638 &session_id,
9639 )
9640 .await
9641 .expect("rejected runtime-run cleanup should forward to inner service");
9642 meerkat_core::service::SessionService::record_live_terminal_error(
9643 &wrapped,
9644 &session_id,
9645 meerkat_core::live_adapter::LiveAdapterErrorCode::ConnectionLost,
9646 )
9647 .await
9648 .expect("live terminal errors should forward to inner service");
9649 meerkat_core::service::SessionService::record_live_output_audio_degraded(
9650 &wrapped,
9651 &session_id,
9652 3,
9653 )
9654 .await
9655 .expect("live output degradation should forward to inner service");
9656 let staged = meerkat_core::service::SessionServiceControlExt::stage_tool_results(
9657 &wrapped,
9658 &session_id,
9659 meerkat_core::service::StageToolResultsRequest {
9660 results: Vec::new(),
9661 },
9662 )
9663 .await
9664 .expect("stage_tool_results should forward to inner service");
9665 let _ = meerkat_core::service::SessionServiceHistoryExt::read_transcript_revision(
9666 &wrapped,
9667 &session_id,
9668 meerkat_core::service::SessionTranscriptRevisionQuery {
9669 revision: "rev-1".to_string(),
9670 offset: 0,
9671 limit: None,
9672 },
9673 )
9674 .await;
9675 let _ = meerkat_core::service::SessionServiceHistoryExt::list_transcript_revisions(
9676 &wrapped,
9677 &session_id,
9678 meerkat_core::service::SessionTranscriptRevisionListQuery::default(),
9679 )
9680 .await;
9681
9682 assert_eq!(staged.accepted_result_count, 7);
9683 let preparation_error = wrapped
9684 .prepare_transient_turn_context_for_active_turn(
9685 &session_id,
9686 &meerkat_core::lifecycle::RunId::new(),
9687 vec![
9688 meerkat_core::lifecycle::run_primitive::TurnRequestContext::new("steer")
9689 .expect("non-empty transient turn context"),
9690 ],
9691 )
9692 .await
9693 .expect_err("probe preparation error should forward unchanged");
9694 assert!(preparation_error.is_unavailable());
9695 let known = wrapped
9699 .session_known_to_archive_authority(&session_id)
9700 .await
9701 .expect("archive-authority probe should forward");
9702 assert!(known, "probe answers true");
9703 assert_eq!(
9704 probe.calls(),
9705 vec![
9706 "cancel_after_boundary_with_machine_authority",
9707 "archive_with_mob_lifecycle_authority",
9708 "reconcile_runtime_compaction_projections",
9709 "abort_uncommitted_compaction_projections",
9710 "abort_rejected_runtime_run_projections",
9711 "record_live_terminal_error",
9712 "record_live_output_audio_degraded",
9713 "stage_tool_results",
9714 "read_transcript_revision",
9715 "list_transcript_revisions",
9716 "prepare_transient_turn_context_for_active_turn",
9717 "session_known_to_archive_authority",
9718 ]
9719 );
9720 }
9721
9722 #[tokio::test]
9723 async fn machine_authorized_boundary_cancel_only_normalizes_quiesced_liveness() {
9724 let probe = Arc::new(ForwardingProbe::default());
9725 let wrapped = PreBuildMobSessionService {
9726 inner: probe.clone(),
9727 hook: no_op_pre_build_hook(),
9728 dispatch_taint: None,
9729 after_create_hook: None,
9730 runtime_adapter_override: Some(Arc::new(meerkat_runtime::MeerkatMachine::ephemeral())),
9731 session_read_absorber: None,
9732 archived_terminal_authority: None,
9733 };
9734 let session_id = meerkat_core::types::SessionId::new();
9735 let run_id = meerkat_core::lifecycle::RunId::new();
9736
9737 for quiesced_outcome in [0, 1] {
9738 probe.set_cancel_outcome(quiesced_outcome);
9739 MobSessionService::cancel_after_boundary_with_machine_authority(
9740 &wrapped,
9741 &session_id,
9742 &run_id,
9743 wrapped
9744 .runtime_adapter()
9745 .expect("wrapper should expose runtime adapter")
9746 .session_control_authority(),
9747 )
9748 .await
9749 .expect("NotFound and NotRunning both prove lower-plane quiescence");
9750 }
9751
9752 probe.set_cancel_outcome(2);
9753 let error = MobSessionService::cancel_after_boundary_with_machine_authority(
9754 &wrapped,
9755 &session_id,
9756 &run_id,
9757 wrapped
9758 .runtime_adapter()
9759 .expect("wrapper should expose runtime adapter")
9760 .session_control_authority(),
9761 )
9762 .await
9763 .expect_err("non-liveness cancellation failures must remain fatal");
9764 assert!(
9765 matches!(error, SessionError::Unsupported(ref detail) if detail == "synthetic cancel rejection"),
9766 "unexpected fail-closed cancellation error: {error}"
9767 );
9768 }
9769
9770 #[tokio::test]
9776 async fn cold_activation_mints_single_runtime_authority_from_durable_session() {
9777 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
9778 let session_store: Arc<dyn SessionStore> = Arc::new(
9779 meerkat_store::SqliteSessionStore::open(dir.path().join("sessions.db"))
9780 .unwrap_or_else(|error| panic!("{error}")),
9781 );
9782 let mut session = meerkat_core::Session::new();
9783 session.push(meerkat_core::Message::User(
9784 meerkat_core::types::UserMessage::text("durable turn"),
9785 ));
9786 session_store
9787 .save(&session)
9788 .await
9789 .unwrap_or_else(|error| panic!("{error}"));
9790 let inner: Arc<dyn meerkat_runtime::RuntimeStore> =
9791 Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
9792 let store = Arc::new(SessionStoreBackedRuntimeStore::new(inner, session_store));
9793 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(session.id());
9794
9795 let authority_read = {
9796 let store = Arc::clone(&store);
9797 let runtime_id = runtime_id.clone();
9798 async move {
9799 meerkat_runtime::RuntimeStore::load_whole_blob_store_authority(&*store, &runtime_id)
9800 .await
9801 }
9802 };
9803 let snapshot_read = {
9804 let store = Arc::clone(&store);
9805 let runtime_id = runtime_id.clone();
9806 async move {
9807 meerkat_runtime::RuntimeStore::load_committed_whole_blob_snapshot(
9808 &*store,
9809 &runtime_id,
9810 )
9811 .await
9812 }
9813 };
9814 let (authority, snapshot) = tokio::join!(authority_read, snapshot_read);
9815 let authority = authority
9816 .unwrap_or_else(|error| panic!("{error}"))
9817 .unwrap_or_else(|| panic!("racer A must observe the minted authority"));
9818 let snapshot = snapshot
9819 .unwrap_or_else(|error| panic!("{error}"))
9820 .unwrap_or_else(|| panic!("racer B must observe the minted snapshot"));
9821 assert_eq!(
9822 &authority,
9823 snapshot.authority(),
9824 "both racers must converge on ONE store-issued authority"
9825 );
9826 }
9827
9828 #[tokio::test]
9836 async fn failed_durable_projection_is_never_reported_durable_and_retry_converges() {
9837 struct FailingProjectionStore {
9838 inner: meerkat_store::MemoryStore,
9839 fail: std::sync::atomic::AtomicBool,
9840 }
9841
9842 impl FailingProjectionStore {
9843 fn outage(&self) -> Option<meerkat_store::SessionStoreError> {
9844 self.fail
9845 .load(std::sync::atomic::Ordering::SeqCst)
9846 .then(|| {
9847 meerkat_store::SessionStoreError::Internal(
9848 "injected projection outage".to_string(),
9849 )
9850 })
9851 }
9852 }
9853
9854 #[async_trait]
9855 impl SessionStore for FailingProjectionStore {
9856 async fn save(
9857 &self,
9858 session: &meerkat_core::Session,
9859 ) -> Result<(), meerkat_store::SessionStoreError> {
9860 if let Some(outage) = self.outage() {
9861 return Err(outage);
9862 }
9863 self.inner.save(session).await
9864 }
9865
9866 async fn save_authoritative_projection(
9867 &self,
9868 session: &meerkat_core::Session,
9869 ) -> Result<(), meerkat_store::SessionStoreError> {
9870 if let Some(outage) = self.outage() {
9871 return Err(outage);
9872 }
9873 self.inner.save_authoritative_projection(session).await
9874 }
9875
9876 async fn load(
9877 &self,
9878 id: &meerkat_core::types::SessionId,
9879 ) -> Result<Option<meerkat_core::Session>, meerkat_store::SessionStoreError>
9880 {
9881 self.inner.load(id).await
9882 }
9883
9884 async fn list(
9885 &self,
9886 filter: meerkat_store::SessionFilter,
9887 ) -> Result<Vec<meerkat_core::SessionMeta>, meerkat_store::SessionStoreError>
9888 {
9889 self.inner.list(filter).await
9890 }
9891
9892 async fn delete(
9893 &self,
9894 id: &meerkat_core::types::SessionId,
9895 ) -> Result<(), meerkat_store::SessionStoreError> {
9896 self.inner.delete(id).await
9897 }
9898
9899 async fn delete_if_current_revision(
9900 &self,
9901 id: &meerkat_core::types::SessionId,
9902 expected_current_revision: &str,
9903 ) -> Result<bool, meerkat_store::SessionStoreError> {
9904 self.inner
9905 .delete_if_current_revision(id, expected_current_revision)
9906 .await
9907 }
9908 }
9909
9910 let failing = Arc::new(FailingProjectionStore {
9911 inner: meerkat_store::MemoryStore::new(),
9912 fail: std::sync::atomic::AtomicBool::new(false),
9913 });
9914 let inner: Arc<dyn meerkat_runtime::RuntimeStore> =
9915 Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
9916 let store = SessionStoreBackedRuntimeStore::new(
9917 inner,
9918 Arc::clone(&failing) as Arc<dyn SessionStore>,
9919 );
9920 let mut session = meerkat_core::Session::new();
9921 session.push(meerkat_core::Message::User(
9922 meerkat_core::types::UserMessage::text("first turn"),
9923 ));
9924 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(session.id());
9925
9926 let bytes = session
9928 .to_persisted_bytes()
9929 .unwrap_or_else(|error| panic!("{error}"));
9930 meerkat_runtime::RuntimeStore::commit_session_snapshot(
9931 &store,
9932 &runtime_id,
9933 meerkat_runtime::store::SerializedSessionSnapshot {
9934 session_snapshot: Arc::new(bytes),
9935 },
9936 )
9937 .await
9938 .unwrap_or_else(|error| panic!("{error}"));
9939 assert!(
9940 failing
9941 .inner
9942 .load(session.id())
9943 .await
9944 .unwrap_or_else(|error| panic!("{error}"))
9945 .is_some(),
9946 "the baseline boundary must project durably"
9947 );
9948
9949 let expected =
9951 meerkat_runtime::RuntimeStore::load_whole_blob_store_authority(&store, &runtime_id)
9952 .await
9953 .unwrap_or_else(|error| panic!("{error}"))
9954 .unwrap_or_else(|| panic!("baseline boundary must issue authority"));
9955 let mut successor = session.clone();
9956 successor.push(meerkat_core::Message::User(
9957 meerkat_core::types::UserMessage::text("second turn"),
9958 ));
9959 let prepared = meerkat_runtime::store::PreparedWholeBlobSnapshotCas::prepare(
9960 expected,
9961 meerkat_core::lifecycle::core_executor::BoundSessionCommit::sealed(Arc::new(
9962 successor.clone(),
9963 ))
9964 .unwrap_or_else(|error| panic!("{error}")),
9965 )
9966 .unwrap_or_else(|error| panic!("{error}"));
9967 failing
9968 .fail
9969 .store(true, std::sync::atomic::Ordering::SeqCst);
9970 let error = meerkat_runtime::RuntimeStore::commit_prepared_whole_blob_snapshot_cas(
9971 &store,
9972 &runtime_id,
9973 prepared.clone(),
9974 )
9975 .await
9976 .expect_err("a failed durable projection must fail the committing verb");
9977 assert!(
9978 error.to_string().contains("durable session projection"),
9979 "the failure must name the projection write: {error}"
9980 );
9981 let after_failure = failing
9982 .inner
9983 .load(session.id())
9984 .await
9985 .unwrap_or_else(|error| panic!("{error}"))
9986 .unwrap_or_else(|| panic!("the previous boundary must survive the outage"));
9987 assert_eq!(
9988 after_failure.messages().len(),
9989 session.messages().len(),
9990 "a boundary whose projection failed must never be reported durable"
9991 );
9992
9993 failing
10000 .fail
10001 .store(false, std::sync::atomic::Ordering::SeqCst);
10002 let observed =
10003 meerkat_runtime::RuntimeStore::load_whole_blob_store_authority(&store, &runtime_id)
10004 .await
10005 .unwrap_or_else(|error| panic!("{error}"))
10006 .unwrap_or_else(|| panic!("the inner successor must be current"));
10007 assert!(
10008 prepared.accepts_committed_authority(&observed),
10009 "the already-current inner successor must prove the retried candidate committed"
10010 );
10011
10012 let mut third = successor.clone();
10017 third.push(meerkat_core::Message::User(
10018 meerkat_core::types::UserMessage::text("third turn"),
10019 ));
10020 let prepared_third = meerkat_runtime::store::PreparedWholeBlobSnapshotCas::prepare(
10021 observed,
10022 meerkat_core::lifecycle::core_executor::BoundSessionCommit::sealed(Arc::new(
10023 third.clone(),
10024 ))
10025 .unwrap_or_else(|error| panic!("{error}")),
10026 )
10027 .unwrap_or_else(|error| panic!("{error}"));
10028 let outcome = meerkat_runtime::RuntimeStore::commit_prepared_whole_blob_snapshot_cas(
10029 &store,
10030 &runtime_id,
10031 prepared_third,
10032 )
10033 .await
10034 .unwrap_or_else(|error| panic!("the healed boundary must commit, not wedge: {error}"));
10035 assert!(
10036 matches!(
10037 outcome,
10038 meerkat_runtime::store::WholeBlobSnapshotCasOutcome::Committed(_)
10039 ),
10040 "the next boundary against the current successor must commit"
10041 );
10042 let converged = failing
10043 .inner
10044 .load(session.id())
10045 .await
10046 .unwrap_or_else(|error| panic!("{error}"))
10047 .unwrap_or_else(|| panic!("the healed boundary must project durably"));
10048 assert_eq!(
10049 converged.messages().len(),
10050 third.messages().len(),
10051 "the durable projection must converge on the latest committed document"
10052 );
10053 }
10054
10055 #[tokio::test]
10064 async fn rewrite_advanced_boundary_projects_missing_commit_into_durable_row() {
10065 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
10066 let session_store: Arc<dyn SessionStore> = Arc::new(
10067 meerkat_store::SqliteSessionStore::open(dir.path().join("sessions.db"))
10068 .unwrap_or_else(|error| panic!("{error}")),
10069 );
10070 let inner: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
10071 meerkat_runtime::store::SqliteRuntimeStore::new(dir.path().join("runtime.db"))
10072 .unwrap_or_else(|error| panic!("{error}")),
10073 );
10074 let mut gen0 = meerkat_core::Session::new();
10076 gen0.push(meerkat_core::Message::User(
10077 meerkat_core::types::UserMessage::text("original opening"),
10078 ));
10079 session_store
10080 .save(&gen0)
10081 .await
10082 .unwrap_or_else(|error| panic!("{error}"));
10083 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(gen0.id());
10084 inner
10085 .commit_session_snapshot(
10086 &runtime_id,
10087 meerkat_runtime::store::SerializedSessionSnapshot {
10088 session_snapshot: Arc::new(
10089 gen0.to_persisted_bytes()
10090 .unwrap_or_else(|error| panic!("{error}")),
10091 ),
10092 },
10093 )
10094 .await
10095 .unwrap_or_else(|error| panic!("{error}"));
10096
10097 let parent_revision = gen0
10100 .transcript_revision()
10101 .unwrap_or_else(|error| panic!("{error}"));
10102 let mut successor = gen0.clone();
10103 successor
10104 .commit_transcript_rewrite(
10105 meerkat_core::TranscriptRewriteSelection::MessageRange { start: 0, end: 1 },
10106 vec![meerkat_core::Message::User(
10107 meerkat_core::types::UserMessage::text("rewritten opening"),
10108 )],
10109 meerkat_core::TranscriptRewriteReason::new("wedged-turn retire"),
10110 Some("task-56-regression".to_string()),
10111 Some(parent_revision),
10112 )
10113 .unwrap_or_else(|error| panic!("{error}"));
10114 successor.push(meerkat_core::Message::User(
10115 meerkat_core::types::UserMessage::text("post-rewrite turn"),
10116 ));
10117
10118 let store = Arc::new(SessionStoreBackedRuntimeStore::new(
10121 Arc::clone(&inner),
10122 Arc::clone(&session_store),
10123 ));
10124 let authority =
10125 meerkat_runtime::RuntimeStore::load_whole_blob_store_authority(&*store, &runtime_id)
10126 .await
10127 .unwrap_or_else(|error| panic!("{error}"))
10128 .unwrap_or_else(|| panic!("the seeded runtime authority must exist"));
10129 let prepared = meerkat_runtime::store::PreparedWholeBlobSnapshotCas::prepare(
10130 authority,
10131 meerkat_core::lifecycle::core_executor::BoundSessionCommit::sealed(Arc::new(
10132 successor.clone(),
10133 ))
10134 .unwrap_or_else(|error| panic!("{error}")),
10135 )
10136 .unwrap_or_else(|error| panic!("{error}"));
10137 let outcome = meerkat_runtime::RuntimeStore::commit_prepared_whole_blob_snapshot_cas(
10138 &*store,
10139 &runtime_id,
10140 prepared,
10141 )
10142 .await
10143 .unwrap_or_else(|error| {
10144 panic!("a rewrite-advanced boundary must project, not tear: {error}")
10145 });
10146 assert!(
10147 matches!(
10148 outcome,
10149 meerkat_runtime::store::WholeBlobSnapshotCasOutcome::Committed(_)
10150 ),
10151 "the boundary must commit"
10152 );
10153
10154 let durable = session_store
10156 .load(gen0.id())
10157 .await
10158 .unwrap_or_else(|error| panic!("the projected row must load cleanly: {error}"))
10159 .unwrap_or_else(|| panic!("the durable row must exist"));
10160 assert_eq!(
10161 durable
10162 .transcript_rewrite_generation()
10163 .unwrap_or_else(|error| panic!("{error}")),
10164 1,
10165 "the durable row must carry the installed rewrite generation"
10166 );
10167 assert_eq!(
10168 durable.messages().len(),
10169 successor.messages().len(),
10170 "the durable document must match the committed successor"
10171 );
10172 assert_eq!(
10173 durable
10174 .transcript_revision()
10175 .unwrap_or_else(|error| panic!("{error}")),
10176 successor
10177 .transcript_revision()
10178 .unwrap_or_else(|error| panic!("{error}")),
10179 "the durable live revision must match the committed successor"
10180 );
10181 let graph = durable
10182 .validated_transcript_history_state()
10183 .unwrap_or_else(|error| panic!("{error}"))
10184 .unwrap_or_else(|| panic!("the durable row must carry the proved graph"));
10185 assert_eq!(
10186 graph.commit_count(),
10187 1,
10188 "the durable graph must retain the installed rewrite commit"
10189 );
10190 }
10191
10192 #[tokio::test]
10199 async fn parent_1_torn_durable_row_heals_on_plain_freshness_pass() {
10200 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
10201 let session_store: Arc<dyn SessionStore> = Arc::new(
10202 meerkat_store::SqliteSessionStore::open(dir.path().join("sessions.db"))
10203 .unwrap_or_else(|error| panic!("{error}")),
10204 );
10205 let inner: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
10206 meerkat_runtime::store::SqliteRuntimeStore::new(dir.path().join("runtime.db"))
10207 .unwrap_or_else(|error| panic!("{error}")),
10208 );
10209 let mut gen0 = meerkat_core::Session::new();
10210 gen0.push(meerkat_core::Message::User(
10211 meerkat_core::types::UserMessage::text("original opening"),
10212 ));
10213 session_store
10214 .save(&gen0)
10215 .await
10216 .unwrap_or_else(|error| panic!("{error}"));
10217 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(gen0.id());
10218 let parent_revision = gen0
10221 .transcript_revision()
10222 .unwrap_or_else(|error| panic!("{error}"));
10223 let mut successor = gen0.clone();
10224 successor
10225 .commit_transcript_rewrite(
10226 meerkat_core::TranscriptRewriteSelection::MessageRange { start: 0, end: 1 },
10227 vec![meerkat_core::Message::User(
10228 meerkat_core::types::UserMessage::text("rewritten opening"),
10229 )],
10230 meerkat_core::TranscriptRewriteReason::new("wedged-turn retire"),
10231 Some("task-56-regression".to_string()),
10232 Some(parent_revision),
10233 )
10234 .unwrap_or_else(|error| panic!("{error}"));
10235 inner
10236 .commit_session_snapshot(
10237 &runtime_id,
10238 meerkat_runtime::store::SerializedSessionSnapshot {
10239 session_snapshot: Arc::new(
10240 successor
10241 .to_persisted_bytes()
10242 .unwrap_or_else(|error| panic!("{error}")),
10243 ),
10244 },
10245 )
10246 .await
10247 .unwrap_or_else(|error| panic!("{error}"));
10248
10249 let store = Arc::new(SessionStoreBackedRuntimeStore::new(
10252 Arc::clone(&inner),
10253 Arc::clone(&session_store),
10254 ));
10255 let snapshot =
10256 meerkat_runtime::RuntimeStore::load_committed_whole_blob_snapshot(&*store, &runtime_id)
10257 .await
10258 .unwrap_or_else(|error| {
10259 panic!("the freshness pass must reconcile, not fail: {error}")
10260 })
10261 .unwrap_or_else(|| panic!("the committed snapshot must remain readable"));
10262 assert_eq!(
10263 snapshot
10264 .session()
10265 .transcript_rewrite_generation()
10266 .unwrap_or_else(|error| panic!("{error}")),
10267 1,
10268 "the committed authority is the gen1 successor"
10269 );
10270
10271 let healed = session_store
10273 .load(gen0.id())
10274 .await
10275 .unwrap_or_else(|error| panic!("{error}"))
10276 .unwrap_or_else(|| panic!("the durable row must exist"));
10277 assert_eq!(
10278 healed
10279 .transcript_rewrite_generation()
10280 .unwrap_or_else(|error| panic!("{error}")),
10281 1,
10282 "a plain freshness pass must heal the torn durable row"
10283 );
10284 assert_eq!(
10285 healed
10286 .transcript_revision()
10287 .unwrap_or_else(|error| panic!("{error}")),
10288 successor
10289 .transcript_revision()
10290 .unwrap_or_else(|error| panic!("{error}")),
10291 "the healed row must match the committed successor exactly"
10292 );
10293 }
10294
10295 #[tokio::test]
10302 async fn append_before_compact_projects_append_then_replays_rewrite() {
10303 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
10304 let session_store: Arc<dyn SessionStore> = Arc::new(
10305 meerkat_store::SqliteSessionStore::open(dir.path().join("sessions.db"))
10306 .unwrap_or_else(|error| panic!("{error}")),
10307 );
10308 let inner: Arc<dyn meerkat_runtime::RuntimeStore> =
10309 Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
10310 let mut gen1 = meerkat_core::Session::new();
10311 gen1.push(meerkat_core::Message::User(
10312 meerkat_core::types::UserMessage::text("original opening"),
10313 ));
10314 let parent_revision = gen1
10315 .transcript_revision()
10316 .unwrap_or_else(|error| panic!("{error}"));
10317 gen1.commit_transcript_rewrite(
10318 meerkat_core::TranscriptRewriteSelection::MessageRange { start: 0, end: 1 },
10319 vec![meerkat_core::Message::User(
10320 meerkat_core::types::UserMessage::text("gen1 head"),
10321 )],
10322 meerkat_core::TranscriptRewriteReason::new("first compaction"),
10323 Some("task-56-regression".to_string()),
10324 Some(parent_revision),
10325 )
10326 .unwrap_or_else(|error| panic!("{error}"));
10327 session_store
10328 .save_authoritative_projection(&gen1)
10329 .await
10330 .unwrap_or_else(|error| panic!("{error}"));
10331 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(gen1.id());
10332 let mut successor = gen1.clone();
10333 successor.push(meerkat_core::Message::User(
10334 meerkat_core::types::UserMessage::text("appended C"),
10335 ));
10336 let parent_revision = successor
10337 .transcript_revision()
10338 .unwrap_or_else(|error| panic!("{error}"));
10339 successor
10340 .commit_transcript_rewrite(
10341 meerkat_core::TranscriptRewriteSelection::MessageRange { start: 0, end: 2 },
10342 vec![meerkat_core::Message::User(
10343 meerkat_core::types::UserMessage::text("gen2 head"),
10344 )],
10345 meerkat_core::TranscriptRewriteReason::new("append-before-compact"),
10346 Some("task-56-regression".to_string()),
10347 Some(parent_revision),
10348 )
10349 .unwrap_or_else(|error| panic!("{error}"));
10350 inner
10351 .commit_session_snapshot(
10352 &runtime_id,
10353 meerkat_runtime::store::SerializedSessionSnapshot {
10354 session_snapshot: Arc::new(
10355 successor
10356 .to_persisted_bytes()
10357 .unwrap_or_else(|error| panic!("{error}")),
10358 ),
10359 },
10360 )
10361 .await
10362 .unwrap_or_else(|error| panic!("{error}"));
10363
10364 let store = Arc::new(SessionStoreBackedRuntimeStore::new(
10365 Arc::clone(&inner),
10366 Arc::clone(&session_store),
10367 ));
10368 store
10369 .project_committed_session_to_durable(&runtime_id)
10370 .await
10371 .unwrap_or_else(|error| {
10372 panic!("append-then-rewrite reconciliation must converge: {error}")
10373 });
10374 let converged = session_store
10375 .load(gen1.id())
10376 .await
10377 .unwrap_or_else(|error| panic!("{error}"))
10378 .unwrap_or_else(|| panic!("the durable row must exist"));
10379 assert_eq!(
10380 converged
10381 .transcript_rewrite_generation()
10382 .unwrap_or_else(|error| panic!("{error}")),
10383 2,
10384 "the gen2 rewrite must be installed after the append projection"
10385 );
10386 assert_eq!(
10387 converged
10388 .transcript_revision()
10389 .unwrap_or_else(|error| panic!("{error}")),
10390 successor
10391 .transcript_revision()
10392 .unwrap_or_else(|error| panic!("{error}")),
10393 "the durable row must converge on the exact committed digests"
10394 );
10395 }
10396
10397 #[tokio::test]
10404 async fn cold_activation_with_no_durable_row_projects_committed_authority() {
10405 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
10406 let session_store: Arc<dyn SessionStore> = Arc::new(
10407 meerkat_store::SqliteSessionStore::open(dir.path().join("sessions.db"))
10408 .unwrap_or_else(|error| panic!("{error}")),
10409 );
10410 let inner: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
10411 meerkat_runtime::store::SqliteRuntimeStore::new(dir.path().join("runtime.db"))
10412 .unwrap_or_else(|error| panic!("{error}")),
10413 );
10414 let mut successor = meerkat_core::Session::new();
10417 successor.push(meerkat_core::Message::User(
10418 meerkat_core::types::UserMessage::text("original opening"),
10419 ));
10420 let parent_revision = successor
10421 .transcript_revision()
10422 .unwrap_or_else(|error| panic!("{error}"));
10423 successor
10424 .commit_transcript_rewrite(
10425 meerkat_core::TranscriptRewriteSelection::MessageRange { start: 0, end: 1 },
10426 vec![meerkat_core::Message::User(
10427 meerkat_core::types::UserMessage::text("rewritten opening"),
10428 )],
10429 meerkat_core::TranscriptRewriteReason::new("first boundary"),
10430 Some("task-56-regression".to_string()),
10431 Some(parent_revision),
10432 )
10433 .unwrap_or_else(|error| panic!("{error}"));
10434 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(successor.id());
10435 inner
10436 .commit_session_snapshot(
10437 &runtime_id,
10438 meerkat_runtime::store::SerializedSessionSnapshot {
10439 session_snapshot: Arc::new(
10440 successor
10441 .to_persisted_bytes()
10442 .unwrap_or_else(|error| panic!("{error}")),
10443 ),
10444 },
10445 )
10446 .await
10447 .unwrap_or_else(|error| panic!("{error}"));
10448
10449 let store = Arc::new(SessionStoreBackedRuntimeStore::new(
10451 Arc::clone(&inner),
10452 Arc::clone(&session_store),
10453 ));
10454 meerkat_runtime::RuntimeStore::load_committed_whole_blob_snapshot(&*store, &runtime_id)
10455 .await
10456 .unwrap_or_else(|error| panic!("the freshness pass must project, not fail: {error}"))
10457 .unwrap_or_else(|| panic!("the committed snapshot must remain readable"));
10458
10459 let projected = session_store
10460 .load(successor.id())
10461 .await
10462 .unwrap_or_else(|error| panic!("{error}"))
10463 .unwrap_or_else(|| {
10464 panic!("the plain freshness pass must create the missing durable row")
10465 });
10466 assert_eq!(
10467 projected
10468 .transcript_rewrite_generation()
10469 .unwrap_or_else(|error| panic!("{error}")),
10470 1,
10471 "the projected row must carry the committed rewrite generation"
10472 );
10473 assert_eq!(
10474 projected
10475 .transcript_revision()
10476 .unwrap_or_else(|error| panic!("{error}")),
10477 successor
10478 .transcript_revision()
10479 .unwrap_or_else(|error| panic!("{error}")),
10480 "the projected row must match the committed authority exactly"
10481 );
10482 }
10483
10484 #[tokio::test]
10495 async fn parked_unregistered_torn_head_heals_on_plain_freshness_pass() {
10496 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
10497 let continuity: Arc<dyn crate::identity_first::ContinuityStore> = Arc::new(
10498 crate::identity_first::LocalContinuityStore::open(dir.path().join("continuity.db"))
10499 .unwrap_or_else(|error| panic!("{error}")),
10500 );
10501 let adapter = Arc::new(crate::identity_first::ContinuitySessionStoreAdapter::new(
10502 Arc::clone(&continuity),
10503 ));
10504 let inner: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
10505 meerkat_runtime::store::SqliteRuntimeStore::new(dir.path().join("runtime.db"))
10506 .unwrap_or_else(|error| panic!("{error}")),
10507 );
10508
10509 let mut gen0 = meerkat_core::Session::new();
10512 gen0.push(meerkat_core::Message::User(
10513 meerkat_core::types::UserMessage::text("original opening"),
10514 ));
10515 let identity = crate::identity_first::AgentIdentity::parse("domain:parked")
10516 .unwrap_or_else(|error| panic!("{error}"));
10517 crate::identity_first::ContinuityStore::upsert_continuity_record(
10521 continuity.as_ref(),
10522 &crate::identity_first::ContinuityRecord {
10523 identity: identity.clone(),
10524 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
10525 "rt:domain:parked:0",
10526 )
10527 .unwrap_or_else(|error| panic!("{error}")),
10528 session_id: gen0.id().clone(),
10529 generation: crate::identity_first::ContinuityGeneration::new(0),
10530 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
10531 },
10532 crate::identity_first::FencingToken::new(1),
10533 )
10534 .await
10535 .unwrap_or_else(|error| panic!("{error}"));
10536 adapter
10537 .register_session(
10538 gen0.id(),
10539 crate::identity_first::SessionRuntimeState {
10540 identity,
10541 generation: crate::identity_first::ContinuityGeneration::new(0),
10542 fencing_token: crate::identity_first::FencingToken::new(1),
10543 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
10544 },
10545 )
10546 .await
10547 .unwrap_or_else(|error| panic!("{error}"));
10548 meerkat::SessionStore::save(adapter.as_ref(), &gen0)
10549 .await
10550 .unwrap_or_else(|error| panic!("{error}"));
10551
10552 adapter
10554 .unregister_session(gen0.id())
10555 .await
10556 .unwrap_or_else(|error| panic!("{error}"));
10557
10558 let parent_revision = gen0
10561 .transcript_revision()
10562 .unwrap_or_else(|error| panic!("{error}"));
10563 let mut successor = gen0.clone();
10564 successor
10565 .commit_transcript_rewrite(
10566 meerkat_core::TranscriptRewriteSelection::MessageRange { start: 0, end: 1 },
10567 vec![meerkat_core::Message::User(
10568 meerkat_core::types::UserMessage::text("rewritten opening"),
10569 )],
10570 meerkat_core::TranscriptRewriteReason::new("wedged-turn retire"),
10571 Some("task-56-corpus-regression".to_string()),
10572 Some(parent_revision),
10573 )
10574 .unwrap_or_else(|error| panic!("{error}"));
10575 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(gen0.id());
10576 inner
10577 .commit_session_snapshot(
10578 &runtime_id,
10579 meerkat_runtime::store::SerializedSessionSnapshot {
10580 session_snapshot: Arc::new(
10581 successor
10582 .to_persisted_bytes()
10583 .unwrap_or_else(|error| panic!("{error}")),
10584 ),
10585 },
10586 )
10587 .await
10588 .unwrap_or_else(|error| panic!("{error}"));
10589
10590 let store = Arc::new(SessionStoreBackedRuntimeStore::new(
10594 Arc::clone(&inner),
10595 Arc::clone(&adapter) as Arc<dyn SessionStore>,
10596 ));
10597 meerkat_runtime::RuntimeStore::load_committed_whole_blob_snapshot(&*store, &runtime_id)
10598 .await
10599 .unwrap_or_else(|error| {
10600 panic!("the parked repair must converge, not deadlock on registration: {error}")
10601 })
10602 .unwrap_or_else(|| panic!("the committed snapshot must remain readable"));
10603
10604 let channel = continuity
10610 .as_incremental_sessions()
10611 .unwrap_or_else(|| panic!("the local continuity store provides the delta channel"));
10612 let healed_head = channel
10613 .load_canonical_head(gen0.id())
10614 .await
10615 .unwrap_or_else(|error| panic!("{error}"))
10616 .unwrap_or_else(|| panic!("the healed head row must exist"));
10617 assert_eq!(
10618 healed_head.rewrite_count, 1,
10619 "the parked member's torn durable head must carry the committed rewrite"
10620 );
10621 let successor_revision = successor
10622 .transcript_revision()
10623 .unwrap_or_else(|error| panic!("{error}"));
10624 assert_eq!(
10625 healed_head.head_revision, successor_revision,
10626 "the healed head row must sit at the committed successor's revision"
10627 );
10628 let healed = meerkat::SessionStore::load(adapter.as_ref(), gen0.id())
10629 .await
10630 .unwrap_or_else(|error| panic!("{error}"))
10631 .unwrap_or_else(|| panic!("the durable row must exist"));
10632 assert_eq!(
10633 healed
10634 .transcript_revision()
10635 .unwrap_or_else(|error| panic!("{error}")),
10636 successor_revision,
10637 "the healed row must match the committed successor exactly"
10638 );
10639
10640 let (record, fencing_token, fence_current) =
10647 crate::identity_first::ContinuityStore::resolve_record_by_session(
10648 continuity.as_ref(),
10649 gen0.id(),
10650 )
10651 .await
10652 .unwrap_or_else(|error| panic!("{error}"))
10653 .unwrap_or_else(|| panic!("the continuity record must still bind the session"));
10654 adapter
10655 .register_session(
10656 gen0.id(),
10657 crate::identity_first::SessionRuntimeState {
10658 identity: record.identity.clone(),
10659 generation: record.generation,
10660 fencing_token,
10661 checkpoint_version: fence_current,
10662 },
10663 )
10664 .await
10665 .unwrap_or_else(|error| panic!("{error}"));
10666 let second_boot = Arc::new(SessionStoreBackedRuntimeStore::new(
10667 Arc::clone(&inner),
10668 Arc::clone(&adapter) as Arc<dyn SessionStore>,
10669 ));
10670 meerkat_runtime::RuntimeStore::load_committed_whole_blob_snapshot(
10671 &*second_boot,
10672 &runtime_id,
10673 )
10674 .await
10675 .unwrap_or_else(|error| {
10676 panic!("the healed row must stay resumable on the next boot: {error}")
10677 })
10678 .unwrap_or_else(|| panic!("the committed snapshot must remain readable on the next boot"));
10679 let head_after_second_boot = channel
10680 .load_canonical_head(gen0.id())
10681 .await
10682 .unwrap_or_else(|error| panic!("{error}"))
10683 .unwrap_or_else(|| panic!("the head row must survive the second boot"));
10684 assert_eq!(
10685 head_after_second_boot.rewrite_count, 1,
10686 "the second boot must not re-replay or regress the healed rewrite"
10687 );
10688 assert_eq!(
10689 head_after_second_boot.head_revision, successor_revision,
10690 "the second boot must leave the healed head at the committed revision"
10691 );
10692 }
10693
10694 #[tokio::test]
10706 async fn parked_inverse_append_durable_rebases_suffix_over_compaction() {
10707 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
10708 let continuity: Arc<dyn crate::identity_first::ContinuityStore> = Arc::new(
10709 crate::identity_first::LocalContinuityStore::open(dir.path().join("continuity.db"))
10710 .unwrap_or_else(|error| panic!("{error}")),
10711 );
10712 let adapter = Arc::new(crate::identity_first::ContinuitySessionStoreAdapter::new(
10713 Arc::clone(&continuity),
10714 ));
10715 let inner: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
10716 meerkat_runtime::store::SqliteRuntimeStore::new(dir.path().join("runtime.db"))
10717 .unwrap_or_else(|error| panic!("{error}")),
10718 );
10719
10720 let mut gen0 = meerkat_core::Session::new();
10721 for text in ["opening", "second", "third"] {
10722 gen0.push(meerkat_core::Message::User(
10723 meerkat_core::types::UserMessage::text(text),
10724 ));
10725 }
10726 let identity = crate::identity_first::AgentIdentity::parse("domain:parked-rebase")
10727 .unwrap_or_else(|error| panic!("{error}"));
10728 crate::identity_first::ContinuityStore::upsert_continuity_record(
10729 continuity.as_ref(),
10730 &crate::identity_first::ContinuityRecord {
10731 identity: identity.clone(),
10732 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
10733 "rt:domain:parked-rebase:0",
10734 )
10735 .unwrap_or_else(|error| panic!("{error}")),
10736 session_id: gen0.id().clone(),
10737 generation: crate::identity_first::ContinuityGeneration::new(0),
10738 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
10739 },
10740 crate::identity_first::FencingToken::new(1),
10741 )
10742 .await
10743 .unwrap_or_else(|error| panic!("{error}"));
10744 adapter
10745 .register_session(
10746 gen0.id(),
10747 crate::identity_first::SessionRuntimeState {
10748 identity,
10749 generation: crate::identity_first::ContinuityGeneration::new(0),
10750 fencing_token: crate::identity_first::FencingToken::new(1),
10751 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
10752 },
10753 )
10754 .await
10755 .unwrap_or_else(|error| panic!("{error}"));
10756 meerkat::SessionStore::save(adapter.as_ref(), &gen0)
10757 .await
10758 .unwrap_or_else(|error| panic!("{error}"));
10759
10760 let mut extended = gen0.clone();
10763 extended.push(meerkat_core::Message::User(
10764 meerkat_core::types::UserMessage::text("unacknowledged tail"),
10765 ));
10766 meerkat::SessionStore::save(adapter.as_ref(), &extended)
10767 .await
10768 .unwrap_or_else(|error| panic!("{error}"));
10769
10770 let parent_revision = gen0
10773 .transcript_revision()
10774 .unwrap_or_else(|error| panic!("{error}"));
10775 let mut successor = gen0.clone();
10776 successor
10777 .commit_transcript_rewrite(
10778 meerkat_core::TranscriptRewriteSelection::MessageRange { start: 0, end: 3 },
10779 vec![meerkat_core::Message::User(
10780 meerkat_core::types::UserMessage::text("compacted summary"),
10781 )],
10782 meerkat_core::TranscriptRewriteReason::new("wedged-turn retire compaction"),
10783 Some("task-56-inverse-append-regression".to_string()),
10784 Some(parent_revision),
10785 )
10786 .unwrap_or_else(|error| panic!("{error}"));
10787
10788 adapter
10790 .unregister_session(gen0.id())
10791 .await
10792 .unwrap_or_else(|error| panic!("{error}"));
10793 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(gen0.id());
10794 inner
10795 .commit_session_snapshot(
10796 &runtime_id,
10797 meerkat_runtime::store::SerializedSessionSnapshot {
10798 session_snapshot: Arc::new(
10799 successor
10800 .to_persisted_bytes()
10801 .unwrap_or_else(|error| panic!("{error}")),
10802 ),
10803 },
10804 )
10805 .await
10806 .unwrap_or_else(|error| panic!("{error}"));
10807
10808 let store = Arc::new(SessionStoreBackedRuntimeStore::new(
10809 Arc::clone(&inner),
10810 Arc::clone(&adapter) as Arc<dyn SessionStore>,
10811 ));
10812 meerkat_runtime::RuntimeStore::load_committed_whole_blob_snapshot(&*store, &runtime_id)
10813 .await
10814 .unwrap_or_else(|error| panic!("the inverse-append repair must converge: {error}"))
10815 .unwrap_or_else(|| panic!("the committed snapshot must remain readable"));
10816
10817 let mut expected = successor.clone();
10820 expected.push(meerkat_core::Message::User(
10821 meerkat_core::types::UserMessage::text("unacknowledged tail"),
10822 ));
10823 let expected_revision = expected
10824 .transcript_revision()
10825 .unwrap_or_else(|error| panic!("{error}"));
10826
10827 let channel = continuity
10828 .as_incremental_sessions()
10829 .unwrap_or_else(|| panic!("the local continuity store provides the delta channel"));
10830 let healed_head = channel
10831 .load_canonical_head(gen0.id())
10832 .await
10833 .unwrap_or_else(|error| panic!("{error}"))
10834 .unwrap_or_else(|| panic!("the healed head row must exist"));
10835 assert_eq!(
10836 healed_head.rewrite_count, 1,
10837 "the compaction rewrite must be installed on the durable head"
10838 );
10839 assert_eq!(
10840 healed_head.head_revision, expected_revision,
10841 "the healed head must carry the compacted head PLUS the preserved suffix"
10842 );
10843 let healed = meerkat::SessionStore::load(adapter.as_ref(), gen0.id())
10844 .await
10845 .unwrap_or_else(|error| panic!("{error}"))
10846 .unwrap_or_else(|| panic!("the durable row must exist"));
10847 assert_eq!(
10848 healed.messages().len(),
10849 2,
10850 "compacted summary plus the preserved unacknowledged tail"
10851 );
10852 let committed_after = inner
10853 .load_committed_whole_blob_snapshot(&runtime_id)
10854 .await
10855 .unwrap_or_else(|error| panic!("{error}"))
10856 .unwrap_or_else(|| panic!("the committed snapshot must survive the repair"));
10857 assert_eq!(
10858 committed_after
10859 .session()
10860 .transcript_revision()
10861 .unwrap_or_else(|error| panic!("{error}")),
10862 expected_revision,
10863 "the committed runtime snapshot must converge to the rebased state"
10864 );
10865
10866 let (record, fencing_token, fence_current) =
10869 crate::identity_first::ContinuityStore::resolve_record_by_session(
10870 continuity.as_ref(),
10871 gen0.id(),
10872 )
10873 .await
10874 .unwrap_or_else(|error| panic!("{error}"))
10875 .unwrap_or_else(|| panic!("the continuity record must still bind the session"));
10876 adapter
10877 .register_session(
10878 gen0.id(),
10879 crate::identity_first::SessionRuntimeState {
10880 identity: record.identity.clone(),
10881 generation: record.generation,
10882 fencing_token,
10883 checkpoint_version: fence_current,
10884 },
10885 )
10886 .await
10887 .unwrap_or_else(|error| panic!("{error}"));
10888 let second_boot = Arc::new(SessionStoreBackedRuntimeStore::new(
10889 Arc::clone(&inner),
10890 Arc::clone(&adapter) as Arc<dyn SessionStore>,
10891 ));
10892 meerkat_runtime::RuntimeStore::load_committed_whole_blob_snapshot(
10893 &*second_boot,
10894 &runtime_id,
10895 )
10896 .await
10897 .unwrap_or_else(|error| panic!("the rebased row must stay resumable: {error}"))
10898 .unwrap_or_else(|| panic!("the committed snapshot must remain readable on boot two"));
10899 let head_after = channel
10900 .load_canonical_head(gen0.id())
10901 .await
10902 .unwrap_or_else(|error| panic!("{error}"))
10903 .unwrap_or_else(|| panic!("the head row must survive the second boot"));
10904 assert_eq!(
10905 head_after.rewrite_count, 1,
10906 "no re-replay on the second boot"
10907 );
10908 assert_eq!(
10909 head_after.head_revision, expected_revision,
10910 "the preserved suffix must survive the second boot untouched"
10911 );
10912 }
10913
10914 #[tokio::test]
10924 async fn parked_durable_prefix_of_parent_heals_to_committed_head() {
10925 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
10926 let continuity: Arc<dyn crate::identity_first::ContinuityStore> = Arc::new(
10927 crate::identity_first::LocalContinuityStore::open(dir.path().join("continuity.db"))
10928 .unwrap_or_else(|error| panic!("{error}")),
10929 );
10930 let adapter = Arc::new(crate::identity_first::ContinuitySessionStoreAdapter::new(
10931 Arc::clone(&continuity),
10932 ));
10933 let inner: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
10934 meerkat_runtime::store::SqliteRuntimeStore::new(dir.path().join("runtime.db"))
10935 .unwrap_or_else(|error| panic!("{error}")),
10936 );
10937
10938 let mut gen0 = meerkat_core::Session::new();
10939 for text in ["opening", "second", "third"] {
10940 gen0.push(meerkat_core::Message::User(
10941 meerkat_core::types::UserMessage::text(text),
10942 ));
10943 }
10944 let identity = crate::identity_first::AgentIdentity::parse("domain:parked-prefix")
10945 .unwrap_or_else(|error| panic!("{error}"));
10946 crate::identity_first::ContinuityStore::upsert_continuity_record(
10947 continuity.as_ref(),
10948 &crate::identity_first::ContinuityRecord {
10949 identity: identity.clone(),
10950 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
10951 "rt:domain:parked-prefix:0",
10952 )
10953 .unwrap_or_else(|error| panic!("{error}")),
10954 session_id: gen0.id().clone(),
10955 generation: crate::identity_first::ContinuityGeneration::new(0),
10956 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
10957 },
10958 crate::identity_first::FencingToken::new(1),
10959 )
10960 .await
10961 .unwrap_or_else(|error| panic!("{error}"));
10962 adapter
10963 .register_session(
10964 gen0.id(),
10965 crate::identity_first::SessionRuntimeState {
10966 identity,
10967 generation: crate::identity_first::ContinuityGeneration::new(0),
10968 fencing_token: crate::identity_first::FencingToken::new(1),
10969 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
10970 },
10971 )
10972 .await
10973 .unwrap_or_else(|error| panic!("{error}"));
10974 meerkat::SessionStore::save(adapter.as_ref(), &gen0)
10977 .await
10978 .unwrap_or_else(|error| panic!("{error}"));
10979
10980 let mut successor = gen0.clone();
10983 successor.push(meerkat_core::Message::User(
10984 meerkat_core::types::UserMessage::text("never projected"),
10985 ));
10986 let parent_revision = successor
10987 .transcript_revision()
10988 .unwrap_or_else(|error| panic!("{error}"));
10989 successor
10990 .commit_transcript_rewrite(
10991 meerkat_core::TranscriptRewriteSelection::MessageRange { start: 0, end: 4 },
10992 vec![meerkat_core::Message::User(
10993 meerkat_core::types::UserMessage::text("compacted summary"),
10994 )],
10995 meerkat_core::TranscriptRewriteReason::new("wedged-turn retire compaction"),
10996 Some("task-56-durable-prefix-regression".to_string()),
10997 Some(parent_revision),
10998 )
10999 .unwrap_or_else(|error| panic!("{error}"));
11000
11001 adapter
11002 .unregister_session(gen0.id())
11003 .await
11004 .unwrap_or_else(|error| panic!("{error}"));
11005 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(gen0.id());
11006 inner
11007 .commit_session_snapshot(
11008 &runtime_id,
11009 meerkat_runtime::store::SerializedSessionSnapshot {
11010 session_snapshot: Arc::new(
11011 successor
11012 .to_persisted_bytes()
11013 .unwrap_or_else(|error| panic!("{error}")),
11014 ),
11015 },
11016 )
11017 .await
11018 .unwrap_or_else(|error| panic!("{error}"));
11019
11020 let store = Arc::new(SessionStoreBackedRuntimeStore::new(
11021 Arc::clone(&inner),
11022 Arc::clone(&adapter) as Arc<dyn SessionStore>,
11023 ));
11024 meerkat_runtime::RuntimeStore::load_committed_whole_blob_snapshot(&*store, &runtime_id)
11025 .await
11026 .unwrap_or_else(|error| panic!("the durable-prefix repair must converge: {error}"))
11027 .unwrap_or_else(|| panic!("the committed snapshot must remain readable"));
11028
11029 let successor_revision = successor
11030 .transcript_revision()
11031 .unwrap_or_else(|error| panic!("{error}"));
11032 let channel = continuity
11033 .as_incremental_sessions()
11034 .unwrap_or_else(|| panic!("the local continuity store provides the delta channel"));
11035 let healed_head = channel
11036 .load_canonical_head(gen0.id())
11037 .await
11038 .unwrap_or_else(|error| panic!("{error}"))
11039 .unwrap_or_else(|| panic!("the healed head row must exist"));
11040 assert_eq!(
11041 healed_head.rewrite_count, 1,
11042 "the compaction rewrite must be installed on the durable head"
11043 );
11044 assert_eq!(
11045 healed_head.head_revision, successor_revision,
11046 "the healed head must land at the committed head exactly (no suffix exists)"
11047 );
11048 let healed = meerkat::SessionStore::load(adapter.as_ref(), gen0.id())
11049 .await
11050 .unwrap_or_else(|error| panic!("{error}"))
11051 .unwrap_or_else(|| panic!("the durable row must exist"));
11052 assert_eq!(
11053 healed
11054 .transcript_revision()
11055 .unwrap_or_else(|error| panic!("{error}")),
11056 successor_revision,
11057 "the healed row must match the committed successor exactly"
11058 );
11059 }
11060
11061 #[tokio::test]
11066 async fn durable_behind_prefix_chain_proves_exact_prefix_and_refuses_divergence() {
11067 let mut base = meerkat_core::Session::new();
11068 for text in ["opening", "second"] {
11069 base.push(meerkat_core::Message::User(
11070 meerkat_core::types::UserMessage::text(text),
11071 ));
11072 }
11073 let durable = base.clone();
11074 let mut divergent = base.clone();
11075 divergent.push(meerkat_core::Message::User(
11076 meerkat_core::types::UserMessage::text("divergent third"),
11077 ));
11078 let mut successor = base;
11079 successor.push(meerkat_core::Message::User(
11080 meerkat_core::types::UserMessage::text("third"),
11081 ));
11082 let parent_revision = successor
11083 .transcript_revision()
11084 .unwrap_or_else(|error| panic!("{error}"));
11085 successor
11086 .commit_transcript_rewrite(
11087 meerkat_core::TranscriptRewriteSelection::MessageRange { start: 0, end: 3 },
11088 vec![meerkat_core::Message::User(
11089 meerkat_core::types::UserMessage::text("compacted summary"),
11090 )],
11091 meerkat_core::TranscriptRewriteReason::new("wedged-turn retire compaction"),
11092 Some("task-56-behind-proof-unit".to_string()),
11093 Some(parent_revision),
11094 )
11095 .unwrap_or_else(|error| panic!("{error}"));
11096 let sealed = successor
11097 .validated_transcript_history_state()
11098 .unwrap_or_else(|error| panic!("{error}"))
11099 .unwrap_or_else(|| panic!("the successor carries a sealed graph"));
11100
11101 let proven = SessionStoreBackedRuntimeStore::durable_behind_prefix_chain(
11102 &successor, &sealed, &durable,
11103 )
11104 .unwrap_or_else(|error| panic!("{error}"))
11105 .unwrap_or_else(|| panic!("a strict prefix of the parent must prove the chain"));
11106 assert_eq!(proven.len(), 1);
11107 assert_eq!(proven[0].rewrite_generation, 1);
11108
11109 let refused = SessionStoreBackedRuntimeStore::durable_behind_prefix_chain(
11110 &successor, &sealed, &divergent,
11111 )
11112 .unwrap_or_else(|error| panic!("{error}"));
11113 assert!(
11114 refused.is_none(),
11115 "a divergent row must not prove the durable-behind chain"
11116 );
11117 }
11118
11119 #[tokio::test]
11124 async fn parked_repair_refuses_foreign_lineage_durable_row_typed() {
11125 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
11126 let continuity: Arc<dyn crate::identity_first::ContinuityStore> = Arc::new(
11127 crate::identity_first::LocalContinuityStore::open(dir.path().join("continuity.db"))
11128 .unwrap_or_else(|error| panic!("{error}")),
11129 );
11130 let adapter = Arc::new(crate::identity_first::ContinuitySessionStoreAdapter::new(
11131 Arc::clone(&continuity),
11132 ));
11133 let inner: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
11134 meerkat_runtime::store::SqliteRuntimeStore::new(dir.path().join("runtime.db"))
11135 .unwrap_or_else(|error| panic!("{error}")),
11136 );
11137
11138 let mut base = meerkat_core::Session::new();
11139 for text in ["opening", "second"] {
11140 base.push(meerkat_core::Message::User(
11141 meerkat_core::types::UserMessage::text(text),
11142 ));
11143 }
11144 let mut committed_parent = base.clone();
11147 committed_parent.push(meerkat_core::Message::User(
11148 meerkat_core::types::UserMessage::text("third"),
11149 ));
11150 let mut divergent = base;
11151 divergent.push(meerkat_core::Message::User(
11152 meerkat_core::types::UserMessage::text("divergent third"),
11153 ));
11154
11155 let identity = crate::identity_first::AgentIdentity::parse("domain:parked-fork")
11156 .unwrap_or_else(|error| panic!("{error}"));
11157 crate::identity_first::ContinuityStore::upsert_continuity_record(
11158 continuity.as_ref(),
11159 &crate::identity_first::ContinuityRecord {
11160 identity: identity.clone(),
11161 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
11162 "rt:domain:parked-fork:0",
11163 )
11164 .unwrap_or_else(|error| panic!("{error}")),
11165 session_id: divergent.id().clone(),
11166 generation: crate::identity_first::ContinuityGeneration::new(0),
11167 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
11168 },
11169 crate::identity_first::FencingToken::new(1),
11170 )
11171 .await
11172 .unwrap_or_else(|error| panic!("{error}"));
11173 adapter
11174 .register_session(
11175 divergent.id(),
11176 crate::identity_first::SessionRuntimeState {
11177 identity,
11178 generation: crate::identity_first::ContinuityGeneration::new(0),
11179 fencing_token: crate::identity_first::FencingToken::new(1),
11180 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
11181 },
11182 )
11183 .await
11184 .unwrap_or_else(|error| panic!("{error}"));
11185 meerkat::SessionStore::save(adapter.as_ref(), &divergent)
11186 .await
11187 .unwrap_or_else(|error| panic!("{error}"));
11188 let divergent_revision = divergent
11189 .transcript_revision()
11190 .unwrap_or_else(|error| panic!("{error}"));
11191
11192 let parent_revision = committed_parent
11193 .transcript_revision()
11194 .unwrap_or_else(|error| panic!("{error}"));
11195 let mut successor = committed_parent;
11196 successor
11197 .commit_transcript_rewrite(
11198 meerkat_core::TranscriptRewriteSelection::MessageRange { start: 0, end: 3 },
11199 vec![meerkat_core::Message::User(
11200 meerkat_core::types::UserMessage::text("compacted summary"),
11201 )],
11202 meerkat_core::TranscriptRewriteReason::new("wedged-turn retire compaction"),
11203 Some("task-56-fork-refusal-regression".to_string()),
11204 Some(parent_revision),
11205 )
11206 .unwrap_or_else(|error| panic!("{error}"));
11207
11208 adapter
11209 .unregister_session(divergent.id())
11210 .await
11211 .unwrap_or_else(|error| panic!("{error}"));
11212 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(divergent.id());
11213 inner
11214 .commit_session_snapshot(
11215 &runtime_id,
11216 meerkat_runtime::store::SerializedSessionSnapshot {
11217 session_snapshot: Arc::new(
11218 successor
11219 .to_persisted_bytes()
11220 .unwrap_or_else(|error| panic!("{error}")),
11221 ),
11222 },
11223 )
11224 .await
11225 .unwrap_or_else(|error| panic!("{error}"));
11226
11227 let store = Arc::new(SessionStoreBackedRuntimeStore::new(
11228 Arc::clone(&inner),
11229 Arc::clone(&adapter) as Arc<dyn SessionStore>,
11230 ));
11231 let refusal =
11232 meerkat_runtime::RuntimeStore::load_committed_whole_blob_snapshot(&*store, &runtime_id)
11233 .await;
11234 assert!(
11235 refusal.is_err(),
11236 "a foreign-lineage durable row must refuse typed, not heal or overwrite"
11237 );
11238
11239 let untouched = meerkat::SessionStore::load(adapter.as_ref(), divergent.id())
11241 .await
11242 .unwrap_or_else(|error| panic!("{error}"))
11243 .unwrap_or_else(|| panic!("the durable row must survive the refusal"));
11244 assert_eq!(
11245 untouched
11246 .transcript_revision()
11247 .unwrap_or_else(|error| panic!("{error}")),
11248 divergent_revision,
11249 "the refusal must not mutate the foreign-lineage durable row"
11250 );
11251 assert_eq!(untouched.messages().len(), 3);
11252 }
11253
11254 #[tokio::test]
11262 async fn freshen_refuses_equal_order_divergent_durable_row_typed() {
11263 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
11264 let session_store: Arc<dyn SessionStore> = Arc::new(
11265 meerkat_store::SqliteSessionStore::open(dir.path().join("sessions.db"))
11266 .unwrap_or_else(|error| panic!("{error}")),
11267 );
11268 let inner: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
11269 meerkat_runtime::store::SqliteRuntimeStore::new(dir.path().join("runtime.db"))
11270 .unwrap_or_else(|error| panic!("{error}")),
11271 );
11272 let seed = meerkat_core::Session::new();
11276 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(seed.id());
11277 let mut committed = seed.clone();
11278 committed.push(meerkat_core::Message::User(
11279 meerkat_core::types::UserMessage::text("committed turn"),
11280 ));
11281 let mut divergent = seed;
11282 divergent.push(meerkat_core::Message::User(
11283 meerkat_core::types::UserMessage::text("divergent turn"),
11284 ));
11285 inner
11286 .commit_session_snapshot(
11287 &runtime_id,
11288 meerkat_runtime::store::SerializedSessionSnapshot {
11289 session_snapshot: Arc::new(
11290 committed
11291 .to_persisted_bytes()
11292 .unwrap_or_else(|error| panic!("{error}")),
11293 ),
11294 },
11295 )
11296 .await
11297 .unwrap_or_else(|error| panic!("{error}"));
11298 session_store
11299 .save(&divergent)
11300 .await
11301 .unwrap_or_else(|error| panic!("{error}"));
11302
11303 let store = Arc::new(SessionStoreBackedRuntimeStore::new(
11304 Arc::clone(&inner),
11305 Arc::clone(&session_store),
11306 ));
11307 for attempt in ["first read", "retry"] {
11308 let refused = meerkat_runtime::RuntimeStore::load_committed_whole_blob_snapshot(
11309 &*store,
11310 &runtime_id,
11311 )
11312 .await
11313 .expect_err("an equal-order divergent durable row must refuse typed, not adopt");
11314 assert!(
11315 refused.to_string().contains("DIVERGES in content"),
11316 "the {attempt} refusal must name the fork: {refused}"
11317 );
11318 }
11319 let retained = inner
11321 .load_committed_whole_blob_snapshot(&runtime_id)
11322 .await
11323 .unwrap_or_else(|error| panic!("{error}"))
11324 .unwrap_or_else(|| panic!("the committed runtime authority must survive"));
11325 assert_eq!(
11326 retained
11327 .session()
11328 .transcript_revision()
11329 .unwrap_or_else(|error| panic!("{error}")),
11330 committed
11331 .transcript_revision()
11332 .unwrap_or_else(|error| panic!("{error}")),
11333 "the committed runtime document must be untouched"
11334 );
11335 let durable = session_store
11336 .load(committed.id())
11337 .await
11338 .unwrap_or_else(|error| panic!("{error}"))
11339 .unwrap_or_else(|| panic!("the durable row must survive"));
11340 assert_eq!(
11341 durable
11342 .transcript_revision()
11343 .unwrap_or_else(|error| panic!("{error}")),
11344 divergent
11345 .transcript_revision()
11346 .unwrap_or_else(|error| panic!("{error}")),
11347 "the durable document must be untouched"
11348 );
11349 }
11350
11351 #[tokio::test]
11358 async fn equal_revision_envelope_debt_clears_on_plain_freshness_pass() {
11359 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
11360 let session_store: Arc<dyn SessionStore> = Arc::new(
11361 meerkat_store::SqliteSessionStore::open(dir.path().join("sessions.db"))
11362 .unwrap_or_else(|error| panic!("{error}")),
11363 );
11364 let inner: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
11365 meerkat_runtime::store::SqliteRuntimeStore::new(dir.path().join("runtime.db"))
11366 .unwrap_or_else(|error| panic!("{error}")),
11367 );
11368 let mut session = meerkat_core::Session::new();
11369 session.push(meerkat_core::Message::User(
11370 meerkat_core::types::UserMessage::text("original opening"),
11371 ));
11372 session_store
11374 .save(&session)
11375 .await
11376 .unwrap_or_else(|error| panic!("{error}"));
11377 let mut committed = session.clone();
11379 committed.set_metadata(
11380 "mobkit:task56:envelope-probe",
11381 serde_json::Value::String("current".to_string()),
11382 );
11383 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(session.id());
11384 inner
11385 .commit_session_snapshot(
11386 &runtime_id,
11387 meerkat_runtime::store::SerializedSessionSnapshot {
11388 session_snapshot: Arc::new(
11389 committed
11390 .to_persisted_bytes()
11391 .unwrap_or_else(|error| panic!("{error}")),
11392 ),
11393 },
11394 )
11395 .await
11396 .unwrap_or_else(|error| panic!("{error}"));
11397
11398 let store = Arc::new(SessionStoreBackedRuntimeStore::new(
11400 Arc::clone(&inner),
11401 Arc::clone(&session_store),
11402 ));
11403 meerkat_runtime::RuntimeStore::load_committed_whole_blob_snapshot(&*store, &runtime_id)
11404 .await
11405 .unwrap_or_else(|error| {
11406 panic!("the freshness pass must complete the envelope, not fail: {error}")
11407 })
11408 .unwrap_or_else(|| panic!("the committed snapshot must remain readable"));
11409
11410 let healed = session_store
11411 .load(session.id())
11412 .await
11413 .unwrap_or_else(|error| panic!("{error}"))
11414 .unwrap_or_else(|| panic!("the durable row must exist"));
11415 assert_eq!(
11416 healed
11417 .metadata()
11418 .get("mobkit:task56:envelope-probe")
11419 .and_then(|value| value.as_str()),
11420 Some("current"),
11421 "a plain freshness pass must complete the lagging envelope"
11422 );
11423 assert_eq!(
11424 healed
11425 .transcript_revision()
11426 .unwrap_or_else(|error| panic!("{error}")),
11427 committed
11428 .transcript_revision()
11429 .unwrap_or_else(|error| panic!("{error}")),
11430 "content stays identical - only the envelope was owed"
11431 );
11432 }
11433
11434 #[tokio::test]
11440 async fn rewrite_replay_partial_failure_keeps_progress_and_exact_retry_converges() {
11441 struct FailNthRewriteStore {
11442 inner: Arc<dyn SessionStore>,
11443 rewrite_calls: std::sync::atomic::AtomicUsize,
11444 fail_on_call: std::sync::atomic::AtomicUsize,
11445 }
11446
11447 #[async_trait]
11448 impl SessionStore for FailNthRewriteStore {
11449 async fn save(
11450 &self,
11451 session: &meerkat_core::Session,
11452 ) -> Result<(), meerkat_store::SessionStoreError> {
11453 self.inner.save(session).await
11454 }
11455
11456 async fn save_transcript_rewrite(
11457 &self,
11458 session: &meerkat_core::Session,
11459 commit: &meerkat_core::TranscriptRewriteCommit,
11460 ) -> Result<(), meerkat_store::SessionStoreError> {
11461 let call = self
11462 .rewrite_calls
11463 .fetch_add(1, std::sync::atomic::Ordering::SeqCst)
11464 + 1;
11465 if call == self.fail_on_call.load(std::sync::atomic::Ordering::SeqCst) {
11466 return Err(meerkat_store::SessionStoreError::Internal(
11467 "injected mid-chain outage".to_string(),
11468 ));
11469 }
11470 self.inner.save_transcript_rewrite(session, commit).await
11471 }
11472
11473 async fn save_authoritative_projection(
11474 &self,
11475 session: &meerkat_core::Session,
11476 ) -> Result<(), meerkat_store::SessionStoreError> {
11477 self.inner.save_authoritative_projection(session).await
11478 }
11479
11480 async fn load(
11481 &self,
11482 id: &meerkat_core::types::SessionId,
11483 ) -> Result<Option<meerkat_core::Session>, meerkat_store::SessionStoreError>
11484 {
11485 self.inner.load(id).await
11486 }
11487
11488 async fn list(
11489 &self,
11490 filter: meerkat_store::SessionFilter,
11491 ) -> Result<Vec<meerkat_core::SessionMeta>, meerkat_store::SessionStoreError>
11492 {
11493 self.inner.list(filter).await
11494 }
11495
11496 async fn delete(
11497 &self,
11498 id: &meerkat_core::types::SessionId,
11499 ) -> Result<(), meerkat_store::SessionStoreError> {
11500 self.inner.delete(id).await
11501 }
11502
11503 async fn delete_if_current_revision(
11504 &self,
11505 id: &meerkat_core::types::SessionId,
11506 expected_current_revision: &str,
11507 ) -> Result<bool, meerkat_store::SessionStoreError> {
11508 self.inner
11509 .delete_if_current_revision(id, expected_current_revision)
11510 .await
11511 }
11512 }
11513
11514 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
11515 let failing = Arc::new(FailNthRewriteStore {
11516 inner: Arc::new(
11517 meerkat_store::SqliteSessionStore::open(dir.path().join("sessions.db"))
11518 .unwrap_or_else(|error| panic!("{error}")),
11519 ),
11520 rewrite_calls: std::sync::atomic::AtomicUsize::new(0),
11521 fail_on_call: std::sync::atomic::AtomicUsize::new(0),
11522 });
11523 let inner: Arc<dyn meerkat_runtime::RuntimeStore> =
11524 Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
11525
11526 let mut gen0 = meerkat_core::Session::new();
11529 gen0.push(meerkat_core::Message::User(
11530 meerkat_core::types::UserMessage::text("original opening"),
11531 ));
11532 failing
11533 .save(&gen0)
11534 .await
11535 .unwrap_or_else(|error| panic!("{error}"));
11536 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(gen0.id());
11537 inner
11538 .commit_session_snapshot(
11539 &runtime_id,
11540 meerkat_runtime::store::SerializedSessionSnapshot {
11541 session_snapshot: Arc::new(
11542 gen0.to_persisted_bytes()
11543 .unwrap_or_else(|error| panic!("{error}")),
11544 ),
11545 },
11546 )
11547 .await
11548 .unwrap_or_else(|error| panic!("{error}"));
11549 let mut successor = gen0.clone();
11550 for (generation, replacement) in [(1u64, "first rewrite"), (2, "second rewrite")] {
11551 let parent_revision = successor
11552 .transcript_revision()
11553 .unwrap_or_else(|error| panic!("{error}"));
11554 let commit = successor
11555 .commit_transcript_rewrite(
11556 meerkat_core::TranscriptRewriteSelection::MessageRange { start: 0, end: 1 },
11557 vec![meerkat_core::Message::User(
11558 meerkat_core::types::UserMessage::text(replacement),
11559 )],
11560 meerkat_core::TranscriptRewriteReason::new("task-56 chain"),
11561 Some("task-56-regression".to_string()),
11562 Some(parent_revision),
11563 )
11564 .unwrap_or_else(|error| panic!("{error}"));
11565 assert_eq!(commit.rewrite_generation, generation);
11566 }
11567
11568 let store = Arc::new(SessionStoreBackedRuntimeStore::new(
11569 Arc::clone(&inner),
11570 Arc::clone(&failing) as Arc<dyn SessionStore>,
11571 ));
11572 failing
11574 .fail_on_call
11575 .store(2, std::sync::atomic::Ordering::SeqCst);
11576 let authority =
11577 meerkat_runtime::RuntimeStore::load_whole_blob_store_authority(&*store, &runtime_id)
11578 .await
11579 .unwrap_or_else(|error| panic!("{error}"))
11580 .unwrap_or_else(|| panic!("the seeded runtime authority must exist"));
11581 let prepared = meerkat_runtime::store::PreparedWholeBlobSnapshotCas::prepare(
11582 authority,
11583 meerkat_core::lifecycle::core_executor::BoundSessionCommit::sealed(Arc::new(
11584 successor.clone(),
11585 ))
11586 .unwrap_or_else(|error| panic!("{error}")),
11587 )
11588 .unwrap_or_else(|error| panic!("{error}"));
11589 let error = meerkat_runtime::RuntimeStore::commit_prepared_whole_blob_snapshot_cas(
11590 &*store,
11591 &runtime_id,
11592 prepared,
11593 )
11594 .await
11595 .expect_err("a mid-chain projection outage must fail the committing verb");
11596 assert!(
11597 error.to_string().contains("generation 2"),
11598 "the failure must name the refused step: {error}"
11599 );
11600 assert_eq!(
11601 failing
11602 .rewrite_calls
11603 .load(std::sync::atomic::Ordering::SeqCst),
11604 2,
11605 "generation 1 installed, generation 2 attempted and refused"
11606 );
11607 let after_failure = failing
11608 .load(gen0.id())
11609 .await
11610 .unwrap_or_else(|error| panic!("{error}"))
11611 .unwrap_or_else(|| panic!("the durable row must survive the outage"));
11612 assert_eq!(
11613 after_failure
11614 .transcript_rewrite_generation()
11615 .unwrap_or_else(|error| panic!("{error}")),
11616 1,
11617 "the monotonic progress before the outage must stand"
11618 );
11619
11620 failing
11624 .fail_on_call
11625 .store(0, std::sync::atomic::Ordering::SeqCst);
11626 store
11627 .project_committed_session_to_durable(&runtime_id)
11628 .await
11629 .unwrap_or_else(|error| panic!("the exact retry must converge: {error}"));
11630 assert_eq!(
11631 failing
11632 .rewrite_calls
11633 .load(std::sync::atomic::Ordering::SeqCst),
11634 3,
11635 "the retry must install only the remaining commit"
11636 );
11637 let converged = failing
11638 .load(gen0.id())
11639 .await
11640 .unwrap_or_else(|error| panic!("{error}"))
11641 .unwrap_or_else(|| panic!("the durable row must exist after retry"));
11642 assert_eq!(
11643 converged
11644 .transcript_rewrite_generation()
11645 .unwrap_or_else(|error| panic!("{error}")),
11646 2,
11647 "the retried projection must install the remaining generation"
11648 );
11649 assert_eq!(
11650 converged
11651 .transcript_revision()
11652 .unwrap_or_else(|error| panic!("{error}")),
11653 successor
11654 .transcript_revision()
11655 .unwrap_or_else(|error| panic!("{error}")),
11656 "the durable row must converge on the committed successor"
11657 );
11658 }
11659
11660 #[tokio::test]
11669 async fn freshen_refuses_divergent_durable_row_typed_instead_of_adopting() {
11670 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
11671 let session_store: Arc<dyn SessionStore> = Arc::new(
11672 meerkat_store::SqliteSessionStore::open(dir.path().join("sessions.db"))
11673 .unwrap_or_else(|error| panic!("{error}")),
11674 );
11675 let inner: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
11679 meerkat_runtime::store::SqliteRuntimeStore::new(dir.path().join("runtime.db"))
11680 .unwrap_or_else(|error| panic!("{error}")),
11681 );
11682 let seed = meerkat_core::Session::new();
11687 let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(seed.id());
11688 let mut committed = seed.clone();
11689 committed.push(meerkat_core::Message::User(
11690 meerkat_core::types::UserMessage::text("committed turn"),
11691 ));
11692 let mut divergent = seed;
11693 divergent.push(meerkat_core::Message::User(
11694 meerkat_core::types::UserMessage::text("divergent turn"),
11695 ));
11696 divergent.push(meerkat_core::Message::User(
11697 meerkat_core::types::UserMessage::text("divergent follow-up"),
11698 ));
11699 inner
11700 .commit_session_snapshot(
11701 &runtime_id,
11702 meerkat_runtime::store::SerializedSessionSnapshot {
11703 session_snapshot: Arc::new(
11704 committed
11705 .to_persisted_bytes()
11706 .unwrap_or_else(|error| panic!("{error}")),
11707 ),
11708 },
11709 )
11710 .await
11711 .unwrap_or_else(|error| panic!("{error}"));
11712 session_store
11713 .save(&divergent)
11714 .await
11715 .unwrap_or_else(|error| panic!("{error}"));
11716
11717 let store = Arc::new(SessionStoreBackedRuntimeStore::new(
11718 Arc::clone(&inner),
11719 Arc::clone(&session_store),
11720 ));
11721 for attempt in ["first read", "retry"] {
11722 let refused = meerkat_runtime::RuntimeStore::load_committed_whole_blob_snapshot(
11723 &*store,
11724 &runtime_id,
11725 )
11726 .await
11727 .expect_err("a divergent durable row must refuse the freshen typed, not adopt");
11728 assert!(
11729 refused.to_string().contains("not a continuation"),
11730 "the {attempt} refusal must be the boundary guard's continuity violation: {refused}"
11731 );
11732 }
11733 let retained = inner
11735 .load_committed_whole_blob_snapshot(&runtime_id)
11736 .await
11737 .unwrap_or_else(|error| panic!("{error}"))
11738 .unwrap_or_else(|| panic!("the committed runtime authority must survive"));
11739 assert_eq!(
11740 retained.session().messages().len(),
11741 1,
11742 "the committed runtime document must be untouched"
11743 );
11744 let durable = session_store
11745 .load(committed.id())
11746 .await
11747 .unwrap_or_else(|error| panic!("{error}"))
11748 .unwrap_or_else(|| panic!("the durable row must survive"));
11749 assert_eq!(
11750 durable.messages().len(),
11751 2,
11752 "the durable document must be untouched"
11753 );
11754 }
11755
11756 #[tokio::test]
11757 async fn session_store_backed_runtime_store_forwards_defaulted_authority_seams() {
11758 let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
11759 let session_store: Arc<dyn SessionStore> = Arc::new(
11760 meerkat_store::SqliteSessionStore::open(dir.path().join("sessions.db"))
11761 .unwrap_or_else(|error| panic!("{error}")),
11762 );
11763 let inner: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
11764 meerkat_runtime::store::SqliteRuntimeStore::new(dir.path().join("runtime.db"))
11765 .unwrap_or_else(|error| panic!("{error}")),
11766 );
11767 let store = SessionStoreBackedRuntimeStore::new(inner, session_store);
11768 let runtime_id = meerkat_runtime::LogicalRuntimeId::new("runtime-store-facade-proof");
11769 let registry = meerkat_runtime::ops_lifecycle::RuntimeOpsLifecycleRegistry::new();
11770 let candidate = registry
11771 .capture_persistence_snapshot(
11772 meerkat_core::RuntimeEpochId::new(),
11773 &meerkat_core::EpochCursorState::new(),
11774 )
11775 .unwrap_or_else(|error| panic!("{error}"));
11776
11777 let initialized = meerkat_runtime::RuntimeStore::initialize_ops_lifecycle_if_absent(
11778 &store,
11779 &runtime_id,
11780 &candidate,
11781 )
11782 .await
11783 .unwrap_or_else(|error| panic!("{error}"));
11784 assert_eq!(initialized.epoch_id, candidate.epoch_id);
11785
11786 assert!(
11787 meerkat_runtime::RuntimeStore::put_mob_host_binding_if_absent(
11788 &store,
11789 "mob-a",
11790 b"binding-v1",
11791 )
11792 .await
11793 .unwrap_or_else(|error| panic!("{error}"))
11794 );
11795 assert!(
11796 meerkat_runtime::RuntimeStore::compare_and_put_mob_host_binding(
11797 &store,
11798 "mob-a",
11799 b"binding-v1",
11800 b"binding-v2",
11801 )
11802 .await
11803 .unwrap_or_else(|error| panic!("{error}"))
11804 );
11805 assert_eq!(
11806 meerkat_runtime::RuntimeStore::load_mob_host_binding(&store, "mob-a")
11807 .await
11808 .unwrap_or_else(|error| panic!("{error}")),
11809 Some(b"binding-v2".to_vec())
11810 );
11811 assert!(
11812 meerkat_runtime::RuntimeStore::delete_mob_host_binding(&store, "mob-a", b"binding-v2",)
11813 .await
11814 .unwrap_or_else(|error| panic!("{error}"))
11815 );
11816
11817 assert!(
11818 meerkat_runtime::RuntimeStore::put_mob_host_binding_if_absent(
11819 &store, "mob-b", b"binding",
11820 )
11821 .await
11822 .unwrap_or_else(|error| panic!("{error}"))
11823 );
11824 assert!(
11825 meerkat_runtime::RuntimeStore::revoke_mob_host_binding(
11826 &store, "mob-b", b"binding", b"receipt",
11827 )
11828 .await
11829 .unwrap_or_else(|error| panic!("{error}"))
11830 );
11831 assert_eq!(
11832 meerkat_runtime::RuntimeStore::load_mob_host_revocation(&store, "mob-b")
11833 .await
11834 .unwrap_or_else(|error| panic!("{error}")),
11835 Some(b"receipt".to_vec())
11836 );
11837 assert_eq!(
11838 meerkat_runtime::RuntimeStore::list_mob_host_revocations(&store)
11839 .await
11840 .unwrap_or_else(|error| panic!("{error}")),
11841 vec![("mob-b".to_string(), b"receipt".to_vec())]
11842 );
11843 assert!(
11844 meerkat_runtime::RuntimeStore::list_mob_host_bindings(&store)
11845 .await
11846 .unwrap_or_else(|error| panic!("{error}"))
11847 .is_empty()
11848 );
11849 assert_eq!(
11850 meerkat_runtime::RuntimeStore::list_mob_host_revocations(&store)
11851 .await
11852 .unwrap_or_else(|error| panic!("{error}")),
11853 vec![("mob-b".to_string(), b"receipt".to_vec())]
11854 );
11855 }
11856
11857 #[test]
11877 fn persistent_bootstrap_uses_sqlite_runtime_store() {
11878 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
11879 let store_path = dir.path().to_path_buf();
11880 let Ok(sqlite) = meerkat_store::SqliteSessionStore::open(store_path.join("sessions.db"))
11881 else {
11882 panic!("failed to open sqlite session store");
11883 };
11884 let session_store: Arc<dyn SessionStore> = Arc::new(sqlite);
11885 let Ok(definition) = meerkat_mob::MobDefinition::from_toml("[mob]\nid = \"test\"\n") else {
11886 panic!("failed to parse minimal mob definition");
11887 };
11888 let spec = MobBootstrapSpec::persistent(
11889 definition,
11890 meerkat_mob::MobStorage::in_memory(),
11891 store_path.clone(),
11892 4,
11893 session_store,
11894 )
11895 .unwrap_or_else(|e| panic!("{e}"));
11896 assert!(
11897 spec.runtime_adapter.is_some(),
11898 "persistent bootstrap must provide its own runtime adapter via spec.runtime_adapter"
11899 );
11900 assert!(
11901 spec.session_service.runtime_adapter().is_some(),
11902 "session service must own a runtime_store so archive/retire don't \
11903 hit the store-only-projection rejection"
11904 );
11905 assert!(
11906 store_path.join("runtime.sqlite").exists(),
11907 "persistent_inner must open a SqliteRuntimeStore at <store_path>/runtime.sqlite"
11908 );
11909 }
11910
11911 #[test]
11914 fn persistent_spec_fails_closed_when_blob_dir_unopenable() {
11915 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
11916 let store_path = dir.path().to_path_buf();
11917 std::fs::write(store_path.join("blobs"), b"not a directory")
11919 .unwrap_or_else(|e| panic!("{e}"));
11920 let session_store: Arc<dyn SessionStore> = Arc::new(
11921 meerkat_store::SqliteSessionStore::open(store_path.join("sessions.db"))
11922 .unwrap_or_else(|e| panic!("{e}")),
11923 );
11924 let definition = meerkat_mob::MobDefinition::from_toml("[mob]\nid = \"test\"\n")
11925 .unwrap_or_else(|e| panic!("{e}"));
11926
11927 match MobBootstrapSpec::persistent(
11928 definition,
11929 meerkat_mob::MobStorage::in_memory(),
11930 store_path.clone(),
11931 4,
11932 session_store,
11933 ) {
11934 Err(StorageResolutionError::Blob(BlobStoreResolutionError::OpenFailed {
11935 path,
11936 ..
11937 })) => {
11938 assert_eq!(path, store_path.join("blobs"));
11939 }
11940 Err(other) => panic!("expected OpenFailed, got: {other}"),
11941 Ok(_) => {
11942 panic!("blob-dir open failure must fail closed, not fall back to in-memory blobs")
11943 }
11944 }
11945 }
11946
11947 #[test]
11951 fn persistent_spec_fails_closed_when_runtime_store_unopenable() {
11952 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
11953 let store_path = dir.path().to_path_buf();
11954 std::fs::create_dir_all(store_path.join(crate::storage_layout::RUNTIME_DB_FILE_NAME))
11956 .unwrap_or_else(|e| panic!("{e}"));
11957 let session_store: Arc<dyn SessionStore> = Arc::new(
11958 meerkat_store::SqliteSessionStore::open(store_path.join("sessions.db"))
11959 .unwrap_or_else(|e| panic!("{e}")),
11960 );
11961 let definition = meerkat_mob::MobDefinition::from_toml("[mob]\nid = \"test\"\n")
11962 .unwrap_or_else(|e| panic!("{e}"));
11963
11964 match MobBootstrapSpec::persistent(
11965 definition.clone(),
11966 meerkat_mob::MobStorage::in_memory(),
11967 store_path.clone(),
11968 4,
11969 session_store.clone(),
11970 ) {
11971 Err(StorageResolutionError::RuntimeStore(error)) => {
11972 assert_eq!(
11973 error.path,
11974 store_path.join(crate::storage_layout::RUNTIME_DB_FILE_NAME)
11975 );
11976 assert!(
11977 error.to_string().contains("ephemeral_runtime_store"),
11978 "the error must name the declaration remediation: {error}"
11979 );
11980 }
11981 Err(other) => panic!("expected a runtime-store resolution error, got: {other}"),
11982 Ok(_) => panic!(
11983 "runtime-store open failure must fail closed, not fall back to \
11984 InMemoryRuntimeStore"
11985 ),
11986 }
11987
11988 let spec = MobBootstrapSpec::persistent_inner(
11991 definition,
11992 meerkat_mob::MobStorage::in_memory(),
11993 store_path,
11994 4,
11995 session_store,
11996 "SqliteSessionStore",
11997 None,
11998 false,
11999 true,
12000 None,
12001 None,
12002 CapabilityFlags::default(),
12003 None,
12004 None,
12005 )
12006 .unwrap_or_else(|e| panic!("declared ephemeral runtime store must compose: {e}"));
12007 let summary = spec.resolved_storage.unwrap_or_else(|| panic!("summary"));
12008 let runtime_slot = summary
12009 .slots
12010 .iter()
12011 .find(|slot| slot.declaration.domain == "runtime")
12012 .unwrap_or_else(|| panic!("runtime slot recorded"));
12013 assert_eq!(
12014 runtime_slot.declaration.resolution,
12015 meerkat_core::DurabilityResolution::DeclaredEphemeral
12016 );
12017 assert_eq!(runtime_slot.backend, "InMemoryRuntimeStore");
12018 }
12019
12020 #[test]
12023 fn persistent_spec_reports_persistent_disk_and_incremental_sessions() {
12024 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
12025 let store_path = dir.path().to_path_buf();
12026 let session_store: Arc<dyn SessionStore> = Arc::new(
12027 meerkat_store::SqliteSessionStore::open(store_path.join("sessions.db"))
12028 .unwrap_or_else(|e| panic!("{e}")),
12029 );
12030 let definition = meerkat_mob::MobDefinition::from_toml("[mob]\nid = \"test\"\n")
12031 .unwrap_or_else(|e| panic!("{e}"));
12032
12033 let spec = MobBootstrapSpec::persistent(
12034 definition,
12035 meerkat_mob::MobStorage::in_memory(),
12036 store_path,
12037 4,
12038 session_store,
12039 )
12040 .unwrap_or_else(|e| panic!("{e}"));
12041 let summary = spec.resolved_storage.unwrap_or_else(|| panic!("summary"));
12042 assert_eq!(summary.blob_durability, BlobDurability::PersistentDisk);
12043 assert_eq!(summary.session_store_incremental, Some(true));
12044 for (domain, backend) in [
12047 ("runtime", "SqliteRuntimeStore"),
12048 ("workgraph", "SqliteWorkGraphStore"),
12049 ] {
12050 let slot = summary
12051 .slots
12052 .iter()
12053 .find(|slot| slot.declaration.domain == domain)
12054 .unwrap_or_else(|| panic!("{domain} slot recorded"));
12055 assert_eq!(slot.backend, backend);
12056 assert_eq!(
12057 slot.declaration.resolution,
12058 meerkat_core::DurabilityResolution::Persistent
12059 );
12060 }
12061 for domain in ["gating_audit", "delivery_history", "routing_resolutions"] {
12062 let slot = summary
12063 .slots
12064 .iter()
12065 .find(|slot| slot.declaration.domain == domain)
12066 .unwrap_or_else(|| panic!("{domain} ring buffer classified"));
12067 assert_eq!(
12068 slot.declaration.class,
12069 meerkat_core::DurabilityClass::Scratch
12070 );
12071 }
12072 }
12073
12074 #[test]
12078 fn persistent_spec_gates_non_persistent_custom_blob_store_on_declaration() {
12079 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
12080 let store_path = dir.path().to_path_buf();
12081 let session_store: Arc<dyn SessionStore> = Arc::new(
12082 meerkat_store::SqliteSessionStore::open(store_path.join("sessions.db"))
12083 .unwrap_or_else(|e| panic!("{e}")),
12084 );
12085 let memory_blobs: Arc<dyn meerkat_core::BlobStore> = Arc::new(Base64BlobStoreAdapter::new(
12086 Arc::new(ObjectStoreBlobStore::memory()),
12087 ));
12088 let definition = meerkat_mob::MobDefinition::from_toml("[mob]\nid = \"test\"\n")
12089 .unwrap_or_else(|e| panic!("{e}"));
12090
12091 match MobBootstrapSpec::persistent_inner(
12092 definition.clone(),
12093 meerkat_mob::MobStorage::in_memory(),
12094 store_path.clone(),
12095 4,
12096 session_store.clone(),
12097 "SqliteSessionStore",
12098 Some(BlobStoreInjection::Core(memory_blobs.clone())),
12099 false,
12100 false,
12101 None,
12102 None,
12103 CapabilityFlags::default(),
12104 None,
12105 None,
12106 ) {
12107 Err(StorageResolutionError::Blob(
12108 BlobStoreResolutionError::NonPersistentUndeclared,
12109 )) => {}
12110 Err(other) => panic!("expected NonPersistentUndeclared, got: {other}"),
12111 Ok(_) => panic!("undeclared non-persistent custom blob store must fail composition"),
12112 }
12113
12114 let spec = MobBootstrapSpec::persistent_inner(
12115 definition,
12116 meerkat_mob::MobStorage::in_memory(),
12117 store_path,
12118 4,
12119 session_store,
12120 "SqliteSessionStore",
12121 Some(BlobStoreInjection::Core(memory_blobs)),
12122 true,
12123 false,
12124 None,
12125 None,
12126 CapabilityFlags::default(),
12127 None,
12128 None,
12129 )
12130 .unwrap_or_else(|e| panic!("declared ephemeral blobs must compose: {e}"));
12131 let summary = spec.resolved_storage.unwrap_or_else(|| panic!("summary"));
12132 assert_eq!(
12133 summary.blob_durability,
12134 BlobDurability::Custom { persistent: false }
12135 );
12136 assert_eq!(summary.session_store_incremental, Some(true));
12137 }
12138
12139 #[test]
12142 fn persistent_spec_declared_ephemeral_blobs_skips_disk() {
12143 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
12144 let store_path = dir.path().to_path_buf();
12145 let session_store: Arc<dyn SessionStore> = Arc::new(
12146 meerkat_store::SqliteSessionStore::open(store_path.join("sessions.db"))
12147 .unwrap_or_else(|e| panic!("{e}")),
12148 );
12149 let definition = meerkat_mob::MobDefinition::from_toml("[mob]\nid = \"test\"\n")
12150 .unwrap_or_else(|e| panic!("{e}"));
12151
12152 let spec = MobBootstrapSpec::persistent_inner(
12153 definition,
12154 meerkat_mob::MobStorage::in_memory(),
12155 store_path.clone(),
12156 4,
12157 session_store,
12158 "SqliteSessionStore",
12159 None,
12160 true,
12161 false,
12162 None,
12163 None,
12164 CapabilityFlags::default(),
12165 None,
12166 None,
12167 )
12168 .unwrap_or_else(|e| panic!("{e}"));
12169 assert_eq!(
12170 spec.resolved_storage.map(|summary| summary.blob_durability),
12171 Some(BlobDurability::DeclaredEphemeral)
12172 );
12173 assert!(
12174 !store_path.join("blobs").exists(),
12175 "declared-ephemeral blobs must not touch the disk blob root"
12176 );
12177 }
12178
12179 #[test]
12182 fn ephemeral_runtime_backed_spec_reports_declared_ephemeral() {
12183 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
12184 let definition = meerkat_mob::MobDefinition::from_toml("[mob]\nid = \"test\"\n")
12185 .unwrap_or_else(|e| panic!("{e}"));
12186
12187 let spec = MobBootstrapSpec::ephemeral_runtime_backed_inner(
12188 definition,
12189 meerkat_mob::MobStorage::in_memory(),
12190 dir.path().to_path_buf(),
12191 4,
12192 None,
12193 "test session store",
12194 None,
12195 None,
12196 None,
12197 CapabilityFlags::default(),
12198 None,
12199 None,
12200 );
12201 let summary = spec.resolved_storage.unwrap_or_else(|| panic!("summary"));
12202 assert_eq!(summary.blob_durability, BlobDurability::DeclaredEphemeral);
12203 assert_eq!(summary.session_store_incremental, None);
12204 for domain in ["sessions", "runtime", "blobs", "workgraph"] {
12206 let slot = summary
12207 .slots
12208 .iter()
12209 .find(|slot| slot.declaration.domain == domain)
12210 .unwrap_or_else(|| panic!("{domain} slot recorded"));
12211 assert_eq!(
12212 slot.declaration.resolution,
12213 meerkat_core::DurabilityResolution::DeclaredEphemeral,
12214 "{domain} must be a declared ephemeral choice"
12215 );
12216 }
12217 }
12218
12219 #[test]
12223 fn ephemeral_runtime_backed_uses_session_service_runtime_adapter() {
12224 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
12225 let store_path = dir.path().to_path_buf();
12226 let Ok(definition) = meerkat_mob::MobDefinition::from_toml("[mob]\nid = \"test\"\n") else {
12227 panic!("failed to parse minimal mob definition");
12228 };
12229 let spec = MobBootstrapSpec::ephemeral_runtime_backed_inner(
12230 definition,
12231 meerkat_mob::MobStorage::in_memory(),
12232 store_path,
12233 4,
12234 None,
12235 "test session store",
12236 None,
12237 None,
12238 None,
12239 CapabilityFlags::default(),
12240 None,
12241 None,
12242 );
12243 assert!(
12244 spec.runtime_adapter.is_some(),
12245 "ephemeral_runtime_backed_inner must expose the shared runtime authority"
12246 );
12247 assert!(
12248 spec.session_service.runtime_adapter().is_some(),
12249 "session service must still expose a runtime adapter so autonomous-host comms can wire"
12250 );
12251 }
12252
12253 #[tokio::test]
12254 async fn agent_mob_tools_expose_definition_profiles_as_realm_profiles() {
12255 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
12256 let store_path = dir.path().to_path_buf();
12257 let Ok(definition) = meerkat_mob::MobDefinition::from_toml(
12258 "[mob]\nid = \"test\"\n\n[profiles.investigation-worker]\nmodel = \"gpt-5.5\"\n[profiles.investigation-worker.tools]\ncomms = true\nmob = true\n\n[profiles.person-worker]\nmodel = \"gpt-5.5\"\n[profiles.person-worker.tools]\ncomms = true\n",
12259 ) else {
12260 panic!("failed to parse mob definition with worker profiles");
12261 };
12262
12263 let spec = MobBootstrapSpec::ephemeral_runtime_backed_inner(
12264 definition,
12265 meerkat_mob::MobStorage::in_memory(),
12266 store_path,
12267 4,
12268 None,
12269 "test session store",
12270 None,
12271 None,
12272 None,
12273 CapabilityFlags::default(),
12274 None,
12275 None,
12276 );
12277 let state = spec
12278 .agent_mob_mcp_state
12279 .expect("agent mob MCP state should be installed");
12280
12281 let profiles = state
12282 .realm_profile_list()
12283 .await
12284 .expect("definition profiles should list through agent mob tools");
12285 let names = profiles
12286 .iter()
12287 .map(|profile| profile.name.as_str())
12288 .collect::<Vec<_>>();
12289 assert!(
12290 names.contains(&"investigation-worker"),
12291 "definition profiles must be visible to mob_profile_list so agents can create mobs that reference them"
12292 );
12293 assert!(names.contains(&"person-worker"));
12294
12295 let worker = state
12296 .realm_profile_get("investigation-worker")
12297 .await
12298 .expect("definition profile lookup should succeed")
12299 .expect("definition profile should exist");
12300 assert_eq!(worker.profile.model, "gpt-5.5");
12301 assert_eq!(
12302 worker.revision, 0,
12303 "definition-backed profiles are immutable runtime seeds, not persisted realm revisions"
12304 );
12305 }
12306
12307 #[tokio::test]
12308 async fn agent_created_mobs_can_spawn_definition_seeded_realm_profiles() {
12309 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
12310 let store_path = dir.path().to_path_buf();
12311 let Ok(parent_definition) = meerkat_mob::MobDefinition::from_toml(
12312 "[mob]\nid = \"parent\"\n\n[profiles.investigation-worker]\nmodel = \"gpt-5.5\"\n[profiles.investigation-worker.tools]\ncomms = true\nmob = true\n",
12313 ) else {
12314 panic!("failed to parse parent mob definition");
12315 };
12316
12317 let mut spec = MobBootstrapSpec::ephemeral_runtime_backed_inner(
12318 parent_definition,
12319 meerkat_mob::MobStorage::in_memory(),
12320 store_path,
12321 4,
12322 None,
12323 "test session store",
12324 None,
12325 None,
12326 None,
12327 CapabilityFlags::default(),
12328 None,
12329 None,
12330 );
12331 spec.options.default_llm_client = Some(Arc::new(meerkat_client::TestClient::default()));
12332 let runtime = MobRuntime::bootstrap(spec)
12333 .await
12334 .unwrap_or_else(|e| panic!("{e}"));
12335 let state = runtime
12336 .agent_mob_mcp_state
12337 .clone()
12338 .expect("agent mob MCP state should be installed");
12339 let Ok(child_definition) = meerkat_mob::MobDefinition::from_toml(
12340 "[mob]\nid = \"child\"\n\n[profiles.investigation-worker]\nrealm_profile = \"investigation-worker\"\n",
12341 ) else {
12342 panic!("failed to parse child mob definition");
12343 };
12344
12345 let mob_id = Box::pin(state.mob_create_definition(child_definition))
12346 .await
12347 .expect("child mob should be created");
12348 Box::pin(state.mob_spawn_spec(
12349 &mob_id,
12350 SpawnMemberSpec::new(
12351 ProfileName::from("investigation-worker"),
12352 meerkat_mob::AgentIdentity::from("investigation-worker-one"),
12355 ),
12356 ))
12357 .await
12358 .expect("created mob should resolve definition-seeded realm profile at spawn time");
12359 }
12360
12361 #[tokio::test]
12369 async fn ephemeral_runtime_backed_custom_session_store_persists_created_session() {
12370 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
12371 let store_path = dir.path().to_path_buf();
12372 let Ok(definition) = meerkat_mob::MobDefinition::from_toml(
12373 "[mob]\nid = \"test\"\n\n[profiles.worker]\nmodel = \"gpt-5.5\"\n[profiles.worker.tools]\ncomms = true\n",
12374 ) else {
12375 panic!("failed to parse minimal mob definition");
12376 };
12377 let custom_store: Arc<dyn SessionStore> = Arc::new(meerkat_store::MemoryStore::new());
12378 let mut spec = MobBootstrapSpec::ephemeral_runtime_backed_inner(
12379 definition,
12380 meerkat_mob::MobStorage::in_memory(),
12381 store_path,
12382 4,
12383 Some(custom_store.clone()),
12384 "test session store",
12385 None,
12386 None,
12387 None,
12388 CapabilityFlags::default(),
12389 None,
12390 None,
12391 );
12392 spec.options.default_llm_client = Some(Arc::new(meerkat_client::TestClient::default()));
12393
12394 let runtime = MobRuntime::bootstrap(spec)
12395 .await
12396 .unwrap_or_else(|e| panic!("{e}"));
12397 let mid = meerkat_mob::ids::AgentIdentity::from("worker-one");
12400 Box::pin(runtime.handle.spawn_spec(SpawnMemberSpec::new(
12401 ProfileName::from("worker"),
12402 mid.clone(),
12403 )))
12404 .await
12405 .unwrap_or_else(|e| panic!("{e}"));
12406 let session_id = runtime
12407 .handle
12408 .resolve_bridge_session_id(&mid)
12409 .await
12410 .unwrap_or_else(|| panic!("spawned worker has no bridge session id"));
12411
12412 let stored = custom_store
12413 .load(&session_id)
12414 .await
12415 .unwrap_or_else(|e| panic!("{e}"));
12416 assert!(
12417 stored.is_some(),
12418 "ephemeral runtime-backed builds with a custom store must persist through that store"
12419 );
12420 }
12421
12422 #[tokio::test]
12429 async fn ephemeral_runtime_backed_custom_session_store_resumes_after_runtime_restart() {
12430 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
12431 let store_path = dir.path().to_path_buf();
12432 let definition_toml = "[mob]\nid = \"test\"\n\n[profiles.worker]\nmodel = \"gpt-5.5\"\n[profiles.worker.tools]\ncomms = true\n";
12433 let Ok(definition) = meerkat_mob::MobDefinition::from_toml(definition_toml) else {
12434 panic!("failed to parse minimal mob definition");
12435 };
12436 let custom_store: Arc<dyn SessionStore> = Arc::new(meerkat_store::MemoryStore::new());
12437 let mut spec = MobBootstrapSpec::ephemeral_runtime_backed_inner(
12438 definition,
12439 meerkat_mob::MobStorage::in_memory(),
12440 store_path.clone(),
12441 4,
12442 Some(custom_store.clone()),
12443 "test session store",
12444 None,
12445 None,
12446 None,
12447 CapabilityFlags::default(),
12448 None,
12449 None,
12450 );
12451 spec.options.default_llm_client = Some(Arc::new(meerkat_client::TestClient::default()));
12452
12453 let runtime = MobRuntime::bootstrap(spec)
12454 .await
12455 .unwrap_or_else(|e| panic!("{e}"));
12456 let mid = meerkat_mob::ids::AgentIdentity::from("worker-one");
12459 Box::pin(runtime.handle.spawn_spec(SpawnMemberSpec::new(
12460 ProfileName::from("worker"),
12461 mid.clone(),
12462 )))
12463 .await
12464 .unwrap_or_else(|e| panic!("{e}"));
12465 let session_id = runtime
12466 .handle
12467 .resolve_bridge_session_id(&mid)
12468 .await
12469 .unwrap_or_else(|| panic!("spawned worker has no bridge session id"));
12470 runtime
12476 .handle
12477 .shutdown()
12478 .await
12479 .unwrap_or_else(|e| panic!("failed to quiesce pre-restart runtime: {e}"));
12480 let before_restart = custom_store
12481 .load(&session_id)
12482 .await
12483 .unwrap_or_else(|e| panic!("failed to load pre-restart session: {e}"))
12484 .unwrap_or_else(|| panic!("spawned worker was not projected before restart"));
12485 let before_restart_revision =
12486 meerkat_core::transcript_messages_digest(before_restart.messages())
12487 .unwrap_or_else(|e| panic!("failed to digest pre-restart transcript: {e}"));
12488 let before_restart_message_count = before_restart.messages().len();
12489 assert!(
12490 before_restart_message_count > 1,
12491 "restart fixture must hold a nontrivial transcript before resume"
12492 );
12493 drop(runtime);
12494
12495 let Ok(definition) = meerkat_mob::MobDefinition::from_toml(definition_toml) else {
12496 panic!("failed to parse minimal mob definition");
12497 };
12498 let mut restarted_spec = MobBootstrapSpec::ephemeral_runtime_backed_inner(
12499 definition,
12500 meerkat_mob::MobStorage::in_memory(),
12501 store_path,
12502 4,
12503 Some(custom_store.clone()),
12504 "test session store",
12505 None,
12506 None,
12507 None,
12508 CapabilityFlags::default(),
12509 None,
12510 None,
12511 );
12512 restarted_spec.options.default_llm_client =
12513 Some(Arc::new(meerkat_client::TestClient::default()));
12514
12515 let restarted = MobRuntime::bootstrap(restarted_spec)
12516 .await
12517 .unwrap_or_else(|e| panic!("{e}"));
12518 let mut resume_spec = SpawnMemberSpec::new(ProfileName::from("worker"), mid.clone());
12519 resume_spec.launch_mode = meerkat_mob::MemberLaunchMode::Resume {
12520 bridge_session_id: session_id.clone(),
12521 };
12522 Box::pin(restarted.handle.spawn_spec(resume_spec))
12523 .await
12524 .unwrap_or_else(|e| panic!("resume should load the external session snapshot: {e}"));
12525
12526 let resumed_session_id = restarted
12527 .handle
12528 .resolve_bridge_session_id(&mid)
12529 .await
12530 .unwrap_or_else(|| panic!("resumed worker has no bridge session id"));
12531 assert_eq!(resumed_session_id, session_id);
12532 let after_restart = custom_store
12533 .load(&session_id)
12534 .await
12535 .unwrap_or_else(|e| panic!("failed to load resumed session: {e}"))
12536 .unwrap_or_else(|| panic!("resumed worker lost its durable projection"));
12537 assert_eq!(
12538 after_restart.messages().len(),
12539 before_restart_message_count,
12540 "turnless resume must not shrink the durable transcript"
12541 );
12542 assert_eq!(
12543 meerkat_core::transcript_messages_digest(after_restart.messages())
12544 .unwrap_or_else(|e| panic!("failed to digest resumed transcript: {e}")),
12545 before_restart_revision,
12546 "turnless resume must preserve the exact durable transcript"
12547 );
12548 restarted
12549 .handle
12550 .shutdown()
12551 .await
12552 .unwrap_or_else(|e| panic!("failed to quiesce resumed runtime: {e}"));
12553 }
12554
12555 #[test]
12558 fn ephemeral_bootstrap_without_image_generation_retains_runtime_host() {
12559 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
12560 let store_path = dir.path().to_path_buf();
12561 let Ok(definition) = meerkat_mob::MobDefinition::from_toml(
12562 "[mob]\nid = \"test\"\n\n[profiles.worker]\nmodel = \"gpt-5.5\"\nruntime_mode = \"autonomous_host\"\n[profiles.worker.tools]\ncomms = true\n",
12563 ) else {
12564 panic!("failed to parse definition");
12565 };
12566 let spec = MobBootstrapSpec::ephemeral_inner(
12567 definition,
12568 meerkat_mob::MobStorage::in_memory(),
12569 store_path,
12570 4,
12571 None,
12572 None,
12573 CapabilityFlags::default(),
12574 None,
12575 None,
12576 );
12577 let spec_adapter = spec
12578 .runtime_adapter
12579 .as_ref()
12580 .unwrap_or_else(|| panic!("public ephemeral builds must retain their runtime adapter"));
12581 let service_adapter = spec
12582 .session_service
12583 .runtime_adapter()
12584 .unwrap_or_else(|| panic!("session service must expose the retained runtime adapter"));
12585 assert!(
12586 Arc::ptr_eq(spec_adapter, &service_adapter),
12587 "the spec and session service must share one exact runtime authority"
12588 );
12589 assert!(
12590 spec_adapter.has_session_llm_reconfigure_host(),
12591 "the retained runtime authority must carry the live LLM reconfiguration host"
12592 );
12593 }
12594
12595 #[test]
12599 fn auto_mark_marks_declared_profile_fields() {
12600 let Ok(mut definition) = meerkat_mob::MobDefinition::from_toml(
12601 "[mob]\nid = \"auto-mark\"\n\n[profiles.worker]\nmodel = \"claude-opus-4-8\"\nprovider = \"anthropic\"\n[profiles.worker.provider_params]\nthinking_budget_tokens = 1024\n",
12602 ) else {
12603 panic!("failed to parse definition");
12604 };
12605 auto_mark_declared_resume_overrides(&mut definition);
12606 let profile = definition
12607 .profiles
12608 .get(&ProfileName::from("worker"))
12609 .and_then(|binding| binding.as_inline())
12610 .unwrap_or_else(|| panic!("worker profile must be inline"));
12611 for field in [
12612 meerkat_mob::ResumeOverrideField::Model,
12613 meerkat_mob::ResumeOverrideField::Provider,
12614 meerkat_mob::ResumeOverrideField::ProviderParams,
12615 ] {
12616 assert!(
12617 profile.resume_overrides.contains(&field),
12618 "declared field {field:?} must be auto-marked resume-overridden"
12619 );
12620 }
12621 }
12622
12623 #[test]
12630 fn auto_mark_derives_provider_from_catalog_for_declared_model() {
12631 let Ok(mut definition) = meerkat_mob::MobDefinition::from_toml(
12632 "[mob]\nid = \"auto-mark\"\n\n[profiles.worker]\nmodel = \"gpt-5.5\"\n",
12633 ) else {
12634 panic!("failed to parse definition");
12635 };
12636 auto_mark_declared_resume_overrides(&mut definition);
12637 let profile = definition
12638 .profiles
12639 .get(&ProfileName::from("worker"))
12640 .and_then(|binding| binding.as_inline())
12641 .unwrap_or_else(|| panic!("worker profile must be inline"));
12642 assert_eq!(
12643 profile.provider,
12644 Some(Provider::OpenAI),
12645 "the pair's provider must be derived from the catalog and written onto the profile"
12646 );
12647 assert!(
12648 profile
12649 .resume_overrides
12650 .contains(&meerkat_mob::ResumeOverrideField::Model)
12651 && profile
12652 .resume_overrides
12653 .contains(&meerkat_mob::ResumeOverrideField::Provider),
12654 "model and provider must be masked together, never independently"
12655 );
12656 assert!(
12657 !profile
12658 .resume_overrides
12659 .contains(&meerkat_mob::ResumeOverrideField::ProviderParams),
12660 "undeclared provider_params must keep durable-wins"
12661 );
12662 }
12663
12664 #[test]
12672 fn auto_mark_completes_the_pair_for_explicit_model_provider_mask() {
12673 let Ok(mut definition) = meerkat_mob::MobDefinition::from_toml(
12674 "[mob]\nid = \"auto-mark\"\n\n[profiles.worker]\nmodel = \"claude-opus-4-8\"\nresume_overrides = [\"model\", \"provider\"]\n",
12675 ) else {
12676 panic!("failed to parse definition");
12677 };
12678 auto_mark_declared_resume_overrides(&mut definition);
12679 let profile = definition
12680 .profiles
12681 .get(&ProfileName::from("worker"))
12682 .and_then(|binding| binding.as_inline())
12683 .unwrap_or_else(|| panic!("worker profile must be inline"));
12684 assert_eq!(
12685 profile.provider,
12686 Some(Provider::Anthropic),
12687 "an explicit model+provider mask with no provider key must gain the catalog \
12688 provider, or the mask applies the model against the durable provider"
12689 );
12690 assert_eq!(
12691 profile
12692 .resume_overrides
12693 .iter()
12694 .filter(|field| **field == meerkat_mob::ResumeOverrideField::Provider)
12695 .count(),
12696 1,
12697 "the explicit mask entry must not be duplicated"
12698 );
12699 }
12700
12701 #[test]
12706 fn auto_mark_unknown_model_uses_config_entry_or_stays_durable_wins() {
12707 let Ok(mut definition) = meerkat_mob::MobDefinition::from_toml(
12708 "[mob]\nid = \"auto-mark\"\n\n[models.house-llm]\nprovider = \"openai\"\n\n[profiles.custom]\nmodel = \"house-llm\"\n\n[profiles.orphan]\nmodel = \"nobody-knows-this-model\"\n[profiles.orphan.provider_params]\nthinking_budget_tokens = 64\n",
12709 ) else {
12710 panic!("failed to parse definition");
12711 };
12712 auto_mark_declared_resume_overrides(&mut definition);
12713 let custom = definition
12714 .profiles
12715 .get(&ProfileName::from("custom"))
12716 .and_then(|binding| binding.as_inline())
12717 .unwrap_or_else(|| panic!("custom profile must be inline"));
12718 assert_eq!(
12719 custom.provider,
12720 Some(Provider::OpenAI),
12721 "a [models.<id>] entry owns the pair's provider for uncatalogued models"
12722 );
12723 assert!(
12724 custom
12725 .resume_overrides
12726 .contains(&meerkat_mob::ResumeOverrideField::Model)
12727 && custom
12728 .resume_overrides
12729 .contains(&meerkat_mob::ResumeOverrideField::Provider),
12730 "config-entry models mask the pair together"
12731 );
12732 let orphan = definition
12733 .profiles
12734 .get(&ProfileName::from("orphan"))
12735 .and_then(|binding| binding.as_inline())
12736 .unwrap_or_else(|| panic!("orphan profile must be inline"));
12737 assert_eq!(
12738 orphan.provider, None,
12739 "no coherent provider source: the profile must not gain one"
12740 );
12741 assert!(
12742 !orphan
12743 .resume_overrides
12744 .contains(&meerkat_mob::ResumeOverrideField::Model)
12745 && !orphan
12746 .resume_overrides
12747 .contains(&meerkat_mob::ResumeOverrideField::Provider),
12748 "without a coherent pair neither field is masked — durable truth wins whole"
12749 );
12750 assert!(
12751 orphan
12752 .resume_overrides
12753 .contains(&meerkat_mob::ResumeOverrideField::ProviderParams),
12754 "provider_params declaration is independent of the LLM-identity pair"
12755 );
12756 }
12757
12758 #[test]
12763 fn auto_mark_honors_a_declared_pair_as_written() {
12764 let Ok(mut definition) = meerkat_mob::MobDefinition::from_toml(
12765 "[mob]\nid = \"auto-mark\"\n\n[profiles.worker]\nmodel = \"claude-opus-4-8\"\nprovider = \"openai\"\n",
12766 ) else {
12767 panic!("failed to parse definition");
12768 };
12769 auto_mark_declared_resume_overrides(&mut definition);
12770 let profile = definition
12771 .profiles
12772 .get(&ProfileName::from("worker"))
12773 .and_then(|binding| binding.as_inline())
12774 .unwrap_or_else(|| panic!("worker profile must be inline"));
12775 assert_eq!(
12776 profile.provider,
12777 Some(Provider::OpenAI),
12778 "a declared provider is honored as written, never catalog-corrected"
12779 );
12780 assert!(
12781 profile
12782 .resume_overrides
12783 .contains(&meerkat_mob::ResumeOverrideField::Model)
12784 && profile
12785 .resume_overrides
12786 .contains(&meerkat_mob::ResumeOverrideField::Provider),
12787 "a declared pair masks together"
12788 );
12789 }
12790
12791 #[test]
12797 fn auto_mark_adopts_self_hosted_for_server_binding() {
12798 let Ok(mut definition) = meerkat_mob::MobDefinition::from_toml(
12799 "[mob]\nid = \"auto-mark\"\n\n[profiles.worker]\nmodel = \"house-llm\"\nself_hosted_server_id = \"srv-1\"\n",
12800 ) else {
12801 panic!("failed to parse definition");
12802 };
12803 auto_mark_declared_resume_overrides(&mut definition);
12804 let profile = definition
12805 .profiles
12806 .get(&ProfileName::from("worker"))
12807 .and_then(|binding| binding.as_inline())
12808 .unwrap_or_else(|| panic!("worker profile must be inline"));
12809 assert_eq!(
12810 profile.provider,
12811 Some(Provider::SelfHosted),
12812 "a server binding pins the pair to self_hosted"
12813 );
12814 assert!(
12815 profile
12816 .resume_overrides
12817 .contains(&meerkat_mob::ResumeOverrideField::Model)
12818 && profile
12819 .resume_overrides
12820 .contains(&meerkat_mob::ResumeOverrideField::Provider),
12821 "the self-hosted pair masks together"
12822 );
12823 }
12824
12825 #[test]
12832 fn auto_mark_completes_explicit_provider_only_mask_to_the_pair() {
12833 let Ok(mut definition) = meerkat_mob::MobDefinition::from_toml(
12834 "[mob]\nid = \"auto-mark\"\n\n[profiles.worker]\nmodel = \"gpt-5.5\"\nresume_overrides = [\"provider\"]\n",
12835 ) else {
12836 panic!("failed to parse definition");
12837 };
12838 auto_mark_declared_resume_overrides(&mut definition);
12839 let profile = definition
12840 .profiles
12841 .get(&ProfileName::from("worker"))
12842 .and_then(|binding| binding.as_inline())
12843 .unwrap_or_else(|| panic!("worker profile must be inline"));
12844 assert_eq!(
12845 profile.provider,
12846 Some(Provider::OpenAI),
12847 "the masked provider must exist on the profile, derived from the declared model"
12848 );
12849 assert!(
12850 profile
12851 .resume_overrides
12852 .contains(&meerkat_mob::ResumeOverrideField::Model),
12853 "the model joins the explicit provider mask: the pair is never split"
12854 );
12855 assert_eq!(
12856 profile
12857 .resume_overrides
12858 .iter()
12859 .filter(|field| **field == meerkat_mob::ResumeOverrideField::Provider)
12860 .count(),
12861 1,
12862 "the explicit provider entry must not be duplicated"
12863 );
12864 }
12865
12866 #[test]
12869 fn auto_mark_mixed_preserves_explicit_list_without_duplicates() {
12870 let Ok(mut definition) = meerkat_mob::MobDefinition::from_toml(
12871 "[mob]\nid = \"auto-mark\"\n\n[profiles.worker]\nmodel = \"gpt-5.5\"\nprovider = \"openai\"\nresume_overrides = [\"model\"]\n",
12872 ) else {
12873 panic!("failed to parse definition");
12874 };
12875 auto_mark_declared_resume_overrides(&mut definition);
12876 auto_mark_declared_resume_overrides(&mut definition);
12877 let profile = definition
12878 .profiles
12879 .get(&ProfileName::from("worker"))
12880 .and_then(|binding| binding.as_inline())
12881 .unwrap_or_else(|| panic!("worker profile must be inline"));
12882 let model_entries = profile
12883 .resume_overrides
12884 .iter()
12885 .filter(|field| **field == meerkat_mob::ResumeOverrideField::Model)
12886 .count();
12887 assert_eq!(
12888 model_entries, 1,
12889 "an explicitly listed field must not be duplicated"
12890 );
12891 assert!(
12892 profile
12893 .resume_overrides
12894 .contains(&meerkat_mob::ResumeOverrideField::Provider),
12895 "declared provider must be added alongside the explicit list"
12896 );
12897 assert!(
12898 !profile
12899 .resume_overrides
12900 .contains(&meerkat_mob::ResumeOverrideField::ProviderParams),
12901 "undeclared provider_params must stay durable-wins"
12902 );
12903 }
12904
12905 #[tokio::test]
12909 async fn bootstrap_auto_marks_declared_resume_overrides() {
12910 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
12911 let store_path = dir.path().to_path_buf();
12912 let Ok(definition) = meerkat_mob::MobDefinition::from_toml(
12913 "[mob]\nid = \"auto-mark\"\n\n[profiles.worker]\nmodel = \"claude-opus-4-8\"\nprovider = \"anthropic\"\n",
12914 ) else {
12915 panic!("failed to parse definition");
12916 };
12917 let spec = MobBootstrapSpec::ephemeral_inner(
12918 definition,
12919 meerkat_mob::MobStorage::in_memory(),
12920 store_path,
12921 4,
12922 None,
12923 None,
12924 CapabilityFlags::default(),
12925 None,
12926 None,
12927 );
12928 let runtime = MobRuntime::bootstrap(spec)
12929 .await
12930 .unwrap_or_else(|e| panic!("{e}"));
12931 let profile = runtime
12932 .handle
12933 .definition()
12934 .profiles
12935 .get(&ProfileName::from("worker"))
12936 .and_then(|binding| binding.as_inline())
12937 .unwrap_or_else(|| panic!("worker profile must be inline"));
12938 assert!(
12939 profile
12940 .resume_overrides
12941 .contains(&meerkat_mob::ResumeOverrideField::Model)
12942 && profile
12943 .resume_overrides
12944 .contains(&meerkat_mob::ResumeOverrideField::Provider),
12945 "bootstrap must install the definition with declared fields auto-marked"
12946 );
12947 runtime
12948 .handle
12949 .shutdown()
12950 .await
12951 .unwrap_or_else(|e| panic!("failed to quiesce runtime: {e}"));
12952 }
12953
12954 #[test]
12959 fn ephemeral_bootstrap_with_image_generation_shares_runtime_adapter() {
12960 let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("{e}"));
12961 let store_path = dir.path().to_path_buf();
12962 let Ok(definition) = meerkat_mob::MobDefinition::from_toml(
12963 r#"
12964[mob]
12965id = "test"
12966
12967[profiles.commander]
12968model = "gpt-5.5"
12969
12970[profiles.commander.tools]
12971builtins = true
12972image_generation = true
12973"#,
12974 ) else {
12975 panic!("failed to parse image-generation definition");
12976 };
12977 let spec = MobBootstrapSpec::ephemeral(
12978 definition,
12979 meerkat_mob::MobStorage::in_memory(),
12980 store_path,
12981 4,
12982 None,
12983 );
12984 let spec_adapter = spec
12985 .runtime_adapter
12986 .as_ref()
12987 .expect("image-generation ephemeral builds must expose a runtime adapter");
12988 let service_adapter = spec
12989 .session_service
12990 .runtime_adapter()
12991 .expect("session service must expose the same runtime adapter");
12992 assert!(
12993 spec_adapter.shares_runtime_persistence_with(&service_adapter),
12994 "image-generation tool state and session state must share one runtime authority"
12995 );
12996 }
12997
12998 #[test]
13001 fn normalize_runtime_turn_request_strips_runtime_owned_semantics() {
13002 let req = meerkat_core::service::StartTurnRequest {
13003 prompt: meerkat_core::ContentInput::Text("checkpoint".to_string()),
13004 injected_context: Vec::new(),
13005 system_prompt: Some("system".to_string()),
13006 event_tx: None,
13007 runtime: meerkat_core::service::StartTurnRuntimeSemantics {
13008 input_identity: None,
13009 handling_mode: meerkat_core::types::HandlingMode::Steer,
13010 turn_tool_overlay: None,
13011 typed_turn_appends: Vec::new(),
13012 turn_metadata: Some(
13015 meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
13016 render_metadata: Some(meerkat_core::types::RenderMetadata {
13017 class: meerkat_core::types::RenderClass::OpsProgress,
13018 salience: meerkat_core::types::RenderSalience::Urgent,
13019 }),
13020 ..Default::default()
13021 },
13022 ),
13023 },
13024 };
13025
13026 let expected_prompt = req.prompt.clone();
13027 let expected_system_prompt = req.system_prompt.clone();
13028
13029 let normalized = normalize_runtime_turn_request(req);
13030
13031 assert_eq!(
13032 normalized.runtime.handling_mode,
13033 meerkat_core::types::HandlingMode::Queue,
13034 "runtime-applied turns must downgrade Steer before reaching direct session services"
13035 );
13036 assert!(
13037 normalized
13038 .runtime
13039 .turn_metadata
13040 .as_ref()
13041 .is_none_or(|metadata| metadata.render_metadata.is_none()),
13042 "runtime-owned render metadata must not be forwarded through the direct agent path"
13043 );
13044 assert_eq!(normalized.prompt, expected_prompt);
13045 assert_eq!(normalized.system_prompt, expected_system_prompt);
13046 }
13047
13048 #[test]
13050 fn session_created_context_fields() {
13051 let ctx = SessionCreatedContext {
13052 model: "claude-sonnet-4-5".to_string(),
13053 labels: std::collections::BTreeMap::from([(
13054 "agent_type".to_string(),
13055 "lead".to_string(),
13056 )]),
13057 system_prompt: Some("You are a lead agent.".to_string()),
13058 };
13059 assert_eq!(ctx.model, "claude-sonnet-4-5");
13060 assert_eq!(ctx.labels["agent_type"], "lead");
13061 assert_eq!(ctx.system_prompt.as_deref(), Some("You are a lead agent."));
13062 }
13063
13064 #[tokio::test]
13066 async fn session_hook_default_impls_are_noop() {
13067 struct EmptyHook;
13068 #[async_trait]
13069 impl SessionHook for EmptyHook {}
13070
13071 let hook = EmptyHook;
13072 let mut req = CreateSessionRequest {
13073 model: "test".to_string(),
13074 prompt: meerkat_core::ContentInput::Text("test".to_string()),
13075 system_prompt: meerkat_core::config::SystemPromptOverride::Inherit,
13076 max_tokens: None,
13077 event_tx: None,
13078 initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
13079 build: None,
13080 labels: None,
13081 deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::default(),
13082 injected_context: Vec::new(),
13083 };
13084 hook.before_create(&mut req).await.unwrap();
13086 let ctx = SessionCreatedContext {
13088 model: "test".to_string(),
13089 labels: Default::default(),
13090 system_prompt: None,
13091 };
13092 hook.after_create(&meerkat_core::types::SessionId::new(), &ctx)
13093 .await;
13094 }
13095
13096 #[tokio::test]
13098 async fn session_hook_before_create_can_abort() {
13099 struct AbortHook;
13100 #[async_trait]
13101 impl SessionHook for AbortHook {
13102 async fn before_create(
13103 &self,
13104 _req: &mut CreateSessionRequest,
13105 ) -> Result<(), SessionError> {
13106 Err(SessionError::Unsupported("hook abort".into()))
13107 }
13108 }
13109
13110 let hook = AbortHook;
13111 let mut req = CreateSessionRequest {
13112 model: "test".to_string(),
13113 prompt: meerkat_core::ContentInput::Text("test".to_string()),
13114 system_prompt: meerkat_core::config::SystemPromptOverride::Inherit,
13115 max_tokens: None,
13116 event_tx: None,
13117 initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
13118 build: None,
13119 labels: None,
13120 deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::default(),
13121 injected_context: Vec::new(),
13122 };
13123 let result = hook.before_create(&mut req).await;
13124 assert!(result.is_err());
13125 }
13126
13127 #[tokio::test]
13129 async fn session_hook_before_create_mutates_request() {
13130 struct MutatingHook;
13131 #[async_trait]
13132 impl SessionHook for MutatingHook {
13133 async fn before_create(
13134 &self,
13135 req: &mut CreateSessionRequest,
13136 ) -> Result<(), SessionError> {
13137 req.model = "hook-overridden".to_string();
13138 req.system_prompt =
13139 meerkat_core::config::SystemPromptOverride::Set("injected by hook".to_string());
13140 Ok(())
13141 }
13142 }
13143
13144 let hook = MutatingHook;
13145 let mut req = CreateSessionRequest {
13146 model: "original".to_string(),
13147 prompt: meerkat_core::ContentInput::Text("test".to_string()),
13148 system_prompt: meerkat_core::config::SystemPromptOverride::Inherit,
13149 max_tokens: None,
13150 event_tx: None,
13151 initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
13152 build: None,
13153 labels: None,
13154 deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::default(),
13155 injected_context: Vec::new(),
13156 };
13157 hook.before_create(&mut req).await.unwrap();
13158 assert_eq!(req.model, "hook-overridden");
13159 assert_eq!(req.system_prompt.as_set_prompt(), Some("injected by hook"));
13160 }
13161
13162 #[test]
13163 fn recoverable_lifecycle_cleanup_accepts_ambiguous_member_cleanup() {
13164 let error = "previous member cleanup ambiguous for member rt:deep-investigator:singleton:0";
13165
13166 assert!(is_previous_member_cleanup_ambiguous_error(error));
13167 assert!(is_recoverable_lifecycle_cleanup_error(error));
13168 }
13169
13170 #[test]
13171 fn topology_restore_failed_peer_ids_extracts_tolerated_peers() {
13172 let identity = meerkat_mob::AgentIdentity::from("rt:review:singleton:0");
13173 let receipt = meerkat_mob::MemberRespawnReceipt::new(
13174 identity.clone(),
13175 meerkat_mob::AgentRuntimeId::new(identity, meerkat_mob::ids::Generation::INITIAL),
13176 meerkat_mob::FenceToken::new(1),
13177 meerkat_mob::FenceToken::new(2),
13178 );
13179 let err = meerkat_mob::MobRespawnError::TopologyRestoreFailed {
13180 receipt,
13181 failed_peer_ids: vec![meerkat_mob::RespawnTopologyPeerId::from(
13182 "initiative:broken",
13183 )],
13184 };
13185
13186 assert_eq!(
13187 topology_restore_failed_peer_ids(&err),
13188 Some(vec!["initiative:broken".to_string()])
13189 );
13190 assert_eq!(
13191 topology_restore_failed_peer_ids(&meerkat_mob::MobRespawnError::NoRuntimeControl {
13192 identity: meerkat_mob::AgentIdentity::from("rt:review:singleton:0"),
13193 }),
13194 None
13195 );
13196 }
13197
13198 #[test]
13199 fn topology_restore_warning_json_surfaces_isolated_peers() {
13200 let failed_peer_ids = vec!["initiative:broken".to_string(), "helper:cold".to_string()];
13201
13202 assert_eq!(
13203 topology_restore_warning_json(&failed_peer_ids),
13204 serde_json::json!({
13205 "kind": "topology_restore_degraded",
13206 "failed_peer_ids": ["initiative:broken", "helper:cold"],
13207 })
13208 );
13209 }
13210
13211 #[test]
13212 fn recoverable_lifecycle_cleanup_preserves_archive_cancel_race() {
13213 let error = "internal error: disposal completed but ArchiveSession failed: \
13214 session error: agent error: Internal error: runtime cancel-before-retire failed \
13215 for 019e3c52-0f1b-73d3-a5c7-4b21c2bbf131: Runtime not ready: running";
13216
13217 assert!(is_recoverable_lifecycle_cleanup_error(error));
13218 }
13219
13220 #[test]
13221 fn recoverable_lifecycle_cleanup_rejects_unrelated_errors() {
13222 assert!(!is_recoverable_lifecycle_cleanup_error(
13223 "actor task dropped"
13224 ));
13225 assert!(!is_recoverable_lifecycle_cleanup_error(
13226 "model provider returned rate limit"
13227 ));
13228 }
13229
13230 #[test]
13235 fn recoverable_lifecycle_cleanup_accepts_stopped_guard_archive_retire() {
13236 let error = "internal error: disposal completed but ArchiveSession failed: \
13237 session error: agent error: Internal error: machine archive retire failed \
13238 after registration: Internal error: DSL authority (Retire): guard rejected \
13239 transition from Stopped for input::Retire";
13240
13241 assert!(is_recoverable_lifecycle_cleanup_error(error));
13242 }
13243
13244 #[test]
13250 fn recoverable_lifecycle_cleanup_accepts_stale_continuity_save_on_retire() {
13251 let record_gone = "internal error: disposal completed but ArchiveSession failed: \
13252 session error: agent error: Internal error: continuity save: \
13253 continuity record not found for identity:luka";
13254 let stale_fence = "internal error: disposal completed but ArchiveSession failed: \
13255 session error: agent error: Internal error: continuity save: \
13256 stale fencing token for identity:luka: presented 1, current 6";
13257
13258 assert!(is_recoverable_lifecycle_cleanup_error(record_gone));
13259 assert!(is_recoverable_lifecycle_cleanup_error(stale_fence));
13260 }
13261
13262 #[test]
13269 fn stopped_session_archive_retire_rejection_matches_only_stopped_guard() {
13270 assert!(is_stopped_session_archive_retire_rejection(
13271 "session error: agent error: Internal error: machine archive retire failed \
13272 after registration: Internal error: DSL authority (Retire): guard rejected \
13273 transition from Stopped for input::Retire"
13274 ));
13275 assert!(is_stopped_session_archive_retire_rejection(
13277 "agent error: Internal error: machine archive retire failed: Internal error: \
13278 DSL authority (Retire): guard rejected transition from Stopped for input::Retire"
13279 ));
13280 assert!(!is_stopped_session_archive_retire_rejection(
13282 "machine archive retire failed after registration: Internal error: \
13283 DSL authority (Retire): guard rejected transition from Running for input::Retire"
13284 ));
13285 assert!(!is_stopped_session_archive_retire_rejection(
13286 "guard rejected transition from Stopped for input::Retire"
13287 ));
13288 assert!(!is_stopped_session_archive_retire_rejection(
13289 "machine archive retire failed: store unavailable"
13290 ));
13291 }
13292
13293 #[test]
13297 fn recoverable_lifecycle_cleanup_requires_completed_disposal() {
13298 assert!(!is_recoverable_lifecycle_cleanup_error(
13299 "continuity save: continuity record not found for identity:luka"
13300 ));
13301 assert!(!is_recoverable_lifecycle_cleanup_error(
13302 "DSL authority (Retire): guard rejected transition from Stopped for input::Retire"
13303 ));
13304 assert!(!is_recoverable_lifecycle_cleanup_error(
13305 "disposal aborted at ArchiveSession: continuity save: stale fencing token"
13306 ));
13307 }
13308
13309 #[test]
13316 fn session_owned_retire_cleanup_accepts_archive_not_found_for_registered_runtime_session() {
13317 let reset_error = "internal error: disposal completed but ArchiveSession failed: \
13321 session error: agent error: Internal error: mob archive authority returned \
13322 NotFound for registered runtime session 019ee136-33bc-7bc3-80f9-2aac38736291";
13323 let delete_error = "internal error: disposal completed but ArchiveSession failed: \
13324 session error: agent error: Internal error: mob archive authority returned \
13325 NotFound for registered runtime session 019ee136-340d-7203-a089-c3357835c824";
13326
13327 assert!(is_recoverable_session_owned_retire_cleanup_error(
13328 reset_error
13329 ));
13330 assert!(is_recoverable_session_owned_retire_cleanup_error(
13331 delete_error
13332 ));
13333 }
13334
13335 #[test]
13340 fn mob_member_lifecycle_gate_still_rejects_archive_not_found() {
13341 let error = "internal error: disposal completed but ArchiveSession failed: \
13342 session error: agent error: Internal error: mob archive authority returned \
13343 NotFound for registered runtime session 019ee136-33bc-7bc3-80f9-2aac38736291";
13344
13345 assert!(!is_recoverable_lifecycle_cleanup_error(error));
13346 assert!(is_session_owned_archive_absent_cleanup_error(error));
13348 }
13349
13350 #[test]
13352 fn session_owned_retire_cleanup_still_accepts_shared_recoverable_cases() {
13353 let cancel_race = "internal error: disposal completed but ArchiveSession failed: \
13354 session error: agent error: Internal error: runtime cancel-before-retire failed \
13355 for 019e3c52-0f1b-73d3-a5c7-4b21c2bbf131: Runtime not ready: running";
13356
13357 assert!(is_recoverable_session_owned_retire_cleanup_error(
13358 cancel_race
13359 ));
13360 }
13361
13362 #[test]
13365 fn session_owned_retire_cleanup_requires_completed_disposal() {
13366 assert!(!is_recoverable_session_owned_retire_cleanup_error(
13367 "disposal aborted at ArchiveSession: mob archive authority returned NotFound \
13368 for registered runtime session 019ee136-33bc-7bc3-80f9-2aac38736291"
13369 ));
13370 assert!(!is_recoverable_session_owned_retire_cleanup_error(
13371 "mob archive authority returned NotFound for registered runtime session \
13372 019ee136-33bc-7bc3-80f9-2aac38736291"
13373 ));
13374 assert!(!is_recoverable_session_owned_retire_cleanup_error(
13375 "model provider returned rate limit"
13376 ));
13377 }
13378
13379 mod console_spawn_projection {
13380 use super::*;
13381 use crate::console_spawn::new_console_spawn_sink_slot;
13382 use crate::unified_runtime::ConsoleEventStore;
13383 use meerkat_core::AgentToolDispatcher;
13384
13385 struct CannedDispatcher {
13388 payload: Value,
13389 is_error: bool,
13390 }
13391
13392 #[async_trait::async_trait]
13393 impl meerkat_core::AgentToolDispatcher for CannedDispatcher {
13394 fn tools(&self) -> Arc<[Arc<meerkat_core::types::ToolDef>]> {
13395 Vec::<Arc<meerkat_core::types::ToolDef>>::new().into()
13396 }
13397
13398 async fn dispatch(
13399 &self,
13400 call: meerkat_core::types::ToolCallView<'_>,
13401 ) -> Result<meerkat_core::ToolDispatchOutcome, meerkat_core::ToolError> {
13402 Ok(meerkat_core::ToolDispatchOutcome::sync_result(
13403 meerkat_core::types::ToolResult::new(
13404 call.id.to_string(),
13405 self.payload.to_string(),
13406 self.is_error,
13407 ),
13408 ))
13409 }
13410
13411 fn capabilities(&self) -> meerkat_core::agent::DispatcherCapabilities {
13412 meerkat_core::agent::DispatcherCapabilities::default()
13413 }
13414 }
13415
13416 fn console_wrapper(
13417 payload: Value,
13418 is_error: bool,
13419 store: Option<&ConsoleEventStore>,
13420 ) -> AutoWireParentMobToolDispatcher {
13421 let console_spawn_sink = new_console_spawn_sink_slot();
13422 if let Some(store) = store {
13423 *console_spawn_sink
13424 .write()
13425 .unwrap_or_else(std::sync::PoisonError::into_inner) =
13426 Some(ConsoleSpawnSink::new(store.clone()));
13427 }
13428 AutoWireParentMobToolDispatcher {
13429 inner: Arc::new(CannedDispatcher { payload, is_error }),
13430 implicit_delegate_retirement_overrides:
13431 ImplicitDelegateRetirementOverrides::default(),
13432 console_spawn_sink,
13433 identity_runtime: Arc::new(std::sync::RwLock::new(None)),
13434 protected_mob_id: "test-mob".to_string(),
13435 spawner_comms_name: Some("ob3/orchestrator/ops-lead".to_string()),
13436 }
13437 }
13438
13439 async fn dispatch_tool(
13440 dispatcher: &AutoWireParentMobToolDispatcher,
13441 name: &str,
13442 args: Value,
13443 ) -> meerkat_core::ToolDispatchOutcome {
13444 let raw = serde_json::value::RawValue::from_string(args.to_string()).expect("raw args");
13445 dispatcher
13446 .dispatch(meerkat_core::types::ToolCallView {
13447 id: "call-1",
13448 name,
13449 args: &raw,
13450 })
13451 .await
13452 .expect("dispatch succeeds")
13453 }
13454
13455 async fn kickoff_events(
13456 store: &ConsoleEventStore,
13457 ) -> Vec<crate::console_contracts::ConsoleIdentityEventEnvelope> {
13458 store
13459 .replay_all(None)
13460 .await
13461 .expect("replay")
13462 .into_iter()
13463 .filter(|event| event.event_type == "user_input")
13464 .collect()
13465 }
13466
13467 #[tokio::test]
13468 async fn mob_spawn_member_projects_kickoff_into_console() {
13469 let store = ConsoleEventStore::new();
13470 let dispatcher = console_wrapper(
13471 serde_json::json!({
13472 "agent_identity": "worker-3",
13473 "member_ref": "opaque-ref"
13474 }),
13475 false,
13476 Some(&store),
13477 );
13478
13479 let outcome = dispatch_tool(
13480 &dispatcher,
13481 "mob_spawn_member",
13482 serde_json::json!({
13483 "mob_id": "ob3",
13484 "profile": "person-worker",
13485 "member_id": "worker-3",
13486 "initial_message": "Find the person",
13487 "labels": { "group": "workers" }
13488 }),
13489 )
13490 .await;
13491 assert!(!outcome.result.is_error);
13492
13493 let kickoffs = kickoff_events(&store).await;
13494 assert_eq!(kickoffs.len(), 1, "spawn must project one kickoff");
13495 let kickoff = &kickoffs[0];
13496 assert_eq!(kickoff.identity, "worker-3");
13497 assert!(kickoff.event_id.starts_with("spawn-kickoff:ob3:worker-3:"));
13498 assert_eq!(kickoff.data["content"][0]["text"], "Find the person");
13499 assert_eq!(kickoff.data["via_tool"], "mob_spawn_member");
13500 assert_eq!(kickoff.data["parent_identity"], "ops-lead");
13501
13502 let labels = store
13503 .identity_labels("worker-3")
13504 .await
13505 .expect("spawn registers console identity labels");
13506 assert_eq!(labels.get("group").map(String::as_str), Some("workers"));
13507 assert_eq!(
13508 labels.get("spawned_by").map(String::as_str),
13509 Some("ops-lead")
13510 );
13511 }
13512
13513 #[tokio::test]
13514 async fn repeated_spawn_dispatch_keeps_one_kickoff() {
13515 let store = ConsoleEventStore::new();
13516 let dispatcher = console_wrapper(
13517 serde_json::json!({ "agent_identity": "worker-3" }),
13518 false,
13519 Some(&store),
13520 );
13521 let args = serde_json::json!({
13522 "mob_id": "ob3",
13523 "profile": "person-worker",
13524 "member_id": "worker-3",
13525 "initial_message": "Find the person"
13526 });
13527
13528 dispatch_tool(&dispatcher, "mob_spawn_member", args.clone()).await;
13529 dispatch_tool(&dispatcher, "mob_spawn_member", args).await;
13530
13531 assert_eq!(
13532 kickoff_events(&store).await.len(),
13533 1,
13534 "retry/double-spawn must not duplicate the kickoff frame"
13535 );
13536 }
13537
13538 #[tokio::test]
13539 async fn delegate_projects_task_kickoff_for_generated_helper() {
13540 let store = ConsoleEventStore::new();
13541 let dispatcher = console_wrapper(
13542 serde_json::json!({
13543 "agent_identity": "helper-3f2a",
13544 "member_ref": "opaque",
13545 "mob_id": "implicit-1",
13546 "wired": true
13547 }),
13548 false,
13549 Some(&store),
13550 );
13551
13552 dispatch_tool(
13553 &dispatcher,
13554 "delegate",
13555 serde_json::json!({ "task": "Review the diff" }),
13556 )
13557 .await;
13558
13559 let kickoffs = kickoff_events(&store).await;
13560 assert_eq!(kickoffs.len(), 1);
13561 assert_eq!(kickoffs[0].identity, "helper-3f2a");
13562 assert_eq!(kickoffs[0].data["content"][0]["text"], "Review the diff");
13563 assert_eq!(kickoffs[0].data["via_tool"], "delegate");
13564 }
13565
13566 #[tokio::test]
13567 async fn spawn_without_console_sink_leaves_outcome_unchanged() {
13568 let dispatcher = console_wrapper(
13569 serde_json::json!({ "agent_identity": "worker-3" }),
13570 false,
13571 None,
13572 );
13573
13574 let outcome = dispatch_tool(
13575 &dispatcher,
13576 "mob_spawn_member",
13577 serde_json::json!({
13578 "mob_id": "ob3",
13579 "profile": "person-worker",
13580 "member_id": "worker-3",
13581 "initial_message": "Find the person"
13582 }),
13583 )
13584 .await;
13585
13586 assert!(!outcome.result.is_error);
13587 assert!(
13588 outcome
13589 .result
13590 .text_content()
13591 .contains("\"agent_identity\":\"worker-3\""),
13592 "no console store → spawn outcome passes through untouched"
13593 );
13594 }
13595
13596 #[tokio::test]
13597 async fn failed_spawn_projects_nothing() {
13598 let store = ConsoleEventStore::new();
13599 let dispatcher = console_wrapper(
13600 serde_json::json!({ "error": "spawn rejected" }),
13601 true,
13602 Some(&store),
13603 );
13604
13605 dispatch_tool(
13606 &dispatcher,
13607 "mob_spawn_member",
13608 serde_json::json!({
13609 "mob_id": "ob3",
13610 "profile": "person-worker",
13611 "member_id": "worker-3",
13612 "initial_message": "Find the person"
13613 }),
13614 )
13615 .await;
13616
13617 assert!(
13618 kickoff_events(&store).await.is_empty(),
13619 "failed spawns must not seed console chats"
13620 );
13621 assert!(store.identity_labels("worker-3").await.is_none());
13622 }
13623 }
13624}