1#![allow(clippy::large_futures)]
9
10use std::collections::BTreeSet;
11use std::sync::Arc;
12use std::time::Duration;
13
14use meerkat_core::agent::CommsRuntime;
15#[allow(unused_imports)]
16use meerkat_core::comms::{
17 CommsCommand, CommsTrustMutation, CommsTrustMutationAuthority, CommsTrustMutationResult,
18 PeerId, PeerRoute, SendError, TrustedPeerDescriptor,
19};
20#[cfg(test)]
21use meerkat_core::comms::{PeerAddress, SendReceipt};
22use meerkat_core::event::AgentEvent;
23use meerkat_core::interaction::{
24 InteractionContent, PeerIngressFact, PeerInputCandidate, PeerInputClass,
25};
26use meerkat_core::time_compat::Instant;
27use meerkat_core::types::SessionId;
28
29use meerkat_contracts::wire::WireMemberHistoryPageBody;
30use meerkat_contracts::wire::supervisor_bridge::{
31 BRIDGE_TURN_OUTCOME_ACK_MAX, BridgeAck, BridgeBindResponse, BridgeCapabilities, BridgeCommand,
32 BridgeDeliveryOutcome, BridgeDeliveryPayload, BridgeDeliveryRejectionCause,
33 BridgeDeliveryResponse, BridgeDestroyResponse, BridgeEventCursor, BridgeHardCancelPayload,
34 BridgeLiveControlledResponse, BridgeLiveOpenedResponse, BridgeMemberEventsPage,
35 BridgeMemberHistoryPage, BridgeMemberIncarnation, BridgeMemberRuntimeState,
36 BridgeMobPeerOverlayHandoff, BridgeObservationResponse, BridgeOutboundTaintTarget,
37 BridgeOutcomeTracking, BridgePeerConnectivity, BridgePeerIdentity, BridgePeerSpec,
38 BridgeProtocolVersion, BridgeRejectionCause, BridgeReply, BridgeRetireResponse,
39 BridgeSupervisorPayload, BridgeSupervisorRotationObservation,
40 BridgeSupervisorRotationOperationReceipt, BridgeSupervisorRotationPendingPhase,
41 BridgeSupervisorRotationRejectionCause, BridgeSupervisorRotationRejectionReceipt,
42 BridgeSupervisorRotationState, BridgeSupervisorRotationTargetReceipt,
43 BridgeTrackedInputCancelPayload, BridgeTrackedInputCancelResponse, SUPERVISOR_BRIDGE_INTENT,
44 SupervisorRotationOperationId, WireEventRow, canonicalize_bridge_address,
45 decode_bridge_command,
46};
47#[cfg(test)]
48use meerkat_contracts::wire::supervisor_bridge::{
49 BridgeHostRuntimeIncarnation, SUPERVISOR_BRIDGE_BOOTSTRAP_TOKEN_PARAM,
50 SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
51};
52
53use crate::comms_bridge::classified_interaction_to_runtime_input;
54#[cfg(test)]
55use crate::completion::CompletionOutcome;
56use crate::completion::interaction_terminal_event;
57use crate::identifiers::IdempotencyKey;
58#[cfg(test)]
59use crate::identifiers::LogicalRuntimeId;
60use crate::input::{
61 Input, InputDurability, InputHeader, InputOrigin, InputVisibility, PeerConvention, PeerInput,
62};
63use crate::member_live::{MemberLiveError, MemberLiveHost};
64use crate::member_observation::{
65 DirectedTurnAdmission, DirectedTurnTracking, DirectedTurnWindow, MemberEventsPollRequest,
66 MemberEventsWindow, MemberHistoryWindow, MemberObservationCursor, MemberObservationError,
67 MemberObservationHost,
68};
69
70use crate::meerkat_machine::SupervisorBinding;
71use crate::meerkat_machine::{
72 DrainExitReason, GeneratedSupervisorBinding, GeneratedSupervisorRotationSubmit, MeerkatMachine,
73 SupervisorAuthorizeAdmission, SupervisorBindAdmission, SupervisorBindingStageError,
74 SupervisorRotationObservation, SupervisorRotationSubmission,
75};
76use crate::service_ext::SessionServiceRuntimeExt as _;
77use crate::tokio::sync::mpsc;
78use crate::traits::RuntimeControlPlane;
79
80pub const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
82const SUPERVISOR_RESPONSE_ROUTE_IO_TIMEOUT: Duration = Duration::from_secs(5);
83
84pub fn spawn_comms_drain(
91 adapter: Arc<MeerkatMachine>,
92 session_id: SessionId,
93 comms_runtime: Arc<dyn CommsRuntime>,
94 idle_timeout: Option<Duration>,
95) -> crate::tokio::task::JoinHandle<()> {
96 let timeout_dur = idle_timeout.unwrap_or(DEFAULT_IDLE_TIMEOUT);
97 let runtime_id = MeerkatMachine::logical_runtime_id(&session_id);
98
99 crate::tokio::spawn(async move {
100 if std::env::var_os("RKAT_TRACE_COMMS_DRAIN_BIND").is_some() {
101 tracing::info!(
102 %session_id,
103 comms_ptr = ?Arc::as_ptr(&comms_runtime),
104 "comms_drain task started"
105 );
106 }
107 let inbox_notify = comms_runtime.inbox_notify();
108 match adapter
109 .peer_ingress_runtime_is_current(&session_id, &comms_runtime)
110 .await
111 {
112 Ok(true) => {}
113 Ok(false) => {
114 tracing::warn!(%session_id, "stale comms drain refused startup");
115 return;
116 }
117 Err(error) => {
118 tracing::warn!(%session_id, %error, "comms drain lost session before startup");
119 return;
120 }
121 }
122 if let Err(error) = async {
123 adapter
124 .stage_local_endpoint_for_comms_runtime(&session_id, comms_runtime.as_ref())
125 .await
126 .map_err(|error| error.to_string())?;
127 hydrate_or_resume_supervisor_rotation_runtime(&adapter, &session_id, &comms_runtime)
128 .await
129 }
130 .await
131 {
132 tracing::warn!(%session_id, %error, "unable to hydrate supervisor authority on comms runtime");
133 }
134 loop {
140 let mut notified = std::pin::pin!(inbox_notify.notified());
150 notified.as_mut().enable();
151
152 let candidates = match comms_runtime.drain_classified_inbox_interactions().await {
160 Ok(candidates) => candidates,
161 Err(err) => {
162 tracing::error!(
166 session_id = %session_id,
167 error = %err,
168 "comms_drain: classified inbox drain failed; exiting via typed Failed terminal (not idle/dismiss)"
169 );
170 if let Err(notify_err) = adapter
171 .notify_comms_drain_exited(&session_id, DrainExitReason::Failed)
172 .await
173 {
174 tracing::error!(
179 session_id = %session_id,
180 error = %notify_err,
181 "comms_drain: NotifyDrainExited(Failed) rejected by machine authority"
182 );
183 adapter
184 .project_comms_drain_failed_safety_net(&session_id)
185 .await;
186 }
187 return;
188 }
189 };
190 if candidates.is_empty() {
191 if crate::tokio::time::timeout(timeout_dur, notified.as_mut())
200 .await
201 .is_err()
202 {
203 tracing::info!("comms_drain: idle timeout expired, stopping");
204 if let Err(notify_err) = adapter
205 .notify_comms_drain_exited(&session_id, DrainExitReason::IdleTimeout)
206 .await
207 {
208 tracing::error!(
212 session_id = %session_id,
213 error = %notify_err,
214 "comms_drain: NotifyDrainExited(IdleTimeout) rejected by machine authority"
215 );
216 adapter
217 .project_comms_drain_failed_safety_net(&session_id)
218 .await;
219 }
220 return;
221 }
222 continue;
223 }
224
225 for candidate in candidates {
227 if try_handle_supervisor_bridge_command(
228 &adapter,
229 &session_id,
230 &comms_runtime,
231 &candidate,
232 )
233 .await
234 {
235 continue;
236 }
237 let candidate_class = candidate.class();
238 match candidate_class {
239 PeerInputClass::Ack => {
240 }
242 PeerInputClass::PeerLifecycleAdded
243 | PeerInputClass::PeerLifecycleRetired
244 | PeerInputClass::PeerLifecycleUnwired => {
245 tracing::debug!(
252 session_id = %session_id,
253 class = ?candidate_class,
254 from_peer_id = ?candidate.from_peer_id(),
255 lifecycle_peer = ?candidate.lifecycle_peer,
256 "comms_drain: consumed silent peer lifecycle notice"
257 );
258 }
259 PeerInputClass::ResponseProgress | PeerInputClass::ResponseTerminal => {
260 let terminal_status = match (
264 candidate_class,
265 candidate.response_terminality,
266 ) {
267 (
268 PeerInputClass::ResponseTerminal,
269 Some(meerkat_core::interaction::TerminalityClass::Terminal {
270 disposition,
271 }),
272 ) => match disposition {
273 meerkat_core::interaction::TerminalDisposition::Completed => {
274 Some(meerkat_core::handles::PeerTerminalDisposition::Completed)
275 }
276 meerkat_core::interaction::TerminalDisposition::Failed => {
277 Some(meerkat_core::handles::PeerTerminalDisposition::Failed)
278 }
279 _ => {
280 reject_peer_response_observation_via_authority(
281 &comms_runtime,
282 &candidate,
283 "unsupported terminal disposition",
284 );
285 tracing::warn!(
286 class = ?candidate_class,
287 disposition = ?disposition,
288 interaction_id = %candidate.interaction.id,
289 "comms_drain: rejected peer response with unsupported terminal disposition"
290 );
291 continue;
292 }
293 },
294 (
295 PeerInputClass::ResponseProgress,
296 Some(meerkat_core::interaction::TerminalityClass::Progress),
297 ) => None,
298 (class, terminality) => {
299 reject_peer_response_observation_via_authority(
300 &comms_runtime,
301 &candidate,
302 "missing or inconsistent machine terminality",
303 );
304 tracing::warn!(
305 class = ?class,
306 terminality = ?terminality,
307 interaction_id = %candidate.interaction.id,
308 "comms_drain: rejected peer response with missing or inconsistent machine terminality"
309 );
310 continue;
311 }
312 };
313 let is_terminal = terminal_status.is_some();
314
315 if is_terminal {
316 let interaction_id = match &candidate.interaction.content {
323 meerkat_core::interaction::InteractionContent::Response {
324 in_reply_to,
325 ..
326 } => *in_reply_to,
327 other => {
328 tracing::warn!(
329 content = ?other,
330 "comms_drain: terminal response candidate missing response content"
331 );
332 continue;
333 }
334 };
335
336 let subscriber = comms_runtime.interaction_subscriber(&interaction_id);
348 let peer_interaction_handle = comms_runtime.peer_interaction_handle();
349 let dsl_installed = peer_interaction_handle.is_some();
350 if let (Some(handle), Some(disposition)) =
351 (peer_interaction_handle.as_ref(), terminal_status)
352 {
353 let corr_id =
354 meerkat_core::PeerCorrelationId::from_uuid(interaction_id.0);
355 if let Err(err) = handle.response_terminal(corr_id, disposition) {
356 tracing::warn!(
357 error = %err,
358 corr_id = %corr_id,
359 "PeerInteractionHandle::response_terminal rejected (no DSL entry — classified drain saw an unknown corr_id)"
360 );
361 reject_peer_response_observation_via_authority(
362 &comms_runtime,
363 &candidate,
364 "generated peer terminal transition rejected",
365 );
366 continue;
367 }
368 }
369
370 if comms_runtime.has_bridge_reply_waiter(&interaction_id) {
380 match comms_runtime.take_bridge_reply_waiter(&interaction_id) {
381 Some(waiter) => {
382 let _ = waiter.send(candidate.clone());
383 }
384 None => {
385 tracing::debug!(
386 interaction_id = %interaction_id,
387 "comms_drain: discarded late bridge reply for tombstoned waiter"
388 );
389 }
390 }
391 comms_runtime.mark_interaction_complete(&interaction_id);
392 continue;
393 }
394
395 let content_input = match classified_interaction_to_runtime_input(
396 &candidate,
397 &runtime_id,
398 ) {
399 Ok(input) => input,
400 Err(err) => {
401 tracing::warn!(
402 error = %err,
403 interaction_id = %interaction_id,
404 "comms_drain: rejected malformed terminal peer ingress"
405 );
406 if dsl_installed {
407 comms_runtime.abandon_interaction_stream(
408 &interaction_id,
409 meerkat_core::InteractionStreamAbandonReason::ResponseRejected,
410 );
411 } else {
412 comms_runtime.mark_interaction_complete(&interaction_id);
413 }
414 continue;
415 }
416 };
417 let result = adapter
418 .accept_input_with_completion(&session_id, content_input)
419 .await;
420 match result {
421 Ok((_outcome, handle)) => {
422 if subscriber.is_some() || handle.is_some() {
423 spawn_completion_bridge(
424 Some(comms_runtime.clone()),
425 interaction_id,
426 subscriber,
427 handle,
428 );
429 } else if !dsl_installed {
430 comms_runtime.mark_interaction_complete(&interaction_id);
431 }
432 }
433 Err(err) => {
434 tracing::warn!(
435 error = %err,
436 "comms_drain: failed to inject terminal response"
437 );
438 if dsl_installed {
439 comms_runtime.abandon_interaction_stream(
440 &interaction_id,
441 meerkat_core::InteractionStreamAbandonReason::ResponseRejected,
442 );
443 } else {
444 comms_runtime.mark_interaction_complete(&interaction_id);
445 }
446 }
447 }
448 } else {
449 if let Some(handle) = comms_runtime.peer_interaction_handle()
454 && let meerkat_core::interaction::InteractionContent::Response {
455 in_reply_to,
456 ..
457 } = &candidate.interaction.content
458 {
459 let corr_id =
460 meerkat_core::PeerCorrelationId::from_uuid(in_reply_to.0);
461 if let Err(err) = handle.response_progress(corr_id) {
462 tracing::warn!(
463 error = %err,
464 corr_id = %corr_id,
465 "PeerInteractionHandle::response_progress rejected"
466 );
467 reject_peer_response_observation_via_authority(
468 &comms_runtime,
469 &candidate,
470 "generated peer progress transition rejected",
471 );
472 continue;
473 }
474 }
475
476 if let meerkat_core::interaction::InteractionContent::Response {
484 in_reply_to,
485 ..
486 } = &candidate.interaction.content
487 && comms_runtime.has_bridge_reply_waiter(in_reply_to)
488 {
489 continue;
490 }
491
492 let input = match classified_interaction_to_runtime_input(
493 &candidate,
494 &runtime_id,
495 ) {
496 Ok(input) => input,
497 Err(err) => {
498 tracing::warn!(
499 error = %err,
500 interaction_id = %candidate.interaction.id,
501 "comms_drain: rejected malformed progress peer ingress"
502 );
503 continue;
504 }
505 };
506 if let Err(err) = adapter.accept_input(&session_id, input).await {
507 tracing::warn!(
508 error = %err,
509 "comms_drain: failed to inject progress response"
510 );
511 }
512 }
513 }
514 PeerInputClass::SilentRequest
515 | PeerInputClass::PeerLifecycleKickoffFailed
516 | PeerInputClass::PeerLifecycleKickoffCancelled
517 | PeerInputClass::ActionableMessage
518 | PeerInputClass::ActionableRequest
519 | PeerInputClass::PlainEvent => {
520 let interaction_id = candidate.interaction.id;
522
523 let fenced_member = if let meerkat_core::interaction::InteractionContent::IncarnationFencedMessage {
524 expected_recipient,
525 ..
526 } = &candidate.interaction.content
527 {
528 let expected_member = BridgeMemberIncarnation {
529 mob_id: expected_recipient.mob_id.clone(),
530 agent_identity: expected_recipient.agent_identity.clone(),
531 host_id: expected_recipient.host_id.clone(),
532 binding_generation: expected_recipient.binding_generation,
533 member_session_id: expected_recipient.member_session_id.clone(),
534 generation: expected_recipient.generation,
535 fence_token: expected_recipient.fence_token,
536 };
537 Some(expected_member)
538 } else {
539 None
540 };
541
542 let is_inbound_peer_request = matches!(
552 candidate_class,
553 PeerInputClass::SilentRequest | PeerInputClass::ActionableRequest
554 );
555 if is_inbound_peer_request {
556 let Some(handle) =
557 comms_runtime.peer_request_response_authority_handle()
558 else {
559 tracing::warn!(
560 interaction_id = %interaction_id,
561 class = ?candidate_class,
562 "comms_drain: rejected inbound peer request without complete peer request authority"
563 );
564 comms_runtime.abandon_interaction_stream(
565 &interaction_id,
566 meerkat_core::InteractionStreamAbandonReason::AdmissionRejected,
567 );
568 continue;
569 };
570 let corr_id =
571 meerkat_core::PeerCorrelationId::from_uuid(interaction_id.0);
572 if handle.inbound_state(corr_id).is_none()
573 && let Err(err) = handle
574 .request_received(corr_id, candidate.interaction.handling_mode)
575 {
576 tracing::warn!(
577 error = %err,
578 corr_id = %corr_id,
579 "PeerInteractionHandle::request_received rejected"
580 );
581 comms_runtime.abandon_interaction_stream(
582 &interaction_id,
583 meerkat_core::InteractionStreamAbandonReason::AdmissionRejected,
584 );
585 continue;
586 }
587 }
588
589 let subscriber = comms_runtime.interaction_subscriber(&interaction_id);
590 let input = match classified_interaction_to_runtime_input(
591 &candidate,
592 &runtime_id,
593 ) {
594 Ok(input) => input,
595 Err(err) => {
596 tracing::warn!(
597 error = %err,
598 interaction_id = %interaction_id,
599 "comms_drain: rejected malformed peer ingress"
600 );
601 comms_runtime.abandon_interaction_stream(
602 &interaction_id,
603 meerkat_core::InteractionStreamAbandonReason::AdmissionRejected,
604 );
605 continue;
606 }
607 };
608 let result = match fenced_member.as_ref() {
609 Some(expected_member) => {
610 adapter
611 .accept_input_with_completion_for_member_residency(
612 &session_id,
613 input,
614 Some(expected_member),
615 )
616 .await
617 }
618 None => {
619 adapter
620 .accept_input_with_completion(&session_id, input)
621 .await
622 }
623 };
624
625 match result {
626 Ok((_outcome, handle)) => {
627 if subscriber.is_some() || handle.is_some() {
628 spawn_completion_bridge(
629 Some(comms_runtime.clone()),
630 interaction_id,
631 subscriber,
632 handle,
633 );
634 } else {
635 comms_runtime.mark_interaction_complete(&interaction_id);
636 }
637 }
638 Err(err) => {
639 tracing::warn!(
640 error = %err,
641 "comms_drain: failed to accept peer input"
642 );
643 comms_runtime.abandon_interaction_stream(
644 &interaction_id,
645 meerkat_core::InteractionStreamAbandonReason::AdmissionRejected,
646 );
647 }
648 }
649 }
650 }
651 }
652 }
653 })
654}
655
656fn reject_peer_response_observation_via_authority(
657 comms_runtime: &Arc<dyn CommsRuntime>,
658 candidate: &PeerInputCandidate,
659 reason: &'static str,
660) {
661 let InteractionContent::Response { in_reply_to, .. } = &candidate.interaction.content else {
662 tracing::warn!(
663 reason = reason,
664 interaction_id = %candidate.interaction.id,
665 "comms_drain: cannot reject malformed peer response observation without response correlation"
666 );
667 return;
668 };
669 let corr_id = meerkat_core::PeerCorrelationId::from_uuid(in_reply_to.0);
670 let Some(handle) = comms_runtime.peer_interaction_handle() else {
671 tracing::warn!(
672 reason = reason,
673 corr_id = %corr_id,
674 interaction_id = %candidate.interaction.id,
675 "comms_drain: malformed peer response observation has no peer-interaction authority"
676 );
677 return;
678 };
679 if let Err(err) = handle.response_rejected(corr_id) {
680 tracing::warn!(
681 error = %err,
682 reason = reason,
683 corr_id = %corr_id,
684 interaction_id = %candidate.interaction.id,
685 "PeerInteractionHandle::response_rejected rejected malformed peer response observation"
686 );
687 return;
688 }
689 comms_runtime.abandon_interaction_stream(
690 in_reply_to,
691 meerkat_core::InteractionStreamAbandonReason::ResponseRejected,
692 );
693}
694
695fn bridge_peer_identity(
696 peer: &BridgePeerSpec,
697 context: &str,
698) -> Result<BridgePeerIdentity, (BridgeRejectionCause, String)> {
699 BridgePeerIdentity::try_from(peer).map_err(|error| {
700 (
701 BridgeRejectionCause::InvalidSupervisorSpec,
702 format!("{context}: invalid supervisor peer spec: {error}"),
703 )
704 })
705}
706
707fn sender_peer_label(sender: &PeerIngressFact) -> String {
708 sender.diagnostic_label()
709}
710
711fn sender_matches_bridge_peer(sender: &PeerIngressFact, peer: &BridgePeerIdentity) -> bool {
712 sender
713 .canonical_peer_id
714 .is_some_and(|sender_peer_id| sender_peer_id == peer.peer_id)
715 || (!peer.pubkey.is_zero() && sender.signing_pubkey == Some(*peer.pubkey.as_bytes()))
716}
717
718fn generated_sender_peer_id(sender: &PeerIngressFact) -> Option<String> {
719 sender.canonical_peer_id.map(|peer_id| peer_id.as_str())
720}
721
722fn generated_sender_signing_public_key(sender: &PeerIngressFact) -> Option<String> {
723 sender
724 .signing_pubkey
725 .map(encode_supervisor_signing_public_key)
726}
727
728fn supervisor_bind_admission_rejection_cause(
729 rejection: crate::meerkat_machine::dsl::SupervisorBindRejectionKind,
730) -> BridgeRejectionCause {
731 match rejection {
732 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::AlreadyBound => {
733 BridgeRejectionCause::AlreadyBound
734 }
735 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::SenderMismatch => {
736 BridgeRejectionCause::SenderMismatch
737 }
738 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::RevocationPending => {
739 BridgeRejectionCause::AlreadyBound
740 }
741 }
742}
743
744fn supervisor_bind_admission_rejection_reason(
745 rejection: crate::meerkat_machine::dsl::SupervisorBindRejectionKind,
746 sender: &PeerIngressFact,
747 supervisor: &BridgePeerIdentity,
748) -> String {
749 match rejection {
750 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::AlreadyBound => {
751 "bind member admission rejected by MeerkatMachine: supervisor already bound; use authorize_supervisor to rotate"
752 .to_string()
753 }
754 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::SenderMismatch => {
755 let sender_label = sender_peer_label(sender);
756 format!(
757 "bind member admission rejected by MeerkatMachine: request sender '{sender_label}' does not match authorized supervisor '{}'",
758 supervisor.peer_id
759 )
760 }
761 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::RevocationPending => {
762 "bind member admission rejected by MeerkatMachine: prior supervisor revocation is still pending; retry revoke_supervisor before binding"
763 .to_string()
764 }
765 }
766}
767
768fn supervisor_authorize_admission_rejection_cause(
769 rejection: crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind,
770) -> BridgeRejectionCause {
771 match rejection {
772 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::NotBound => {
773 BridgeRejectionCause::NotBound
774 }
775 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::StaleSupervisor => {
776 BridgeRejectionCause::StaleSupervisor
777 }
778 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::SenderMismatch => {
779 BridgeRejectionCause::SenderMismatch
780 }
781 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::RotationNotAllowed => {
782 BridgeRejectionCause::Unsupported
783 }
784 }
785}
786
787fn supervisor_authorize_admission_rejection_reason(
788 rejection: crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind,
789 sender: &PeerIngressFact,
790 supervisor: &BridgePeerIdentity,
791) -> String {
792 match rejection {
793 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::NotBound => {
794 "authorize supervisor admission rejected by MeerkatMachine: use bind_member to establish initial supervisor authority"
795 .to_string()
796 }
797 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::StaleSupervisor => {
798 "authorize supervisor admission rejected by MeerkatMachine: stale supervisor binding"
799 .to_string()
800 }
801 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::SenderMismatch => {
802 let sender_label = sender_peer_label(sender);
803 format!(
804 "authorize supervisor admission rejected by MeerkatMachine: request sender '{sender_label}' does not match authorized supervisor '{}'",
805 supervisor.peer_id
806 )
807 }
808 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::RotationNotAllowed => {
809 "authorize supervisor admission rejected by MeerkatMachine: supervisor rotation is not allowed after the member runtime has stopped accepting ordinary commands"
810 .to_string()
811 }
812 }
813}
814
815fn supervisor_bridge_admission_rejection_cause(
816 rejection: crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind,
817) -> BridgeRejectionCause {
818 match rejection {
819 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::NotBound => {
820 BridgeRejectionCause::NotBound
821 }
822 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::StaleSupervisor => {
823 BridgeRejectionCause::StaleSupervisor
824 }
825 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::SenderMismatch => {
826 BridgeRejectionCause::SenderMismatch
827 }
828 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::CommandNotAllowed => {
829 BridgeRejectionCause::Unsupported
830 }
831 }
832}
833
834fn supervisor_bridge_admission_rejection_reason(
835 rejection: crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind,
836 sender: &PeerIngressFact,
837 supervisor: &BridgePeerIdentity,
838) -> String {
839 match rejection {
840 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::NotBound => {
841 "supervisor bridge admission rejected by MeerkatMachine: no authorized supervisor registered"
842 .to_string()
843 }
844 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::StaleSupervisor => {
845 "supervisor bridge admission rejected by MeerkatMachine: stale supervisor binding"
846 .to_string()
847 }
848 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::SenderMismatch => {
849 let sender_label = sender_peer_label(sender);
850 format!(
851 "supervisor bridge admission rejected by MeerkatMachine: request sender '{sender_label}' does not match authorized supervisor '{}'",
852 supervisor.peer_id
853 )
854 }
855 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::CommandNotAllowed => {
856 "supervisor bridge admission rejected by MeerkatMachine: command is not allowed in the current runtime phase"
857 .to_string()
858 }
859 }
860}
861
862struct BridgeMobPeerOverlay {
863 endpoints: BTreeSet<crate::meerkat_machine::dsl::PeerEndpoint>,
864 endpoint_count: u64,
865}
866
867fn bridge_mob_peer_overlay(
868 handoff: &BridgeMobPeerOverlayHandoff,
869) -> Result<BridgeMobPeerOverlay, String> {
870 let mut endpoints = BTreeSet::new();
871 let endpoint_count = u64::try_from(handoff.peer_specs.len())
872 .map_err(|_| "MobMachine peer overlay contains too many endpoints".to_string())?;
873 for peer_spec in &handoff.peer_specs {
874 let peer = TrustedPeerDescriptor::try_from(peer_spec)?;
875 let endpoint = crate::meerkat_machine::dsl::PeerEndpoint::from(&peer);
876 endpoints.insert(endpoint);
877 }
878 Ok(BridgeMobPeerOverlay {
879 endpoints,
880 endpoint_count,
881 })
882}
883
884async fn resolve_authorized_supervisor(
889 adapter: &MeerkatMachine,
890 session_id: &SessionId,
891 sender: &PeerIngressFact,
892 payload: &BridgeSupervisorPayload,
893) -> Result<BridgePeerIdentity, (BridgeRejectionCause, String)> {
894 let supervisor = bridge_peer_identity(&payload.supervisor, "supervisor bridge request")?;
895 let sender_peer_id = sender
896 .canonical_peer_id
897 .map(|sender_peer_id| sender_peer_id.to_string());
898 match adapter
899 .resolve_supervisor_bridge_command_admission(
900 session_id,
901 supervisor.peer_id.to_string(),
902 payload.epoch,
903 sender_peer_id,
904 )
905 .await
906 {
907 Ok(crate::meerkat_machine::SupervisorBridgeCommandAdmission::Accepted) => Ok(supervisor),
908 Ok(crate::meerkat_machine::SupervisorBridgeCommandAdmission::ResumePendingRevoke) => Err((
909 BridgeRejectionCause::Internal,
910 "ordinary supervisor admission returned a cleanup-only pending-revoke disposition"
911 .to_string(),
912 )),
913 Ok(crate::meerkat_machine::SupervisorBridgeCommandAdmission::Rejected(rejection)) => Err((
914 supervisor_bridge_admission_rejection_cause(rejection),
915 supervisor_bridge_admission_rejection_reason(rejection, sender, &supervisor),
916 )),
917 Err(error) => Err((
918 BridgeRejectionCause::Internal,
919 format!("supervisor bridge admission failed: {error}"),
920 )),
921 }
922}
923
924fn supervisor_cleanup_command_label(
925 command_kind: crate::meerkat_machine::dsl::SupervisorCleanupCommandKind,
926) -> &'static str {
927 match command_kind {
928 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Retire => "retire_member",
929 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Observe => "observe_member",
930 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Destroy => "destroy_member",
931 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Revoke => "revoke_supervisor",
932 }
933}
934
935struct AuthorizedSupervisorCleanup {
939 supervisor: BridgePeerIdentity,
940 resume_pending_revoke: bool,
941}
942
943async fn resolve_authorized_supervisor_cleanup(
944 adapter: &MeerkatMachine,
945 session_id: &SessionId,
946 sender: &PeerIngressFact,
947 payload: &BridgeSupervisorPayload,
948 command_kind: crate::meerkat_machine::dsl::SupervisorCleanupCommandKind,
949) -> Result<AuthorizedSupervisorCleanup, (BridgeRejectionCause, String)> {
950 let supervisor = bridge_peer_identity(&payload.supervisor, "supervisor cleanup request")?;
951 let sender_peer_id = sender
952 .canonical_peer_id
953 .map(|sender_peer_id| sender_peer_id.to_string());
954 match adapter
955 .resolve_supervisor_cleanup_command_admission(
956 session_id,
957 command_kind,
958 supervisor.peer_id.to_string(),
959 payload.epoch,
960 sender_peer_id,
961 )
962 .await
963 {
964 Ok((crate::meerkat_machine::SupervisorBridgeCommandAdmission::Accepted, _)) => {
965 Ok(AuthorizedSupervisorCleanup {
966 supervisor,
967 resume_pending_revoke: false,
968 })
969 }
970 Ok((crate::meerkat_machine::SupervisorBridgeCommandAdmission::ResumePendingRevoke, _)) => {
971 Ok(AuthorizedSupervisorCleanup {
972 supervisor,
973 resume_pending_revoke: true,
974 })
975 }
976 Ok((
977 crate::meerkat_machine::SupervisorBridgeCommandAdmission::Rejected(rejection),
978 phase,
979 )) => {
980 let reason = if rejection
981 == crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::CommandNotAllowed
982 {
983 format!(
984 "supervisor cleanup command '{}' is not allowed while the member runtime is {phase:?}",
985 supervisor_cleanup_command_label(command_kind)
986 )
987 } else {
988 supervisor_bridge_admission_rejection_reason(rejection, sender, &supervisor)
989 };
990 Err((
991 supervisor_bridge_admission_rejection_cause(rejection),
992 reason,
993 ))
994 }
995 Err(error) => Err((
996 BridgeRejectionCause::Internal,
997 format!("supervisor cleanup admission failed: {error}"),
998 )),
999 }
1000}
1001
1002async fn resolve_authorized_supervisor_with_response_route(
1007 adapter: &Arc<MeerkatMachine>,
1008 session_id: &SessionId,
1009 comms_runtime: &Arc<dyn CommsRuntime>,
1010 sender: &PeerIngressFact,
1011 payload: &BridgeSupervisorPayload,
1012 context: &str,
1013) -> Result<BridgePeerIdentity, (BridgeRejectionCause, String)> {
1014 let supervisor = resolve_authorized_supervisor(adapter, session_id, sender, payload).await?;
1015 let response_peer = supervisor.clone().into_trusted_peer_descriptor();
1016 refresh_and_publish_bound_supervisor_route(
1017 adapter,
1018 session_id,
1019 comms_runtime.as_ref(),
1020 &response_peer,
1021 payload.epoch,
1022 generated_sender_peer_id(sender),
1023 context,
1024 )
1025 .await
1026 .map_err(|reason| (BridgeRejectionCause::Internal, reason))?;
1027 Ok(supervisor)
1028}
1029
1030async fn resolve_authorized_supervisor_cleanup_with_response_route(
1031 adapter: &Arc<MeerkatMachine>,
1032 session_id: &SessionId,
1033 comms_runtime: &Arc<dyn CommsRuntime>,
1034 sender: &PeerIngressFact,
1035 payload: &BridgeSupervisorPayload,
1036 command_kind: crate::meerkat_machine::dsl::SupervisorCleanupCommandKind,
1037 context: &str,
1038) -> Result<BridgePeerIdentity, (BridgeRejectionCause, String)> {
1039 let supervisor =
1040 resolve_authorized_supervisor_cleanup(adapter, session_id, sender, payload, command_kind)
1041 .await?;
1042 if supervisor.resume_pending_revoke {
1043 return Err((
1044 BridgeRejectionCause::Internal,
1045 format!("{context}: pending revoke disposition is invalid for this command"),
1046 ));
1047 }
1048 let response_peer = supervisor.supervisor.clone().into_trusted_peer_descriptor();
1049 refresh_and_publish_bound_supervisor_route(
1050 adapter,
1051 session_id,
1052 comms_runtime.as_ref(),
1053 &response_peer,
1054 payload.epoch,
1055 generated_sender_peer_id(sender),
1056 context,
1057 )
1058 .await
1059 .map_err(|reason| (BridgeRejectionCause::Internal, reason))?;
1060 Ok(supervisor.supervisor)
1061}
1062
1063fn bridge_capabilities(
1064 origin_protocol: meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion,
1065) -> BridgeCapabilities {
1066 let mut capabilities = BridgeCapabilities {
1067 deliver_member_input: true,
1068 observe_member: true,
1069 interrupt_member: true,
1070 hard_cancel_member: origin_protocol.supports_multi_host(),
1073 tracked_input_cancel: origin_protocol.supports_multi_host(),
1074 retire_member: true,
1075 destroy_member: true,
1076 wire_member: true,
1077 unwire_member: true,
1078 ..BridgeCapabilities::default()
1079 };
1080 if !origin_protocol.supports_multi_host() {
1081 capabilities.current_protocol_version = origin_protocol;
1082 capabilities.default_protocol_version = origin_protocol;
1083 capabilities.supported_protocol_versions = match origin_protocol {
1084 version
1085 if version
1086 == meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V2 =>
1087 {
1088 vec![meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V2]
1089 }
1090 _ => vec![
1091 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V2,
1092 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V3,
1093 ],
1094 };
1095 }
1096 capabilities
1097}
1098
1099fn peer_input_from_delivery_payload(
1100 session_id: &SessionId,
1101 sender_peer_id: PeerId,
1102 payload: BridgeDeliveryPayload,
1103) -> Result<Input, String> {
1104 if payload.turn.is_some() {
1105 return Err(
1106 "directive-bearing member delivery must use the tracked flow-step lowering path"
1107 .to_string(),
1108 );
1109 }
1110 let stable_uuid = canonical_non_nil_delivery_uuid("input_id", &payload.input_id)?;
1111 let transcript_uuid = payload
1112 .transcript_interaction_id
1113 .as_deref()
1114 .map(|raw| canonical_non_nil_delivery_uuid("transcript_interaction_id", raw))
1115 .transpose()?;
1116 let is_placed = payload.expected_member.is_some();
1117 let directed_interaction_id = delivery_requests_tracked_interaction(&payload)
1118 .then_some(meerkat_core::interaction::InteractionId(stable_uuid));
1119 Ok(Input::Peer(PeerInput {
1120 directed_interaction_id,
1121 objective_id: payload.objective_id,
1122 header: InputHeader {
1123 id: meerkat_core::lifecycle::InputId::from_uuid(stable_uuid),
1124 timestamp: chrono::Utc::now(),
1125 source: InputOrigin::Peer {
1126 peer_id: sender_peer_id.as_str(),
1127 display_identity: Some(sender_peer_id.as_str()),
1128 runtime_id: Some(MeerkatMachine::logical_runtime_id(session_id)),
1129 },
1130 durability: InputDurability::Durable,
1131 visibility: InputVisibility {
1132 transcript_eligible: true,
1133 operator_eligible: true,
1134 },
1135 idempotency_key: Some(IdempotencyKey::new(payload.input_id)),
1136 supersession_key: None,
1137 correlation_id: transcript_uuid
1142 .or_else(|| (!is_placed).then_some(stable_uuid))
1143 .map(crate::identifiers::CorrelationId::from_uuid),
1144 },
1145 convention: Some(PeerConvention::Message),
1146 content: payload.content,
1147 payload: None,
1148 handling_mode: Some(payload.handling_mode),
1149 sender_taint: None,
1152 injected_context: payload.injected_context,
1156 }))
1157}
1158
1159fn delivery_requests_tracked_interaction(payload: &BridgeDeliveryPayload) -> bool {
1164 match payload.outcome_tracking {
1165 None => false,
1166 Some(BridgeOutcomeTracking::Interaction) => true,
1167 }
1168}
1169
1170fn delivery_requires_tracked_turn_custody(payload: &BridgeDeliveryPayload) -> bool {
1171 payload.turn.is_some() || delivery_requests_tracked_interaction(payload)
1172}
1173
1174fn validate_delivery_tracking_request(payload: &BridgeDeliveryPayload) -> Result<(), String> {
1175 canonical_non_nil_delivery_uuid("input_id", &payload.input_id)?;
1176 if payload.turn.is_some() && payload.outcome_tracking.is_some() {
1177 return Err(
1178 "turn and outcome_tracking are mutually exclusive terminal-custody requests"
1179 .to_string(),
1180 );
1181 }
1182 if payload.outcome_tracking.is_some() && !payload.protocol_version.supports_multi_host() {
1183 return Err(
1184 "outcome_tracking requires a V4 supervisor bridge protocol version".to_string(),
1185 );
1186 }
1187 if payload.outcome_tracking.is_some() && payload.expected_member.is_none() {
1188 return Err("outcome_tracking requires an exact placed-member incarnation".to_string());
1189 }
1190 if let Some(transcript_id) = payload.transcript_interaction_id.as_deref() {
1191 if !payload.protocol_version.supports_multi_host() {
1192 return Err(
1193 "transcript_interaction_id requires a V4 supervisor bridge protocol version"
1194 .to_string(),
1195 );
1196 }
1197 if payload.expected_member.is_none() {
1198 return Err(
1199 "transcript_interaction_id requires an exact placed-member incarnation".to_string(),
1200 );
1201 }
1202 if payload.turn.is_some() {
1203 return Err(
1204 "turn and transcript_interaction_id are mutually exclusive identity carriers"
1205 .to_string(),
1206 );
1207 }
1208 canonical_non_nil_delivery_uuid("transcript_interaction_id", transcript_id)?;
1209 }
1210 if payload.outcome_tracking == Some(BridgeOutcomeTracking::Interaction)
1211 && payload.transcript_interaction_id.as_deref() != Some(payload.input_id.as_str())
1212 {
1213 return Err(
1214 "tracked interaction requires transcript_interaction_id to equal input_id".to_string(),
1215 );
1216 }
1217 Ok(())
1218}
1219
1220fn canonical_non_nil_delivery_uuid(field: &str, raw: &str) -> Result<uuid::Uuid, String> {
1221 let parsed = uuid::Uuid::parse_str(raw)
1222 .map_err(|error| format!("delivery {field} '{raw}' is not a UUID: {error}"))?;
1223 if parsed.is_nil() || parsed.to_string() != raw {
1224 return Err(format!(
1225 "delivery {field} '{raw}' must be a canonical non-nil UUID"
1226 ));
1227 }
1228 Ok(parsed)
1229}
1230
1231fn bridge_delivery_rejection_cause(
1232 reason: &crate::accept::RejectReason,
1233) -> BridgeDeliveryRejectionCause {
1234 match reason {
1235 crate::accept::RejectReason::NotReady { state } => BridgeDeliveryRejectionCause::NotReady {
1236 state: runtime_state_to_bridge(*state),
1237 },
1238 crate::accept::RejectReason::DurabilityViolation { detail } => {
1239 BridgeDeliveryRejectionCause::DurabilityViolation {
1240 detail: detail.clone(),
1241 }
1242 }
1243 crate::accept::RejectReason::PeerHandlingModeInvalid { detail } => {
1244 BridgeDeliveryRejectionCause::PeerHandlingModeInvalid {
1245 detail: detail.clone(),
1246 }
1247 }
1248 crate::accept::RejectReason::PeerResponseTerminalInvalid { detail } => {
1249 BridgeDeliveryRejectionCause::Internal {
1250 detail: detail.clone(),
1251 }
1252 }
1253 }
1254}
1255
1256fn advertised_bind_bootstrap_token(
1257 comms_runtime: &Arc<dyn CommsRuntime>,
1258) -> Result<String, (BridgeRejectionCause, String)> {
1259 if let Some(token) = comms_runtime.bridge_bootstrap_token()
1260 && !token.is_empty()
1261 {
1262 return Ok(token);
1263 }
1264 Err((
1265 BridgeRejectionCause::InvalidBootstrapToken,
1266 "runtime does not expose a typed bridge bootstrap token".to_string(),
1267 ))
1268}
1269
1270async fn validate_bind_request(
1286 adapter: &Arc<MeerkatMachine>,
1287 session_id: &SessionId,
1288 comms_runtime: &Arc<dyn CommsRuntime>,
1289 sender: &PeerIngressFact,
1290 payload: &meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload,
1291) -> Result<(TrustedPeerDescriptor, String), (BridgeRejectionCause, String)> {
1292 let expected_bootstrap_token = advertised_bind_bootstrap_token(comms_runtime)?;
1293 let advertised_address = comms_runtime.advertised_address().ok_or_else(|| {
1294 (
1295 BridgeRejectionCause::Internal,
1296 "runtime does not expose an advertised address for bind bootstrap".to_string(),
1297 )
1298 })?;
1299 let supervisor = bridge_peer_identity(&payload.supervisor, "bind member failed")?;
1300 let Some(actual_peer_id) = comms_runtime.peer_id().map(|peer_id| peer_id.as_str()) else {
1305 return Err((
1306 BridgeRejectionCause::Internal,
1307 "bind member failed: runtime peer_id unavailable".to_string(),
1308 ));
1309 };
1310
1311 let address_matches = canonicalize_bridge_address(&payload.expected_address)
1314 == canonicalize_bridge_address(&advertised_address);
1315 let sender_matches_supervisor = sender_matches_bridge_peer(sender, &supervisor);
1316 let expected_peer_id_matches = actual_peer_id == payload.expected_peer_id;
1317 let bootstrap_token_matches = payload.bootstrap_token.as_str() == expected_bootstrap_token;
1318
1319 let verdict = adapter
1320 .resolve_supervisor_bind_material_admission(
1321 session_id,
1322 address_matches,
1323 sender_matches_supervisor,
1324 expected_peer_id_matches,
1325 bootstrap_token_matches,
1326 )
1327 .await
1328 .map_err(|error| {
1329 (
1330 BridgeRejectionCause::Internal,
1331 format!("bind member material admission failed: {error}"),
1332 )
1333 })?;
1334
1335 match verdict {
1336 crate::meerkat_machine::dsl::SupervisorBindMaterialAdmissionKind::Accept => Ok((
1337 supervisor.into_trusted_peer_descriptor(),
1338 advertised_address,
1339 )),
1340 crate::meerkat_machine::dsl::SupervisorBindMaterialAdmissionKind::AddressMismatch => Err((
1341 BridgeRejectionCause::AddressMismatch,
1342 format!(
1343 "bind address mismatch: expected '{}', actual '{}'",
1344 payload.expected_address, advertised_address
1345 ),
1346 )),
1347 crate::meerkat_machine::dsl::SupervisorBindMaterialAdmissionKind::SenderMismatch => {
1348 let sender_label = sender_peer_label(sender);
1349 Err((
1350 BridgeRejectionCause::SenderMismatch,
1351 format!(
1352 "request sender '{sender_label}' does not match supervisor '{}'",
1353 supervisor.peer_id
1354 ),
1355 ))
1356 }
1357 crate::meerkat_machine::dsl::SupervisorBindMaterialAdmissionKind::InvalidPeerSpec => Err((
1358 BridgeRejectionCause::InvalidPeerSpec,
1359 format!(
1360 "bind peer_id mismatch: expected '{}', actual '{actual_peer_id}'",
1361 payload.expected_peer_id
1362 ),
1363 )),
1364 crate::meerkat_machine::dsl::SupervisorBindMaterialAdmissionKind::InvalidBootstrapToken => {
1365 Err((
1366 BridgeRejectionCause::InvalidBootstrapToken,
1367 "bind member failed: invalid bootstrap token".to_string(),
1368 ))
1369 }
1370 }
1371}
1372
1373#[derive(Clone, Debug)]
1374struct BoundSupervisorState {
1375 name: String,
1376 peer_id: String,
1377 address: String,
1378 signing_public_key: String,
1379 epoch: u64,
1380}
1381
1382impl BoundSupervisorState {
1383 fn from_binding(binding: &SupervisorBinding) -> Option<Self> {
1386 let SupervisorBinding::Bound {
1387 name,
1388 peer_id,
1389 address,
1390 signing_public_key,
1391 epoch,
1392 } = binding
1393 else {
1394 return None;
1395 };
1396 Some(Self {
1397 name: name.clone(),
1398 peer_id: peer_id.clone(),
1399 address: address.clone(),
1400 signing_public_key: signing_public_key.clone(),
1401 epoch: *epoch,
1402 })
1403 }
1404}
1405
1406pub fn encode_supervisor_signing_public_key(pubkey: [u8; 32]) -> String {
1407 const HEX: &[u8; 16] = b"0123456789abcdef";
1408 let mut encoded = String::with_capacity(64);
1409 for byte in pubkey {
1410 encoded.push(HEX[(byte >> 4) as usize] as char);
1411 encoded.push(HEX[(byte & 0x0f) as usize] as char);
1412 }
1413 encoded
1414}
1415
1416pub fn decode_supervisor_signing_public_key(encoded: &str) -> Result<[u8; 32], String> {
1417 fn hex_nibble(byte: u8) -> Option<u8> {
1418 match byte {
1419 b'0'..=b'9' => Some(byte - b'0'),
1420 b'a'..=b'f' => Some(byte - b'a' + 10),
1421 b'A'..=b'F' => Some(byte - b'A' + 10),
1422 _ => None,
1423 }
1424 }
1425
1426 let bytes = encoded.as_bytes();
1427 if bytes.len() != 64 {
1428 return Err(format!(
1429 "supervisor signing public key must be 64 hex characters, got {}",
1430 bytes.len()
1431 ));
1432 }
1433 let mut decoded = [0u8; 32];
1434 for (index, chunk) in bytes.chunks_exact(2).enumerate() {
1435 let high = hex_nibble(chunk[0]).ok_or_else(|| {
1436 format!(
1437 "invalid supervisor signing public key hex at byte {}",
1438 index * 2
1439 )
1440 })?;
1441 let low = hex_nibble(chunk[1]).ok_or_else(|| {
1442 format!(
1443 "invalid supervisor signing public key hex at byte {}",
1444 index * 2 + 1
1445 )
1446 })?;
1447 decoded[index] = (high << 4) | low;
1448 }
1449 Ok(decoded)
1450}
1451
1452pub fn trusted_peer_descriptor_from_supervisor_publish_obligation(
1453 obligation: &crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
1454) -> Result<TrustedPeerDescriptor, String> {
1455 let signing_public_key = obligation.signing_public_key().as_ref().ok_or_else(|| {
1456 "generated supervisor trust publish obligation omitted signing public key".to_string()
1457 })?;
1458 let pubkey = decode_supervisor_signing_public_key(signing_public_key)?;
1459 TrustedPeerDescriptor::unsigned_with_pubkey(
1460 obligation.name().clone(),
1461 obligation.peer_id().clone(),
1462 pubkey,
1463 obligation.address().clone(),
1464 )
1465}
1466
1467async fn single_supervisor_publish_obligation(
1468 adapter: &MeerkatMachine,
1469 session_id: &SessionId,
1470 transition: &crate::meerkat_machine::dsl::MeerkatMachineTransition,
1471 context: &str,
1472) -> Result<crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation, String> {
1473 let freshness_authority = adapter
1474 .supervisor_trust_publish_freshness_authority(session_id)
1475 .await
1476 .map_err(|error| {
1477 format!("{context}: generated supervisor publish freshness unavailable: {error}")
1478 })?;
1479 let obligations = crate::protocol_supervisor_trust_publish::extract_obligations_with_freshness(
1480 transition,
1481 freshness_authority,
1482 );
1483 match obligations.as_slice() {
1484 [obligation] => Ok(obligation.clone()),
1485 [] => Err(format!("{context}: generated publish effect was absent")),
1486 _ => Err(format!(
1487 "{context}: generated multiple supervisor trust publish effects"
1488 )),
1489 }
1490}
1491
1492async fn matching_supervisor_revoke_obligation(
1493 adapter: &Arc<MeerkatMachine>,
1494 session_id: &SessionId,
1495 transition: &crate::meerkat_machine::dsl::MeerkatMachineTransition,
1496 peer_id: &str,
1497 epoch: u64,
1498 context: &str,
1499) -> Result<crate::protocol_supervisor_trust_revoke::SupervisorTrustRevokeObligation, String> {
1500 let freshness_authority = adapter
1501 .supervisor_trust_revoke_freshness_authority(session_id)
1502 .await
1503 .map_err(|error| {
1504 format!("{context}: generated supervisor revoke freshness unavailable: {error}")
1505 })?;
1506 crate::protocol_supervisor_trust_revoke::extract_obligations_with_freshness(
1507 transition,
1508 freshness_authority,
1509 )
1510 .into_iter()
1511 .find(|obligation| obligation.peer_id().as_str() == peer_id && obligation.epoch() == epoch)
1512 .ok_or_else(|| format!("{context}: generated revoke effect was absent"))
1513}
1514
1515fn validate_supervisor_publish_obligation(
1516 obligation: &crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
1517 expected: &TrustedPeerDescriptor,
1518 expected_epoch: u64,
1519 context: &str,
1520) -> Result<(), String> {
1521 let expected_signing_key = encode_supervisor_signing_public_key(expected.pubkey);
1522 if obligation.name() != expected.name.as_str()
1523 || obligation.peer_id().as_str() != expected.peer_id.as_str()
1524 || obligation.address().as_str() != expected.address.to_string()
1525 || obligation.signing_public_key().as_deref() != Some(expected_signing_key.as_str())
1526 || obligation.epoch() != expected_epoch
1527 {
1528 return Err(format!(
1529 "{context}: generated publish obligation did not match the staged supervisor binding"
1530 ));
1531 }
1532 Ok(())
1533}
1534
1535async fn apply_generated_private_trust_add(
1536 comms_runtime: &dyn CommsRuntime,
1537 peer: TrustedPeerDescriptor,
1538 authority: CommsTrustMutationAuthority,
1539) -> Result<(), String> {
1540 match comms_runtime
1541 .apply_trust_mutation(CommsTrustMutation::AddPrivateTrustedPeer { peer, authority })
1542 .await
1543 .map_err(|error| error.to_string())?
1544 {
1545 CommsTrustMutationResult::Added { .. } => Ok(()),
1546 other => Err(format!(
1547 "generated trust add returned unexpected result: {other:?}"
1548 )),
1549 }
1550}
1551
1552async fn apply_generated_private_trust_remove(
1553 comms_runtime: &dyn CommsRuntime,
1554 peer_id: String,
1555 authority: CommsTrustMutationAuthority,
1556) -> Result<bool, String> {
1557 match comms_runtime
1558 .apply_trust_mutation(CommsTrustMutation::RemovePrivateTrustedPeer { peer_id, authority })
1559 .await
1560 .map_err(|error| error.to_string())?
1561 {
1562 CommsTrustMutationResult::Removed { removed } => Ok(removed),
1563 other => Err(format!(
1564 "generated trust remove returned unexpected result: {other:?}"
1565 )),
1566 }
1567}
1568
1569fn supervisor_publish_authority(
1570 obligation: &crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
1571) -> Result<CommsTrustMutationAuthority, String> {
1572 crate::protocol_supervisor_trust_publish::publish_authority_for_peer(
1573 obligation,
1574 obligation.peer_id(),
1575 )
1576}
1577
1578fn supervisor_publish_cleanup_authority(
1579 obligation: &crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
1580) -> Result<CommsTrustMutationAuthority, String> {
1581 crate::protocol_supervisor_trust_publish::cleanup_authority_for_peer(
1582 obligation,
1583 obligation.peer_id(),
1584 )
1585}
1586
1587fn supervisor_revoke_authority(
1588 obligation: &crate::protocol_supervisor_trust_revoke::SupervisorTrustRevokeObligation,
1589) -> Result<CommsTrustMutationAuthority, String> {
1590 crate::protocol_supervisor_trust_revoke::revoke_authority_for_peer(
1591 obligation,
1592 obligation.peer_id(),
1593 )
1594}
1595
1596async fn publish_supervisor_trust_from_generated_obligation(
1597 adapter: &MeerkatMachine,
1598 session_id: &SessionId,
1599 comms_runtime: &dyn CommsRuntime,
1600 obligation: &crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
1601) -> Result<(), String> {
1602 let trusted_peer = trusted_peer_descriptor_from_supervisor_publish_obligation(obligation)?;
1603 let failure_cleanup_obligation = supervisor_route_publish_obligation(
1613 adapter,
1614 session_id,
1615 &trusted_peer,
1616 "supervisor trust route replacement failure cleanup",
1617 )
1618 .await?;
1619 let replacement_cleanup_authority = supervisor_publish_cleanup_authority(obligation)?;
1620 let failure_cleanup_authority =
1621 supervisor_publish_cleanup_authority(&failure_cleanup_obligation)?;
1622 apply_generated_private_trust_remove(
1623 comms_runtime,
1624 obligation.peer_id().clone(),
1625 replacement_cleanup_authority,
1626 )
1627 .await?;
1628 apply_generated_private_trust_add(
1629 comms_runtime,
1630 trusted_peer,
1631 supervisor_publish_authority(obligation)?,
1632 )
1633 .await?;
1634 if let Err(error) = adapter
1635 .stage_supervisor_trust_published(
1636 session_id,
1637 obligation.peer_id().clone(),
1638 obligation.epoch(),
1639 )
1640 .await
1641 {
1642 let cleanup_result = apply_generated_private_trust_remove(
1643 comms_runtime,
1644 obligation.peer_id().clone(),
1645 failure_cleanup_authority,
1646 )
1647 .await;
1648 let mut reason = format!("generated trust publish ack rejected: {error}");
1649 if let Err(cleanup_error) = cleanup_result {
1650 reason.push_str(&format!("; cleanup remove failed: {cleanup_error}"));
1651 }
1652 return Err(reason);
1653 }
1654 Ok(())
1655}
1656
1657async fn refresh_and_publish_bound_supervisor_route(
1658 adapter: &MeerkatMachine,
1659 session_id: &SessionId,
1660 comms_runtime: &dyn CommsRuntime,
1661 descriptor: &TrustedPeerDescriptor,
1662 epoch: u64,
1663 sender_peer_id: Option<String>,
1664 context: &str,
1665) -> Result<(), String> {
1666 adapter
1667 .stage_local_endpoint_for_comms_runtime(session_id, comms_runtime)
1668 .await
1669 .map_err(|error| format!("{context}: local endpoint rejected: {error}"))?;
1670 let transition = adapter
1671 .stage_supervisor_binding_route_refresh(
1672 session_id,
1673 GeneratedSupervisorBinding {
1674 name: descriptor.name.as_str().to_owned(),
1675 peer_id: descriptor.peer_id.as_str(),
1676 address: descriptor.address.to_string(),
1677 signing_public_key: encode_supervisor_signing_public_key(descriptor.pubkey),
1678 epoch,
1679 },
1680 sender_peer_id,
1681 )
1682 .await
1683 .map_err(|error| format!("{context}: route refresh rejected: {error}"))?;
1684 let obligation =
1685 single_supervisor_publish_obligation(adapter, session_id, &transition, context).await?;
1686 validate_supervisor_publish_obligation(&obligation, descriptor, epoch, context)?;
1687 publish_supervisor_trust_from_generated_obligation(
1688 adapter,
1689 session_id,
1690 comms_runtime,
1691 &obligation,
1692 )
1693 .await
1694}
1695
1696async fn republish_private_supervisor_trust(
1707 adapter: &MeerkatMachine,
1708 session_id: &SessionId,
1709 comms_runtime: &dyn CommsRuntime,
1710 supervisor: &TrustedPeerDescriptor,
1711 epoch: u64,
1712 sender_peer_id: Option<String>,
1713) -> Result<(), String> {
1714 refresh_and_publish_bound_supervisor_route(
1715 adapter,
1716 session_id,
1717 comms_runtime,
1718 supervisor,
1719 epoch,
1720 sender_peer_id,
1721 "bind member idempotent ack trust repair",
1722 )
1723 .await
1724}
1725
1726async fn rollback_bind_after_trust_publication_failure(
1727 adapter: &MeerkatMachine,
1728 session_id: &SessionId,
1729 peer_id: &str,
1730 epoch: u64,
1731) -> Result<(), SupervisorBindingStageError> {
1732 let transition = adapter
1733 .stage_supervisor_revoke(session_id, peer_id.to_string(), epoch)
1734 .await?;
1735 if let Some(obligation) =
1736 crate::protocol_supervisor_trust_revoke::extract_obligations(&transition)
1737 .into_iter()
1738 .find(|obligation| {
1739 obligation.peer_id().as_str() == peer_id && obligation.epoch() == epoch
1740 })
1741 {
1742 adapter
1743 .stage_supervisor_trust_revoked(
1744 session_id,
1745 obligation.peer_id().clone(),
1746 obligation.epoch(),
1747 )
1748 .await?;
1749 }
1750 Ok(())
1751}
1752
1753async fn commit_supervisor_bind_with_rollback(
1763 adapter: &MeerkatMachine,
1764 session_id: &SessionId,
1765 comms_runtime: &dyn CommsRuntime,
1766 supervisor_spec: &TrustedPeerDescriptor,
1767 epoch: u64,
1768 context: &str,
1769) -> Result<(), String> {
1770 adapter
1771 .stage_local_endpoint_for_comms_runtime(session_id, comms_runtime)
1772 .await
1773 .map_err(|error| format!("{context} failed: local endpoint rejected: {error}"))?;
1774 let bind_transition = adapter
1778 .stage_supervisor_bind(
1779 session_id,
1780 supervisor_spec.name.as_str().to_owned(),
1781 supervisor_spec.peer_id.as_str(),
1782 supervisor_spec.address.to_string(),
1783 encode_supervisor_signing_public_key(supervisor_spec.pubkey),
1784 epoch,
1785 )
1786 .await
1787 .map_err(|error| format!("{context} failed: DSL rejected binding: {error}"))?;
1788 let publish_obligation =
1789 match single_supervisor_publish_obligation(adapter, session_id, &bind_transition, context)
1790 .await
1791 {
1792 Ok(obligation) => obligation,
1793 Err(error) => {
1794 let _ = rollback_bind_after_trust_publication_failure(
1795 adapter,
1796 session_id,
1797 &supervisor_spec.peer_id.as_str(),
1798 epoch,
1799 )
1800 .await;
1801 return Err(error);
1802 }
1803 };
1804 if let Err(error) =
1805 validate_supervisor_publish_obligation(&publish_obligation, supervisor_spec, epoch, context)
1806 {
1807 let _ = rollback_bind_after_trust_publication_failure(
1808 adapter,
1809 session_id,
1810 publish_obligation.peer_id(),
1811 publish_obligation.epoch(),
1812 )
1813 .await;
1814 return Err(error);
1815 }
1816 if let Err(error) = publish_supervisor_trust_from_generated_obligation(
1817 adapter,
1818 session_id,
1819 comms_runtime,
1820 &publish_obligation,
1821 )
1822 .await
1823 {
1824 let _ = adapter
1825 .stage_supervisor_trust_publish_failed(
1826 session_id,
1827 publish_obligation.peer_id().clone(),
1828 publish_obligation.epoch(),
1829 error.clone(),
1830 )
1831 .await;
1832 let reason = match rollback_bind_after_trust_publication_failure(
1833 adapter,
1834 session_id,
1835 publish_obligation.peer_id(),
1836 publish_obligation.epoch(),
1837 )
1838 .await
1839 {
1840 Ok(()) => {
1841 format!("{context} failed: trust publication failed after DSL commit: {error}")
1842 }
1843 Err(rollback_error) => format!(
1844 "{context} failed: trust publication failed after DSL commit: {error}; rollback failed: {rollback_error}"
1845 ),
1846 };
1847 return Err(reason);
1848 }
1849 Ok(())
1850}
1851
1852#[derive(Debug, thiserror::Error)]
1858pub enum SupervisorMaterializeBindError {
1859 #[error("supervisor bind admission failed: {0}")]
1862 Admission(String),
1863 #[error("supervisor bind admission rejected: {0}")]
1865 AdmissionRejected(String),
1866 #[error("supervisor trust repair failed for idempotent rebind: {0}")]
1869 TrustRepair(String),
1870 #[error("supervisor bind commit failed: {0}")]
1873 Commit(String),
1874}
1875
1876pub async fn bind_supervisor_for_materialized_session(
1893 adapter: &MeerkatMachine,
1894 session_id: &SessionId,
1895 comms_runtime: &dyn CommsRuntime,
1896 supervisor: &TrustedPeerDescriptor,
1897 epoch: u64,
1898) -> Result<(), SupervisorMaterializeBindError> {
1899 let admission = adapter
1900 .resolve_supervisor_bind_admission(
1901 session_id,
1902 supervisor.peer_id.as_str(),
1903 epoch,
1904 Some(supervisor.peer_id.as_str()),
1908 )
1909 .await
1910 .map_err(|error| SupervisorMaterializeBindError::Admission(error.to_string()))?;
1911 match admission {
1912 SupervisorBindAdmission::Bootstrap => commit_supervisor_bind_with_rollback(
1913 adapter,
1914 session_id,
1915 comms_runtime,
1916 supervisor,
1917 epoch,
1918 "materialized member supervisor bind",
1919 )
1920 .await
1921 .map_err(SupervisorMaterializeBindError::Commit),
1922 SupervisorBindAdmission::IdempotentAck => republish_private_supervisor_trust(
1923 adapter,
1924 session_id,
1925 comms_runtime,
1926 supervisor,
1927 epoch,
1928 Some(supervisor.peer_id.as_str()),
1929 )
1930 .await
1931 .map_err(SupervisorMaterializeBindError::TrustRepair),
1932 SupervisorBindAdmission::Rejected(rejection) => Err(
1933 SupervisorMaterializeBindError::AdmissionRejected(format!("{rejection:?}")),
1934 ),
1935 }
1936}
1937
1938#[derive(Debug, thiserror::Error)]
1941pub enum SupervisorMaterializeAuthorizeError {
1942 #[error("supervisor authorize admission rejected: {0}")]
1945 AdmissionRejected(String),
1946 #[error("supervisor authorize apply failed: {0}")]
1949 Apply(String),
1950}
1951
1952pub async fn authorize_supervisor_for_materialized_session(
1959 adapter: &MeerkatMachine,
1960 session_id: &SessionId,
1961 comms_runtime: &dyn CommsRuntime,
1962 supervisor: &TrustedPeerDescriptor,
1963 epoch: u64,
1964) -> Result<(), SupervisorMaterializeAuthorizeError> {
1965 apply_supervisor_authorization(
1966 adapter,
1967 session_id,
1968 comms_runtime,
1969 supervisor,
1970 epoch,
1971 Some(supervisor.peer_id.as_str()),
1972 Some(encode_supervisor_signing_public_key(supervisor.pubkey)),
1973 "materialized member supervisor authorize",
1974 )
1975 .await
1976 .map_err(|error| match error {
1977 SupervisorAuthorizeApplyError::Rejected(rejection) => {
1978 SupervisorMaterializeAuthorizeError::AdmissionRejected(format!("{rejection:?}"))
1979 }
1980 SupervisorAuthorizeApplyError::Internal(reason) => {
1981 SupervisorMaterializeAuthorizeError::Apply(reason)
1982 }
1983 })
1984}
1985
1986#[cfg(test)]
1987async fn register_member_incarnation(
1988 adapter: &Arc<MeerkatMachine>,
1989 session_id: SessionId,
1990 incarnation: meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation,
1991) -> Result<(), crate::traits::RuntimeDriverError> {
1992 let update = adapter.begin_member_residency_update(session_id).await;
1993 let publication = update.commit(incarnation, None).await?;
1994 drop(publication);
1995 Ok(())
1996}
1997
1998fn require_registered_member_incarnation(
2003 adapter: &MeerkatMachine,
2004 session_id: &SessionId,
2005 expected: &BridgeMemberIncarnation,
2006 context: &str,
2007) -> Result<(), (BridgeRejectionCause, String)> {
2008 match adapter.member_incarnation(session_id) {
2009 Some(current) if ¤t == expected => Ok(()),
2010 Some(current) => Err((
2011 BridgeRejectionCause::StaleFence,
2012 format!("{context}: expected member incarnation {expected:?}; current is {current:?}"),
2013 )),
2014 None => Err((
2015 BridgeRejectionCause::Unavailable,
2016 format!(
2017 "{context}: target session '{session_id}' has no registered host-member incarnation"
2018 ),
2019 )),
2020 }
2021}
2022
2023#[cfg(test)]
2024fn peer_recipient_incarnation_matches(
2025 expected: &meerkat_core::comms::PeerRecipientIncarnation,
2026 current: &BridgeMemberIncarnation,
2027) -> bool {
2028 expected.mob_id == current.mob_id
2029 && expected.agent_identity == current.agent_identity
2030 && expected.host_id == current.host_id
2031 && expected.binding_generation == current.binding_generation
2032 && expected.member_session_id == current.member_session_id
2033 && expected.generation == current.generation
2034 && expected.fence_token == current.fence_token
2035}
2036
2037fn require_optional_registered_member_incarnation(
2038 adapter: &MeerkatMachine,
2039 session_id: &SessionId,
2040 expected: Option<&BridgeMemberIncarnation>,
2041 context: &str,
2042) -> Result<(), (BridgeRejectionCause, String)> {
2043 match expected {
2044 Some(expected) => {
2045 require_registered_member_incarnation(adapter, session_id, expected, context)
2046 }
2047 None if adapter.member_incarnation(session_id).is_some() => Err((
2048 BridgeRejectionCause::StaleFence,
2049 format!(
2050 "{context}: host-materialized target requires an exact expected_member incarnation"
2051 ),
2052 )),
2053 None => Ok(()),
2054 }
2055}
2056
2057async fn acquire_registered_member_effect_authority(
2058 adapter: &MeerkatMachine,
2059 session_id: &SessionId,
2060 expected: &BridgeMemberIncarnation,
2061 context: &str,
2062) -> Result<crate::meerkat_machine::MemberEffectAuthorityGuard, (BridgeRejectionCause, String)> {
2063 adapter
2064 .lock_member_effect_authority(session_id, expected)
2065 .await
2066 .map_err(|error| match error {
2067 crate::traits::RuntimeDriverError::StaleAuthority { reason } => (
2068 BridgeRejectionCause::StaleFence,
2069 format!("{context}: {reason}"),
2070 ),
2071 other => (
2072 BridgeRejectionCause::Unavailable,
2073 format!("{context}: failed to acquire member effect authority: {other}"),
2074 ),
2075 })
2076}
2077
2078async fn acquire_optional_member_effect_authority(
2079 adapter: &MeerkatMachine,
2080 session_id: &SessionId,
2081 expected: Option<&BridgeMemberIncarnation>,
2082 context: &str,
2083) -> Result<crate::meerkat_machine::MemberEffectAuthorityGuard, (BridgeRejectionCause, String)> {
2084 adapter
2085 .lock_optional_member_effect_authority(session_id, expected)
2086 .await
2087 .map_err(|error| match error {
2088 crate::traits::RuntimeDriverError::StaleAuthority { reason } => (
2089 BridgeRejectionCause::StaleFence,
2090 format!("{context}: {reason}"),
2091 ),
2092 other => (
2093 BridgeRejectionCause::Unavailable,
2094 format!("{context}: failed to acquire member effect authority: {other}"),
2095 ),
2096 })
2097}
2098
2099const MEMBER_POLL_WAIT_MAX_MS: u32 = 20_000;
2103
2104const MEMBER_POLL_DEFAULT_MAX_ROWS: u32 = 256;
2106const MEMBER_POLL_MAX_ROWS: u32 = 1024;
2107
2108const MEMBER_POLL_DEFAULT_MAX_OUTCOMES: u32 = 8;
2111const MEMBER_POLL_MAX_OUTCOMES: u32 = 64;
2112const MEMBER_EVENTS_REPLY_MAX_BYTES: usize = 512 * 1024;
2118const MEMBER_HISTORY_REPLY_MAX_BYTES: usize = 512 * 1024;
2119
2120const HARD_CANCEL_SETTLE_TIMEOUT: Duration = Duration::from_secs(4);
2125const HARD_CANCEL_REASSERT_INTERVAL: Duration = Duration::from_millis(10);
2126
2127fn observation_error_to_bridge_rejection(
2129 error: &MemberObservationError,
2130) -> (BridgeRejectionCause, String) {
2131 match error {
2132 MemberObservationError::StaleIncarnation { .. } => {
2133 (BridgeRejectionCause::StaleFence, error.to_string())
2134 }
2135 MemberObservationError::StaleCursor {
2136 watermark,
2137 generation,
2138 } => (
2139 BridgeRejectionCause::StaleCursor {
2140 watermark: *watermark,
2141 generation: *generation,
2142 },
2143 error.to_string(),
2144 ),
2145 MemberObservationError::FutureGenerationCursor { .. } => {
2148 (BridgeRejectionCause::Internal, error.to_string())
2149 }
2150 MemberObservationError::Unavailable { .. } => {
2151 (BridgeRejectionCause::Unavailable, error.to_string())
2152 }
2153 MemberObservationError::OutcomeRecordTooLarge { .. } => {
2154 (BridgeRejectionCause::Internal, error.to_string())
2155 }
2156 MemberObservationError::Internal { .. } => {
2157 (BridgeRejectionCause::Internal, error.to_string())
2158 }
2159 }
2160}
2161
2162fn resolve_member_observation_host(
2166 adapter: &Arc<MeerkatMachine>,
2167) -> Result<Arc<dyn MemberObservationHost>, (BridgeRejectionCause, String)> {
2168 adapter.member_observation_host().ok_or_else(|| {
2169 (
2170 BridgeRejectionCause::Unavailable,
2171 "member host serves no observation substrate".to_string(),
2172 )
2173 })
2174}
2175
2176fn resolve_member_live_host(
2182 adapter: &Arc<MeerkatMachine>,
2183) -> Result<Arc<dyn MemberLiveHost>, (BridgeRejectionCause, String)> {
2184 adapter.note_live_command_served();
2188 adapter.member_live_host().ok_or_else(|| {
2189 (
2190 BridgeRejectionCause::LiveTransportUnavailable,
2191 "member host serves no live substrate".to_string(),
2192 )
2193 })
2194}
2195
2196fn member_live_error_to_bridge_rejection(
2201 error: &MemberLiveError,
2202) -> (BridgeRejectionCause, String) {
2203 (error.to_bridge_rejection(), error.to_string())
2204}
2205
2206async fn stage_correlated_reply_endpoint(
2215 comms_runtime: &Arc<dyn CommsRuntime>,
2216 candidate: &PeerInputCandidate,
2217) {
2218 if let (Some(endpoint), Some(sender_peer_id), Some(signing_pubkey)) = (
2219 candidate.ingress.declared_reply_endpoint.clone(),
2220 candidate.ingress.canonical_peer_id,
2221 candidate.ingress.signing_pubkey,
2222 ) {
2223 match comms_runtime
2224 .stage_correlated_reply_endpoint(
2225 sender_peer_id,
2226 candidate.interaction.id,
2227 signing_pubkey,
2228 endpoint,
2229 )
2230 .await
2231 {
2232 Ok(()) | Err(SendError::Unsupported(_)) => {}
2233 Err(error) => {
2234 tracing::warn!(
2235 interaction_id = %candidate.interaction.id,
2236 error = %error,
2237 "comms_drain: failed to stage authenticated correlated reply endpoint"
2238 );
2239 }
2240 }
2241 }
2242}
2243
2244async fn send_bridge_response(
2245 comms_runtime: &Arc<dyn CommsRuntime>,
2246 candidate: &PeerInputCandidate,
2247 status: meerkat_core::interaction::ResponseStatus,
2248 reply: BridgeReply,
2249 _declared_reply_address: Option<&str>,
2253) {
2254 let result = serde_json::to_value(&reply).unwrap_or_else(|error| {
2255 tracing::error!(
2256 interaction_id = %candidate.interaction.id,
2257 error = %error,
2258 "comms_drain: BridgeReply serialization failed; falling back to minimal rejection"
2259 );
2260 serde_json::json!({
2261 "result": "rejected",
2262 "cause": "internal",
2263 "reason": "bridge reply serialization failed",
2264 })
2265 });
2266 let to = match resolve_bridge_response_route(comms_runtime, candidate).await {
2267 Some(route) => route,
2268 None => {
2269 tracing::warn!(
2270 from = %candidate.ingress.diagnostic_label(),
2271 interaction_id = %candidate.interaction.id,
2272 "comms_drain: failed to resolve bridge response peer route"
2273 );
2274 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
2275 return;
2276 }
2277 };
2278
2279 stage_correlated_reply_endpoint(comms_runtime, candidate).await;
2280
2281 if let Err(error) = comms_runtime
2282 .send(CommsCommand::PeerResponse {
2283 to,
2284 in_reply_to: candidate.interaction.id,
2285 status,
2286 result,
2287 blocks: None,
2288 content_taint: None,
2289 handling_mode: None,
2290 objective_id: candidate.interaction.objective_id,
2291 })
2292 .await
2293 {
2294 if let Some(sender_peer_id) = candidate.ingress.canonical_peer_id
2295 && candidate.ingress.declared_reply_endpoint.is_some()
2296 && let Err(cleanup_error) = comms_runtime
2297 .unstage_correlated_reply_endpoint(sender_peer_id, candidate.interaction.id)
2298 .await
2299 && !matches!(cleanup_error, SendError::Unsupported(_))
2300 {
2301 tracing::warn!(
2302 from = %candidate.ingress.diagnostic_label(),
2303 interaction_id = %candidate.interaction.id,
2304 error = %cleanup_error,
2305 "comms_drain: failed to clear correlated reply endpoint after response failure"
2306 );
2307 }
2308 tracing::warn!(
2309 from = %candidate.ingress.diagnostic_label(),
2310 interaction_id = %candidate.interaction.id,
2311 error = %error,
2312 "comms_drain: failed to send bridge response"
2313 );
2314 }
2315 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
2316}
2317
2318async fn resolve_bridge_response_route(
2319 comms_runtime: &Arc<dyn CommsRuntime>,
2320 candidate: &PeerInputCandidate,
2321) -> Option<PeerRoute> {
2322 if let Some(sender_route) = candidate.ingress.route.clone() {
2323 if let Some(route) = resolve_peer_route(comms_runtime, sender_route.peer_id).await {
2324 return Some(route);
2325 }
2326 return Some(sender_route);
2327 }
2328
2329 if let Some(sender_peer_id) = candidate.ingress.canonical_peer_id {
2330 return resolve_peer_route(comms_runtime, sender_peer_id)
2331 .await
2332 .or_else(|| Some(PeerRoute::new(sender_peer_id)));
2333 }
2334 None
2335}
2336
2337async fn resolve_peer_route(
2338 comms_runtime: &Arc<dyn CommsRuntime>,
2339 peer_id: PeerId,
2340) -> Option<PeerRoute> {
2341 let peers = comms_runtime.peers().await;
2342 peers
2343 .iter()
2344 .find(|entry| entry.peer_id == peer_id)
2345 .map(|entry| PeerRoute::with_display_name(entry.peer_id, entry.name.clone()))
2346}
2347
2348fn record_bridge_inbound_peer_request(
2349 comms_runtime: &Arc<dyn CommsRuntime>,
2350 candidate: &PeerInputCandidate,
2351) -> bool {
2352 let Some(handle) = comms_runtime.peer_request_response_authority_handle() else {
2353 tracing::warn!(
2354 interaction_id = %candidate.interaction.id,
2355 "comms_drain: rejected supervisor bridge request without complete peer request authority"
2356 );
2357 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
2358 return false;
2359 };
2360 let corr_id = meerkat_core::PeerCorrelationId::from_uuid(candidate.interaction.id.0);
2361 if handle.inbound_state(corr_id).is_some() {
2362 return true;
2363 }
2364 if let Err(err) = handle.request_received(corr_id, candidate.interaction.handling_mode) {
2365 tracing::warn!(
2366 error = %err,
2367 corr_id = %corr_id,
2368 "PeerInteractionHandle::request_received rejected for supervisor bridge command"
2369 );
2370 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
2371 return false;
2372 }
2373 true
2374}
2375
2376async fn send_bridge_failure(
2377 comms_runtime: &Arc<dyn CommsRuntime>,
2378 candidate: &PeerInputCandidate,
2379 cause: BridgeRejectionCause,
2380 message: impl Into<String>,
2381 declared_reply_address: Option<&str>,
2382) {
2383 send_bridge_response(
2384 comms_runtime,
2385 candidate,
2386 meerkat_core::interaction::ResponseStatus::Failed,
2387 BridgeReply::Rejected {
2388 cause,
2389 reason: message.into(),
2390 },
2391 declared_reply_address,
2392 )
2393 .await;
2394}
2395
2396async fn ensure_bridge_response_route_for_supervisor(
2397 adapter: &Arc<MeerkatMachine>,
2398 session_id: &SessionId,
2399 comms_runtime: &Arc<dyn CommsRuntime>,
2400 supervisor: &BridgePeerIdentity,
2401 context: &str,
2402) -> Result<(), (BridgeRejectionCause, String)> {
2403 ensure_bridge_response_route_for_descriptor(
2404 adapter,
2405 session_id,
2406 comms_runtime,
2407 supervisor.clone().into_trusted_peer_descriptor(),
2408 context,
2409 )
2410 .await
2411}
2412
2413async fn ensure_bridge_response_route_for_descriptor(
2414 adapter: &Arc<MeerkatMachine>,
2415 session_id: &SessionId,
2416 comms_runtime: &Arc<dyn CommsRuntime>,
2417 descriptor: TrustedPeerDescriptor,
2418 context: &str,
2419) -> Result<(), (BridgeRejectionCause, String)> {
2420 let obligation = supervisor_route_publish_obligation(adapter, session_id, &descriptor, context)
2421 .await
2422 .map_err(|reason| (BridgeRejectionCause::Internal, reason))?;
2423 let authority = supervisor_publish_authority(&obligation)
2424 .map_err(|reason| (BridgeRejectionCause::Internal, reason))?;
2425 apply_generated_private_trust_add(comms_runtime.as_ref(), descriptor, authority)
2426 .await
2427 .map_err(|error| {
2428 (
2429 BridgeRejectionCause::Internal,
2430 format!("{context}: failed to install supervisor response route: {error}"),
2431 )
2432 })
2433}
2434
2435async fn supervisor_route_publish_obligation(
2441 adapter: &MeerkatMachine,
2442 session_id: &SessionId,
2443 descriptor: &TrustedPeerDescriptor,
2444 context: &str,
2445) -> Result<crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation, String> {
2446 let binding = adapter.supervisor_binding(session_id).await;
2447 let bound = BoundSupervisorState::from_binding(&binding).ok_or_else(|| {
2448 format!("{context}: supervisor binding absent for response-route authority")
2449 })?;
2450 let transition = adapter
2451 .stage_supervisor_trust_publish_request(
2452 session_id,
2453 bound.name.clone(),
2454 bound.peer_id.clone(),
2455 bound.address.clone(),
2456 bound.signing_public_key.clone(),
2457 bound.epoch,
2458 )
2459 .await
2460 .map_err(|error| {
2461 format!("{context}: generated supervisor route publish request rejected: {error}")
2462 })?;
2463 let obligation =
2464 single_supervisor_publish_obligation(adapter, session_id, &transition, context).await?;
2465 if obligation.peer_id().as_str() != descriptor.peer_id.as_str() {
2466 return Err(format!(
2467 "{context}: generated supervisor route publish obligation peer_id mismatch"
2468 ));
2469 }
2470 Ok(obligation)
2471}
2472
2473struct SupervisorRotationDriveContext<'a> {
2488 operation_id: &'a str,
2489 previous: &'a BoundSupervisorState,
2490 next: &'a TrustedPeerDescriptor,
2491 next_epoch: u64,
2492 phase: crate::meerkat_machine::dsl::SupervisorRotationPhase,
2493}
2494
2495async fn drive_supervisor_rotation(
2496 adapter: &Arc<MeerkatMachine>,
2497 session_id: &SessionId,
2498 comms_runtime: &Arc<dyn CommsRuntime>,
2499 context: SupervisorRotationDriveContext<'_>,
2500) -> Result<(), String> {
2501 use crate::meerkat_machine::dsl::SupervisorRotationPhase;
2502 let SupervisorRotationDriveContext {
2503 operation_id,
2504 previous,
2505 next,
2506 next_epoch,
2507 mut phase,
2508 } = context;
2509
2510 if matches!(
2511 phase,
2512 SupervisorRotationPhase::Completed | SupervisorRotationPhase::Rejected
2513 ) {
2514 return Ok(());
2515 }
2516 let mut transition = adapter
2517 .stage_supervisor_rotation_resume(session_id, operation_id.to_string())
2518 .await
2519 .map_err(|error| format!("rotation resume rejected: {error}"))?;
2520
2521 loop {
2522 match phase {
2523 SupervisorRotationPhase::PreviousRevokePending => {
2524 let obligation = matching_supervisor_revoke_obligation(
2525 adapter,
2526 session_id,
2527 &transition,
2528 &previous.peer_id,
2529 previous.epoch,
2530 "supervisor rotation operation",
2531 )
2532 .await?;
2533 let authority = supervisor_revoke_authority(&obligation)?;
2534 let removal = crate::tokio::time::timeout(
2535 SUPERVISOR_RESPONSE_ROUTE_IO_TIMEOUT,
2536 apply_generated_private_trust_remove(
2537 comms_runtime.as_ref(),
2538 obligation.peer_id().clone(),
2539 authority,
2540 ),
2541 )
2542 .await;
2543 if let Err(error) = match removal {
2544 Ok(result) => result.map(|_| ()),
2545 Err(_) => Err("previous supervisor trust removal timed out".to_string()),
2546 } {
2547 let _ = adapter
2548 .stage_supervisor_trust_revoke_failed(
2549 session_id,
2550 obligation.peer_id().clone(),
2551 obligation.epoch(),
2552 error.clone(),
2553 )
2554 .await;
2555 return Err(format!(
2556 "supervisor rotation operation left previous revoke pending: {error}"
2557 ));
2558 }
2559 transition = adapter
2560 .stage_supervisor_rotation_previous_revoked(
2561 session_id,
2562 operation_id.to_string(),
2563 obligation.peer_id().clone(),
2564 obligation.epoch(),
2565 )
2566 .await
2567 .map_err(|error| {
2568 format!(
2569 "previous supervisor route is absent but rotation checkpoint persistence failed: {error}"
2570 )
2571 })?;
2572 phase = SupervisorRotationPhase::NextPublishPending;
2573 }
2574 SupervisorRotationPhase::NextPublishPending => {
2575 let obligation = single_supervisor_publish_obligation(
2576 adapter,
2577 session_id,
2578 &transition,
2579 "supervisor rotation operation",
2580 )
2581 .await?;
2582 validate_supervisor_publish_obligation(
2583 &obligation,
2584 next,
2585 next_epoch,
2586 "supervisor rotation operation",
2587 )?;
2588 let authority = supervisor_publish_authority(&obligation)?;
2589 let publication = crate::tokio::time::timeout(
2590 SUPERVISOR_RESPONSE_ROUTE_IO_TIMEOUT,
2591 apply_generated_private_trust_add(
2592 comms_runtime.as_ref(),
2593 next.clone(),
2594 authority,
2595 ),
2596 )
2597 .await;
2598 if let Err(error) = match publication {
2599 Ok(result) => result,
2600 Err(_) => Err("next supervisor trust publication timed out".to_string()),
2601 } {
2602 let _ = adapter
2603 .stage_supervisor_trust_publish_failed(
2604 session_id,
2605 obligation.peer_id().clone(),
2606 obligation.epoch(),
2607 error.clone(),
2608 )
2609 .await;
2610 return Err(format!(
2611 "supervisor rotation operation left next publication pending: {error}"
2612 ));
2613 }
2614 adapter
2619 .stage_supervisor_rotation_next_published(
2620 session_id,
2621 operation_id.to_string(),
2622 obligation.peer_id().clone(),
2623 obligation.epoch(),
2624 )
2625 .await
2626 .map_err(|error| {
2627 format!(
2628 "next supervisor route is live but rotation checkpoint persistence failed: {error}"
2629 )
2630 })?;
2631 phase = SupervisorRotationPhase::Completed;
2632 }
2633 SupervisorRotationPhase::Completed | SupervisorRotationPhase::Rejected => return Ok(()),
2634 }
2635 }
2636}
2637
2638fn trusted_peer_from_generated_binding(
2639 binding: &crate::meerkat_machine::GeneratedSupervisorBinding,
2640) -> Result<TrustedPeerDescriptor, String> {
2641 TrustedPeerDescriptor::unsigned_with_pubkey(
2642 binding.name.clone(),
2643 &binding.peer_id,
2644 decode_supervisor_signing_public_key(&binding.signing_public_key)?,
2645 &binding.address,
2646 )
2647}
2648
2649async fn start_supervisor_rotation_worker_if_pending(
2650 adapter: &Arc<MeerkatMachine>,
2651 session_id: &SessionId,
2652 comms_runtime: &Arc<dyn CommsRuntime>,
2653) -> Result<bool, String> {
2654 let Some(receipt) = adapter
2655 .active_supervisor_rotation(session_id)
2656 .await
2657 .map_err(|error| error.to_string())?
2658 else {
2659 return Ok(false);
2660 };
2661 if matches!(
2662 receipt.phase,
2663 crate::meerkat_machine::dsl::SupervisorRotationPhase::Completed
2664 | crate::meerkat_machine::dsl::SupervisorRotationPhase::Rejected
2665 ) {
2666 return Ok(false);
2667 }
2668 let operation_id = receipt.operation_id.clone();
2669 let worker_adapter = Arc::clone(adapter);
2670 let worker_session_id = session_id.clone();
2671 let worker_runtime = Arc::clone(comms_runtime);
2672 let slot_runtime = Arc::clone(comms_runtime);
2673 let task_operation_id = operation_id.clone();
2674 let (start_tx, start_rx) = crate::tokio::sync::oneshot::channel::<()>();
2675 let task = crate::tokio::spawn(async move {
2676 if start_rx.await.is_err() {
2677 return;
2678 }
2679 let mut retry_delay = std::time::Duration::from_millis(25);
2680 loop {
2681 match worker_adapter
2682 .peer_ingress_runtime_is_current(&worker_session_id, &worker_runtime)
2683 .await
2684 {
2685 Ok(true) => {}
2686 Ok(false) => {
2687 tracing::warn!(
2688 session_id = %worker_session_id,
2689 operation_id = %task_operation_id,
2690 "supervisor rotation worker refused stale runtime retry"
2691 );
2692 return;
2693 }
2694 Err(error) => {
2695 tracing::warn!(
2696 session_id = %worker_session_id,
2697 operation_id = %task_operation_id,
2698 %error,
2699 "supervisor rotation worker lost runtime authority before retry"
2700 );
2701 return;
2702 }
2703 }
2704 let receipt = match worker_adapter
2705 .active_supervisor_rotation(&worker_session_id)
2706 .await
2707 {
2708 Ok(Some(receipt)) if receipt.operation_id == task_operation_id => receipt,
2709 Ok(_) => return,
2710 Err(error) => {
2711 tracing::warn!(
2712 session_id = %worker_session_id,
2713 operation_id = %task_operation_id,
2714 %error,
2715 "supervisor rotation worker lost its registered session"
2716 );
2717 return;
2718 }
2719 };
2720 if matches!(
2721 receipt.phase,
2722 crate::meerkat_machine::dsl::SupervisorRotationPhase::Completed
2723 | crate::meerkat_machine::dsl::SupervisorRotationPhase::Rejected
2724 ) {
2725 return;
2726 }
2727 let next = match trusted_peer_from_generated_binding(&receipt.next) {
2728 Ok(next) => next,
2729 Err(error) => {
2730 tracing::error!(
2731 session_id = %worker_session_id,
2732 operation_id = %task_operation_id,
2733 %error,
2734 "supervisor rotation receipt carried an invalid target"
2735 );
2736 return;
2737 }
2738 };
2739 let previous = BoundSupervisorState {
2740 name: receipt.previous.name,
2741 peer_id: receipt.previous.peer_id,
2742 address: receipt.previous.address,
2743 signing_public_key: receipt.previous.signing_public_key,
2744 epoch: receipt.previous.epoch,
2745 };
2746 match drive_supervisor_rotation(
2747 &worker_adapter,
2748 &worker_session_id,
2749 &worker_runtime,
2750 SupervisorRotationDriveContext {
2751 operation_id: &task_operation_id,
2752 previous: &previous,
2753 next: &next,
2754 next_epoch: receipt.next.epoch,
2755 phase: receipt.phase,
2756 },
2757 )
2758 .await
2759 {
2760 Ok(()) => return,
2761 Err(error) => {
2762 tracing::warn!(
2763 session_id = %worker_session_id,
2764 operation_id = %task_operation_id,
2765 %error,
2766 ?retry_delay,
2767 "supervisor rotation remains pending; session-owned worker will retry"
2768 );
2769 crate::tokio::time::sleep(retry_delay).await;
2770 retry_delay = std::cmp::min(
2771 retry_delay.saturating_mul(2),
2772 std::time::Duration::from_secs(1),
2773 );
2774 }
2775 }
2776 }
2777 });
2778 let installed = adapter
2779 .install_supervisor_rotation_task_if_current(session_id, operation_id, slot_runtime, task)
2780 .await
2781 .map_err(|error| error.to_string())?;
2782 if installed {
2783 start_tx
2784 .send(())
2785 .map_err(|()| "supervisor rotation worker exited before start".to_string())?;
2786 }
2787 Ok(installed)
2788}
2789
2790async fn hydrate_current_bound_supervisor_route(
2791 adapter: &Arc<MeerkatMachine>,
2792 session_id: &SessionId,
2793 comms_runtime: &Arc<dyn CommsRuntime>,
2794) -> Result<bool, String> {
2795 let binding = adapter.supervisor_binding(session_id).await;
2796 let Some(bound) = BoundSupervisorState::from_binding(&binding) else {
2797 return Ok(false);
2798 };
2799 let descriptor = TrustedPeerDescriptor::unsigned_with_pubkey(
2800 bound.name,
2801 &bound.peer_id,
2802 decode_supervisor_signing_public_key(&bound.signing_public_key)?,
2803 &bound.address,
2804 )?;
2805 ensure_bridge_response_route_for_descriptor(
2806 adapter,
2807 session_id,
2808 comms_runtime,
2809 descriptor,
2810 "supervisor runtime hydration",
2811 )
2812 .await
2813 .map_err(|(_, reason)| reason)?;
2814 Ok(true)
2815}
2816
2817async fn hydrate_or_resume_supervisor_rotation_runtime(
2818 adapter: &Arc<MeerkatMachine>,
2819 session_id: &SessionId,
2820 comms_runtime: &Arc<dyn CommsRuntime>,
2821) -> Result<(), String> {
2822 match adapter
2823 .active_supervisor_rotation(session_id)
2824 .await
2825 .map_err(|error| error.to_string())?
2826 .map(|receipt| receipt.phase)
2827 {
2828 Some(
2829 crate::meerkat_machine::dsl::SupervisorRotationPhase::PreviousRevokePending
2830 | crate::meerkat_machine::dsl::SupervisorRotationPhase::NextPublishPending,
2831 ) => {
2832 start_supervisor_rotation_worker_if_pending(adapter, session_id, comms_runtime).await?;
2833 }
2834 Some(
2835 crate::meerkat_machine::dsl::SupervisorRotationPhase::Completed
2836 | crate::meerkat_machine::dsl::SupervisorRotationPhase::Rejected,
2837 )
2838 | None => {
2839 let _ =
2845 hydrate_current_bound_supervisor_route(adapter, session_id, comms_runtime).await?;
2846 }
2847 }
2848 Ok(())
2849}
2850
2851fn bridge_supervisor_rotation_state(
2852 receipt: crate::meerkat_machine::GeneratedSupervisorRotationReceipt,
2853) -> Result<BridgeSupervisorRotationState, String> {
2854 let operation_id = receipt
2855 .operation_id
2856 .parse()
2857 .map_err(|error| format!("stored supervisor rotation operation id is invalid: {error}"))?;
2858 let target = BridgePeerSpec {
2859 name: receipt.next.name,
2860 peer_id: receipt.next.peer_id,
2861 address: receipt.next.address,
2862 pubkey: decode_supervisor_signing_public_key(&receipt.next.signing_public_key)?,
2863 };
2864 let operation = BridgeSupervisorRotationOperationReceipt {
2865 operation_id,
2866 target: BridgeSupervisorRotationTargetReceipt {
2867 target,
2868 target_epoch: receipt.next.epoch,
2869 },
2870 };
2871 match receipt.phase {
2872 crate::meerkat_machine::dsl::SupervisorRotationPhase::PreviousRevokePending => {
2873 Ok(BridgeSupervisorRotationState::Pending {
2874 operation,
2875 phase: BridgeSupervisorRotationPendingPhase::PreviousRevokePending,
2876 })
2877 }
2878 crate::meerkat_machine::dsl::SupervisorRotationPhase::NextPublishPending => {
2879 Ok(BridgeSupervisorRotationState::Pending {
2880 operation,
2881 phase: BridgeSupervisorRotationPendingPhase::NextPublishPending,
2882 })
2883 }
2884 crate::meerkat_machine::dsl::SupervisorRotationPhase::Completed => {
2885 Ok(BridgeSupervisorRotationState::Completed { receipt: operation })
2886 }
2887 crate::meerkat_machine::dsl::SupervisorRotationPhase::Rejected => {
2888 let rejection = receipt.rejection.ok_or_else(|| {
2889 "stored rejected supervisor rotation omitted its typed cause".to_string()
2890 })?;
2891 let cause = match rejection {
2892 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::OperationConflict => {
2893 BridgeSupervisorRotationRejectionCause::OperationConflict
2894 }
2895 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::NotBound => {
2896 BridgeSupervisorRotationRejectionCause::Internal
2897 }
2898 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::SenderMismatch => {
2899 BridgeSupervisorRotationRejectionCause::SenderMismatch
2900 }
2901 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::TargetEpochNotAdvanced => {
2902 BridgeSupervisorRotationRejectionCause::StaleTargetEpoch
2903 }
2904 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::InvalidTarget => {
2905 BridgeSupervisorRotationRejectionCause::InvalidTarget
2906 }
2907 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::UnsupportedProtocolVersion => {
2908 BridgeSupervisorRotationRejectionCause::UnsupportedProtocolVersion
2909 }
2910 };
2911 Ok(BridgeSupervisorRotationState::Rejected {
2912 receipt: BridgeSupervisorRotationRejectionReceipt {
2913 operation,
2914 cause,
2915 reason: format!("supervisor rotation rejected: {rejection:?}"),
2916 },
2917 })
2918 }
2919 }
2920}
2921
2922#[derive(Debug)]
2923enum SupervisorAuthorizeApplyError {
2924 Rejected(crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind),
2925 Internal(String),
2926}
2927
2928#[allow(clippy::too_many_arguments)]
2932async fn apply_supervisor_authorization(
2933 adapter: &MeerkatMachine,
2934 session_id: &SessionId,
2935 comms_runtime: &dyn CommsRuntime,
2936 supervisor: &TrustedPeerDescriptor,
2937 epoch: u64,
2938 sender_peer_id: Option<String>,
2939 sender_signing_public_key: Option<String>,
2940 context: &str,
2941) -> Result<(), SupervisorAuthorizeApplyError> {
2942 adapter
2943 .stage_local_endpoint_for_comms_runtime(session_id, comms_runtime)
2944 .await
2945 .map_err(|error| {
2946 SupervisorAuthorizeApplyError::Internal(format!(
2947 "{context}: local endpoint rejected: {error}"
2948 ))
2949 })?;
2950
2951 let admission = adapter
2952 .resolve_supervisor_authorize_admission(
2953 session_id,
2954 GeneratedSupervisorBinding {
2955 name: supervisor.name.as_str().to_owned(),
2956 peer_id: supervisor.peer_id.as_str(),
2957 address: supervisor.address.to_string(),
2958 signing_public_key: encode_supervisor_signing_public_key(supervisor.pubkey),
2959 epoch,
2960 },
2961 sender_peer_id.clone(),
2962 sender_signing_public_key,
2963 )
2964 .await
2965 .map_err(|error| {
2966 SupervisorAuthorizeApplyError::Internal(format!(
2967 "{context}: generated admission failed: {error}"
2968 ))
2969 })?;
2970
2971 match admission {
2972 SupervisorAuthorizeAdmission::IdempotentAck => refresh_and_publish_bound_supervisor_route(
2973 adapter,
2974 session_id,
2975 comms_runtime,
2976 supervisor,
2977 epoch,
2978 sender_peer_id,
2979 context,
2980 )
2981 .await
2982 .map_err(SupervisorAuthorizeApplyError::Internal),
2983 SupervisorAuthorizeAdmission::Proceed(_) => {
2984 let transition = adapter
2985 .stage_supervisor_authorize(
2986 session_id,
2987 supervisor.name.as_str().to_owned(),
2988 supervisor.peer_id.as_str(),
2989 supervisor.address.to_string(),
2990 encode_supervisor_signing_public_key(supervisor.pubkey),
2991 epoch,
2992 )
2993 .await
2994 .map_err(|error| {
2995 SupervisorAuthorizeApplyError::Internal(format!(
2996 "{context}: version advance failed: {error}"
2997 ))
2998 })?;
2999 let obligation =
3000 single_supervisor_publish_obligation(adapter, session_id, &transition, context)
3001 .await
3002 .map_err(SupervisorAuthorizeApplyError::Internal)?;
3003 validate_supervisor_publish_obligation(&obligation, supervisor, epoch, context)
3004 .map_err(SupervisorAuthorizeApplyError::Internal)?;
3005 publish_supervisor_trust_from_generated_obligation(
3006 adapter,
3007 session_id,
3008 comms_runtime,
3009 &obligation,
3010 )
3011 .await
3012 .map_err(|error| {
3013 SupervisorAuthorizeApplyError::Internal(format!(
3014 "{context}: version publish failed: {error}"
3015 ))
3016 })
3017 }
3018 SupervisorAuthorizeAdmission::Rejected(rejection) => {
3019 Err(SupervisorAuthorizeApplyError::Rejected(rejection))
3020 }
3021 }
3022}
3023
3024async fn handle_authorize_supervisor_command(
3025 adapter: &Arc<MeerkatMachine>,
3026 session_id: &SessionId,
3027 comms_runtime: &Arc<dyn CommsRuntime>,
3028 candidate: &PeerInputCandidate,
3029 sender: &PeerIngressFact,
3030 payload: BridgeSupervisorPayload,
3031) -> bool {
3032 let supervisor = match bridge_peer_identity(&payload.supervisor, "authorize supervisor failed")
3033 {
3034 Ok(supervisor) => supervisor,
3035 Err((cause, reason)) => {
3036 send_bridge_failure(
3037 comms_runtime,
3038 candidate,
3039 cause,
3040 reason,
3041 Some(payload.supervisor.address.as_str()),
3042 )
3043 .await;
3044 return true;
3045 }
3046 };
3047 let supervisor_spec = match TrustedPeerDescriptor::try_from(payload.supervisor.clone()) {
3048 Ok(supervisor_spec) => supervisor_spec,
3049 Err(error) => {
3050 send_bridge_failure(
3051 comms_runtime,
3052 candidate,
3053 BridgeRejectionCause::InvalidSupervisorSpec,
3054 format!("authorize supervisor failed: invalid supervisor peer spec: {error}"),
3055 Some(payload.supervisor.address.as_str()),
3056 )
3057 .await;
3058 return true;
3059 }
3060 };
3061 match apply_supervisor_authorization(
3062 adapter,
3063 session_id,
3064 comms_runtime.as_ref(),
3065 &supervisor_spec,
3066 payload.epoch,
3067 generated_sender_peer_id(sender),
3068 generated_sender_signing_public_key(sender),
3069 "authorize supervisor",
3070 )
3071 .await
3072 {
3073 Ok(()) => {
3074 if crate::tokio::time::timeout(
3075 SUPERVISOR_RESPONSE_ROUTE_IO_TIMEOUT,
3076 send_bridge_response(
3077 comms_runtime,
3078 candidate,
3079 meerkat_core::interaction::ResponseStatus::Completed,
3080 BridgeReply::Ack(BridgeAck { ok: true }),
3081 Some(payload.supervisor.address.as_str()),
3082 ),
3083 )
3084 .await
3085 .is_err()
3086 {
3087 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
3088 }
3089 }
3090 Err(SupervisorAuthorizeApplyError::Rejected(rejection)) => {
3091 send_bridge_failure(
3092 comms_runtime,
3093 candidate,
3094 supervisor_authorize_admission_rejection_cause(rejection),
3095 supervisor_authorize_admission_rejection_reason(rejection, sender, &supervisor),
3096 Some(payload.supervisor.address.as_str()),
3097 )
3098 .await;
3099 }
3100 Err(SupervisorAuthorizeApplyError::Internal(error)) => {
3101 send_bridge_failure(
3102 comms_runtime,
3103 candidate,
3104 BridgeRejectionCause::Internal,
3105 error,
3106 Some(payload.supervisor.address.as_str()),
3107 )
3108 .await;
3109 }
3110 }
3111 true
3112}
3113
3114#[derive(serde::Deserialize)]
3115#[serde(rename_all = "snake_case")]
3116enum SupervisorDeliveryMarker {
3117 SubmitSupervisorRotation,
3118}
3119
3120#[derive(serde::Deserialize)]
3121#[serde(deny_unknown_fields)]
3122struct RawSupervisorRotationSubmitDelivery {
3123 #[serde(rename = "delivery")]
3124 _delivery: SupervisorDeliveryMarker,
3125 operation_id: SupervisorRotationOperationId,
3126 target: BridgePeerSpec,
3127 target_epoch: u64,
3128 protocol_version: u32,
3129}
3130
3131#[derive(Default)]
3132struct SupervisorDeliveryMarkerProbe {
3133 delivery_key_count: usize,
3134 reserved_rotation_marker_seen: bool,
3135}
3136
3137impl<'de> serde::Deserialize<'de> for SupervisorDeliveryMarkerProbe {
3138 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
3139 where
3140 D: serde::Deserializer<'de>,
3141 {
3142 struct ProbeVisitor;
3143
3144 impl<'de> serde::de::Visitor<'de> for ProbeVisitor {
3145 type Value = SupervisorDeliveryMarkerProbe;
3146
3147 fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
3148 formatter.write_str("a supervisor delivery JSON object")
3149 }
3150
3151 fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
3152 where
3153 A: serde::de::MapAccess<'de>,
3154 {
3155 let mut probe = SupervisorDeliveryMarkerProbe::default();
3156 while let Some(key) = map.next_key::<String>()? {
3157 if key == "delivery" {
3158 probe.delivery_key_count += 1;
3159 let value = map.next_value::<serde_json::Value>()?;
3160 if value.as_str() == Some("submit_supervisor_rotation") {
3161 probe.reserved_rotation_marker_seen = true;
3162 }
3163 } else {
3164 map.next_value::<serde::de::IgnoredAny>()?;
3165 }
3166 }
3167 Ok(probe)
3168 }
3169 }
3170
3171 deserializer.deserialize_map(ProbeVisitor)
3172 }
3173}
3174
3175async fn try_handle_supervisor_bridge_delivery(
3176 adapter: &Arc<MeerkatMachine>,
3177 session_id: &SessionId,
3178 comms_runtime: &Arc<dyn CommsRuntime>,
3179 candidate: &PeerInputCandidate,
3180) -> bool {
3181 let InteractionContent::Message { body, .. } = &candidate.interaction.content else {
3182 return false;
3183 };
3184 let Ok(marker_probe) = serde_json::from_str::<SupervisorDeliveryMarkerProbe>(body) else {
3185 return false;
3186 };
3187 if !marker_probe.reserved_rotation_marker_seen {
3188 return false;
3189 }
3190 if marker_probe.delivery_key_count != 1 {
3191 tracing::warn!(
3192 %session_id,
3193 delivery_key_count = marker_probe.delivery_key_count,
3194 "duplicate/conflicting supervisor rotation delivery markers consumed fail-closed"
3195 );
3196 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
3197 return true;
3198 }
3199 let delivery = match serde_json::from_str::<RawSupervisorRotationSubmitDelivery>(body) {
3200 Ok(delivery) => delivery,
3201 Err(error) => {
3202 tracing::warn!(%session_id, %error, "malformed supervisor rotation delivery consumed fail-closed");
3203 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
3204 return true;
3205 }
3206 };
3207 let sender_peer_id = generated_sender_peer_id(&candidate.ingress);
3208 let sender_signing_public_key = generated_sender_signing_public_key(&candidate.ingress);
3209 let raw_target = delivery.target;
3210 let (next, preflight_rejection) = if BridgeProtocolVersion::try_from(delivery.protocol_version)
3211 .ok()
3212 == Some(BridgeProtocolVersion::V4)
3213 {
3214 match TrustedPeerDescriptor::try_from(raw_target.clone()) {
3215 Ok(target) => (
3216 GeneratedSupervisorBinding {
3217 name: target.name.as_str().to_owned(),
3218 peer_id: target.peer_id.as_str(),
3219 address: target.address.to_string(),
3220 signing_public_key: encode_supervisor_signing_public_key(target.pubkey),
3221 epoch: delivery.target_epoch,
3222 },
3223 None,
3224 ),
3225 Err(error) => {
3226 tracing::warn!(%session_id, %error, "durably rejecting one-way supervisor rotation with invalid target");
3227 (
3228 GeneratedSupervisorBinding {
3229 name: raw_target.name,
3230 peer_id: raw_target.peer_id,
3231 address: raw_target.address,
3232 signing_public_key: encode_supervisor_signing_public_key(raw_target.pubkey),
3233 epoch: delivery.target_epoch,
3234 },
3235 Some(
3236 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::InvalidTarget,
3237 ),
3238 )
3239 }
3240 }
3241 } else {
3242 (
3243 GeneratedSupervisorBinding {
3244 name: raw_target.name,
3245 peer_id: raw_target.peer_id,
3246 address: raw_target.address,
3247 signing_public_key: encode_supervisor_signing_public_key(raw_target.pubkey),
3248 epoch: delivery.target_epoch,
3249 },
3250 Some(
3251 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::UnsupportedProtocolVersion,
3252 ),
3253 )
3254 };
3255 if preflight_rejection.is_none()
3256 && let Err(error) = adapter
3257 .stage_local_endpoint_for_comms_runtime(session_id, comms_runtime.as_ref())
3258 .await
3259 {
3260 tracing::warn!(%session_id, %error, "unable to stage local endpoint for supervisor rotation submission");
3261 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
3262 return true;
3263 }
3264 let submission = GeneratedSupervisorRotationSubmit {
3265 operation_id: delivery.operation_id.to_string(),
3266 next,
3267 preflight_rejection,
3268 sender_peer_id,
3269 sender_signing_public_key,
3270 };
3271 match adapter
3272 .submit_supervisor_rotation(session_id, submission)
3273 .await
3274 {
3275 Ok(
3276 SupervisorRotationSubmission::New(_) | SupervisorRotationSubmission::ExistingPending(_),
3277 ) => {
3278 if let Err(error) =
3279 start_supervisor_rotation_worker_if_pending(adapter, session_id, comms_runtime)
3280 .await
3281 {
3282 tracing::warn!(%session_id, %error, "supervisor rotation was durably accepted but worker start failed");
3283 }
3284 }
3285 Ok(SupervisorRotationSubmission::ExistingTerminal) => {}
3286 Ok(
3287 SupervisorRotationSubmission::Rejected(rejection)
3288 | SupervisorRotationSubmission::Conflict(rejection),
3289 ) => {
3290 tracing::warn!(%session_id, ?rejection, "one-way supervisor rotation submission rejected");
3291 }
3292 Err(error) => {
3293 tracing::warn!(%session_id, %error, "one-way supervisor rotation submission failed");
3294 }
3295 }
3296 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
3300 true
3301}
3302
3303async fn serve_hard_cancel_member(
3313 adapter: &Arc<MeerkatMachine>,
3314 session_id: &SessionId,
3315 comms_runtime: &Arc<dyn CommsRuntime>,
3316 sender: &PeerIngressFact,
3317 candidate: &PeerInputCandidate,
3318 payload: &BridgeHardCancelPayload,
3319) {
3320 let sup_payload = BridgeSupervisorPayload {
3325 supervisor: payload.supervisor.clone(),
3326 epoch: payload.epoch,
3327 protocol_version: payload.protocol_version,
3328 };
3329 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
3330 adapter,
3331 session_id,
3332 comms_runtime,
3333 sender,
3334 &sup_payload,
3335 "hard cancel member failed",
3336 )
3337 .await
3338 {
3339 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3340 return;
3341 }
3342 if let Err((cause, reason)) = require_registered_member_incarnation(
3343 adapter,
3344 session_id,
3345 &payload.expected_member,
3346 "hard cancel member",
3347 ) {
3348 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3349 return;
3350 }
3351 let deadline = Instant::now() + HARD_CANCEL_SETTLE_TIMEOUT;
3352 loop {
3353 match adapter
3354 .hard_cancel_run_if_current_for_member_incarnation(
3355 session_id,
3356 &payload.expected_run_id,
3357 &payload.expected_member,
3358 payload.reason.clone(),
3359 )
3360 .await
3361 {
3362 Ok(false) => {
3367 if let Err((cause, reason)) = require_registered_member_incarnation(
3368 adapter,
3369 session_id,
3370 &payload.expected_member,
3371 "hard cancel member completion",
3372 ) {
3373 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3374 return;
3375 }
3376 tracing::debug!(
3377 operation_id = %payload.operation_id,
3378 expected_run_id = %payload.expected_run_id,
3379 "hard-cancel operation reached exact-run terminal truth"
3380 );
3381 send_bridge_response(
3382 comms_runtime,
3383 candidate,
3384 meerkat_core::interaction::ResponseStatus::Completed,
3385 BridgeReply::Ack(BridgeAck { ok: true }),
3386 None,
3387 )
3388 .await;
3389 return;
3390 }
3391 Ok(true) => {}
3392 Err(crate::traits::RuntimeDriverError::StaleAuthority { reason }) => {
3393 send_bridge_failure(
3394 comms_runtime,
3395 candidate,
3396 BridgeRejectionCause::StaleFence,
3397 reason,
3398 None,
3399 )
3400 .await;
3401 return;
3402 }
3403 Err(crate::traits::RuntimeDriverError::NotReady { state }) => {
3406 send_bridge_failure(
3407 comms_runtime,
3408 candidate,
3409 BridgeRejectionCause::Unavailable,
3410 format!(
3411 "hard-cancel operation {} could not cancel expected run {} in state {state:?}",
3412 payload.operation_id, payload.expected_run_id
3413 ),
3414 None,
3415 )
3416 .await;
3417 return;
3418 }
3419 Err(crate::traits::RuntimeDriverError::Destroyed) => {
3420 send_bridge_response(
3425 comms_runtime,
3426 candidate,
3427 meerkat_core::interaction::ResponseStatus::Completed,
3428 BridgeReply::Ack(BridgeAck { ok: true }),
3429 None,
3430 )
3431 .await;
3432 return;
3433 }
3434 Err(error) => {
3435 send_bridge_failure(
3436 comms_runtime,
3437 candidate,
3438 BridgeRejectionCause::Internal,
3439 format!(
3440 "hard-cancel operation {} for expected run {} failed: {error}",
3441 payload.operation_id, payload.expected_run_id
3442 ),
3443 None,
3444 )
3445 .await;
3446 return;
3447 }
3448 }
3449
3450 if Instant::now() >= deadline {
3451 send_bridge_failure(
3452 comms_runtime,
3453 candidate,
3454 BridgeRejectionCause::Unavailable,
3455 format!(
3456 "hard-cancel operation {} timed out waiting for expected run {} to become terminal",
3457 payload.operation_id, payload.expected_run_id
3458 ),
3459 None,
3460 )
3461 .await;
3462 return;
3463 }
3464 crate::tokio::time::sleep(HARD_CANCEL_REASSERT_INTERVAL).await;
3465 }
3466}
3467
3468async fn serve_cancel_tracked_member_input(
3469 adapter: &Arc<MeerkatMachine>,
3470 session_id: &SessionId,
3471 comms_runtime: &Arc<dyn CommsRuntime>,
3472 sender: &PeerIngressFact,
3473 candidate: &PeerInputCandidate,
3474 payload: &BridgeTrackedInputCancelPayload,
3475) {
3476 let sup_payload = BridgeSupervisorPayload {
3477 supervisor: payload.supervisor.clone(),
3478 epoch: payload.epoch,
3479 protocol_version: payload.protocol_version,
3480 };
3481 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
3482 adapter,
3483 session_id,
3484 comms_runtime,
3485 sender,
3486 &sup_payload,
3487 "cancel tracked member input failed",
3488 )
3489 .await
3490 {
3491 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3492 return;
3493 }
3494 if let Err((cause, reason)) = require_registered_member_incarnation(
3495 adapter,
3496 session_id,
3497 &payload.expected_member,
3498 "cancel tracked member input",
3499 ) {
3500 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3501 return;
3502 }
3503 match uuid::Uuid::parse_str(&payload.input_id) {
3504 Ok(input_uuid) if !input_uuid.is_nil() && input_uuid.to_string() == payload.input_id => {}
3505 Ok(_) => {
3506 send_bridge_failure(
3507 comms_runtime,
3508 candidate,
3509 BridgeRejectionCause::Unsupported,
3510 "tracked-input cancel input_id must be a canonical non-nil UUID".to_string(),
3511 None,
3512 )
3513 .await;
3514 return;
3515 }
3516 Err(error) => {
3517 send_bridge_failure(
3518 comms_runtime,
3519 candidate,
3520 BridgeRejectionCause::Unsupported,
3521 format!("tracked-input cancel input_id is not a UUID: {error}"),
3522 None,
3523 )
3524 .await;
3525 return;
3526 }
3527 }
3528 let observation = match resolve_member_observation_host(adapter) {
3529 Ok(observation) => observation,
3530 Err((cause, reason)) => {
3531 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3532 return;
3533 }
3534 };
3535 match observation
3536 .cancel_tracked_member_input(session_id, &payload.expected_member, &payload.input_id)
3537 .await
3538 {
3539 Ok(outcome) => {
3540 send_bridge_response(
3541 comms_runtime,
3542 candidate,
3543 meerkat_core::interaction::ResponseStatus::Completed,
3544 BridgeReply::TrackedInputCancelled(BridgeTrackedInputCancelResponse {
3545 expected_member: payload.expected_member.clone(),
3546 input_id: payload.input_id.clone(),
3547 outcome,
3548 }),
3549 None,
3550 )
3551 .await;
3552 }
3553 Err(error) => {
3554 let (cause, reason) = observation_error_to_bridge_rejection(&error);
3555 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3556 }
3557 }
3558}
3559
3560async fn try_handle_supervisor_bridge_command(
3561 adapter: &Arc<MeerkatMachine>,
3562 session_id: &SessionId,
3563 comms_runtime: &Arc<dyn CommsRuntime>,
3564 candidate: &PeerInputCandidate,
3565) -> bool {
3566 if try_handle_supervisor_bridge_delivery(adapter, session_id, comms_runtime, candidate).await {
3567 return true;
3568 }
3569 let InteractionContent::Request { intent, params, .. } = &candidate.interaction.content else {
3570 return false;
3571 };
3572
3573 if intent != SUPERVISOR_BRIDGE_INTENT {
3577 return false;
3578 }
3579
3580 if !record_bridge_inbound_peer_request(comms_runtime, candidate) {
3586 return true;
3587 }
3588
3589 let command: BridgeCommand = match decode_bridge_command(params.clone()) {
3590 Ok(cmd) => cmd,
3591 Err(error) => {
3592 let cause = match &error {
3593 meerkat_contracts::wire::supervisor_bridge::BridgeCommandDecodeError::UnsupportedProtocolVersion(_) => {
3594 BridgeRejectionCause::UnsupportedProtocolVersion
3595 }
3596 meerkat_contracts::wire::supervisor_bridge::BridgeCommandDecodeError::Invalid(_) => {
3597 BridgeRejectionCause::Unsupported
3598 }
3599 };
3600 send_bridge_failure(
3601 comms_runtime,
3602 candidate,
3603 cause,
3604 format!("invalid bridge command: {error}"),
3605 None,
3606 )
3607 .await;
3608 return true;
3609 }
3610 };
3611
3612 let sender = &candidate.ingress;
3613
3614 match command {
3615 BridgeCommand::BindMember(payload) => {
3616 let supervisor = match bridge_peer_identity(&payload.supervisor, "bind member failed") {
3617 Ok(supervisor) => supervisor,
3618 Err((cause, reason)) => {
3619 send_bridge_failure(
3620 comms_runtime,
3621 candidate,
3622 cause,
3623 reason,
3624 Some(payload.supervisor.address.as_str()),
3625 )
3626 .await;
3627 return true;
3628 }
3629 };
3630 let admission = match adapter
3631 .resolve_supervisor_bind_admission(
3632 session_id,
3633 supervisor.peer_id.as_str().clone(),
3634 payload.epoch,
3635 generated_sender_peer_id(sender),
3636 )
3637 .await
3638 {
3639 Ok(admission) => admission,
3640 Err(error) => {
3641 send_bridge_failure(
3642 comms_runtime,
3643 candidate,
3644 BridgeRejectionCause::Internal,
3645 format!("bind member admission failed: {error}"),
3646 Some(payload.supervisor.address.as_str()),
3647 )
3648 .await;
3649 return true;
3650 }
3651 };
3652 match admission {
3653 SupervisorBindAdmission::IdempotentAck => {
3654 let Some(advertised) = comms_runtime.advertised_address() else {
3661 send_bridge_failure(
3662 comms_runtime,
3663 candidate,
3664 BridgeRejectionCause::Internal,
3665 "idempotent ack invariant violated",
3666 Some(payload.supervisor.address.as_str()),
3667 )
3668 .await;
3669 return true;
3670 };
3671 let Some(peer_id) = comms_runtime.peer_id().map(|peer_id| peer_id.as_str())
3672 else {
3673 send_bridge_failure(
3674 comms_runtime,
3675 candidate,
3676 BridgeRejectionCause::Internal,
3677 "idempotent ack invariant violated",
3678 Some(payload.supervisor.address.as_str()),
3679 )
3680 .await;
3681 return true;
3682 };
3683 let supervisor_spec =
3684 match TrustedPeerDescriptor::try_from(payload.supervisor.clone()) {
3685 Ok(spec) => spec,
3686 Err(error) => {
3687 send_bridge_failure(
3688 comms_runtime,
3689 candidate,
3690 BridgeRejectionCause::InvalidSupervisorSpec,
3691 format!(
3692 "bind member failed: invalid supervisor peer spec: {error}"
3693 ),
3694 Some(payload.supervisor.address.as_str()),
3695 )
3696 .await;
3697 return true;
3698 }
3699 };
3700 if let Err(error) = republish_private_supervisor_trust(
3701 adapter,
3702 session_id,
3703 comms_runtime.as_ref(),
3704 &supervisor_spec,
3705 payload.epoch,
3706 generated_sender_peer_id(sender),
3707 )
3708 .await
3709 {
3710 send_bridge_failure(
3711 comms_runtime,
3712 candidate,
3713 BridgeRejectionCause::Internal,
3714 format!(
3715 "bind member failed: trust repair failed before idempotent ack: {error}"
3716 ),
3717 Some(payload.supervisor.address.as_str()),
3718 )
3719 .await;
3720 return true;
3721 }
3722 send_bridge_response(
3723 comms_runtime,
3724 candidate,
3725 meerkat_core::interaction::ResponseStatus::Completed,
3726 BridgeReply::BindMember(BridgeBindResponse {
3727 peer_id,
3728 address: canonicalize_bridge_address(&advertised),
3729 capabilities: bridge_capabilities(payload.protocol_version),
3730 }),
3731 None,
3732 )
3733 .await;
3734 return true;
3735 }
3736 SupervisorBindAdmission::Bootstrap => {}
3737 SupervisorBindAdmission::Rejected(rejection) => {
3738 send_bridge_failure(
3739 comms_runtime,
3740 candidate,
3741 supervisor_bind_admission_rejection_cause(rejection),
3742 supervisor_bind_admission_rejection_reason(rejection, sender, &supervisor),
3743 Some(payload.supervisor.address.as_str()),
3744 )
3745 .await;
3746 return true;
3747 }
3748 }
3749 let (supervisor_spec, advertised_address) =
3750 match validate_bind_request(adapter, session_id, comms_runtime, sender, &payload)
3751 .await
3752 {
3753 Ok(binding) => binding,
3754 Err((cause, reason)) => {
3755 send_bridge_failure(
3756 comms_runtime,
3757 candidate,
3758 cause,
3759 reason,
3760 Some(payload.supervisor.address.as_str()),
3761 )
3762 .await;
3763 return true;
3764 }
3765 };
3766 let Some(peer_id) = comms_runtime.peer_id().map(|peer_id| peer_id.as_str()) else {
3773 send_bridge_failure(
3774 comms_runtime,
3775 candidate,
3776 BridgeRejectionCause::Internal,
3777 "bind member failed: runtime peer_id unavailable",
3778 Some(payload.supervisor.address.as_str()),
3779 )
3780 .await;
3781 return true;
3782 };
3783 if let Err(reason) = commit_supervisor_bind_with_rollback(
3791 adapter,
3792 session_id,
3793 comms_runtime.as_ref(),
3794 &supervisor_spec,
3795 payload.epoch,
3796 "bind member",
3797 )
3798 .await
3799 {
3800 send_bridge_failure(
3801 comms_runtime,
3802 candidate,
3803 BridgeRejectionCause::Internal,
3804 reason,
3805 Some(payload.supervisor.address.as_str()),
3806 )
3807 .await;
3808 return true;
3809 }
3810 send_bridge_response(
3811 comms_runtime,
3812 candidate,
3813 meerkat_core::interaction::ResponseStatus::Completed,
3814 BridgeReply::BindMember(BridgeBindResponse {
3815 peer_id,
3816 address: canonicalize_bridge_address(&advertised_address),
3817 capabilities: bridge_capabilities(payload.protocol_version),
3818 }),
3819 None,
3820 )
3821 .await;
3822 true
3823 }
3824 BridgeCommand::AuthorizeSupervisor(payload) => {
3825 handle_authorize_supervisor_command(
3826 adapter,
3827 session_id,
3828 comms_runtime,
3829 candidate,
3830 sender,
3831 payload,
3832 )
3833 .await
3834 }
3835 BridgeCommand::RevokeSupervisor(payload) => {
3836 let authorized_supervisor = match resolve_authorized_supervisor_cleanup(
3837 adapter,
3838 session_id,
3839 sender,
3840 &payload,
3841 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Revoke,
3842 )
3843 .await
3844 {
3845 Ok(supervisor) => supervisor,
3846 Err((BridgeRejectionCause::NotBound, reason)) => {
3847 send_bridge_failure(
3848 comms_runtime,
3849 candidate,
3850 BridgeRejectionCause::NotBound,
3851 reason,
3852 Some(payload.supervisor.address.as_str()),
3853 )
3854 .await;
3855 return true;
3856 }
3857 Err((cause, reason)) => {
3858 send_bridge_failure(
3859 comms_runtime,
3860 candidate,
3861 cause,
3862 reason,
3863 Some(payload.supervisor.address.as_str()),
3864 )
3865 .await;
3866 return true;
3867 }
3868 };
3869 let resume_pending_revoke = authorized_supervisor.resume_pending_revoke;
3870 let authorized_supervisor = authorized_supervisor.supervisor;
3871 if let Err(error) = adapter
3872 .stage_local_endpoint_for_comms_runtime(session_id, comms_runtime.as_ref())
3873 .await
3874 {
3875 send_bridge_failure(
3876 comms_runtime,
3877 candidate,
3878 BridgeRejectionCause::Internal,
3879 format!("revoke supervisor failed: local endpoint rejected: {error}"),
3880 Some(payload.supervisor.address.as_str()),
3881 )
3882 .await;
3883 return true;
3884 }
3885 let supervisor_peer_id = authorized_supervisor.peer_id.as_str();
3886 let revoke_transition = match adapter
3887 .stage_supervisor_revoke(session_id, supervisor_peer_id.clone(), payload.epoch)
3888 .await
3889 {
3890 Ok(transition) => transition,
3891 Err(error) => {
3892 send_bridge_failure(
3893 comms_runtime,
3894 candidate,
3895 BridgeRejectionCause::Internal,
3896 format!("revoke supervisor failed: DSL rejected revoke: {error}"),
3897 Some(payload.supervisor.address.as_str()),
3898 )
3899 .await;
3900 return true;
3901 }
3902 };
3903 let revoke_obligation = match matching_supervisor_revoke_obligation(
3904 adapter,
3905 session_id,
3906 &revoke_transition,
3907 supervisor_peer_id.as_str(),
3908 payload.epoch,
3909 "revoke supervisor",
3910 )
3911 .await
3912 {
3913 Ok(obligation) => obligation,
3914 Err(reason) => {
3915 if let Err(error) = adapter
3916 .stage_supervisor_trust_revoke_failed(
3917 session_id,
3918 supervisor_peer_id.clone(),
3919 payload.epoch,
3920 reason.clone(),
3921 )
3922 .await
3923 {
3924 tracing::warn!(
3925 %session_id,
3926 epoch = payload.epoch,
3927 %error,
3928 "failed to restore supervisor binding after missing revoke effect"
3929 );
3930 }
3931 send_bridge_failure(
3932 comms_runtime,
3933 candidate,
3934 BridgeRejectionCause::Internal,
3935 reason,
3936 Some(payload.supervisor.address.as_str()),
3937 )
3938 .await;
3939 return true;
3940 }
3941 };
3942 let revoke_authority = match supervisor_revoke_authority(&revoke_obligation) {
3943 Ok(authority) => authority,
3944 Err(error) => {
3945 let _ = adapter
3946 .stage_supervisor_trust_revoke_failed(
3947 session_id,
3948 revoke_obligation.peer_id().clone(),
3949 revoke_obligation.epoch(),
3950 error.clone(),
3951 )
3952 .await;
3953 send_bridge_failure(
3954 comms_runtime,
3955 candidate,
3956 BridgeRejectionCause::Internal,
3957 error,
3958 Some(payload.supervisor.address.as_str()),
3959 )
3960 .await;
3961 return true;
3962 }
3963 };
3964 if resume_pending_revoke {
3969 send_bridge_failure(
3970 comms_runtime,
3971 candidate,
3972 BridgeRejectionCause::NotBound,
3973 "supervisor revoke cleanup was already pending".to_string(),
3974 Some(payload.supervisor.address.as_str()),
3975 )
3976 .await;
3977 } else {
3978 send_bridge_response(
3979 comms_runtime,
3980 candidate,
3981 meerkat_core::interaction::ResponseStatus::Completed,
3982 BridgeReply::Ack(BridgeAck { ok: true }),
3983 Some(payload.supervisor.address.as_str()),
3984 )
3985 .await;
3986 }
3987 if let Err(error) = apply_generated_private_trust_remove(
3988 comms_runtime.as_ref(),
3989 revoke_obligation.peer_id().clone(),
3990 revoke_authority,
3991 )
3992 .await
3993 {
3994 let feedback_result = adapter
3995 .stage_supervisor_trust_revoke_failed(
3996 session_id,
3997 revoke_obligation.peer_id().clone(),
3998 revoke_obligation.epoch(),
3999 error.clone(),
4000 )
4001 .await;
4002 let mut reason = format!(
4003 "revoke supervisor failed: trust removal rejected after DSL commit: {error}"
4004 );
4005 if let Err(feedback_error) = feedback_result {
4006 reason.push_str(&format!("; feedback failed: {feedback_error}"));
4007 }
4008 tracing::warn!(%session_id, %reason, "accepted supervisor revoke remains pending");
4009 return true;
4010 }
4011 if let Err(error) = adapter
4017 .stage_supervisor_trust_revoked(
4018 session_id,
4019 revoke_obligation.peer_id().clone(),
4020 revoke_obligation.epoch(),
4021 )
4022 .await
4023 {
4024 tracing::warn!(%session_id, %error, "accepted supervisor revoke cleanup checkpoint failed");
4025 return true;
4026 }
4027 true
4028 }
4029 BridgeCommand::DeliverMemberInput(payload) => {
4030 let sup_payload = BridgeSupervisorPayload {
4031 supervisor: payload.supervisor.clone(),
4032 epoch: payload.epoch,
4033 protocol_version: payload.protocol_version,
4034 };
4035 let authorized_supervisor = match resolve_authorized_supervisor_with_response_route(
4036 adapter,
4037 session_id,
4038 comms_runtime,
4039 sender,
4040 &sup_payload,
4041 "deliver member input failed",
4042 )
4043 .await
4044 {
4045 Ok(supervisor) => supervisor,
4046 Err((cause, reason)) => {
4047 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4048 return true;
4049 }
4050 };
4051 let request_input_id = payload.input_id.clone();
4052 let registered_incarnation = adapter.member_incarnation(session_id);
4053 match (®istered_incarnation, &payload.expected_member) {
4054 (Some(current), Some(expected)) if current == expected => {}
4055 (Some(current), _) => {
4056 let response = BridgeDeliveryResponse {
4057 input_id: request_input_id,
4058 canonical_input_id: None,
4059 outcome: BridgeDeliveryOutcome::Rejected {
4060 cause: BridgeDeliveryRejectionCause::StaleMemberIncarnation {
4061 current: current.clone(),
4062 },
4063 reason: format!(
4064 "delivery expected member incarnation {:?}; current is {:?}",
4065 payload.expected_member, current
4066 ),
4067 },
4068 };
4069 send_bridge_response(
4070 comms_runtime,
4071 candidate,
4072 meerkat_core::interaction::ResponseStatus::Completed,
4073 BridgeReply::Delivery(response),
4074 None,
4075 )
4076 .await;
4077 return true;
4078 }
4079 (None, Some(expected)) => {
4080 let response = BridgeDeliveryResponse {
4081 input_id: request_input_id,
4082 canonical_input_id: None,
4083 outcome: BridgeDeliveryOutcome::Rejected {
4084 cause: BridgeDeliveryRejectionCause::Internal {
4085 detail: "target session has no registered host-member incarnation"
4086 .to_string(),
4087 },
4088 reason: format!(
4089 "delivery addressed host member {expected:?}, but target session has no host journal"
4090 ),
4091 },
4092 };
4093 send_bridge_response(
4094 comms_runtime,
4095 candidate,
4096 meerkat_core::interaction::ResponseStatus::Completed,
4097 BridgeReply::Delivery(response),
4098 None,
4099 )
4100 .await;
4101 return true;
4102 }
4103 (None, None) => {}
4104 }
4105 if let Err(detail) = validate_delivery_tracking_request(&payload) {
4111 let response = BridgeDeliveryResponse {
4112 input_id: request_input_id,
4113 canonical_input_id: None,
4114 outcome: BridgeDeliveryOutcome::Rejected {
4115 cause: BridgeDeliveryRejectionCause::TurnDirectiveUnsupported {
4116 detail: detail.clone(),
4117 },
4118 reason: detail,
4119 },
4120 };
4121 send_bridge_response(
4122 comms_runtime,
4123 candidate,
4124 meerkat_core::interaction::ResponseStatus::Completed,
4125 BridgeReply::Delivery(response),
4126 None,
4127 )
4128 .await;
4129 return true;
4130 }
4131 if delivery_requires_tracked_turn_custody(&payload) {
4137 serve_tracked_member_delivery(
4138 adapter,
4139 session_id,
4140 comms_runtime,
4141 candidate,
4142 authorized_supervisor.peer_id,
4143 payload,
4144 )
4145 .await;
4146 return true;
4147 }
4148 let expected_member = payload.expected_member.clone();
4149 let input = match peer_input_from_delivery_payload(
4150 session_id,
4151 authorized_supervisor.peer_id,
4152 payload,
4153 ) {
4154 Ok(input) => input,
4155 Err(reason) => {
4156 let response = BridgeDeliveryResponse {
4157 input_id: request_input_id,
4158 canonical_input_id: None,
4159 outcome: BridgeDeliveryOutcome::Rejected {
4160 cause: BridgeDeliveryRejectionCause::Internal {
4161 detail: reason.clone(),
4162 },
4163 reason,
4164 },
4165 };
4166 send_bridge_response(
4167 comms_runtime,
4168 candidate,
4169 meerkat_core::interaction::ResponseStatus::Completed,
4170 BridgeReply::Delivery(response),
4171 None,
4172 )
4173 .await;
4174 return true;
4175 }
4176 };
4177 let accept_result = adapter
4178 .accept_input_with_completion_for_member_residency(
4179 session_id,
4180 input,
4181 expected_member.as_ref(),
4182 )
4183 .await;
4184 match accept_result {
4185 Ok((accept_outcome, _completion_handle)) => {
4186 let response = match accept_outcome {
4187 crate::accept::AcceptOutcome::Accepted { input_id, .. } => {
4188 BridgeDeliveryResponse {
4189 input_id: request_input_id,
4190 canonical_input_id: Some(input_id.to_string()),
4191 outcome: BridgeDeliveryOutcome::Accepted,
4192 }
4193 }
4194 crate::accept::AcceptOutcome::Deduplicated { existing_id, .. } => {
4195 let existing_id = existing_id.to_string();
4196 BridgeDeliveryResponse {
4197 input_id: request_input_id,
4198 canonical_input_id: Some(existing_id.clone()),
4199 outcome: BridgeDeliveryOutcome::Deduplicated {
4200 existing_input_id: existing_id,
4201 },
4202 }
4203 }
4204 crate::accept::AcceptOutcome::Rejected { reason } => {
4205 let cause = bridge_delivery_rejection_cause(&reason);
4206 BridgeDeliveryResponse {
4207 input_id: request_input_id,
4208 canonical_input_id: None,
4209 outcome: BridgeDeliveryOutcome::Rejected {
4210 cause,
4211 reason: reason.to_string(),
4212 },
4213 }
4214 }
4215 };
4216 send_bridge_response(
4217 comms_runtime,
4218 candidate,
4219 meerkat_core::interaction::ResponseStatus::Completed,
4220 BridgeReply::Delivery(response),
4221 None,
4222 )
4223 .await;
4224 }
4225 Err(crate::traits::RuntimeDriverError::StaleAuthority { reason }) => {
4226 if let Some(current) = adapter.member_incarnation(session_id) {
4227 let response = BridgeDeliveryResponse {
4228 input_id: request_input_id,
4229 canonical_input_id: None,
4230 outcome: BridgeDeliveryOutcome::Rejected {
4231 cause: BridgeDeliveryRejectionCause::StaleMemberIncarnation {
4232 current,
4233 },
4234 reason,
4235 },
4236 };
4237 send_bridge_response(
4238 comms_runtime,
4239 candidate,
4240 meerkat_core::interaction::ResponseStatus::Completed,
4241 BridgeReply::Delivery(response),
4242 None,
4243 )
4244 .await;
4245 } else {
4246 send_bridge_failure(
4247 comms_runtime,
4248 candidate,
4249 BridgeRejectionCause::StaleFence,
4250 reason,
4251 None,
4252 )
4253 .await;
4254 }
4255 }
4256 Err(error) => {
4257 send_bridge_failure(
4258 comms_runtime,
4259 candidate,
4260 BridgeRejectionCause::Internal,
4261 format!("deliver member input failed: {error}"),
4262 None,
4263 )
4264 .await;
4265 }
4266 }
4267 true
4268 }
4269 BridgeCommand::InterruptMember(payload) => {
4270 let sup_payload = BridgeSupervisorPayload {
4271 supervisor: payload.supervisor.clone(),
4272 epoch: payload.epoch,
4273 protocol_version: payload.protocol_version,
4274 };
4275 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
4276 adapter,
4277 session_id,
4278 comms_runtime,
4279 sender,
4280 &sup_payload,
4281 "interrupt member failed",
4282 )
4283 .await
4284 {
4285 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4286 return true;
4287 }
4288 if let Err((cause, reason)) = require_optional_registered_member_incarnation(
4289 adapter,
4290 session_id,
4291 payload.expected_member.as_ref(),
4292 "interrupt member",
4293 ) {
4294 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4295 return true;
4296 }
4297 let result = adapter
4298 .cancel_after_boundary_for_optional_member_incarnation(
4299 session_id,
4300 payload.expected_member.as_ref(),
4301 )
4302 .await;
4303 match result {
4304 Ok(()) => {
4305 if let Err((cause, reason)) = require_optional_registered_member_incarnation(
4306 adapter,
4307 session_id,
4308 payload.expected_member.as_ref(),
4309 "interrupt member completion",
4310 ) {
4311 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4312 return true;
4313 }
4314 send_bridge_response(
4315 comms_runtime,
4316 candidate,
4317 meerkat_core::interaction::ResponseStatus::Completed,
4318 BridgeReply::Ack(BridgeAck { ok: true }),
4319 None,
4320 )
4321 .await;
4322 }
4323 Err(crate::traits::RuntimeDriverError::StaleAuthority { reason }) => {
4324 send_bridge_failure(
4325 comms_runtime,
4326 candidate,
4327 BridgeRejectionCause::StaleFence,
4328 reason,
4329 None,
4330 )
4331 .await;
4332 }
4333 Err(error) => {
4334 send_bridge_failure(
4335 comms_runtime,
4336 candidate,
4337 BridgeRejectionCause::Internal,
4338 format!("interrupt member failed: {error}"),
4339 None,
4340 )
4341 .await;
4342 }
4343 }
4344 true
4345 }
4346 BridgeCommand::RetireMember(payload) => {
4347 if let Err((cause, reason)) = resolve_authorized_supervisor_cleanup_with_response_route(
4348 adapter,
4349 session_id,
4350 comms_runtime,
4351 sender,
4352 &payload,
4353 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Retire,
4354 "retire member failed",
4355 )
4356 .await
4357 {
4358 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4359 return true;
4360 }
4361 match adapter.retire_runtime(session_id).await {
4362 Ok(report) => {
4363 send_bridge_response(
4364 comms_runtime,
4365 candidate,
4366 meerkat_core::interaction::ResponseStatus::Completed,
4367 BridgeReply::Retire(BridgeRetireResponse {
4368 inputs_abandoned: report.inputs_abandoned,
4369 inputs_pending_drain: report.inputs_pending_drain,
4370 }),
4371 None,
4372 )
4373 .await;
4374 }
4375 Err(error) => {
4376 send_bridge_failure(
4377 comms_runtime,
4378 candidate,
4379 BridgeRejectionCause::Internal,
4380 format!("retire member failed: {error}"),
4381 None,
4382 )
4383 .await;
4384 }
4385 }
4386 true
4387 }
4388 BridgeCommand::DestroyMember(payload) => {
4389 if let Err((cause, reason)) = resolve_authorized_supervisor_cleanup_with_response_route(
4390 adapter,
4391 session_id,
4392 comms_runtime,
4393 sender,
4394 &payload,
4395 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Destroy,
4396 "destroy member failed",
4397 )
4398 .await
4399 {
4400 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4401 return true;
4402 }
4403 let runtime_id = MeerkatMachine::logical_runtime_id(session_id);
4404 match RuntimeControlPlane::destroy(adapter.as_ref(), &runtime_id).await {
4405 Ok(report) => {
4406 send_bridge_response(
4407 comms_runtime,
4408 candidate,
4409 meerkat_core::interaction::ResponseStatus::Completed,
4410 BridgeReply::Destroy(BridgeDestroyResponse {
4411 inputs_abandoned: report.inputs_abandoned,
4412 }),
4413 None,
4414 )
4415 .await;
4416 }
4417 Err(error) => {
4418 send_bridge_failure(
4419 comms_runtime,
4420 candidate,
4421 BridgeRejectionCause::Internal,
4422 format!("destroy member failed: {error}"),
4423 None,
4424 )
4425 .await;
4426 }
4427 }
4428 true
4429 }
4430 BridgeCommand::ObserveMember(payload) => {
4431 if let Err((cause, reason)) = resolve_authorized_supervisor_cleanup_with_response_route(
4432 adapter,
4433 session_id,
4434 comms_runtime,
4435 sender,
4436 &payload,
4437 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Observe,
4438 "observe member failed",
4439 )
4440 .await
4441 {
4442 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4443 return true;
4444 }
4445 match crate::service_ext::SessionServiceRuntimeExt::runtime_state(
4446 adapter.as_ref(),
4447 session_id,
4448 )
4449 .await
4450 {
4451 Ok(state) => {
4452 let current_run_id = adapter
4453 .meerkat_machine_spine_snapshot(session_id)
4454 .await
4455 .and_then(|snapshot| {
4456 snapshot
4457 .control
4458 .current_run_id
4459 .map(|run_id| run_id.to_string())
4460 });
4461 let bridge_state = runtime_state_to_bridge(state);
4462 let lifecycle_facts =
4463 match crate::meerkat_machine::classify_runtime_lifecycle_state(state) {
4464 Ok(facts) => facts,
4465 Err(error) => {
4466 send_bridge_failure(
4467 comms_runtime,
4468 candidate,
4469 BridgeRejectionCause::Internal,
4470 format!("runtime lifecycle classification failed: {error}"),
4471 None,
4472 )
4473 .await;
4474 return true;
4475 }
4476 };
4477 send_bridge_response(
4478 comms_runtime,
4479 candidate,
4480 meerkat_core::interaction::ResponseStatus::Completed,
4481 BridgeReply::Observation(BridgeObservationResponse::new(
4482 bridge_state,
4483 Some(lifecycle_facts.can_accept_input()),
4484 current_run_id,
4485 Some(BridgePeerConnectivity::Reachable),
4486 None,
4487 chrono::Utc::now().to_rfc3339(),
4488 )),
4489 None,
4490 )
4491 .await;
4492 }
4493 Err(error) => {
4494 send_bridge_failure(
4495 comms_runtime,
4496 candidate,
4497 BridgeRejectionCause::Internal,
4498 format!("observe member failed: {error}"),
4499 None,
4500 )
4501 .await;
4502 }
4503 }
4504 true
4505 }
4506 BridgeCommand::WireMember(payload) => {
4507 let sup_payload = BridgeSupervisorPayload {
4508 supervisor: payload.supervisor.clone(),
4509 epoch: payload.epoch,
4510 protocol_version: payload.protocol_version,
4511 };
4512 let authorized_supervisor = match resolve_authorized_supervisor_with_response_route(
4513 adapter,
4514 session_id,
4515 comms_runtime,
4516 sender,
4517 &sup_payload,
4518 "wire member failed",
4519 )
4520 .await
4521 {
4522 Ok(supervisor) => supervisor,
4523 Err((cause, reason)) => {
4524 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4525 return true;
4526 }
4527 };
4528 let peer_spec = match TrustedPeerDescriptor::try_from(payload.peer_spec) {
4536 Ok(spec) => spec,
4537 Err(error) => {
4538 send_bridge_failure(
4539 comms_runtime,
4540 candidate,
4541 BridgeRejectionCause::InvalidPeerSpec,
4542 format!("wire member failed: invalid trusted peer spec: {error}"),
4543 None,
4544 )
4545 .await;
4546 return true;
4547 }
4548 };
4549 let mob_peer_overlay = match payload.mob_peer_overlay {
4554 Some(overlay) => overlay,
4555 None => {
4556 send_bridge_failure(
4557 comms_runtime,
4558 candidate,
4559 BridgeRejectionCause::UnsupportedProtocolVersion,
4560 format!(
4561 "wire member failed: peer wiring requires supervisor bridge protocol v{} (mob peer overlay absent); upgrade the supervisor",
4562 BridgeProtocolVersion::V3
4563 ),
4564 None,
4565 )
4566 .await;
4567 return true;
4568 }
4569 };
4570 let overlay = match bridge_mob_peer_overlay(&mob_peer_overlay) {
4571 Ok(overlay) => overlay,
4572 Err(error) => {
4573 send_bridge_failure(
4574 comms_runtime,
4575 candidate,
4576 BridgeRejectionCause::InvalidPeerSpec,
4577 format!("wire member failed: invalid MobMachine peer overlay: {error}"),
4578 None,
4579 )
4580 .await;
4581 return true;
4582 }
4583 };
4584 match adapter
4585 .stage_authorized_supervisor_mob_peer_overlay(
4586 session_id,
4587 payload.supervisor.peer_id,
4588 payload.epoch,
4589 mob_peer_overlay.recipient_peer_id,
4590 mob_peer_overlay.topology_epoch,
4591 overlay.endpoints,
4592 overlay.endpoint_count,
4593 peer_spec.peer_id.to_string(),
4594 crate::meerkat_machine::dsl::PeerEndpoint::from(&peer_spec),
4595 crate::meerkat_machine::dsl::MobPeerOverlayCommandKind::Wire,
4596 Arc::clone(comms_runtime),
4597 )
4598 .await
4599 {
4600 Ok(()) => {
4601 if let Err((cause, reason)) = ensure_bridge_response_route_for_supervisor(
4602 adapter,
4603 session_id,
4604 comms_runtime,
4605 &authorized_supervisor,
4606 "wire member failed",
4607 )
4608 .await
4609 {
4610 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4611 return true;
4612 }
4613 send_bridge_response(
4614 comms_runtime,
4615 candidate,
4616 meerkat_core::interaction::ResponseStatus::Completed,
4617 BridgeReply::Ack(BridgeAck { ok: true }),
4618 None,
4619 )
4620 .await;
4621 }
4622 Err(error) => {
4623 let endpoint = crate::meerkat_machine::dsl::PeerEndpoint::from(&peer_spec);
4630 if matches!(
4631 error,
4632 crate::meerkat_machine::PeerEndpointStageError::Dsl(
4633 crate::meerkat_machine::dsl::MeerkatMachineTransitionError::GuardRejected {
4634 ..
4635 }
4636 )
4637 ) && adapter
4638 .direct_peer_endpoints(session_id)
4639 .await
4640 .map(|endpoints| endpoints.contains(&endpoint))
4641 .unwrap_or(false)
4642 {
4643 if let Err((cause, reason)) = ensure_bridge_response_route_for_supervisor(
4644 adapter,
4645 session_id,
4646 comms_runtime,
4647 &authorized_supervisor,
4648 "wire member failed",
4649 )
4650 .await
4651 {
4652 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4653 return true;
4654 }
4655 send_bridge_response(
4656 comms_runtime,
4657 candidate,
4658 meerkat_core::interaction::ResponseStatus::Completed,
4659 BridgeReply::Ack(BridgeAck { ok: true }),
4660 None,
4661 )
4662 .await;
4663 } else {
4664 send_bridge_failure(
4665 comms_runtime,
4666 candidate,
4667 BridgeRejectionCause::Internal,
4668 format!(
4669 "wire member failed: generated peer overlay authority rejected: {error}"
4670 ),
4671 None,
4672 )
4673 .await;
4674 }
4675 }
4676 }
4677 true
4678 }
4679 BridgeCommand::UnwireMember(payload) => {
4680 let sup_payload = BridgeSupervisorPayload {
4681 supervisor: payload.supervisor.clone(),
4682 epoch: payload.epoch,
4683 protocol_version: payload.protocol_version,
4684 };
4685 let authorized_supervisor = match resolve_authorized_supervisor_with_response_route(
4686 adapter,
4687 session_id,
4688 comms_runtime,
4689 sender,
4690 &sup_payload,
4691 "unwire member failed",
4692 )
4693 .await
4694 {
4695 Ok(supervisor) => supervisor,
4696 Err((cause, reason)) => {
4697 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4698 return true;
4699 }
4700 };
4701 let peer_spec = match TrustedPeerDescriptor::try_from(payload.peer_spec) {
4708 Ok(spec) => spec,
4709 Err(error) => {
4710 send_bridge_failure(
4711 comms_runtime,
4712 candidate,
4713 BridgeRejectionCause::InvalidPeerSpec,
4714 format!("unwire member failed: invalid trusted peer spec: {error}"),
4715 None,
4716 )
4717 .await;
4718 return true;
4719 }
4720 };
4721 let mob_peer_overlay = match payload.mob_peer_overlay {
4724 Some(overlay) => overlay,
4725 None => {
4726 send_bridge_failure(
4727 comms_runtime,
4728 candidate,
4729 BridgeRejectionCause::UnsupportedProtocolVersion,
4730 format!(
4731 "unwire member failed: peer wiring requires supervisor bridge protocol v{} (mob peer overlay absent); upgrade the supervisor",
4732 BridgeProtocolVersion::V3
4733 ),
4734 None,
4735 )
4736 .await;
4737 return true;
4738 }
4739 };
4740 let overlay = match bridge_mob_peer_overlay(&mob_peer_overlay) {
4741 Ok(overlay) => overlay,
4742 Err(error) => {
4743 send_bridge_failure(
4744 comms_runtime,
4745 candidate,
4746 BridgeRejectionCause::InvalidPeerSpec,
4747 format!("unwire member failed: invalid MobMachine peer overlay: {error}"),
4748 None,
4749 )
4750 .await;
4751 return true;
4752 }
4753 };
4754 match adapter
4755 .stage_authorized_supervisor_mob_peer_overlay(
4756 session_id,
4757 payload.supervisor.peer_id,
4758 payload.epoch,
4759 mob_peer_overlay.recipient_peer_id,
4760 mob_peer_overlay.topology_epoch,
4761 overlay.endpoints,
4762 overlay.endpoint_count,
4763 peer_spec.peer_id.to_string(),
4764 crate::meerkat_machine::dsl::PeerEndpoint::from(&peer_spec),
4765 crate::meerkat_machine::dsl::MobPeerOverlayCommandKind::Unwire,
4766 Arc::clone(comms_runtime),
4767 )
4768 .await
4769 {
4770 Ok(()) => {
4771 if let Err((cause, reason)) = ensure_bridge_response_route_for_supervisor(
4772 adapter,
4773 session_id,
4774 comms_runtime,
4775 &authorized_supervisor,
4776 "unwire member failed",
4777 )
4778 .await
4779 {
4780 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4781 return true;
4782 }
4783 send_bridge_response(
4784 comms_runtime,
4785 candidate,
4786 meerkat_core::interaction::ResponseStatus::Completed,
4787 BridgeReply::Ack(BridgeAck { ok: true }),
4788 None,
4789 )
4790 .await;
4791 }
4792 Err(error) => {
4793 let endpoint = crate::meerkat_machine::dsl::PeerEndpoint::from(&peer_spec);
4799 if matches!(
4800 error,
4801 crate::meerkat_machine::PeerEndpointStageError::Dsl(
4802 crate::meerkat_machine::dsl::MeerkatMachineTransitionError::GuardRejected {
4803 ..
4804 }
4805 )
4806 ) && !adapter
4807 .direct_peer_endpoints(session_id)
4808 .await
4809 .map(|endpoints| endpoints.contains(&endpoint))
4810 .unwrap_or(false)
4811 {
4812 if let Err((cause, reason)) = ensure_bridge_response_route_for_supervisor(
4813 adapter,
4814 session_id,
4815 comms_runtime,
4816 &authorized_supervisor,
4817 "unwire member failed",
4818 )
4819 .await
4820 {
4821 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4822 return true;
4823 }
4824 send_bridge_response(
4825 comms_runtime,
4826 candidate,
4827 meerkat_core::interaction::ResponseStatus::Completed,
4828 BridgeReply::Ack(BridgeAck { ok: true }),
4829 None,
4830 )
4831 .await;
4832 } else {
4833 send_bridge_failure(
4834 comms_runtime,
4835 candidate,
4836 BridgeRejectionCause::Internal,
4837 format!(
4838 "unwire member failed: generated peer overlay authority rejected: {error}"
4839 ),
4840 None,
4841 )
4842 .await;
4843 }
4844 }
4845 }
4846 true
4847 }
4848 BridgeCommand::ObserveSupervisorRotation(payload) => {
4849 if payload.protocol_version != BridgeProtocolVersion::V4 {
4850 send_bridge_failure(
4851 comms_runtime,
4852 candidate,
4853 BridgeRejectionCause::UnsupportedProtocolVersion,
4854 format!(
4855 "observe supervisor rotation requires protocol v4, received {:?}",
4856 payload.protocol_version
4857 ),
4858 Some(payload.observer.address.as_str()),
4859 )
4860 .await;
4861 return true;
4862 }
4863 let observer =
4864 match bridge_peer_identity(&payload.observer, "observe supervisor rotation failed")
4865 {
4866 Ok(observer) => observer,
4867 Err((cause, reason)) => {
4868 send_bridge_failure(
4869 comms_runtime,
4870 candidate,
4871 cause,
4872 reason,
4873 Some(payload.observer.address.as_str()),
4874 )
4875 .await;
4876 return true;
4877 }
4878 };
4879 let authenticated_peer_id = generated_sender_peer_id(sender);
4880 let authenticated_key = generated_sender_signing_public_key(sender);
4881 let observer_peer_id = observer.peer_id.as_str();
4882 let observer_key = encode_supervisor_signing_public_key(observer.pubkey.into_bytes());
4883 if authenticated_peer_id.as_deref() != Some(observer_peer_id.as_str())
4884 || authenticated_key.as_deref() != Some(observer_key.as_str())
4885 {
4886 send_bridge_failure(
4887 comms_runtime,
4888 candidate,
4889 BridgeRejectionCause::SenderMismatch,
4890 "observe supervisor rotation failed: observer does not match authenticated sender"
4891 .to_string(),
4892 Some(payload.observer.address.as_str()),
4893 )
4894 .await;
4895 return true;
4896 }
4897 let operation_id = payload.operation_id;
4898 let observer = GeneratedSupervisorBinding {
4899 name: observer.name.as_str().to_owned(),
4900 peer_id: observer_peer_id,
4901 address: observer.address.to_string(),
4902 signing_public_key: observer_key,
4903 epoch: payload.observer_epoch,
4904 };
4905 let reply =
4906 match adapter
4907 .observe_supervisor_rotation(session_id, operation_id.to_string(), &observer)
4908 .await
4909 {
4910 Ok(SupervisorRotationObservation::Found(receipt)) => {
4911 match bridge_supervisor_rotation_state(receipt) {
4912 Ok(state) => BridgeReply::SupervisorRotation(
4913 BridgeSupervisorRotationObservation::Found { state },
4914 ),
4915 Err(error) => {
4916 send_bridge_failure(
4917 comms_runtime,
4918 candidate,
4919 BridgeRejectionCause::Internal,
4920 error,
4921 Some(payload.observer.address.as_str()),
4922 )
4923 .await;
4924 return true;
4925 }
4926 }
4927 }
4928 Ok(SupervisorRotationObservation::NotFound) => BridgeReply::SupervisorRotation(
4929 BridgeSupervisorRotationObservation::NotFound { operation_id },
4930 ),
4931 Err(error) => {
4932 send_bridge_failure(
4933 comms_runtime,
4934 candidate,
4935 BridgeRejectionCause::Internal,
4936 format!("observe supervisor rotation failed: {error}"),
4937 Some(payload.observer.address.as_str()),
4938 )
4939 .await;
4940 return true;
4941 }
4942 };
4943 send_bridge_response(
4951 comms_runtime,
4952 candidate,
4953 meerkat_core::interaction::ResponseStatus::Completed,
4954 reply,
4955 Some(payload.observer.address.as_str()),
4956 )
4957 .await;
4958 true
4959 }
4960 BridgeCommand::DeclareMemberOutboundTaint(payload) => {
4961 let sup_payload = BridgeSupervisorPayload {
4962 supervisor: payload.supervisor.clone(),
4963 epoch: payload.epoch,
4964 protocol_version: payload.protocol_version,
4965 };
4966 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
4967 adapter,
4968 session_id,
4969 comms_runtime,
4970 sender,
4971 &sup_payload,
4972 "declare member outbound taint failed",
4973 )
4974 .await
4975 {
4976 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4977 return true;
4978 }
4979 let incarnation_check = match payload.target.as_ref() {
4980 Some(BridgeOutboundTaintTarget::Placed(expected)) => {
4981 require_registered_member_incarnation(
4982 adapter,
4983 session_id,
4984 expected,
4985 "declare member outbound taint failed",
4986 )
4987 }
4988 Some(BridgeOutboundTaintTarget::PeerOnly) | None => {
4989 if let Some(current) = adapter.member_incarnation(session_id) {
4990 Err((
4991 BridgeRejectionCause::StaleFence,
4992 format!(
4993 "declare member outbound taint failed: peer-only or legacy target cannot address registered incarnation {current:?}"
4994 ),
4995 ))
4996 } else {
4997 Ok(())
4998 }
4999 }
5000 };
5001 if let Err((cause, reason)) = incarnation_check {
5002 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5003 return true;
5004 }
5005 let expected_member = match payload.target.as_ref() {
5006 Some(BridgeOutboundTaintTarget::Placed(expected)) => Some(expected),
5007 Some(BridgeOutboundTaintTarget::PeerOnly) | None => None,
5008 };
5009 let effect_authority = match acquire_optional_member_effect_authority(
5010 adapter,
5011 session_id,
5012 expected_member,
5013 "declare member outbound taint",
5014 )
5015 .await
5016 {
5017 Ok(authority) => authority,
5018 Err((cause, reason)) => {
5019 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5020 return true;
5021 }
5022 };
5023 let result = comms_runtime.set_outbound_content_taint(payload.taint);
5028 drop(effect_authority);
5029 match result {
5030 Ok(()) => {
5031 send_bridge_response(
5032 comms_runtime,
5033 candidate,
5034 meerkat_core::interaction::ResponseStatus::Completed,
5035 BridgeReply::Ack(BridgeAck { ok: true }),
5036 None,
5037 )
5038 .await;
5039 }
5040 Err(error) => {
5041 send_bridge_failure(
5042 comms_runtime,
5043 candidate,
5044 BridgeRejectionCause::Internal,
5045 format!("declare member outbound taint failed: {error}"),
5046 None,
5047 )
5048 .await;
5049 }
5050 }
5051 true
5052 }
5053 BridgeCommand::HardCancelMember(payload) => {
5054 serve_hard_cancel_member(
5055 adapter,
5056 session_id,
5057 comms_runtime,
5058 sender,
5059 candidate,
5060 &payload,
5061 )
5062 .await;
5063 true
5064 }
5065 BridgeCommand::CancelTrackedMemberInput(payload) => {
5066 serve_cancel_tracked_member_input(
5067 adapter,
5068 session_id,
5069 comms_runtime,
5070 sender,
5071 candidate,
5072 &payload,
5073 )
5074 .await;
5075 true
5076 }
5077 BridgeCommand::ReadMemberHistory(payload) => {
5078 let sup_payload = BridgeSupervisorPayload {
5081 supervisor: payload.supervisor.clone(),
5082 epoch: payload.epoch,
5083 protocol_version: payload.protocol_version,
5084 };
5085 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
5086 adapter,
5087 session_id,
5088 comms_runtime,
5089 sender,
5090 &sup_payload,
5091 "read member history failed",
5092 )
5093 .await
5094 {
5095 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5096 return true;
5097 }
5098 let observation = match resolve_member_observation_host(adapter) {
5099 Ok(observation) => observation,
5100 Err((cause, reason)) => {
5101 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5102 return true;
5103 }
5104 };
5105 if let Err((cause, reason)) = require_registered_member_incarnation(
5106 adapter,
5107 session_id,
5108 &payload.expected_member,
5109 "read member history",
5110 ) {
5111 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5112 return true;
5113 }
5114 match observation
5115 .read_history(session_id, payload.from_index, payload.limit)
5116 .await
5117 {
5118 Ok(window) => {
5119 if let Err((cause, reason)) = require_registered_member_incarnation(
5120 adapter,
5121 session_id,
5122 &payload.expected_member,
5123 "read member history completion",
5124 ) {
5125 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5126 return true;
5127 }
5128 match bounded_member_history_reply(window) {
5129 Ok(reply) => {
5130 send_bridge_response(
5131 comms_runtime,
5132 candidate,
5133 meerkat_core::interaction::ResponseStatus::Completed,
5134 reply,
5135 None,
5136 )
5137 .await;
5138 }
5139 Err(MemberHistoryReplyError::RowTooLarge {
5140 index,
5141 encoded_bytes,
5142 }) => {
5143 send_bridge_failure(
5144 comms_runtime,
5145 candidate,
5146 BridgeRejectionCause::HistoryRowTooLarge {
5147 index,
5148 encoded_bytes: u64::try_from(encoded_bytes).unwrap_or(u64::MAX),
5149 max_bytes: MEMBER_HISTORY_REPLY_MAX_BYTES as u64,
5150 },
5151 format!(
5152 "member history row at index {index} is {encoded_bytes} bytes (maximum bridge reply budget {MEMBER_HISTORY_REPLY_MAX_BYTES})"
5153 ),
5154 None,
5155 )
5156 .await;
5157 }
5158 Err(MemberHistoryReplyError::Internal(reason)) => {
5159 send_bridge_failure(
5160 comms_runtime,
5161 candidate,
5162 BridgeRejectionCause::Internal,
5163 reason,
5164 None,
5165 )
5166 .await;
5167 }
5168 }
5169 }
5170 Err(error) => {
5171 let (cause, reason) = observation_error_to_bridge_rejection(&error);
5172 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5173 }
5174 }
5175 true
5176 }
5177 BridgeCommand::PollMemberEvents(payload) => {
5178 let sup_payload = BridgeSupervisorPayload {
5186 supervisor: payload.supervisor.clone(),
5187 epoch: payload.epoch,
5188 protocol_version: payload.protocol_version,
5189 };
5190 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
5191 adapter,
5192 session_id,
5193 comms_runtime,
5194 sender,
5195 &sup_payload,
5196 "poll member events failed",
5197 )
5198 .await
5199 {
5200 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5201 return true;
5202 }
5203 if payload.outcome_acks.len() > BRIDGE_TURN_OUTCOME_ACK_MAX {
5204 send_bridge_failure(
5205 comms_runtime,
5206 candidate,
5207 BridgeRejectionCause::Internal,
5208 format!(
5209 "turn-outcome acknowledgement batch has {} rows (maximum {})",
5210 payload.outcome_acks.len(),
5211 BRIDGE_TURN_OUTCOME_ACK_MAX
5212 ),
5213 None,
5214 )
5215 .await;
5216 return true;
5217 }
5218 let observation = match resolve_member_observation_host(adapter) {
5219 Ok(observation) => observation,
5220 Err((cause, reason)) => {
5221 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5222 return true;
5223 }
5224 };
5225 let cursor = match payload.cursor {
5226 BridgeEventCursor::Tail => MemberObservationCursor::Tail,
5227 BridgeEventCursor::At { generation, seq } => {
5228 MemberObservationCursor::At { generation, seq }
5229 }
5230 };
5231 let max = payload
5232 .max
5233 .unwrap_or(MEMBER_POLL_DEFAULT_MAX_ROWS)
5234 .clamp(1, MEMBER_POLL_MAX_ROWS);
5235 let outcome_acks = payload.outcome_acks;
5236 let max_outcomes = payload
5237 .max_outcomes
5238 .unwrap_or(MEMBER_POLL_DEFAULT_MAX_OUTCOMES)
5239 .clamp(1, MEMBER_POLL_MAX_OUTCOMES);
5240 let wait_ms = payload.wait_ms.unwrap_or(0).min(MEMBER_POLL_WAIT_MAX_MS);
5241 let wait = Duration::from_millis(u64::from(wait_ms));
5242 let request = MemberEventsPageRequest {
5243 cursor,
5244 max,
5245 wait,
5246 outcome_acks,
5247 max_outcomes,
5248 expected_member: payload.expected_member,
5249 };
5250 if wait_ms == 0 {
5251 serve_member_events_page(
5253 adapter,
5254 &observation,
5255 session_id,
5256 comms_runtime,
5257 candidate,
5258 request,
5259 )
5260 .await;
5261 return true;
5262 }
5263 let observation = Arc::clone(&observation);
5264 let adapter = Arc::clone(adapter);
5265 let session_id = session_id.clone();
5266 let comms_runtime = Arc::clone(comms_runtime);
5267 let candidate = candidate.clone();
5268 crate::tokio::spawn(async move {
5269 serve_member_events_page(
5270 adapter.as_ref(),
5271 &observation,
5272 &session_id,
5273 &comms_runtime,
5274 &candidate,
5275 request,
5276 )
5277 .await;
5278 });
5279 true
5280 }
5281 BridgeCommand::OpenMemberLiveChannel(payload) => {
5282 let sup_payload = BridgeSupervisorPayload {
5292 supervisor: payload.supervisor.clone(),
5293 epoch: payload.epoch,
5294 protocol_version: payload.protocol_version,
5295 };
5296 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
5297 adapter,
5298 session_id,
5299 comms_runtime,
5300 sender,
5301 &sup_payload,
5302 "open member live channel failed",
5303 )
5304 .await
5305 {
5306 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5307 return true;
5308 }
5309 let live = match resolve_member_live_host(adapter) {
5310 Ok(live) => live,
5311 Err((cause, reason)) => {
5312 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5313 return true;
5314 }
5315 };
5316 let session_id = session_id.clone();
5317 let adapter = Arc::clone(adapter);
5318 let comms_runtime = Arc::clone(comms_runtime);
5319 let candidate = candidate.clone();
5320 let expected_member = payload.expected_member;
5321 let turning_mode = payload.turning_mode;
5322 let transport = payload.transport;
5323 crate::tokio::spawn(async move {
5324 let effect_authority = match acquire_registered_member_effect_authority(
5325 adapter.as_ref(),
5326 &session_id,
5327 &expected_member,
5328 "open member live channel",
5329 )
5330 .await
5331 {
5332 Ok(authority) => authority,
5333 Err((cause, reason)) => {
5334 send_bridge_failure(&comms_runtime, &candidate, cause, reason, None).await;
5335 return;
5336 }
5337 };
5338 let result = live.open(&session_id, turning_mode, transport).await;
5339 drop(effect_authority);
5340 match result {
5341 Ok(open) => {
5342 send_bridge_response(
5343 &comms_runtime,
5344 &candidate,
5345 meerkat_core::interaction::ResponseStatus::Completed,
5346 BridgeReply::MemberLiveChannelOpened(BridgeLiveOpenedResponse { open }),
5347 None,
5348 )
5349 .await;
5350 }
5351 Err(error) => {
5352 let (cause, reason) = member_live_error_to_bridge_rejection(&error);
5353 send_bridge_failure(&comms_runtime, &candidate, cause, reason, None).await;
5354 }
5355 }
5356 });
5357 true
5358 }
5359 BridgeCommand::CloseMemberLiveChannel(payload) => {
5360 let sup_payload = BridgeSupervisorPayload {
5365 supervisor: payload.supervisor.clone(),
5366 epoch: payload.epoch,
5367 protocol_version: payload.protocol_version,
5368 };
5369 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
5370 adapter,
5371 session_id,
5372 comms_runtime,
5373 sender,
5374 &sup_payload,
5375 "close member live channel failed",
5376 )
5377 .await
5378 {
5379 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5380 return true;
5381 }
5382 let live = match resolve_member_live_host(adapter) {
5383 Ok(live) => live,
5384 Err((cause, reason)) => {
5385 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5386 return true;
5387 }
5388 };
5389 let effect_authority = match acquire_registered_member_effect_authority(
5390 adapter,
5391 session_id,
5392 &payload.expected_member,
5393 "close member live channel",
5394 )
5395 .await
5396 {
5397 Ok(authority) => authority,
5398 Err((cause, reason)) => {
5399 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5400 return true;
5401 }
5402 };
5403 let result = live.close(session_id, &payload.channel_id).await;
5404 drop(effect_authority);
5405 match result {
5406 Ok(status) => {
5407 send_bridge_response(
5408 comms_runtime,
5409 candidate,
5410 meerkat_core::interaction::ResponseStatus::Completed,
5411 BridgeReply::MemberLiveChannelClosed { status },
5412 None,
5413 )
5414 .await;
5415 }
5416 Err(error) => {
5417 let (cause, reason) = member_live_error_to_bridge_rejection(&error);
5418 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5419 }
5420 }
5421 true
5422 }
5423 BridgeCommand::MemberLiveChannelStatus(payload) => {
5424 let sup_payload = BridgeSupervisorPayload {
5429 supervisor: payload.supervisor.clone(),
5430 epoch: payload.epoch,
5431 protocol_version: payload.protocol_version,
5432 };
5433 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
5434 adapter,
5435 session_id,
5436 comms_runtime,
5437 sender,
5438 &sup_payload,
5439 "member live channel status failed",
5440 )
5441 .await
5442 {
5443 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5444 return true;
5445 }
5446 let live = match resolve_member_live_host(adapter) {
5447 Ok(live) => live,
5448 Err((cause, reason)) => {
5449 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5450 return true;
5451 }
5452 };
5453 let effect_authority = match acquire_registered_member_effect_authority(
5454 adapter,
5455 session_id,
5456 &payload.expected_member,
5457 "member live channel status",
5458 )
5459 .await
5460 {
5461 Ok(authority) => authority,
5462 Err((cause, reason)) => {
5463 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5464 return true;
5465 }
5466 };
5467 let result = live.status(session_id, payload.channel_id).await;
5468 drop(effect_authority);
5469 match result {
5470 Ok(report) => {
5471 send_bridge_response(
5472 comms_runtime,
5473 candidate,
5474 meerkat_core::interaction::ResponseStatus::Completed,
5475 BridgeReply::MemberLiveChannelStatusReport {
5476 channel_id: report.channel_id,
5477 status: report.status,
5478 },
5479 None,
5480 )
5481 .await;
5482 }
5483 Err(error) => {
5484 let (cause, reason) = member_live_error_to_bridge_rejection(&error);
5485 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5486 }
5487 }
5488 true
5489 }
5490 BridgeCommand::ControlMemberLiveChannel(payload) => {
5491 let sup_payload = BridgeSupervisorPayload {
5498 supervisor: payload.supervisor.clone(),
5499 epoch: payload.epoch,
5500 protocol_version: payload.protocol_version,
5501 };
5502 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
5503 adapter,
5504 session_id,
5505 comms_runtime,
5506 sender,
5507 &sup_payload,
5508 "control member live channel failed",
5509 )
5510 .await
5511 {
5512 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5513 return true;
5514 }
5515 let live = match resolve_member_live_host(adapter) {
5516 Ok(live) => live,
5517 Err((cause, reason)) => {
5518 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5519 return true;
5520 }
5521 };
5522 let effect_authority = match acquire_registered_member_effect_authority(
5523 adapter,
5524 session_id,
5525 &payload.expected_member,
5526 "control member live channel",
5527 )
5528 .await
5529 {
5530 Ok(authority) => authority,
5531 Err((cause, reason)) => {
5532 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5533 return true;
5534 }
5535 };
5536 let result = live
5537 .control(session_id, &payload.channel_id, payload.verb)
5538 .await;
5539 drop(effect_authority);
5540 match result {
5541 Ok(outcome) => {
5542 send_bridge_response(
5543 comms_runtime,
5544 candidate,
5545 meerkat_core::interaction::ResponseStatus::Completed,
5546 BridgeReply::MemberLiveChannelControlled(BridgeLiveControlledResponse {
5547 outcome,
5548 }),
5549 None,
5550 )
5551 .await;
5552 }
5553 Err(error) => {
5554 let (cause, reason) = member_live_error_to_bridge_rejection(&error);
5555 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5556 }
5557 }
5558 true
5559 }
5560 _ => {
5561 send_bridge_failure(
5562 comms_runtime,
5563 candidate,
5564 BridgeRejectionCause::Unsupported,
5565 "unsupported supervisor bridge command".to_string(),
5566 None,
5567 )
5568 .await;
5569 true
5570 }
5571 }
5572}
5573
5574#[derive(Debug)]
5575enum MemberHistoryReplyError {
5576 RowTooLarge { index: u64, encoded_bytes: usize },
5577 Internal(String),
5578}
5579
5580fn bounded_member_history_reply(
5586 window: MemberHistoryWindow,
5587) -> Result<BridgeReply, MemberHistoryReplyError> {
5588 let mut page = BridgeMemberHistoryPage {
5589 generation: window.generation,
5590 page: WireMemberHistoryPageBody::try_from_history_page(&window.page).map_err(|error| {
5591 MemberHistoryReplyError::Internal(format!(
5592 "member history page projection failed: {error}"
5593 ))
5594 })?,
5595 };
5596 let encoded_len = |candidate: &BridgeMemberHistoryPage| {
5597 serde_json::to_vec(&BridgeReply::MemberHistoryPage(candidate.clone()))
5598 .map(|bytes| bytes.len())
5599 .map_err(|error| {
5600 MemberHistoryReplyError::Internal(format!(
5601 "member history reply serialization failed: {error}"
5602 ))
5603 })
5604 };
5605 let full_bytes = encoded_len(&page)?;
5606 if full_bytes <= MEMBER_HISTORY_REPLY_MAX_BYTES {
5607 return Ok(BridgeReply::MemberHistoryPage(page));
5608 }
5609
5610 let original_len = page.page.messages.len();
5611 if original_len == 0 {
5612 return Err(MemberHistoryReplyError::Internal(format!(
5613 "empty member history reply exceeds the {MEMBER_HISTORY_REPLY_MAX_BYTES} byte bridge budget"
5614 )));
5615 }
5616 if original_len == 1 {
5617 return Err(MemberHistoryReplyError::RowTooLarge {
5618 index: page.page.from_index,
5619 encoded_bytes: full_bytes,
5620 });
5621 }
5622
5623 let candidate_with_prefix = |prefix_len: usize| {
5624 let mut candidate = page.clone();
5625 candidate.page.messages.truncate(prefix_len);
5626 if prefix_len < original_len {
5627 let served = u64::try_from(prefix_len).map_err(|_| {
5628 MemberHistoryReplyError::Internal(format!(
5629 "member history prefix length {prefix_len} exceeds the u64 cursor domain"
5630 ))
5631 })?;
5632 candidate.page.next_index = Some(
5633 candidate
5634 .page
5635 .from_index
5636 .checked_add(served)
5637 .ok_or_else(|| {
5638 MemberHistoryReplyError::Internal(format!(
5639 "member history continuation cursor exhausts the u64 domain at {} + {served}",
5640 candidate.page.from_index
5641 ))
5642 })?,
5643 );
5644 candidate.page.complete = false;
5645 }
5646 Ok(candidate)
5647 };
5648
5649 let first = candidate_with_prefix(1)?;
5652 let first_bytes = encoded_len(&first)?;
5653 if first_bytes > MEMBER_HISTORY_REPLY_MAX_BYTES {
5654 return Err(MemberHistoryReplyError::RowTooLarge {
5655 index: page.page.from_index,
5656 encoded_bytes: first_bytes,
5657 });
5658 }
5659
5660 let mut low = 1usize;
5663 let mut high = original_len - 1;
5664 while low < high {
5665 let mid = low + (high - low).div_ceil(2);
5666 let candidate = candidate_with_prefix(mid)?;
5667 if encoded_len(&candidate)? <= MEMBER_HISTORY_REPLY_MAX_BYTES {
5668 low = mid;
5669 } else {
5670 high = mid - 1;
5671 }
5672 }
5673 page = candidate_with_prefix(low)?;
5674 Ok(BridgeReply::MemberHistoryPage(page))
5675}
5676
5677#[derive(Debug)]
5680enum MemberEventsReplyError {
5681 OversizedEvent {
5682 generation: u64,
5683 durable_seq: u64,
5684 encoded_bytes: usize,
5685 },
5686 Internal(String),
5687}
5688
5689fn member_event_frontier_after(sequence: u64) -> Result<u64, MemberEventsReplyError> {
5690 sequence.checked_add(1).ok_or_else(|| {
5691 MemberEventsReplyError::Internal(
5692 "member event sequence space is exhausted; reply cursor cannot advance past u64::MAX"
5693 .to_string(),
5694 )
5695 })
5696}
5697
5698fn bounded_member_events_reply(
5699 window: MemberEventsWindow,
5700) -> Result<BridgeReply, MemberEventsReplyError> {
5701 let MemberEventsWindow {
5702 runtime_incarnation,
5703 generation,
5704 fence_token,
5705 rows,
5706 from_seq,
5707 next_seq,
5708 watermark,
5709 turn_outcomes,
5710 outcomes_complete,
5711 } = window;
5712 let mut page = BridgeMemberEventsPage {
5713 runtime_incarnation,
5714 generation,
5715 fence_token,
5716 events: rows
5717 .into_iter()
5718 .map(|(durable_seq, envelope)| WireEventRow {
5719 durable_seq,
5720 envelope,
5721 })
5722 .collect(),
5723 from_seq,
5724 next_seq,
5725 watermark,
5726 turn_outcomes,
5727 outcomes_complete,
5728 };
5729
5730 if let Some(last) = page.events.last() {
5735 let expected_next = member_event_frontier_after(last.durable_seq)?;
5736 if page.next_seq != expected_next {
5737 return Err(MemberEventsReplyError::Internal(format!(
5738 "member event page next_seq {} does not exactly follow last durable seq {}",
5739 page.next_seq, last.durable_seq
5740 )));
5741 }
5742 } else if page.watermark == u64::MAX && page.next_seq == u64::MAX {
5743 return Err(MemberEventsReplyError::Internal(
5744 "member event sequence space is exhausted at an empty MAX frontier".to_string(),
5745 ));
5746 }
5747
5748 let encoded_len = |candidate: &BridgeMemberEventsPage| {
5749 serde_json::to_vec(&BridgeReply::MemberEventsPage(candidate.clone()))
5750 .map(|bytes| bytes.len())
5751 .map_err(|error| {
5752 MemberEventsReplyError::Internal(format!(
5753 "member events reply serialization failed: {error}"
5754 ))
5755 })
5756 };
5757 if encoded_len(&page)? <= MEMBER_EVENTS_REPLY_MAX_BYTES {
5758 return Ok(BridgeReply::MemberEventsPage(page));
5759 }
5760
5761 if !page.events.is_empty() {
5765 let mut first_only = page.clone();
5766 first_only.events.truncate(1);
5767 first_only.next_seq = member_event_frontier_after(first_only.events[0].durable_seq)?;
5768 first_only.turn_outcomes.clear();
5769 first_only.outcomes_complete = true;
5770 let event_only_bytes = encoded_len(&first_only)?;
5771 if event_only_bytes > MEMBER_EVENTS_REPLY_MAX_BYTES {
5772 return Err(MemberEventsReplyError::OversizedEvent {
5773 generation: page.generation,
5774 durable_seq: page.events[0].durable_seq,
5775 encoded_bytes: event_only_bytes,
5776 });
5777 }
5778 }
5779
5780 let mut low = 0usize;
5784 let mut high = page.events.len();
5785 while low < high {
5786 let mid = low + (high - low).div_ceil(2);
5787 let mut candidate = page.clone();
5788 candidate.events.truncate(mid);
5789 candidate.next_seq = match candidate.events.last() {
5790 Some(row) => member_event_frontier_after(row.durable_seq)?,
5791 None => from_seq,
5792 };
5793 candidate.turn_outcomes.clear();
5794 candidate.outcomes_complete = false;
5795 if encoded_len(&candidate)? <= MEMBER_EVENTS_REPLY_MAX_BYTES {
5796 low = mid;
5797 } else {
5798 high = mid - 1;
5799 }
5800 }
5801 page.events.truncate(low);
5802 page.next_seq = match page.events.last() {
5803 Some(row) => member_event_frontier_after(row.durable_seq)?,
5804 None => from_seq,
5805 };
5806
5807 let original_outcome_len = page.turn_outcomes.len();
5811 page.turn_outcomes
5812 .retain(|record| record.terminal_seq < page.next_seq);
5813 let eligible_outcome_len = page.turn_outcomes.len();
5814 page.outcomes_complete = page.outcomes_complete && eligible_outcome_len == original_outcome_len;
5815 low = 0;
5816 high = eligible_outcome_len;
5817 while low < high {
5818 let mid = low + (high - low).div_ceil(2);
5819 let mut candidate = page.clone();
5820 candidate.turn_outcomes.truncate(mid);
5821 candidate.outcomes_complete = page.outcomes_complete && mid == eligible_outcome_len;
5822 if encoded_len(&candidate)? <= MEMBER_EVENTS_REPLY_MAX_BYTES {
5823 low = mid;
5824 } else {
5825 high = mid - 1;
5826 }
5827 }
5828 if eligible_outcome_len > 0 && low == 0 && page.events.is_empty() {
5829 return Err(MemberEventsReplyError::Internal(format!(
5830 "one bounded turn outcome cannot fit the {MEMBER_EVENTS_REPLY_MAX_BYTES} byte bridge budget"
5831 )));
5832 }
5833 page.turn_outcomes.truncate(low);
5834 if low < eligible_outcome_len {
5835 page.outcomes_complete = false;
5836 }
5837 Ok(BridgeReply::MemberEventsPage(page))
5838}
5839
5840struct MemberEventsPageRequest {
5841 cursor: MemberObservationCursor,
5842 max: u32,
5843 wait: Duration,
5844 outcome_acks: Vec<meerkat_contracts::wire::supervisor_bridge::BridgeTurnOutcomeAck>,
5845 max_outcomes: u32,
5846 expected_member: BridgeMemberIncarnation,
5847}
5848
5849async fn serve_member_events_page(
5850 adapter: &MeerkatMachine,
5851 observation: &Arc<dyn MemberObservationHost>,
5852 session_id: &SessionId,
5853 comms_runtime: &Arc<dyn CommsRuntime>,
5854 candidate: &PeerInputCandidate,
5855 request: MemberEventsPageRequest,
5856) {
5857 let MemberEventsPageRequest {
5858 cursor,
5859 max,
5860 wait,
5861 outcome_acks,
5862 max_outcomes,
5863 expected_member,
5864 } = request;
5865 if let Err((cause, reason)) = require_registered_member_incarnation(
5866 adapter,
5867 session_id,
5868 &expected_member,
5869 "poll member events",
5870 ) {
5871 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5872 return;
5873 }
5874 match observation
5875 .poll_events(
5876 session_id,
5877 MemberEventsPollRequest {
5878 expected_member: &expected_member,
5879 cursor,
5880 max,
5881 wait,
5882 outcome_acks: &outcome_acks,
5883 max_outcomes,
5884 },
5885 )
5886 .await
5887 {
5888 Ok(window) => {
5889 if let Err((cause, reason)) = require_registered_member_incarnation(
5890 adapter,
5891 session_id,
5892 &expected_member,
5893 "poll member events completion",
5894 ) {
5895 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5896 return;
5897 }
5898 match bounded_member_events_reply(window) {
5899 Ok(reply) => {
5900 send_bridge_response(
5901 comms_runtime,
5902 candidate,
5903 meerkat_core::interaction::ResponseStatus::Completed,
5904 reply,
5905 None,
5906 )
5907 .await;
5908 }
5909 Err(MemberEventsReplyError::OversizedEvent {
5910 generation,
5911 durable_seq,
5912 encoded_bytes,
5913 }) => {
5914 let Some(next_seq) = durable_seq.checked_add(1) else {
5915 send_bridge_failure(
5916 comms_runtime,
5917 candidate,
5918 BridgeRejectionCause::Internal,
5919 "member event sequence space is exhausted; oversized-row disposition cannot advance the cursor".to_string(),
5920 None,
5921 )
5922 .await;
5923 return;
5924 };
5925 send_bridge_failure(
5926 comms_runtime,
5927 candidate,
5928 BridgeRejectionCause::OversizedEvent {
5929 generation,
5930 durable_seq,
5931 next_seq,
5932 encoded_bytes: u64::try_from(encoded_bytes).unwrap_or(u64::MAX),
5933 max_bytes: MEMBER_EVENTS_REPLY_MAX_BYTES as u64,
5934 },
5935 format!(
5936 "single member event row at durable seq {durable_seq} is {encoded_bytes} bytes (maximum bridge reply budget {MEMBER_EVENTS_REPLY_MAX_BYTES})"
5937 ),
5938 None,
5939 )
5940 .await;
5941 }
5942 Err(MemberEventsReplyError::Internal(reason)) => {
5943 send_bridge_failure(
5944 comms_runtime,
5945 candidate,
5946 BridgeRejectionCause::Internal,
5947 reason,
5948 None,
5949 )
5950 .await;
5951 }
5952 }
5953 }
5954 Err(error) => {
5955 let (cause, reason) = observation_error_to_bridge_rejection(&error);
5956 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5957 }
5958 }
5959}
5960
5961fn directed_rejection_after_pending_cancel(
5985 input_id: &str,
5986 cause: BridgeDeliveryRejectionCause,
5987 reason: String,
5988 cancellation: Result<(), crate::member_observation::DirectedTurnReject>,
5989) -> Result<BridgeDeliveryResponse, String> {
5990 cancellation.map_err(|error| {
5991 format!(
5992 "runtime proved no admission, but durable Pending cancellation failed: {}",
5993 error.detail
5994 )
5995 })?;
5996 Ok(BridgeDeliveryResponse {
5997 input_id: input_id.to_string(),
5998 canonical_input_id: None,
5999 outcome: BridgeDeliveryOutcome::Rejected { cause, reason },
6000 })
6001}
6002
6003fn definite_no_effect_delivery_rejection(
6004 error: &crate::traits::RuntimeDriverError,
6005 current_member: Option<BridgeMemberIncarnation>,
6006 expected_member: &BridgeMemberIncarnation,
6007) -> Option<BridgeDeliveryRejectionCause> {
6008 match error {
6009 crate::traits::RuntimeDriverError::ValidationFailed { reason } => {
6010 Some(BridgeDeliveryRejectionCause::Internal {
6011 detail: reason.clone(),
6012 })
6013 }
6014 crate::traits::RuntimeDriverError::StaleAuthority { .. } => {
6015 Some(BridgeDeliveryRejectionCause::StaleMemberResidency {
6016 expected: expected_member.clone(),
6017 current: current_member,
6018 })
6019 }
6020 _ => None,
6021 }
6022}
6023
6024#[cfg(test)]
6029async fn open_directed_turn_before_runtime_accept<T, F, Fut>(
6030 observation: &Arc<dyn MemberObservationHost>,
6031 session_id: &SessionId,
6032 expected_member: &BridgeMemberIncarnation,
6033 input_id: &str,
6034 accept: F,
6035) -> Result<
6036 (crate::member_observation::DirectedTurnWindow, T),
6037 crate::member_observation::DirectedTurnReject,
6038>
6039where
6040 F: FnOnce() -> Fut,
6041 Fut: std::future::Future<Output = T>,
6042{
6043 let window = observation
6044 .open_directed_turn_window(session_id, expected_member, input_id)
6045 .await?;
6046 Ok((window, accept().await))
6047}
6048
6049enum DirectedTurnPreAcceptDecision {
6050 SubmitToRuntime,
6051 TerminalReplay(BridgeDeliveryResponse),
6052}
6053
6054fn directed_turn_pre_accept_decision(
6060 request_input_id: &str,
6061 expected_member: &BridgeMemberIncarnation,
6062 window: &DirectedTurnWindow,
6063) -> Result<DirectedTurnPreAcceptDecision, String> {
6064 if matches!(
6065 window.tracking,
6066 DirectedTurnTracking::PendingFresh | DirectedTurnTracking::PendingReplay
6067 ) {
6068 return Ok(DirectedTurnPreAcceptDecision::SubmitToRuntime);
6069 }
6070 if window.input_id != request_input_id || &window.expected_member != expected_member {
6071 return Err(format!(
6072 "terminal replay reservation does not match request input/residency: request input '{request_input_id}', reserved input '{}', request member {expected_member:?}, reserved member {:?}",
6073 window.input_id, window.expected_member
6074 ));
6075 }
6076 Ok(DirectedTurnPreAcceptDecision::TerminalReplay(
6077 BridgeDeliveryResponse {
6078 input_id: request_input_id.to_string(),
6079 canonical_input_id: Some(request_input_id.to_string()),
6080 outcome: BridgeDeliveryOutcome::Deduplicated {
6081 existing_input_id: request_input_id.to_string(),
6082 },
6083 },
6084 ))
6085}
6086
6087async fn serve_tracked_member_delivery(
6088 adapter: &Arc<MeerkatMachine>,
6089 session_id: &SessionId,
6090 comms_runtime: &Arc<dyn CommsRuntime>,
6091 candidate: &PeerInputCandidate,
6092 sender_peer_id: PeerId,
6093 payload: BridgeDeliveryPayload,
6094) {
6095 let request_input_id = payload.input_id.clone();
6096 let reject = |detail: String| BridgeDeliveryResponse {
6097 input_id: request_input_id.clone(),
6098 canonical_input_id: None,
6099 outcome: BridgeDeliveryOutcome::Rejected {
6100 cause: BridgeDeliveryRejectionCause::TurnDirectiveUnsupported {
6101 detail: detail.clone(),
6102 },
6103 reason: detail,
6104 },
6105 };
6106 if let Err(detail) = validate_delivery_tracking_request(&payload) {
6107 let response = reject(detail);
6108 send_bridge_response(
6109 comms_runtime,
6110 candidate,
6111 meerkat_core::interaction::ResponseStatus::Completed,
6112 BridgeReply::Delivery(response),
6113 None,
6114 )
6115 .await;
6116 return;
6117 }
6118 let Some(expected_member) = payload.expected_member.clone() else {
6119 let response =
6120 reject("tracked delivery requires an exact placed-member incarnation".to_string());
6121 send_bridge_response(
6122 comms_runtime,
6123 candidate,
6124 meerkat_core::interaction::ResponseStatus::Completed,
6125 BridgeReply::Delivery(response),
6126 None,
6127 )
6128 .await;
6129 return;
6130 };
6131 let input = if let Some(directive) = payload.turn.clone() {
6132 if !payload.injected_context.is_empty() {
6133 let response = reject(
6137 "injected_context is not supported on directive-bearing deliveries".to_string(),
6138 );
6139 send_bridge_response(
6140 comms_runtime,
6141 candidate,
6142 meerkat_core::interaction::ResponseStatus::Completed,
6143 BridgeReply::Delivery(response),
6144 None,
6145 )
6146 .await;
6147 return;
6148 }
6149 if payload.objective_id.is_some() {
6150 let response =
6154 reject("objective_id is not supported on directive-bearing deliveries".to_string());
6155 send_bridge_response(
6156 comms_runtime,
6157 candidate,
6158 meerkat_core::interaction::ResponseStatus::Completed,
6159 BridgeReply::Delivery(response),
6160 None,
6161 )
6162 .await;
6163 return;
6164 }
6165 let turn_metadata = crate::runtime_loop::for_bridge_turn_directive(
6166 payload.handling_mode,
6167 directive.tool_overlay.clone().map(Into::into),
6168 );
6169 match crate::mob_adapter::create_tracked_flow_step_input(
6170 &directive.correlation.step_id,
6171 payload.content.clone(),
6172 &directive.correlation.run_id,
6173 Some(turn_metadata),
6174 &payload.input_id,
6175 ) {
6176 Ok(input) => input,
6177 Err(detail) => {
6178 let response = reject(detail);
6179 send_bridge_response(
6180 comms_runtime,
6181 candidate,
6182 meerkat_core::interaction::ResponseStatus::Completed,
6183 BridgeReply::Delivery(response),
6184 None,
6185 )
6186 .await;
6187 return;
6188 }
6189 }
6190 } else {
6191 match peer_input_from_delivery_payload(session_id, sender_peer_id, payload) {
6192 Ok(input) => input,
6193 Err(reason) => {
6194 let response = BridgeDeliveryResponse {
6195 input_id: request_input_id.clone(),
6196 canonical_input_id: None,
6197 outcome: BridgeDeliveryOutcome::Rejected {
6198 cause: BridgeDeliveryRejectionCause::Internal {
6199 detail: reason.clone(),
6200 },
6201 reason,
6202 },
6203 };
6204 send_bridge_response(
6205 comms_runtime,
6206 candidate,
6207 meerkat_core::interaction::ResponseStatus::Completed,
6208 BridgeReply::Delivery(response),
6209 None,
6210 )
6211 .await;
6212 return;
6213 }
6214 }
6215 };
6216 let Some(observation) = adapter.member_observation_host() else {
6217 let response = reject("member host serves no observation substrate".to_string());
6218 send_bridge_response(
6219 comms_runtime,
6220 candidate,
6221 meerkat_core::interaction::ResponseStatus::Completed,
6222 BridgeReply::Delivery(response),
6223 None,
6224 )
6225 .await;
6226 return;
6227 };
6228 let window = match observation
6233 .open_directed_turn_window(session_id, &expected_member, &request_input_id)
6234 .await
6235 {
6236 Ok(window) => window,
6237 Err(rejected) => {
6238 if !rejected.definite_no_effect {
6239 send_bridge_failure(
6240 comms_runtime,
6241 candidate,
6242 BridgeRejectionCause::Internal,
6243 rejected.detail,
6244 None,
6245 )
6246 .await;
6247 return;
6248 }
6249 let response = BridgeDeliveryResponse {
6250 input_id: request_input_id.clone(),
6251 canonical_input_id: None,
6252 outcome: BridgeDeliveryOutcome::Rejected {
6253 cause: rejected.cause,
6254 reason: rejected.detail,
6255 },
6256 };
6257 send_bridge_response(
6258 comms_runtime,
6259 candidate,
6260 meerkat_core::interaction::ResponseStatus::Completed,
6261 BridgeReply::Delivery(response),
6262 None,
6263 )
6264 .await;
6265 return;
6266 }
6267 };
6268 if window.expected_member != expected_member {
6269 let cancellation = observation
6270 .cancel_directed_turn_window(session_id, window)
6271 .await;
6272 let reason = format!(
6273 "directed-turn residency changed before accept; controller expected {expected_member:?}"
6274 );
6275 match directed_rejection_after_pending_cancel(
6276 &request_input_id,
6277 BridgeDeliveryRejectionCause::StaleMemberResidency {
6278 expected: expected_member.clone(),
6279 current: adapter.member_incarnation(session_id),
6280 },
6281 reason,
6282 cancellation,
6283 ) {
6284 Ok(response) => {
6285 send_bridge_response(
6286 comms_runtime,
6287 candidate,
6288 meerkat_core::interaction::ResponseStatus::Completed,
6289 BridgeReply::Delivery(response),
6290 None,
6291 )
6292 .await;
6293 }
6294 Err(detail) => {
6295 send_bridge_failure(
6296 comms_runtime,
6297 candidate,
6298 BridgeRejectionCause::Internal,
6299 detail,
6300 None,
6301 )
6302 .await;
6303 }
6304 }
6305 return;
6306 }
6307 match directed_turn_pre_accept_decision(&request_input_id, &expected_member, &window) {
6308 Ok(DirectedTurnPreAcceptDecision::SubmitToRuntime) => {}
6309 Ok(DirectedTurnPreAcceptDecision::TerminalReplay(response)) => {
6310 send_bridge_response(
6311 comms_runtime,
6312 candidate,
6313 meerkat_core::interaction::ResponseStatus::Completed,
6314 BridgeReply::Delivery(response),
6315 None,
6316 )
6317 .await;
6318 return;
6319 }
6320 Err(detail) => {
6321 send_bridge_failure(
6322 comms_runtime,
6323 candidate,
6324 BridgeRejectionCause::Internal,
6325 detail,
6326 None,
6327 )
6328 .await;
6329 return;
6330 }
6331 }
6332 let Some(journal) = adapter.tracked_turn_journal(session_id) else {
6333 let cancellation = observation
6334 .cancel_directed_turn_window(session_id, window)
6335 .await;
6336 let response = match directed_rejection_after_pending_cancel(
6337 &request_input_id,
6338 BridgeDeliveryRejectionCause::TurnDirectiveUnsupported {
6339 detail: "member runtime has no tracked-turn journal registration".to_string(),
6340 },
6341 "member runtime has no tracked-turn journal registration".to_string(),
6342 cancellation,
6343 ) {
6344 Ok(response) => response,
6345 Err(detail) => {
6346 send_bridge_failure(
6347 comms_runtime,
6348 candidate,
6349 BridgeRejectionCause::Internal,
6350 detail,
6351 None,
6352 )
6353 .await;
6354 return;
6355 }
6356 };
6357 send_bridge_response(
6358 comms_runtime,
6359 candidate,
6360 meerkat_core::interaction::ResponseStatus::Completed,
6361 BridgeReply::Delivery(response),
6362 None,
6363 )
6364 .await;
6365 return;
6366 };
6367 if journal.member_incarnation() != &expected_member {
6368 let cancellation = observation
6369 .cancel_directed_turn_window(session_id, window)
6370 .await;
6371 let detail = "tracked-turn journal incarnation differs from reserved residency".to_string();
6372 match directed_rejection_after_pending_cancel(
6373 &request_input_id,
6374 BridgeDeliveryRejectionCause::TurnDirectiveUnsupported {
6375 detail: detail.clone(),
6376 },
6377 detail,
6378 cancellation,
6379 ) {
6380 Ok(response) => {
6381 send_bridge_response(
6382 comms_runtime,
6383 candidate,
6384 meerkat_core::interaction::ResponseStatus::Completed,
6385 BridgeReply::Delivery(response),
6386 None,
6387 )
6388 .await;
6389 }
6390 Err(detail) => {
6391 send_bridge_failure(
6392 comms_runtime,
6393 candidate,
6394 BridgeRejectionCause::Internal,
6395 detail,
6396 None,
6397 )
6398 .await;
6399 }
6400 }
6401 return;
6402 }
6403 let _admission_permit = match observation
6409 .lock_and_revalidate_directed_turn_admission(session_id, window.admission_request())
6410 .await
6411 {
6412 Ok(crate::member_observation::DirectedTurnAdmissionDecision::Admit(permit)) => permit,
6413 Ok(crate::member_observation::DirectedTurnAdmissionDecision::TerminalReplay) => {
6414 let response = BridgeDeliveryResponse {
6415 input_id: request_input_id.clone(),
6416 canonical_input_id: Some(request_input_id.clone()),
6417 outcome: BridgeDeliveryOutcome::Deduplicated {
6418 existing_input_id: request_input_id.clone(),
6419 },
6420 };
6421 send_bridge_response(
6422 comms_runtime,
6423 candidate,
6424 meerkat_core::interaction::ResponseStatus::Completed,
6425 BridgeReply::Delivery(response),
6426 None,
6427 )
6428 .await;
6429 return;
6430 }
6431 Err(error) => {
6432 send_bridge_failure(
6433 comms_runtime,
6434 candidate,
6435 BridgeRejectionCause::Internal,
6436 format!(
6437 "directed-turn final admission revalidation was ambiguous: {}",
6438 error.detail
6439 ),
6440 None,
6441 )
6442 .await;
6443 return;
6444 }
6445 };
6446 let accept_result = adapter
6447 .accept_input_with_completion_for_member_residency(
6448 session_id,
6449 input,
6450 Some(&expected_member),
6451 )
6452 .await;
6453 match accept_result {
6454 Ok((accept_outcome, completion_handle)) => {
6455 let response = match accept_outcome {
6456 crate::accept::AcceptOutcome::Accepted { input_id, .. } => {
6457 let canonical_input_id = input_id.to_string();
6458 if let Err(rejected) = observation
6459 .admit_directed_turn(
6460 session_id,
6461 DirectedTurnAdmission {
6462 input_id: request_input_id.clone(),
6463 window,
6464 completion_handle,
6465 journal,
6466 },
6467 )
6468 .await
6469 {
6470 tracing::error!(
6474 session_id = %session_id,
6475 input_id = %request_input_id,
6476 detail = %rejected.detail,
6477 "directed turn accepted but watcher admission failed; durable Pending retained for recovery"
6478 );
6479 }
6480 BridgeDeliveryResponse {
6481 input_id: request_input_id.clone(),
6482 canonical_input_id: Some(canonical_input_id),
6483 outcome: BridgeDeliveryOutcome::Accepted,
6484 }
6485 }
6486 crate::accept::AcceptOutcome::Deduplicated { existing_id, .. } => {
6487 let existing_id = existing_id.to_string();
6488 if let Err(rejected) = observation
6489 .admit_directed_turn(
6490 session_id,
6491 DirectedTurnAdmission {
6492 input_id: request_input_id.clone(),
6493 window,
6494 completion_handle,
6495 journal,
6496 },
6497 )
6498 .await
6499 {
6500 tracing::error!(
6501 session_id = %session_id,
6502 input_id = %request_input_id,
6503 detail = %rejected.detail,
6504 "deduplicated directed turn could not reattach watcher; durable Pending retained for recovery"
6505 );
6506 }
6507 BridgeDeliveryResponse {
6508 input_id: request_input_id.clone(),
6509 canonical_input_id: Some(existing_id.clone()),
6510 outcome: BridgeDeliveryOutcome::Deduplicated {
6511 existing_input_id: existing_id,
6512 },
6513 }
6514 }
6515 crate::accept::AcceptOutcome::Rejected { reason } => {
6516 let cause = bridge_delivery_rejection_cause(&reason);
6517 let cancellation = observation
6518 .cancel_directed_turn_window(session_id, window)
6519 .await;
6520 match directed_rejection_after_pending_cancel(
6521 &request_input_id,
6522 cause,
6523 reason.to_string(),
6524 cancellation,
6525 ) {
6526 Ok(response) => response,
6527 Err(detail) => {
6528 tracing::error!(
6529 session_id = %session_id,
6530 input_id = %request_input_id,
6531 detail = %detail,
6532 "proven-rejected directed turn could not durably certify no-effect"
6533 );
6534 send_bridge_failure(
6535 comms_runtime,
6536 candidate,
6537 BridgeRejectionCause::Internal,
6538 detail,
6539 None,
6540 )
6541 .await;
6542 return;
6543 }
6544 }
6545 }
6546 };
6547 send_bridge_response(
6548 comms_runtime,
6549 candidate,
6550 meerkat_core::interaction::ResponseStatus::Completed,
6551 BridgeReply::Delivery(response),
6552 None,
6553 )
6554 .await;
6555 }
6556 Err(error) => {
6557 if let Some(cause) = definite_no_effect_delivery_rejection(
6561 &error,
6562 adapter.member_incarnation(session_id),
6563 &expected_member,
6564 ) {
6565 let cancellation = observation
6566 .cancel_directed_turn_window(session_id, window)
6567 .await;
6568 match directed_rejection_after_pending_cancel(
6569 &request_input_id,
6570 cause,
6571 error.to_string(),
6572 cancellation,
6573 ) {
6574 Ok(response) => {
6575 send_bridge_response(
6576 comms_runtime,
6577 candidate,
6578 meerkat_core::interaction::ResponseStatus::Completed,
6579 BridgeReply::Delivery(response),
6580 None,
6581 )
6582 .await;
6583 }
6584 Err(detail) => {
6585 tracing::error!(
6586 session_id = %session_id,
6587 input_id = %request_input_id,
6588 detail = %detail,
6589 "proven validation rejection could not durably certify no-effect"
6590 );
6591 send_bridge_failure(
6592 comms_runtime,
6593 candidate,
6594 BridgeRejectionCause::Internal,
6595 detail,
6596 None,
6597 )
6598 .await;
6599 }
6600 }
6601 return;
6602 }
6603 send_bridge_failure(
6604 comms_runtime,
6605 candidate,
6606 BridgeRejectionCause::Internal,
6607 format!("deliver member input failed: {error}"),
6608 None,
6609 )
6610 .await;
6611 }
6612 }
6613}
6614
6615fn runtime_state_to_bridge(state: crate::RuntimeState) -> BridgeMemberRuntimeState {
6617 match state {
6618 crate::RuntimeState::Initializing => BridgeMemberRuntimeState::Initializing,
6619 crate::RuntimeState::Idle => BridgeMemberRuntimeState::Idle,
6620 crate::RuntimeState::Attached => BridgeMemberRuntimeState::Attached,
6621 crate::RuntimeState::Running => BridgeMemberRuntimeState::Running,
6622 crate::RuntimeState::Retired => BridgeMemberRuntimeState::Retired,
6623 crate::RuntimeState::Stopped => BridgeMemberRuntimeState::Stopped,
6624 crate::RuntimeState::Destroyed => BridgeMemberRuntimeState::Destroyed,
6625 }
6626}
6627
6628#[cfg(test)]
6629#[allow(clippy::items_after_test_module)]
6630#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
6631mod tests {
6632 use super::*;
6633 use meerkat_contracts::wire::supervisor_bridge::{
6634 supervisor_bridge_current_protocol_version, supervisor_bridge_default_protocol_version,
6635 supervisor_bridge_supported_protocol_versions,
6636 };
6637 use meerkat_contracts::{
6638 BridgeMobPeerOverlayHandoff, BridgeOutboundTaintPayload, BridgePeerWiringPayload,
6639 };
6640 use meerkat_core::InteractionId;
6641 use meerkat_core::SendError;
6642 use meerkat_core::interaction::InboxInteraction;
6643 use meerkat_core::interaction::{PeerIngressConvention, PeerIngressIdentity};
6644 use meerkat_core::types::HandlingMode;
6645 use serde_json::json;
6646 use std::collections::{HashMap, HashSet};
6647 use uuid::Uuid;
6648
6649 struct ExhaustedDirectedTurnObservation;
6650
6651 #[async_trait::async_trait]
6652 impl MemberObservationHost for ExhaustedDirectedTurnObservation {
6653 async fn member_generation(
6654 &self,
6655 _session: &SessionId,
6656 ) -> Result<u64, MemberObservationError> {
6657 Err(MemberObservationError::Internal {
6658 reason: "unused test seam".to_string(),
6659 })
6660 }
6661
6662 async fn read_history(
6663 &self,
6664 _session: &SessionId,
6665 _from_index: Option<u64>,
6666 _limit: Option<u32>,
6667 ) -> Result<MemberHistoryWindow, MemberObservationError> {
6668 Err(MemberObservationError::Internal {
6669 reason: "unused test seam".to_string(),
6670 })
6671 }
6672
6673 async fn poll_events(
6674 &self,
6675 _session: &SessionId,
6676 _request: MemberEventsPollRequest<'_>,
6677 ) -> Result<MemberEventsWindow, MemberObservationError> {
6678 Err(MemberObservationError::Internal {
6679 reason: "unused test seam".to_string(),
6680 })
6681 }
6682
6683 async fn open_directed_turn_window(
6684 &self,
6685 _session: &SessionId,
6686 _expected_member: &BridgeMemberIncarnation,
6687 _input_id: &str,
6688 ) -> Result<
6689 crate::member_observation::DirectedTurnWindow,
6690 crate::member_observation::DirectedTurnReject,
6691 > {
6692 Err(crate::member_observation::DirectedTurnReject::unsupported(
6693 "member event sequence space is exhausted before Pending reservation",
6694 ))
6695 }
6696
6697 async fn cancel_directed_turn_window(
6698 &self,
6699 _session: &SessionId,
6700 _window: crate::member_observation::DirectedTurnWindow,
6701 ) -> Result<(), crate::member_observation::DirectedTurnReject> {
6702 Err(crate::member_observation::DirectedTurnReject::unsupported(
6703 "unused test seam",
6704 ))
6705 }
6706
6707 async fn admit_directed_turn(
6708 &self,
6709 _session: &SessionId,
6710 _admission: DirectedTurnAdmission,
6711 ) -> Result<(), crate::member_observation::DirectedTurnReject> {
6712 Err(crate::member_observation::DirectedTurnReject::unsupported(
6713 "unused test seam",
6714 ))
6715 }
6716 }
6717
6718 #[tokio::test]
6719 async fn rejected_exhausted_window_never_invokes_runtime_accept() {
6720 let observation: Arc<dyn MemberObservationHost> =
6721 Arc::new(ExhaustedDirectedTurnObservation);
6722 let session_id = SessionId::new();
6723 let accept_calls = std::sync::atomic::AtomicUsize::new(0);
6724 let result = open_directed_turn_before_runtime_accept(
6725 &observation,
6726 &session_id,
6727 &BridgeMemberIncarnation::default(),
6728 "input-exhausted",
6729 || async {
6730 accept_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6731 },
6732 )
6733 .await;
6734 assert!(result.is_err());
6735 assert_eq!(
6736 accept_calls.load(std::sync::atomic::Ordering::SeqCst),
6737 0,
6738 "runtime acceptance must remain strictly downstream of window reservation"
6739 );
6740 }
6741
6742 #[test]
6743 fn exact_terminal_replay_never_invokes_runtime_accept() {
6744 let expected_member = BridgeMemberIncarnation {
6745 mob_id: "mob".to_string(),
6746 agent_identity: "worker".to_string(),
6747 host_id: "host".to_string(),
6748 binding_generation: 3,
6749 member_session_id: SessionId::new().to_string(),
6750 generation: 7,
6751 fence_token: 11,
6752 };
6753 let input_id = Uuid::new_v4().to_string();
6754 let window = DirectedTurnWindow {
6755 expected_member: expected_member.clone(),
6756 subscription: Box::pin(futures::stream::empty()),
6757 window_start: 42,
6758 generation: expected_member.generation,
6759 fence_token: expected_member.fence_token,
6760 input_id: input_id.clone(),
6761 tracking: DirectedTurnTracking::TerminalReplay,
6762 };
6763 let accept_calls = std::sync::atomic::AtomicUsize::new(0);
6764 let response = match directed_turn_pre_accept_decision(&input_id, &expected_member, &window)
6765 .expect("exact terminal replay is valid")
6766 {
6767 DirectedTurnPreAcceptDecision::SubmitToRuntime => {
6768 accept_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6769 panic!("durable terminal replay must dominate runtime admission")
6770 }
6771 DirectedTurnPreAcceptDecision::TerminalReplay(response) => response,
6772 };
6773
6774 assert_eq!(
6775 accept_calls.load(std::sync::atomic::Ordering::SeqCst),
6776 0,
6777 "terminal replay must never reach the runtime accept closure"
6778 );
6779 assert_eq!(response.input_id, input_id);
6780 assert_eq!(
6781 response.canonical_input_id.as_deref(),
6782 Some(input_id.as_str())
6783 );
6784 assert!(matches!(
6785 response.outcome,
6786 BridgeDeliveryOutcome::Deduplicated {
6787 existing_input_id
6788 } if existing_input_id == input_id
6789 ));
6790 }
6791
6792 #[test]
6793 fn terminal_replay_with_different_request_identity_fails_closed() {
6794 let expected_member = BridgeMemberIncarnation {
6795 mob_id: "mob".to_string(),
6796 agent_identity: "worker".to_string(),
6797 host_id: "host".to_string(),
6798 binding_generation: 3,
6799 member_session_id: SessionId::new().to_string(),
6800 generation: 7,
6801 fence_token: 11,
6802 };
6803 let window = DirectedTurnWindow {
6804 expected_member: expected_member.clone(),
6805 subscription: Box::pin(futures::stream::empty()),
6806 window_start: 42,
6807 generation: expected_member.generation,
6808 fence_token: expected_member.fence_token,
6809 input_id: Uuid::new_v4().to_string(),
6810 tracking: DirectedTurnTracking::TerminalReplay,
6811 };
6812 assert!(
6813 directed_turn_pre_accept_decision(
6814 &Uuid::new_v4().to_string(),
6815 &expected_member,
6816 &window,
6817 )
6818 .is_err(),
6819 "a terminal row for another input must never suppress or submit this request"
6820 );
6821 }
6822
6823 #[test]
6824 fn directed_no_effect_rejection_requires_durable_pending_cancel() {
6825 let cause = BridgeDeliveryRejectionCause::Internal {
6826 detail: "validation failed".to_string(),
6827 };
6828 let response = directed_rejection_after_pending_cancel(
6829 "input-1",
6830 cause.clone(),
6831 "Input validation failed: invalid".to_string(),
6832 Ok(()),
6833 )
6834 .expect("durably canceled Pending certifies definite no-effect");
6835 assert!(matches!(
6836 response.outcome,
6837 BridgeDeliveryOutcome::Rejected {
6838 cause: actual,
6839 ..
6840 } if actual == cause
6841 ));
6842
6843 let ambiguous = directed_rejection_after_pending_cancel(
6844 "input-1",
6845 cause,
6846 "Input validation failed: invalid".to_string(),
6847 Err(crate::member_observation::DirectedTurnReject::unsupported(
6848 "host actor cancellation unavailable",
6849 )),
6850 )
6851 .expect_err("cancel failure must not mint a definite Delivery Rejected");
6852 assert!(ambiguous.contains("durable Pending cancellation failed"));
6853 assert!(ambiguous.contains("host actor cancellation unavailable"));
6854 }
6855
6856 #[test]
6857 fn stale_fenced_accept_maps_to_typed_stale_member_rejection() {
6858 let expected = BridgeMemberIncarnation {
6859 mob_id: "mob".to_string(),
6860 agent_identity: "worker".to_string(),
6861 host_id: "host".to_string(),
6862 binding_generation: 1,
6863 member_session_id: "session-g1".to_string(),
6864 generation: 1,
6865 fence_token: 11,
6866 };
6867 let current = BridgeMemberIncarnation {
6868 binding_generation: 2,
6869 member_session_id: "session-g2".to_string(),
6870 generation: 2,
6871 fence_token: 12,
6872 ..expected.clone()
6873 };
6874 let cause = definite_no_effect_delivery_rejection(
6875 &crate::traits::RuntimeDriverError::StaleAuthority {
6876 reason: "G1 lost the residency cutover".to_string(),
6877 },
6878 Some(current.clone()),
6879 &expected,
6880 )
6881 .expect("stale authority is a definite no-effect rejection");
6882 assert_eq!(
6883 cause,
6884 BridgeDeliveryRejectionCause::StaleMemberResidency {
6885 expected,
6886 current: Some(current),
6887 }
6888 );
6889 }
6890
6891 #[test]
6892 fn stale_fenced_accept_without_replacement_never_fabricates_current_member() {
6893 let expected = BridgeMemberIncarnation {
6894 mob_id: "mob".to_string(),
6895 agent_identity: "worker".to_string(),
6896 host_id: "host".to_string(),
6897 binding_generation: 1,
6898 member_session_id: "session-g1".to_string(),
6899 generation: 1,
6900 fence_token: 11,
6901 };
6902 let cause = definite_no_effect_delivery_rejection(
6903 &crate::traits::RuntimeDriverError::StaleAuthority {
6904 reason: "G1 lost to vacate".to_string(),
6905 },
6906 None,
6907 &expected,
6908 )
6909 .expect("vacated stale authority remains a typed no-effect rejection");
6910 assert_eq!(
6911 cause,
6912 BridgeDeliveryRejectionCause::StaleMemberResidency {
6913 expected,
6914 current: None,
6915 }
6916 );
6917 }
6918
6919 #[test]
6920 fn member_events_reply_trims_outcomes_below_transport_budget() {
6921 let outcomes = (0..16)
6922 .map(|index| {
6923 meerkat_contracts::wire::supervisor_bridge::BridgeTurnOutcomeRecord {
6924 input_id: format!("input-{index}"),
6925 generation: 1,
6926 fence_token: 1,
6927 terminal_seq: index,
6928 outcome: meerkat_contracts::wire::supervisor_bridge::WireFlowTurnOutcome::RunFailed {
6929 detail: meerkat_contracts::wire::supervisor_bridge::WireFlowFailureDetail::complete(
6930 "x".repeat(60_000),
6931 ),
6932 },
6933 }
6934 })
6935 .collect();
6936 let reply = bounded_member_events_reply(MemberEventsWindow {
6937 runtime_incarnation: BridgeHostRuntimeIncarnation::new(),
6938 generation: 1,
6939 fence_token: 1,
6940 rows: Vec::new(),
6941 from_seq: 7,
6942 next_seq: 7,
6943 watermark: 6,
6944 turn_outcomes: outcomes,
6945 outcomes_complete: true,
6946 })
6947 .expect("bounded outcome page");
6948 let encoded = serde_json::to_vec(&reply).expect("serialize bounded reply");
6949 assert!(encoded.len() <= MEMBER_EVENTS_REPLY_MAX_BYTES);
6950 let BridgeReply::MemberEventsPage(page) = reply else {
6951 panic!("expected member events page")
6952 };
6953 assert_eq!(
6954 page.from_seq, 7,
6955 "the wire page preserves the host-resolved read floor even when no event row fits"
6956 );
6957 assert_eq!(page.next_seq, page.from_seq);
6958 assert!(page.turn_outcomes.len() < 16);
6959 assert!(!page.outcomes_complete);
6960 }
6961
6962 #[test]
6963 fn member_events_reply_rejects_one_oversized_event_without_cursor_skip() {
6964 let session_id =
6965 SessionId::parse("0195e9a0-0000-7000-8000-000000000001").expect("session id");
6966 let envelope = meerkat_core::event::EventEnvelope::new(
6967 "worker",
6968 1,
6969 Some("mob-size".to_string()),
6970 AgentEvent::ExtractionFailed {
6971 session_id,
6972 last_output: "x".repeat(MEMBER_EVENTS_REPLY_MAX_BYTES + 1024),
6973 attempts: 1,
6974 reason: "oversized".to_string(),
6975 },
6976 );
6977 let error = bounded_member_events_reply(MemberEventsWindow {
6978 runtime_incarnation: BridgeHostRuntimeIncarnation::new(),
6979 generation: 1,
6980 fence_token: 1,
6981 rows: vec![(7, envelope)],
6982 from_seq: 7,
6983 next_seq: 8,
6984 watermark: 7,
6985 turn_outcomes: Vec::new(),
6986 outcomes_complete: true,
6987 })
6988 .expect_err("single oversized event must reject instead of replay-looping empty pages");
6989 assert!(matches!(
6990 error,
6991 MemberEventsReplyError::OversizedEvent {
6992 generation: 1,
6993 durable_seq: 7,
6994 ..
6995 }
6996 ));
6997 }
6998
6999 #[test]
7000 fn member_events_reply_rejects_exhausted_row_without_max_replay_or_zero_wrap() {
7001 let envelope = meerkat_core::event::EventEnvelope::new(
7002 "worker",
7003 1,
7004 Some("mob-exhausted".to_string()),
7005 AgentEvent::TurnStarted { turn_number: 1 },
7006 );
7007 let error = bounded_member_events_reply(MemberEventsWindow {
7008 runtime_incarnation: BridgeHostRuntimeIncarnation::new(),
7009 generation: 1,
7010 fence_token: 1,
7011 rows: vec![(u64::MAX, envelope)],
7012 from_seq: u64::MAX,
7013 next_seq: u64::MAX,
7014 watermark: u64::MAX,
7015 turn_outcomes: Vec::new(),
7016 outcomes_complete: true,
7017 })
7018 .expect_err("a MAX row has no representable successor cursor");
7019 assert!(matches!(
7020 error,
7021 MemberEventsReplyError::Internal(reason)
7022 if reason.contains("sequence space is exhausted")
7023 ));
7024 }
7025
7026 #[test]
7027 fn member_events_reply_rejects_empty_exhausted_frontier_instead_of_spinning() {
7028 let error = bounded_member_events_reply(MemberEventsWindow {
7029 runtime_incarnation: BridgeHostRuntimeIncarnation::new(),
7030 generation: 1,
7031 fence_token: 1,
7032 rows: Vec::new(),
7033 from_seq: u64::MAX,
7034 next_seq: u64::MAX,
7035 watermark: u64::MAX,
7036 turn_outcomes: Vec::new(),
7037 outcomes_complete: true,
7038 })
7039 .expect_err("an empty MAX frontier cannot make progress");
7040 assert!(matches!(
7041 error,
7042 MemberEventsReplyError::Internal(reason)
7043 if reason.contains("empty MAX frontier")
7044 ));
7045 }
7046
7047 fn member_history_window(
7048 offset: usize,
7049 message_count: usize,
7050 rows: impl IntoIterator<Item = String>,
7051 ) -> MemberHistoryWindow {
7052 let messages: Vec<_> = rows
7053 .into_iter()
7054 .map(|text| {
7055 meerkat_core::types::Message::User(meerkat_core::types::UserMessage::text(text))
7056 })
7057 .collect();
7058 MemberHistoryWindow {
7059 generation: 4,
7060 page: meerkat_core::service::SessionHistoryPage {
7061 session_id: SessionId::parse("0195e9a0-0000-7000-8000-000000000001")
7062 .expect("session id"),
7063 message_count,
7064 offset,
7065 limit: Some(messages.len()),
7066 has_more: offset + messages.len() < message_count,
7067 messages,
7068 },
7069 }
7070 }
7071
7072 #[test]
7073 fn member_history_reply_returns_maximal_transport_safe_prefix() {
7074 let reply = bounded_member_history_reply(member_history_window(
7075 7,
7076 10,
7077 [
7078 "a".repeat(200_000),
7079 "b".repeat(200_000),
7080 "c".repeat(200_000),
7081 ],
7082 ))
7083 .expect("bounded history page");
7084 let encoded = serde_json::to_vec(&reply).expect("serialize bounded history reply");
7085 assert!(encoded.len() <= MEMBER_HISTORY_REPLY_MAX_BYTES);
7086 let BridgeReply::MemberHistoryPage(page) = reply else {
7087 panic!("expected member history page")
7088 };
7089 assert_eq!(page.page.from_index, 7);
7090 assert_eq!(page.page.messages.len(), 2);
7091 assert_eq!(page.page.next_index, Some(9));
7092 assert!(!page.page.complete);
7093 assert_eq!(page.page.message_count, 10);
7094 }
7095
7096 #[test]
7097 fn member_history_reply_rejects_single_poison_row_at_exact_index() {
7098 let error = bounded_member_history_reply(member_history_window(
7099 9,
7100 10,
7101 ["x".repeat(MEMBER_HISTORY_REPLY_MAX_BYTES + 64 * 1024)],
7102 ))
7103 .expect_err("one oversized history row must fail typed");
7104 assert!(matches!(
7105 error,
7106 MemberHistoryReplyError::RowTooLarge { index: 9, .. }
7107 ));
7108 }
7109
7110 #[test]
7111 fn member_history_reply_rejects_non_advancing_domain_page() {
7112 let error = bounded_member_history_reply(MemberHistoryWindow {
7113 generation: 4,
7114 page: meerkat_core::service::SessionHistoryPage {
7115 session_id: SessionId::parse("0195e9a0-0000-7000-8000-000000000001")
7116 .expect("session id"),
7117 message_count: 1,
7118 offset: 0,
7119 limit: Some(1),
7120 has_more: true,
7121 messages: Vec::new(),
7122 },
7123 })
7124 .expect_err("a non-advancing page must fail before it reaches the bridge");
7125 assert!(matches!(
7126 error,
7127 MemberHistoryReplyError::Internal(reason)
7128 if reason.contains("page projection failed") && reason.contains("serves none")
7129 ));
7130 }
7131
7132 const PEER_ID_RECEIVER: &str = "0b86269a-ed69-5e31-a1ce-9552506eebe3"; const PEER_ID_SUPERVISOR: &str = "0adb7b34-edb5-5898-bab4-3f7cb10ba4da"; const PEER_ID_OLD_SUPERVISOR: &str = "d15ad17e-57de-58da-a91a-466c87a3e0c4"; fn test_pubkey_bytes_for_peer_id(peer_id: &str) -> Option<[u8; 32]> {
7147 match peer_id {
7148 PEER_ID_RECEIVER => Some([0xaa; 32]),
7149 PEER_ID_SUPERVISOR => Some([0xbb; 32]),
7150 PEER_ID_OLD_SUPERVISOR => Some([0xdd; 32]),
7151 _ => None,
7152 }
7153 }
7154
7155 fn test_public_key_for_peer_id(peer_id: &str) -> Option<String> {
7156 test_pubkey_bytes_for_peer_id(peer_id)
7157 .map(|bytes| meerkat_comms::PubKey::new(bytes).to_pubkey_string())
7158 }
7159
7160 fn test_comms_name_from_address(address: &str) -> String {
7161 address
7162 .strip_prefix("inproc://")
7163 .unwrap_or(address)
7164 .to_string()
7165 }
7166
7167 fn test_projection_trust_dsl(
7168 runtime: &meerkat_comms::CommsRuntime,
7169 ) -> Arc<crate::handles::HandleDslAuthority> {
7170 let dsl = Arc::new(crate::handles::HandleDslAuthority::ephemeral());
7171 dsl.apply_signal(
7172 crate::meerkat_machine::dsl::MeerkatMachineSignal::Initialize,
7173 "test_projection_trust_initialize",
7174 )
7175 .expect("Initialize signal");
7176 dsl.apply_input(
7177 crate::meerkat_machine::dsl::MeerkatMachineInput::RegisterSession {
7178 session_id: crate::meerkat_machine::dsl::SessionId::from(
7179 "comms-drain-test-projection-trust",
7180 ),
7181 },
7182 "test_projection_trust_register",
7183 )
7184 .expect("RegisterSession input");
7185 crate::handles::RuntimePeerCommsHandle::install_generated_on(Arc::clone(&dsl), runtime)
7186 .expect("install generated peer-comms handle");
7187 dsl
7188 }
7189
7190 async fn add_test_projection_trust(
7191 runtime: &meerkat_comms::CommsRuntime,
7192 peer: TrustedPeerDescriptor,
7193 context: &str,
7194 ) {
7195 let dsl = test_projection_trust_dsl(runtime);
7196 add_test_projection_trust_with_dsl(runtime, dsl, peer, 1, context).await;
7197 }
7198
7199 async fn add_test_projection_trust_with_dsl(
7200 runtime: &meerkat_comms::CommsRuntime,
7201 dsl: Arc<crate::handles::HandleDslAuthority>,
7202 peer: TrustedPeerDescriptor,
7203 epoch: u64,
7204 context: &str,
7205 ) {
7206 let local_peer_id = runtime
7207 .peer_id()
7208 .unwrap_or_else(|| panic!("{context}: runtime peer_id unavailable"));
7209 let endpoint = crate::meerkat_machine::dsl::PeerEndpoint::from(&peer);
7210 dsl.apply_input(
7211 crate::meerkat_machine::dsl::MeerkatMachineInput::PublishLocalEndpoint {
7212 endpoint: crate::meerkat_machine::dsl::PeerEndpoint::new(
7213 "local",
7214 local_peer_id.to_string(),
7215 "inproc://local",
7216 [0x7f; 32],
7217 ),
7218 },
7219 "test_projection_trust_publish_local_endpoint",
7220 )
7221 .expect("PublishLocalEndpoint input");
7222 let transition = dsl
7223 .apply_input_with_transition(
7224 crate::meerkat_machine::dsl::MeerkatMachineInput::ApplyMobPeerOverlay {
7225 epoch,
7226 endpoints: std::collections::BTreeSet::from([endpoint.clone()]),
7227 },
7228 "test_projection_trust_apply_overlay",
7229 )
7230 .expect("ApplyMobPeerOverlay input");
7231 let obligation = crate::protocol_comms_trust_reconcile::extract_obligations_with_freshness(
7232 &transition,
7233 dsl.peer_projection_freshness_authority(),
7234 )
7235 .into_iter()
7236 .next()
7237 .expect("generated reconcile obligation");
7238 let authority =
7239 crate::protocol_comms_trust_reconcile::authority_for_endpoint(&obligation, &endpoint)
7240 .expect("generated authority");
7241 match runtime
7242 .apply_trust_mutation(CommsTrustMutation::AddTrustedPeer { peer, authority })
7243 .await
7244 .unwrap_or_else(|error| panic!("{context}: {error}"))
7245 {
7246 CommsTrustMutationResult::Added { .. } => {}
7247 other => panic!("{context}: unexpected trust mutation result {other:?}"),
7248 }
7249 }
7250
7251 async fn add_member_private_supervisor_trust_via_adapter(
7262 adapter: &Arc<MeerkatMachine>,
7263 session_id: &SessionId,
7264 runtime: &Arc<meerkat_comms::CommsRuntime>,
7265 supervisor: &TrustedPeerDescriptor,
7266 _epoch: u64,
7267 context: &str,
7268 ) {
7269 let runtime_dyn: Arc<dyn CommsRuntime> = Arc::clone(runtime) as Arc<dyn CommsRuntime>;
7270 adapter
7271 .stage_local_endpoint_for_comms_runtime(session_id, runtime_dyn.as_ref())
7272 .await
7273 .unwrap_or_else(|error| panic!("{context}: local endpoint rejected: {error}"));
7274 let obligation =
7275 supervisor_route_publish_obligation(adapter, session_id, supervisor, context)
7276 .await
7277 .unwrap_or_else(|error| panic!("{context}: {error}"));
7278 publish_supervisor_trust_from_generated_obligation(
7279 adapter,
7280 session_id,
7281 runtime_dyn.as_ref(),
7282 &obligation,
7283 )
7284 .await
7285 .unwrap_or_else(|error| panic!("{context}: private trust publish failed: {error}"));
7286 }
7287
7288 fn install_running_test_peer_comms_handle(
7289 runtime: &meerkat_comms::CommsRuntime,
7290 ) -> Arc<crate::handles::HandleDslAuthority> {
7291 let authority = Arc::new(crate::handles::HandleDslAuthority::ephemeral());
7292 let session_id = SessionId::new();
7293 authority
7294 .apply_signal(
7295 crate::meerkat_machine::dsl::MeerkatMachineSignal::Initialize,
7296 "running_test_peer_comms_handle_initialize",
7297 )
7298 .expect("test peer-comms authority should initialize");
7299 authority
7300 .apply_input(
7301 crate::meerkat_machine::dsl::MeerkatMachineInput::RegisterSession {
7302 session_id: crate::meerkat_machine::dsl::SessionId::from_domain(&session_id),
7303 },
7304 "running_test_peer_comms_handle_register",
7305 )
7306 .expect("test peer-comms authority should register a session");
7307 authority
7308 .apply_input(
7309 crate::meerkat_machine::dsl::MeerkatMachineInput::Prepare {
7310 session_id: crate::meerkat_machine::dsl::SessionId::from_domain(&session_id),
7311 run_id: crate::meerkat_machine::dsl::RunId::from_domain(
7312 &meerkat_core::lifecycle::RunId::new(),
7313 ),
7314 },
7315 "running_test_peer_comms_handle_prepare",
7316 )
7317 .expect("test peer-comms authority should enter Running");
7318 crate::handles::RuntimePeerCommsHandle::install_generated_on(
7319 Arc::clone(&authority),
7320 runtime,
7321 )
7322 .expect("install generated peer-comms handle");
7323 authority
7324 }
7325
7326 fn test_pubkey(seed: u8) -> meerkat_comms::PubKey {
7327 assert_ne!(seed, 0, "test pubkey seed must be non-zero");
7328 meerkat_comms::PubKey::new([seed; 32])
7329 }
7330
7331 fn test_supervisor_signing_public_key(seed: u8) -> String {
7332 encode_supervisor_signing_public_key([seed; 32])
7333 }
7334
7335 fn signing_public_key_for_descriptor(peer: &TrustedPeerDescriptor) -> String {
7336 encode_supervisor_signing_public_key(peer.pubkey)
7337 }
7338
7339 fn bridge_peer_spec_with_seed(name: &str, seed: u8, address: &str) -> BridgePeerSpec {
7340 let pubkey = test_pubkey(seed);
7341 BridgePeerSpec {
7342 name: name.to_string(),
7343 peer_id: pubkey.to_peer_id().as_str(),
7344 address: address.to_string(),
7345 pubkey: *pubkey.as_bytes(),
7346 }
7347 }
7348
7349 fn supervisor_bridge_spec() -> BridgePeerSpec {
7350 bridge_peer_spec_with_seed(
7351 "mob/__mob_supervisor__",
7352 0xbb,
7353 "inproc://mob/__mob_supervisor__",
7354 )
7355 }
7356
7357 fn current_supervisor_bridge_spec() -> BridgePeerSpec {
7358 bridge_peer_spec_with_seed(
7359 "mob/__mob_supervisor__",
7360 0xcc,
7361 "inproc://mob/__mob_supervisor__",
7362 )
7363 }
7364
7365 fn pubkey_sender_for_bridge_spec(spec: &BridgePeerSpec) -> String {
7366 meerkat_comms::PubKey::new(spec.pubkey).to_pubkey_string()
7367 }
7368
7369 fn old_supervisor_bridge_spec() -> BridgePeerSpec {
7370 bridge_peer_spec_with_seed(
7371 "mob/__mob_supervisor__",
7372 0xdd,
7373 "inproc://mob/__mob_supervisor__",
7374 )
7375 }
7376
7377 fn trusted_supervisor_descriptor(seed: u8) -> TrustedPeerDescriptor {
7378 TrustedPeerDescriptor::try_from(bridge_peer_spec_with_seed(
7379 "mob/__mob_supervisor__",
7380 seed,
7381 "inproc://mob/__mob_supervisor__",
7382 ))
7383 .expect("valid supervisor spec")
7384 }
7385
7386 fn trusted_peer_from_runtime(
7387 name: &str,
7388 runtime: &meerkat_comms::CommsRuntime,
7389 ) -> TrustedPeerDescriptor {
7390 let pubkey = runtime.public_key();
7391 TrustedPeerDescriptor::unsigned_with_pubkey(
7392 name,
7393 pubkey.to_peer_id().as_str(),
7394 *pubkey.as_bytes(),
7395 format!("inproc://{name}"),
7396 )
7397 .expect("valid non-zero trusted peer descriptor")
7398 }
7399
7400 async fn recorded_bridge_replies(
7401 sent_commands: &Arc<tokio::sync::Mutex<Vec<CommsCommand>>>,
7402 ) -> Vec<(meerkat_core::interaction::ResponseStatus, BridgeReply)> {
7403 sent_commands
7404 .lock()
7405 .await
7406 .iter()
7407 .filter_map(|cmd| match cmd {
7408 CommsCommand::PeerResponse { status, result, .. } => Some((
7409 *status,
7410 serde_json::from_value::<BridgeReply>(result.clone())
7411 .expect("recorded bridge response is typed"),
7412 )),
7413 _ => None,
7414 })
7415 .collect()
7416 }
7417
7418 fn assert_completed_bridge_ack(
7419 reply: &(meerkat_core::interaction::ResponseStatus, BridgeReply),
7420 ) {
7421 assert_eq!(
7422 reply.0,
7423 meerkat_core::interaction::ResponseStatus::Completed
7424 );
7425 assert!(
7426 matches!(reply.1, BridgeReply::Ack(BridgeAck { ok: true })),
7427 "expected completed bridge ack, got {:?}",
7428 reply.1
7429 );
7430 }
7431
7432 fn trusted_tcp_peer_from_runtime(
7433 name: &str,
7434 runtime: &meerkat_comms::CommsRuntime,
7435 ) -> TrustedPeerDescriptor {
7436 let pubkey = runtime.public_key();
7437 TrustedPeerDescriptor::unsigned_with_pubkey(
7438 name,
7439 pubkey.to_peer_id().as_str(),
7440 *pubkey.as_bytes(),
7441 runtime.advertised_address(),
7442 )
7443 .expect("valid non-zero TCP trusted peer descriptor")
7444 }
7445
7446 fn install_test_peer_request_response_authority(
7447 runtime: &Arc<meerkat_comms::CommsRuntime>,
7448 peer_handle: Arc<CountingPeerInteractionHandle>,
7449 ) {
7450 runtime.install_peer_request_response_authority(
7451 meerkat_comms::PeerRequestResponseAuthority::new(
7452 peer_handle,
7453 Arc::new(crate::handles::RuntimeInteractionStreamHandle::ephemeral()),
7454 ),
7455 );
7456 }
7457
7458 #[derive(Default)]
7459 struct CountingPeerInteractionHandle {
7460 inbound: std::sync::Mutex<HashSet<meerkat_core::PeerCorrelationId>>,
7461 inbound_modes: std::sync::Mutex<HashMap<meerkat_core::PeerCorrelationId, HandlingMode>>,
7462 request_received_count: std::sync::atomic::AtomicUsize,
7463 response_rejected_count: std::sync::atomic::AtomicUsize,
7464 response_replied_count: std::sync::atomic::AtomicUsize,
7465 reject_request_received: std::sync::atomic::AtomicBool,
7466 reject_response_progress: std::sync::atomic::AtomicBool,
7467 reject_response_terminal: std::sync::atomic::AtomicBool,
7468 }
7469
7470 impl CountingPeerInteractionHandle {
7471 fn rejecting_request_received() -> Self {
7472 let handle = Self::default();
7473 handle
7474 .reject_request_received
7475 .store(true, std::sync::atomic::Ordering::SeqCst);
7476 handle
7477 }
7478
7479 fn rejecting_response_progress() -> Self {
7480 let handle = Self::default();
7481 handle
7482 .reject_response_progress
7483 .store(true, std::sync::atomic::Ordering::SeqCst);
7484 handle
7485 }
7486
7487 fn rejecting_response_terminal() -> Self {
7488 let handle = Self::default();
7489 handle
7490 .reject_response_terminal
7491 .store(true, std::sync::atomic::Ordering::SeqCst);
7492 handle
7493 }
7494
7495 fn rejected(
7496 context: &'static str,
7497 corr_id: meerkat_core::PeerCorrelationId,
7498 ) -> meerkat_core::handles::DslTransitionError {
7499 meerkat_core::handles::DslTransitionError::guard_rejected(
7500 context,
7501 format!("test authority rejected corr_id {corr_id}"),
7502 )
7503 }
7504
7505 fn request_received_count(&self) -> usize {
7506 self.request_received_count
7507 .load(std::sync::atomic::Ordering::SeqCst)
7508 }
7509
7510 fn response_replied_count(&self) -> usize {
7511 self.response_replied_count
7512 .load(std::sync::atomic::Ordering::SeqCst)
7513 }
7514
7515 fn response_rejected_count(&self) -> usize {
7516 self.response_rejected_count
7517 .load(std::sync::atomic::Ordering::SeqCst)
7518 }
7519 }
7520
7521 impl meerkat_core::handles::PeerInteractionHandle for CountingPeerInteractionHandle {
7522 fn request_sent(
7523 &self,
7524 _corr_id: meerkat_core::PeerCorrelationId,
7525 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7526 Ok(())
7527 }
7528
7529 fn response_progress(
7530 &self,
7531 corr_id: meerkat_core::PeerCorrelationId,
7532 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7533 if self
7534 .reject_response_progress
7535 .load(std::sync::atomic::Ordering::SeqCst)
7536 {
7537 return Err(Self::rejected(
7538 "PeerInteractionHandle::response_progress",
7539 corr_id,
7540 ));
7541 }
7542 Ok(())
7543 }
7544
7545 fn response_terminal(
7546 &self,
7547 corr_id: meerkat_core::PeerCorrelationId,
7548 _disposition: meerkat_core::handles::PeerTerminalDisposition,
7549 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7550 if self
7551 .reject_response_terminal
7552 .load(std::sync::atomic::Ordering::SeqCst)
7553 {
7554 return Err(Self::rejected(
7555 "PeerInteractionHandle::response_terminal",
7556 corr_id,
7557 ));
7558 }
7559 Ok(())
7560 }
7561
7562 fn response_rejected(
7563 &self,
7564 _corr_id: meerkat_core::PeerCorrelationId,
7565 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7566 self.response_rejected_count
7567 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7568 Ok(())
7569 }
7570
7571 fn request_timed_out(
7572 &self,
7573 _corr_id: meerkat_core::PeerCorrelationId,
7574 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7575 Ok(())
7576 }
7577
7578 fn request_send_failed(
7579 &self,
7580 _corr_id: meerkat_core::PeerCorrelationId,
7581 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7582 Ok(())
7583 }
7584
7585 fn classify_response_reply(
7586 &self,
7587 status: meerkat_core::ResponseStatus,
7588 ) -> Result<meerkat_core::TerminalityClass, meerkat_core::handles::DslTransitionError>
7589 {
7590 let generated = crate::handles::RuntimePeerInteractionHandle::ephemeral();
7591 meerkat_core::handles::PeerInteractionHandle::classify_response_reply(
7592 &generated, status,
7593 )
7594 }
7595
7596 fn request_received(
7597 &self,
7598 corr_id: meerkat_core::PeerCorrelationId,
7599 handling_mode: HandlingMode,
7600 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7601 if self
7602 .reject_request_received
7603 .load(std::sync::atomic::Ordering::SeqCst)
7604 {
7605 return Err(Self::rejected(
7606 "PeerInteractionHandle::request_received",
7607 corr_id,
7608 ));
7609 }
7610 let mut inbound = self.inbound.lock().expect("inbound mutex");
7611 if !inbound.insert(corr_id) {
7612 return Err(Self::rejected(
7613 "PeerInteractionHandle::request_received",
7614 corr_id,
7615 ));
7616 }
7617 self.inbound_modes
7618 .lock()
7619 .expect("inbound modes mutex")
7620 .insert(corr_id, handling_mode);
7621 self.request_received_count
7622 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7623 Ok(())
7624 }
7625
7626 fn response_replied(
7627 &self,
7628 corr_id: meerkat_core::PeerCorrelationId,
7629 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7630 let mut inbound = self.inbound.lock().expect("inbound mutex");
7631 if !inbound.remove(&corr_id) {
7632 return Err(Self::rejected(
7633 "PeerInteractionHandle::response_replied",
7634 corr_id,
7635 ));
7636 }
7637 self.inbound_modes
7638 .lock()
7639 .expect("inbound modes mutex")
7640 .remove(&corr_id);
7641 self.response_replied_count
7642 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7643 Ok(())
7644 }
7645
7646 fn outbound_state(
7647 &self,
7648 _corr_id: meerkat_core::PeerCorrelationId,
7649 ) -> Option<meerkat_core::OutboundPeerRequestState> {
7650 None
7651 }
7652
7653 fn inbound_state(
7654 &self,
7655 corr_id: meerkat_core::PeerCorrelationId,
7656 ) -> Option<meerkat_core::InboundPeerRequestState> {
7657 self.inbound
7658 .lock()
7659 .expect("inbound mutex")
7660 .contains(&corr_id)
7661 .then_some(meerkat_core::InboundPeerRequestState::Received)
7662 }
7663
7664 fn inbound_handling_mode(
7665 &self,
7666 corr_id: meerkat_core::PeerCorrelationId,
7667 ) -> Option<HandlingMode> {
7668 self.inbound_modes
7669 .lock()
7670 .expect("inbound modes mutex")
7671 .get(&corr_id)
7672 .copied()
7673 }
7674
7675 fn install_cleanup_observer(
7676 &self,
7677 _observer: Arc<dyn meerkat_core::handles::PeerInteractionCleanupObserver>,
7678 ) {
7679 }
7680 }
7681
7682 struct OneShotPeerRequestRuntime {
7683 notify: Arc<tokio::sync::Notify>,
7684 candidate: std::sync::Mutex<Option<PeerInputCandidate>>,
7685 peer_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
7686 peer_request_response_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
7687 completed_count: std::sync::atomic::AtomicUsize,
7688 abandonments:
7689 std::sync::Mutex<Vec<(InteractionId, meerkat_core::InteractionStreamAbandonReason)>>,
7690 }
7691
7692 impl OneShotPeerRequestRuntime {
7693 fn new(
7694 candidate: PeerInputCandidate,
7695 peer_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
7696 ) -> Self {
7697 Self {
7698 notify: Arc::new(tokio::sync::Notify::new()),
7699 candidate: std::sync::Mutex::new(Some(candidate)),
7700 peer_handle,
7701 peer_request_response_handle: None,
7702 completed_count: std::sync::atomic::AtomicUsize::new(0),
7703 abandonments: std::sync::Mutex::new(Vec::new()),
7704 }
7705 }
7706
7707 fn with_complete_authority(
7708 candidate: PeerInputCandidate,
7709 peer_handle: Arc<dyn meerkat_core::handles::PeerInteractionHandle>,
7710 ) -> Self {
7711 Self {
7712 notify: Arc::new(tokio::sync::Notify::new()),
7713 candidate: std::sync::Mutex::new(Some(candidate)),
7714 peer_handle: Some(peer_handle.clone()),
7715 peer_request_response_handle: Some(peer_handle),
7716 completed_count: std::sync::atomic::AtomicUsize::new(0),
7717 abandonments: std::sync::Mutex::new(Vec::new()),
7718 }
7719 }
7720
7721 fn completed_count(&self) -> usize {
7722 self.completed_count
7723 .load(std::sync::atomic::Ordering::SeqCst)
7724 }
7725
7726 fn abandonment_reasons(&self) -> Vec<meerkat_core::InteractionStreamAbandonReason> {
7727 self.abandonments
7728 .lock()
7729 .expect("abandonments mutex")
7730 .iter()
7731 .map(|(_, reason)| *reason)
7732 .collect()
7733 }
7734 }
7735
7736 #[async_trait::async_trait]
7737 impl CommsRuntime for OneShotPeerRequestRuntime {
7738 async fn drain_messages(&self) -> Vec<String> {
7739 Vec::new()
7740 }
7741
7742 fn inbox_notify(&self) -> Arc<tokio::sync::Notify> {
7743 self.notify.clone()
7744 }
7745
7746 fn dismiss_received(&self) -> bool {
7747 self.candidate.lock().expect("candidate mutex").is_none()
7748 }
7749
7750 fn peer_interaction_handle(
7751 &self,
7752 ) -> Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>> {
7753 self.peer_handle.clone()
7754 }
7755
7756 fn peer_request_response_authority_handle(
7757 &self,
7758 ) -> Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>> {
7759 self.peer_request_response_handle.clone()
7760 }
7761
7762 async fn drain_classified_inbox_interactions(
7763 &self,
7764 ) -> Result<
7765 Vec<meerkat_core::interaction::ClassifiedInboxInteraction>,
7766 meerkat_core::agent::CommsCapabilityError,
7767 > {
7768 Ok(self
7769 .candidate
7770 .lock()
7771 .expect("candidate mutex")
7772 .take()
7773 .into_iter()
7774 .collect())
7775 }
7776
7777 fn mark_interaction_complete(&self, _id: &InteractionId) {
7778 self.completed_count
7779 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7780 }
7781
7782 fn abandon_interaction_stream(
7783 &self,
7784 id: &InteractionId,
7785 reason: meerkat_core::InteractionStreamAbandonReason,
7786 ) {
7787 self.abandonments
7788 .lock()
7789 .expect("abandonments mutex")
7790 .push((*id, reason));
7791 }
7792 }
7793
7794 fn inbound_peer_request_candidate(class: PeerInputClass) -> PeerInputCandidate {
7795 assert!(
7796 matches!(
7797 class,
7798 PeerInputClass::SilentRequest | PeerInputClass::ActionableRequest
7799 ),
7800 "test helper only builds inbound peer requests"
7801 );
7802 let id = InteractionId(Uuid::new_v4());
7803 let intent = format!("test.inbound.{class:?}");
7804 let sender = "partial-authority-peer";
7805 PeerInputCandidate {
7806 interaction: InboxInteraction {
7807 objective_id: None,
7808 sender_taint: None,
7809 id,
7810 from_route: None,
7811 from: sender.to_string(),
7812 content: InteractionContent::Request {
7813 intent: intent.clone(),
7814 params: json!({ "ok": true }),
7815 blocks: None,
7816 },
7817 rendered_text: String::new(),
7818 handling_mode: HandlingMode::Queue,
7819 render_metadata: None,
7820 },
7821 ingress: PeerIngressFact::peer(
7822 id,
7823 class,
7824 meerkat_core::PeerIngressKind::Request,
7825 Some(meerkat_core::PeerIngressAuthDecision::Required),
7826 PeerIngressIdentity::new(
7827 PeerId::new(),
7828 sender,
7829 PeerIngressConvention::Request {
7830 request_id: id.to_string(),
7831 intent,
7832 },
7833 ),
7834 ),
7835 lifecycle_peer: None,
7836 response_terminality: None,
7837 }
7838 }
7839
7840 fn response_candidate(
7841 class: PeerInputClass,
7842 response_terminality: Option<meerkat_core::interaction::TerminalityClass>,
7843 ) -> PeerInputCandidate {
7844 assert!(
7845 matches!(
7846 class,
7847 PeerInputClass::ResponseProgress | PeerInputClass::ResponseTerminal
7848 ),
7849 "test helper only builds peer response candidates"
7850 );
7851 let id = InteractionId(Uuid::new_v4());
7852 let in_reply_to = InteractionId(Uuid::new_v4());
7853 PeerInputCandidate {
7854 interaction: InboxInteraction {
7855 objective_id: None,
7856 sender_taint: None,
7857 id,
7858 from_route: Some(PeerId::new()),
7859 from: "response-peer".to_string(),
7860 content: InteractionContent::Response {
7861 in_reply_to,
7862 status: meerkat_core::interaction::ResponseStatus::Completed,
7863 result: json!({ "ok": true }),
7864 blocks: None,
7865 },
7866 rendered_text: String::new(),
7867 handling_mode: HandlingMode::Queue,
7868 render_metadata: None,
7869 },
7870 ingress: PeerIngressFact::peer(
7871 id,
7872 class,
7873 meerkat_core::PeerIngressKind::Response,
7874 Some(meerkat_core::PeerIngressAuthDecision::Required),
7875 PeerIngressIdentity::new(
7876 PeerId::new(),
7877 "response-peer",
7878 PeerIngressConvention::Response {
7879 in_reply_to,
7880 status: meerkat_core::interaction::ResponseStatus::Completed,
7881 },
7882 ),
7883 ),
7884 lifecycle_peer: None,
7885 response_terminality,
7886 }
7887 }
7888
7889 struct ClassifiedDrainOutcomeRuntime {
7892 notify: Arc<tokio::sync::Notify>,
7893 fail_classified_drain: bool,
7894 }
7895
7896 impl ClassifiedDrainOutcomeRuntime {
7897 fn new(fail_classified_drain: bool) -> Self {
7898 Self {
7899 notify: Arc::new(tokio::sync::Notify::new()),
7900 fail_classified_drain,
7901 }
7902 }
7903 }
7904
7905 #[async_trait::async_trait]
7906 impl CommsRuntime for ClassifiedDrainOutcomeRuntime {
7907 async fn drain_messages(&self) -> Vec<String> {
7908 Vec::new()
7909 }
7910
7911 fn inbox_notify(&self) -> Arc<tokio::sync::Notify> {
7912 self.notify.clone()
7913 }
7914
7915 async fn drain_classified_inbox_interactions(
7916 &self,
7917 ) -> Result<
7918 Vec<meerkat_core::interaction::ClassifiedInboxInteraction>,
7919 meerkat_core::agent::CommsCapabilityError,
7920 > {
7921 if self.fail_classified_drain {
7922 Err(meerkat_core::agent::CommsCapabilityError::Unsupported(
7923 "synthetic classification fault".to_string(),
7924 ))
7925 } else {
7926 Ok(Vec::new())
7927 }
7928 }
7929 }
7930
7931 async fn spawn_authorized_test_comms_drain(
7932 adapter: Arc<MeerkatMachine>,
7933 session_id: SessionId,
7934 runtime: Arc<dyn CommsRuntime>,
7935 idle_timeout: Duration,
7936 ) -> crate::tokio::task::JoinHandle<()> {
7937 adapter
7943 .test_authorize_direct_comms_drain_runtime(&session_id, Arc::clone(&runtime))
7944 .await
7945 .expect("direct-drain test authority should accept");
7946 spawn_comms_drain(adapter, session_id, runtime, Some(idle_timeout))
7947 }
7948
7949 #[tokio::test]
7957 async fn classified_drain_error_fails_closed_not_idle_timeout() {
7958 let adapter = Arc::new(MeerkatMachine::ephemeral());
7959 let session_id = SessionId::new();
7960 adapter
7961 .register_session(session_id.clone())
7962 .await
7963 .expect("register session");
7964
7965 let runtime = Arc::new(ClassifiedDrainOutcomeRuntime::new(true));
7968 let drain = spawn_authorized_test_comms_drain(
7969 adapter.clone(),
7970 session_id.clone(),
7971 runtime.clone(),
7972 Duration::from_secs(60),
7973 )
7974 .await;
7975
7976 tokio::time::timeout(Duration::from_secs(2), drain)
7977 .await
7978 .expect(
7979 "a classified-drain error must fail closed promptly, NOT idle until the 60s timeout",
7980 )
7981 .expect("drain task should not panic");
7982 }
7983
7984 #[tokio::test]
7989 async fn empty_classified_drain_idles_rather_than_exiting() {
7990 let adapter = Arc::new(MeerkatMachine::ephemeral());
7991 let session_id = SessionId::new();
7992 adapter
7993 .register_session(session_id.clone())
7994 .await
7995 .expect("register session");
7996
7997 let runtime = Arc::new(ClassifiedDrainOutcomeRuntime::new(false));
7998 let drain = spawn_authorized_test_comms_drain(
7999 adapter.clone(),
8000 session_id.clone(),
8001 runtime.clone(),
8002 Duration::from_secs(60),
8003 )
8004 .await;
8005
8006 assert!(
8009 tokio::time::timeout(Duration::from_millis(200), drain)
8010 .await
8011 .is_err(),
8012 "an empty (Ok) inbox must idle, not exit promptly"
8013 );
8014 }
8015
8016 #[tokio::test]
8017 async fn response_without_machine_terminality_is_rejected_before_progress_or_admission() {
8018 let adapter = Arc::new(MeerkatMachine::ephemeral());
8019 let session_id = SessionId::new();
8020 adapter
8021 .register_session(session_id.clone())
8022 .await
8023 .expect("register session");
8024
8025 let runtime = Arc::new(OneShotPeerRequestRuntime::new(
8026 response_candidate(PeerInputClass::ResponseTerminal, None),
8027 None,
8028 ));
8029 let drain = spawn_authorized_test_comms_drain(
8030 adapter.clone(),
8031 session_id.clone(),
8032 runtime.clone(),
8033 Duration::from_millis(10),
8034 )
8035 .await;
8036
8037 tokio::time::timeout(Duration::from_secs(1), drain)
8038 .await
8039 .expect("drain should exit after one response candidate")
8040 .expect("drain task should not panic");
8041
8042 let snapshot = adapter
8043 .meerkat_machine_spine_snapshot(&session_id)
8044 .await
8045 .expect("registered session snapshot");
8046 assert_eq!(
8047 snapshot.ledger.input_count, 0,
8048 "missing terminality must fail closed before runtime admission"
8049 );
8050 }
8051
8052 #[tokio::test]
8053 async fn malformed_response_terminality_rejects_pending_peer_truth_via_authority() {
8054 let adapter = Arc::new(MeerkatMachine::ephemeral());
8055 let session_id = SessionId::new();
8056 adapter
8057 .register_session(session_id.clone())
8058 .await
8059 .expect("register session");
8060
8061 let peer_handle = Arc::new(CountingPeerInteractionHandle::default());
8062 let peer_authority: Arc<dyn meerkat_core::handles::PeerInteractionHandle> =
8063 peer_handle.clone();
8064 let runtime = Arc::new(OneShotPeerRequestRuntime::new(
8065 response_candidate(PeerInputClass::ResponseTerminal, None),
8066 Some(peer_authority),
8067 ));
8068 let drain = spawn_authorized_test_comms_drain(
8069 adapter.clone(),
8070 session_id.clone(),
8071 runtime.clone(),
8072 Duration::from_millis(10),
8073 )
8074 .await;
8075
8076 tokio::time::timeout(Duration::from_secs(1), drain)
8077 .await
8078 .expect("drain should exit after one malformed response candidate")
8079 .expect("drain task should not panic");
8080
8081 assert_eq!(
8082 peer_handle.response_rejected_count(),
8083 1,
8084 "malformed response observations must terminalize pending peer truth through generated authority"
8085 );
8086 let snapshot = adapter
8087 .meerkat_machine_spine_snapshot(&session_id)
8088 .await
8089 .expect("registered session snapshot");
8090 assert_eq!(
8091 snapshot.ledger.input_count, 0,
8092 "malformed response terminality must fail closed before runtime admission"
8093 );
8094 }
8095
8096 #[tokio::test]
8097 async fn rejected_peer_response_transition_fails_closed_before_machine_admission() {
8098 for (candidate, peer_handle) in [
8099 (
8100 response_candidate(
8101 PeerInputClass::ResponseTerminal,
8102 Some(meerkat_core::interaction::TerminalityClass::Terminal {
8103 disposition: meerkat_core::interaction::TerminalDisposition::Completed,
8104 }),
8105 ),
8106 Arc::new(CountingPeerInteractionHandle::rejecting_response_terminal()),
8107 ),
8108 (
8109 response_candidate(
8110 PeerInputClass::ResponseProgress,
8111 Some(meerkat_core::interaction::TerminalityClass::Progress),
8112 ),
8113 Arc::new(CountingPeerInteractionHandle::rejecting_response_progress()),
8114 ),
8115 ] {
8116 let adapter = Arc::new(MeerkatMachine::ephemeral());
8117 let session_id = SessionId::new();
8118 adapter
8119 .register_session(session_id.clone())
8120 .await
8121 .expect("register session");
8122
8123 let runtime = Arc::new(OneShotPeerRequestRuntime::new(
8124 candidate,
8125 Some(peer_handle.clone()),
8126 ));
8127 let drain = spawn_authorized_test_comms_drain(
8128 adapter.clone(),
8129 session_id.clone(),
8130 runtime.clone(),
8131 Duration::from_millis(10),
8132 )
8133 .await;
8134
8135 tokio::time::timeout(Duration::from_secs(1), drain)
8136 .await
8137 .expect("drain should exit after one rejected response candidate")
8138 .expect("drain task should not panic");
8139
8140 assert_eq!(
8141 peer_handle.response_rejected_count(),
8142 1,
8143 "rejected generated peer-response transition should terminalize peer truth through response rejection"
8144 );
8145 let snapshot = adapter
8146 .meerkat_machine_spine_snapshot(&session_id)
8147 .await
8148 .expect("registered session snapshot");
8149 assert_eq!(
8150 snapshot.ledger.input_count, 0,
8151 "rejected generated peer-response transition must fail closed before runtime admission"
8152 );
8153 }
8154 }
8155
8156 #[tokio::test]
8157 async fn rejected_peer_request_recording_rejects_inbound_request_before_machine_admission() {
8158 for class in [
8159 PeerInputClass::SilentRequest,
8160 PeerInputClass::ActionableRequest,
8161 ] {
8162 let adapter = Arc::new(MeerkatMachine::ephemeral());
8163 let session_id = SessionId::new();
8164 adapter
8165 .register_session(session_id.clone())
8166 .await
8167 .expect("register session");
8168
8169 let peer_handle = Arc::new(CountingPeerInteractionHandle::rejecting_request_received());
8170 let peer_authority: Arc<dyn meerkat_core::handles::PeerInteractionHandle> = peer_handle;
8171 let runtime = Arc::new(OneShotPeerRequestRuntime::with_complete_authority(
8172 inbound_peer_request_candidate(class),
8173 peer_authority,
8174 ));
8175 let drain = spawn_authorized_test_comms_drain(
8176 adapter.clone(),
8177 session_id.clone(),
8178 runtime.clone(),
8179 Duration::from_millis(10),
8180 )
8181 .await;
8182
8183 tokio::time::timeout(Duration::from_secs(1), drain)
8184 .await
8185 .expect("drain should exit after one candidate")
8186 .expect("drain task should not panic");
8187
8188 let snapshot = adapter
8189 .meerkat_machine_spine_snapshot(&session_id)
8190 .await
8191 .expect("registered session snapshot");
8192 assert_eq!(
8193 snapshot.ledger.input_count, 0,
8194 "rejected PeerRequestReceived must not admit {class:?} into machine work"
8195 );
8196 assert_eq!(runtime.completed_count(), 0);
8197 assert_eq!(
8198 runtime.abandonment_reasons(),
8199 vec![meerkat_core::InteractionStreamAbandonReason::AdmissionRejected],
8200 "rejected {class:?} must use the typed abandonment path"
8201 );
8202 }
8203 }
8204
8205 #[tokio::test]
8206 async fn incomplete_peer_request_authority_rejects_inbound_request_before_machine_admission() {
8207 for class in [
8208 PeerInputClass::SilentRequest,
8209 PeerInputClass::ActionableRequest,
8210 ] {
8211 let adapter = Arc::new(MeerkatMachine::ephemeral());
8212 let session_id = SessionId::new();
8213 adapter
8214 .register_session(session_id.clone())
8215 .await
8216 .expect("register session");
8217
8218 let peer_handle = Arc::new(CountingPeerInteractionHandle::default());
8219 let peer_authority: Arc<dyn meerkat_core::handles::PeerInteractionHandle> =
8220 peer_handle.clone();
8221 let runtime = Arc::new(OneShotPeerRequestRuntime::new(
8222 inbound_peer_request_candidate(class),
8223 Some(peer_authority),
8224 ));
8225 let drain = spawn_authorized_test_comms_drain(
8226 adapter.clone(),
8227 session_id.clone(),
8228 runtime.clone(),
8229 Duration::from_millis(10),
8230 )
8231 .await;
8232
8233 tokio::time::timeout(Duration::from_secs(1), drain)
8234 .await
8235 .expect("drain should exit after one candidate")
8236 .expect("drain task should not panic");
8237
8238 let snapshot = adapter
8239 .meerkat_machine_spine_snapshot(&session_id)
8240 .await
8241 .expect("registered session snapshot");
8242 assert_eq!(
8243 peer_handle.request_received_count(),
8244 0,
8245 "partial authority must not fire PeerRequestReceived for {class:?}"
8246 );
8247 assert_eq!(
8248 snapshot.ledger.input_count, 0,
8249 "partial authority must not admit {class:?} into machine work"
8250 );
8251 assert_eq!(runtime.completed_count(), 0);
8252 assert_eq!(
8253 runtime.abandonment_reasons(),
8254 vec![meerkat_core::InteractionStreamAbandonReason::AdmissionRejected],
8255 "rejected {class:?} must use the typed abandonment path"
8256 );
8257 }
8258 }
8259
8260 #[tokio::test]
8261 async fn absent_peer_request_authority_rejects_inbound_request_before_machine_admission() {
8262 for class in [
8263 PeerInputClass::SilentRequest,
8264 PeerInputClass::ActionableRequest,
8265 ] {
8266 let adapter = Arc::new(MeerkatMachine::ephemeral());
8267 let session_id = SessionId::new();
8268 adapter
8269 .register_session(session_id.clone())
8270 .await
8271 .expect("register session");
8272
8273 let runtime = Arc::new(OneShotPeerRequestRuntime::new(
8274 inbound_peer_request_candidate(class),
8275 None,
8276 ));
8277 let drain = spawn_authorized_test_comms_drain(
8278 adapter.clone(),
8279 session_id.clone(),
8280 runtime.clone(),
8281 Duration::from_millis(10),
8282 )
8283 .await;
8284
8285 tokio::time::timeout(Duration::from_secs(1), drain)
8286 .await
8287 .expect("drain should exit after one candidate")
8288 .expect("drain task should not panic");
8289
8290 let snapshot = adapter
8291 .meerkat_machine_spine_snapshot(&session_id)
8292 .await
8293 .expect("registered session snapshot");
8294 assert_eq!(
8295 snapshot.ledger.input_count, 0,
8296 "missing authority must not admit {class:?} into machine work"
8297 );
8298 assert_eq!(runtime.completed_count(), 0);
8299 assert_eq!(
8300 runtime.abandonment_reasons(),
8301 vec![meerkat_core::InteractionStreamAbandonReason::AdmissionRejected],
8302 "rejected {class:?} must use the typed abandonment path"
8303 );
8304 }
8305 }
8306
8307 fn bridge_sender_fact(sender: &str) -> PeerIngressFact {
8308 let id = InteractionId(Uuid::new_v4());
8309 bridge_sender_fact_with_id(id, sender)
8310 }
8311
8312 fn bridge_sender_fact_with_display(
8313 canonical_peer_id: PeerId,
8314 display_label: &str,
8315 ) -> PeerIngressFact {
8316 let id = InteractionId(Uuid::new_v4());
8317 PeerIngressFact::peer(
8318 id,
8319 PeerInputClass::ActionableRequest,
8320 meerkat_core::PeerIngressKind::Request,
8321 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8322 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8323 )),
8324 PeerIngressIdentity::new(
8325 canonical_peer_id,
8326 display_label,
8327 meerkat_core::PeerIngressConvention::Request {
8328 request_id: id.to_string(),
8329 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8330 },
8331 ),
8332 )
8333 }
8334
8335 fn bridge_sender_fact_with_id(id: InteractionId, sender: &str) -> PeerIngressFact {
8336 if let Ok(pubkey) = meerkat_comms::PubKey::from_pubkey_string(sender) {
8337 return PeerIngressFact::peer(
8338 id,
8339 PeerInputClass::ActionableRequest,
8340 meerkat_core::PeerIngressKind::Request,
8341 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8342 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8343 )),
8344 PeerIngressIdentity::new(
8345 pubkey.to_peer_id(),
8346 sender,
8347 meerkat_core::PeerIngressConvention::Request {
8348 request_id: id.to_string(),
8349 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8350 },
8351 )
8352 .with_signing_pubkey(*pubkey.as_bytes()),
8353 );
8354 }
8355 if let Ok(peer_id) = PeerId::parse(sender) {
8356 return PeerIngressFact::peer(
8357 id,
8358 PeerInputClass::ActionableRequest,
8359 meerkat_core::PeerIngressKind::Request,
8360 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8361 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8362 )),
8363 PeerIngressIdentity::new(
8364 peer_id,
8365 sender,
8366 meerkat_core::PeerIngressConvention::Request {
8367 request_id: id.to_string(),
8368 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8369 },
8370 ),
8371 );
8372 }
8373 PeerIngressFact::peer(
8374 id,
8375 PeerInputClass::ActionableRequest,
8376 meerkat_core::PeerIngressKind::Request,
8377 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8378 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8379 )),
8380 PeerIngressIdentity::new(
8381 PeerId::new(),
8382 sender,
8383 meerkat_core::PeerIngressConvention::Request {
8384 request_id: id.to_string(),
8385 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8386 },
8387 ),
8388 )
8389 }
8390
8391 #[test]
8392 fn generated_sender_peer_id_uses_canonical_peer_id_not_display_name() {
8393 let current_peer_id = PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id");
8394 let attacker_peer_id =
8395 PeerId::parse(PEER_ID_OLD_SUPERVISOR).expect("valid attacker peer id");
8396 let sender = bridge_sender_fact_with_display(attacker_peer_id, "mob/__mob_supervisor__");
8397
8398 assert_ne!(
8399 generated_sender_peer_id(&sender),
8400 Some(current_peer_id.as_str()),
8401 "display label must not become generated admission identity"
8402 );
8403 }
8404
8405 #[test]
8406 fn sender_matches_bridge_peer_rejects_same_display_name_with_different_peer_id() {
8407 let attacker_peer_id =
8408 PeerId::parse(PEER_ID_OLD_SUPERVISOR).expect("valid attacker peer id");
8409 let peer = bridge_peer_identity(&supervisor_bridge_spec(), "test")
8410 .expect("valid bridge peer identity");
8411 let sender = bridge_sender_fact_with_display(attacker_peer_id, peer.name.as_str());
8412
8413 assert!(
8414 !sender_matches_bridge_peer(&sender, &peer),
8415 "display label must not prove bridge peer authority"
8416 );
8417 }
8418
8419 #[tokio::test]
8420 async fn bridge_response_route_requires_known_canonical_peer_id() {
8421 let peer = meerkat_comms::Keypair::generate();
8422 let peer_pubkey = peer.public_key();
8423 let peer_spec = TrustedPeerDescriptor::unsigned_with_pubkey(
8424 "bridge-response-peer",
8425 peer_pubkey.to_peer_id().as_str(),
8426 *peer_pubkey.as_bytes(),
8427 "inproc://bridge-response-peer",
8428 )
8429 .expect("valid peer spec");
8430 let runtime = Arc::new(
8431 meerkat_comms::CommsRuntime::inproc_only("bridge-response-route").expect("runtime"),
8432 );
8433 let runtime_dyn: Arc<dyn CommsRuntime> = runtime.clone();
8434 add_test_projection_trust(runtime.as_ref(), peer_spec.clone(), "trust peer").await;
8435
8436 let route = resolve_peer_route(&runtime_dyn, peer_spec.peer_id)
8437 .await
8438 .expect("canonical peer id should resolve through the peer directory");
8439
8440 assert_eq!(route.peer_id, peer_spec.peer_id);
8441 let unknown_peer_id = PeerId::new();
8442 assert!(
8443 resolve_peer_route(&runtime_dyn, unknown_peer_id)
8444 .await
8445 .is_none(),
8446 "unknown PeerIds must not synthesize bridge response routes"
8447 );
8448 }
8449
8450 #[tokio::test]
8451 async fn bridge_response_route_uses_pubkey_sender_not_spoofed_payload_peer_id() {
8452 let runtime = Arc::new(
8453 meerkat_comms::CommsRuntime::inproc_only("bridge-response-spoof").expect("runtime"),
8454 );
8455 let runtime_dyn: Arc<dyn CommsRuntime> = runtime.clone();
8456 let spoofed_key = meerkat_comms::Keypair::generate();
8457 let spoofed_pubkey = spoofed_key.public_key();
8458 let spoofed_peer_id = spoofed_pubkey.to_peer_id();
8459 let spoofed_target = TrustedPeerDescriptor::unsigned_with_pubkey(
8460 "spoofed-target",
8461 spoofed_peer_id.as_str(),
8462 *spoofed_pubkey.as_bytes(),
8463 "inproc://spoofed-target",
8464 )
8465 .expect("valid spoofed target");
8466 add_test_projection_trust(runtime.as_ref(), spoofed_target, "trust spoofed target").await;
8467 let sender_key = meerkat_comms::Keypair::generate();
8468 let sender_pubkey = sender_key.public_key();
8469 let spoofed_peer_id = spoofed_peer_id.as_str();
8470 let command = BridgeCommand::BindMember(
8471 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
8472 supervisor: BridgePeerSpec {
8473 name: "mob/__mob_supervisor__".to_string(),
8474 peer_id: spoofed_peer_id.clone(),
8475 address: "inproc://mob/__mob_supervisor__".to_string(),
8476 pubkey: *sender_pubkey.as_bytes(),
8477 },
8478 epoch: 1,
8479 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
8480 expected_peer_id: PEER_ID_RECEIVER.to_string(),
8481 expected_address: "inproc://receiver".to_string(),
8482 bootstrap_token: "bootstrap".to_string().into(),
8483 },
8484 );
8485 let id = InteractionId(Uuid::new_v4());
8486 let sender_label = sender_pubkey.to_pubkey_string();
8487 let candidate = PeerInputCandidate {
8488 interaction: InboxInteraction {
8489 objective_id: None,
8490 sender_taint: None,
8491 id,
8492 from_route: None,
8493 from: sender_pubkey.to_pubkey_string(),
8494 content: InteractionContent::Request {
8495 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8496 params: serde_json::to_value(command).expect("serialize bridge command"),
8497 blocks: None,
8498 },
8499 rendered_text: String::new(),
8500 handling_mode: HandlingMode::Queue,
8501 render_metadata: None,
8502 },
8503 ingress: bridge_sender_fact_with_id(id, &sender_label),
8504 lifecycle_peer: None,
8505 response_terminality: None,
8506 };
8507
8508 let route = resolve_bridge_response_route(&runtime_dyn, &candidate)
8509 .await
8510 .expect("raw pubkey sender should resolve to its derived route");
8511
8512 assert_eq!(route.peer_id, sender_pubkey.to_peer_id());
8513 assert_ne!(
8514 route.peer_id.as_str(),
8515 spoofed_peer_id,
8516 "bridge replies must not route to the caller-supplied supervisor.peer_id"
8517 );
8518 }
8519
8520 #[tokio::test]
8521 async fn bridge_response_route_accepts_trusted_display_name_sender() {
8522 let peer = meerkat_comms::Keypair::generate();
8523 let peer_pubkey = peer.public_key();
8524 let peer_spec = TrustedPeerDescriptor::unsigned_with_pubkey(
8525 "mob/__mob_supervisor__",
8526 peer_pubkey.to_peer_id().as_str(),
8527 *peer_pubkey.as_bytes(),
8528 "inproc://mob/__mob_supervisor__",
8529 )
8530 .expect("valid peer spec");
8531 let runtime = Arc::new(
8532 meerkat_comms::CommsRuntime::inproc_only("bridge-response-display").expect("runtime"),
8533 );
8534 let runtime_dyn: Arc<dyn CommsRuntime> = runtime.clone();
8535 add_test_projection_trust(runtime.as_ref(), peer_spec.clone(), "trust peer").await;
8536
8537 let command = BridgeCommand::ObserveMember(BridgeSupervisorPayload {
8538 supervisor: BridgePeerSpec::from(peer_spec.clone()),
8539 epoch: 1,
8540 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
8541 });
8542 let id = InteractionId(Uuid::new_v4());
8543 let ingress = PeerIngressFact::peer(
8544 id,
8545 PeerInputClass::ActionableRequest,
8546 meerkat_core::PeerIngressKind::Request,
8547 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8548 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8549 )),
8550 PeerIngressIdentity::new(
8551 peer_pubkey.to_peer_id(),
8552 peer_spec.name.as_str(),
8553 meerkat_core::PeerIngressConvention::Request {
8554 request_id: id.to_string(),
8555 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8556 },
8557 )
8558 .with_signing_pubkey(*peer_pubkey.as_bytes()),
8559 );
8560 let candidate = bridge_candidate_with_ingress(peer_spec.name.as_str(), &command, ingress);
8561 let route = resolve_bridge_response_route(&runtime_dyn, &candidate)
8562 .await
8563 .expect("trusted display-name sender should resolve through peer directory");
8564
8565 assert_eq!(route.peer_id, peer_spec.peer_id);
8566 assert_eq!(
8567 route.display_name.as_ref().map(|name| name.as_str()),
8568 Some(peer_spec.name.as_str())
8569 );
8570 }
8571
8572 #[tokio::test]
8573 async fn bridge_response_route_uses_typed_sender_peer_id_not_display_or_payload_identity() {
8574 let peer = meerkat_comms::Keypair::generate();
8575 let peer_pubkey = peer.public_key();
8576 let peer_spec = TrustedPeerDescriptor::unsigned_with_pubkey(
8577 "mob/__mob_supervisor__",
8578 peer_pubkey.to_peer_id().as_str(),
8579 *peer_pubkey.as_bytes(),
8580 "inproc://mob/__mob_supervisor__",
8581 )
8582 .expect("valid peer spec");
8583 let runtime = Arc::new(
8584 meerkat_comms::CommsRuntime::inproc_only("bridge-response-typed-sender")
8585 .expect("runtime"),
8586 );
8587 let runtime_dyn: Arc<dyn CommsRuntime> = runtime.clone();
8588 add_test_projection_trust(runtime.as_ref(), peer_spec.clone(), "trust peer").await;
8589
8590 let payload_pubkey = meerkat_comms::Keypair::generate().public_key();
8591 let spoofed_payload_peer_id = PeerId::new();
8592 let command = BridgeCommand::ObserveMember(BridgeSupervisorPayload {
8593 supervisor: BridgePeerSpec {
8594 name: peer_spec.name.to_string(),
8595 peer_id: spoofed_payload_peer_id.as_str(),
8596 address: "inproc://mob/__mob_supervisor__".to_string(),
8597 pubkey: *payload_pubkey.as_bytes(),
8598 },
8599 epoch: 1,
8600 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
8601 });
8602 let id = InteractionId(Uuid::new_v4());
8603 let ingress = PeerIngressFact::peer(
8604 id,
8605 PeerInputClass::ActionableRequest,
8606 meerkat_core::PeerIngressKind::Request,
8607 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8608 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8609 )),
8610 PeerIngressIdentity::new(
8611 peer_pubkey.to_peer_id(),
8612 peer_spec.name.as_str(),
8613 meerkat_core::PeerIngressConvention::Request {
8614 request_id: id.to_string(),
8615 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8616 },
8617 )
8618 .with_signing_pubkey(*peer_pubkey.as_bytes()),
8619 );
8620 let spoofed_candidate =
8621 bridge_candidate_with_ingress(peer_spec.name.as_str(), &command, ingress);
8622 let route = resolve_bridge_response_route(&runtime_dyn, &spoofed_candidate)
8623 .await
8624 .expect("typed ingress route should ignore spoofed payload peer_id");
8625 assert_eq!(route.peer_id, peer_spec.peer_id);
8626 assert_ne!(route.peer_id, spoofed_payload_peer_id);
8627 assert_eq!(
8628 route.display_name.as_ref().map(|name| name.as_str()),
8629 Some(peer_spec.name.as_str())
8630 );
8631 }
8632
8633 #[tokio::test]
8634 async fn bridge_response_seeds_inbound_request_before_peer_response_send() {
8635 let suffix = Uuid::new_v4().simple().to_string();
8636 let member_name = format!("bridge-response-member-{suffix}");
8637 let supervisor_name = format!("bridge-response-supervisor-{suffix}");
8638 let member_runtime =
8639 Arc::new(meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member"));
8640 let supervisor_runtime = Arc::new(
8641 meerkat_comms::CommsRuntime::inproc_only(&supervisor_name).expect("supervisor"),
8642 );
8643 let peer_handle = Arc::new(CountingPeerInteractionHandle::default());
8644 let member_dsl = install_running_test_peer_comms_handle(member_runtime.as_ref());
8645 member_runtime.install_peer_request_response_authority(
8646 meerkat_comms::PeerRequestResponseAuthority::new(
8647 peer_handle.clone(),
8648 Arc::new(crate::handles::RuntimeInteractionStreamHandle::ephemeral()),
8649 ),
8650 );
8651 let supervisor_dsl = install_running_test_peer_comms_handle(supervisor_runtime.as_ref());
8652
8653 let supervisor_pubkey = supervisor_runtime.public_key();
8654 let supervisor_spec = TrustedPeerDescriptor::unsigned_with_pubkey(
8655 &supervisor_name,
8656 supervisor_pubkey.to_peer_id().as_str(),
8657 *supervisor_pubkey.as_bytes(),
8658 format!("inproc://{supervisor_name}"),
8659 )
8660 .expect("valid supervisor spec");
8661 add_test_projection_trust_with_dsl(
8662 member_runtime.as_ref(),
8663 Arc::clone(&member_dsl),
8664 supervisor_spec.clone(),
8665 1,
8666 "member should trust supervisor",
8667 )
8668 .await;
8669
8670 let member_pubkey = member_runtime.public_key();
8671 let member_spec = TrustedPeerDescriptor::unsigned_with_pubkey(
8672 &member_name,
8673 member_pubkey.to_peer_id().as_str(),
8674 *member_pubkey.as_bytes(),
8675 format!("inproc://{member_name}"),
8676 )
8677 .expect("valid member spec");
8678 add_test_projection_trust_with_dsl(
8679 supervisor_runtime.as_ref(),
8680 Arc::clone(&supervisor_dsl),
8681 member_spec,
8682 1,
8683 "supervisor should trust member",
8684 )
8685 .await;
8686
8687 let adapter = Arc::new(MeerkatMachine::ephemeral());
8688 let session_id = SessionId::new();
8689 adapter
8690 .register_session(session_id.clone())
8691 .await
8692 .expect("register session");
8693 adapter
8694 .stage_supervisor_bind(
8695 &session_id,
8696 supervisor_spec.name.to_string(),
8697 supervisor_spec.peer_id.as_str(),
8698 supervisor_spec.address.to_string(),
8699 signing_public_key_for_descriptor(&supervisor_spec),
8700 1,
8701 )
8702 .await
8703 .expect("pre-bind supervisor");
8704 let command = BridgeCommand::ObserveMember(BridgeSupervisorPayload {
8705 supervisor: BridgePeerSpec::from(supervisor_spec.clone()),
8706 epoch: 1,
8707 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
8708 });
8709 let interaction_id = InteractionId(Uuid::new_v4());
8710 let ingress = PeerIngressFact::peer(
8711 interaction_id,
8712 PeerInputClass::ActionableRequest,
8713 meerkat_core::PeerIngressKind::Request,
8714 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8715 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8716 )),
8717 PeerIngressIdentity::new(
8718 supervisor_spec.peer_id,
8719 supervisor_spec.name.as_str(),
8720 meerkat_core::PeerIngressConvention::Request {
8721 request_id: interaction_id.to_string(),
8722 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8723 },
8724 )
8725 .with_signing_pubkey(supervisor_spec.pubkey),
8726 );
8727 let candidate =
8728 bridge_candidate_with_ingress(supervisor_spec.name.as_str(), &command, ingress);
8729
8730 assert!(
8731 try_handle_supervisor_bridge_command(
8732 &adapter,
8733 &session_id,
8734 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
8735 &candidate,
8736 )
8737 .await,
8738 "bridge handler must own ObserveMember"
8739 );
8740
8741 assert_eq!(
8742 peer_handle.request_received_count(),
8743 1,
8744 "bridge handler must seed inbound peer request state before replying"
8745 );
8746 assert_eq!(
8747 peer_handle.response_replied_count(),
8748 1,
8749 "real CommsRuntime::send(PeerResponse) should pass the inbound-state guard"
8750 );
8751 }
8752
8753 #[cfg(not(target_arch = "wasm32"))]
8754 #[tokio::test]
8755 async fn bind_member_bootstrap_stores_supervisor_address_not_member_address() {
8756 let suffix = Uuid::new_v4().simple().to_string();
8757 let member_name = format!("tcp-bootstrap-member-{suffix}");
8758 let supervisor_name = format!("tcp-bootstrap-supervisor-{suffix}");
8759
8760 let mut member_runtime =
8761 meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member runtime");
8762 member_runtime
8763 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
8764 .expect("configure member TCP listener");
8765 member_runtime
8766 .start_listeners()
8767 .await
8768 .expect("start member TCP listener");
8769 let member_runtime = Arc::new(member_runtime);
8770 install_test_peer_request_response_authority(
8771 &member_runtime,
8772 Arc::new(CountingPeerInteractionHandle::default()),
8773 );
8774
8775 let mut supervisor_runtime =
8776 meerkat_comms::CommsRuntime::inproc_only(&supervisor_name).expect("supervisor runtime");
8777 supervisor_runtime
8778 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
8779 .expect("configure supervisor TCP listener");
8780 supervisor_runtime
8781 .start_listeners()
8782 .await
8783 .expect("start supervisor TCP listener");
8784 let supervisor_runtime = Arc::new(supervisor_runtime);
8785 install_test_peer_request_response_authority(
8786 &supervisor_runtime,
8787 Arc::new(CountingPeerInteractionHandle::default()),
8788 );
8789
8790 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
8791 let supervisor_dsl = install_running_test_peer_comms_handle(supervisor_runtime.as_ref());
8792 add_test_projection_trust_with_dsl(
8793 supervisor_runtime.as_ref(),
8794 Arc::clone(&supervisor_dsl),
8795 member_spec.clone(),
8796 1,
8797 "supervisor trusts member target",
8798 )
8799 .await;
8800 let supervisor_spec =
8801 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
8802
8803 let adapter = Arc::new(MeerkatMachine::ephemeral());
8804 let session_id = SessionId::new();
8805 adapter
8806 .register_session(session_id.clone())
8807 .await
8808 .expect("register session");
8809 adapter
8810 .test_install_session_peer_comms_handle_on_runtime(&session_id, member_runtime.as_ref())
8811 .await
8812 .expect("install adapter session peer-comms handle on member runtime");
8813
8814 let command = BridgeCommand::BindMember(
8815 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
8816 supervisor: BridgePeerSpec::from(supervisor_spec.clone()),
8817 epoch: 1,
8818 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
8819 expected_peer_id: member_spec.peer_id.as_str(),
8820 expected_address: member_spec.address.to_string(),
8821 bootstrap_token: member_runtime.bridge_bootstrap_token().to_owned().into(),
8822 },
8823 );
8824 let request_receipt = supervisor_runtime
8825 .send(CommsCommand::PeerRequest {
8826 content_taint: None,
8827 to: PeerRoute::with_display_name(member_spec.peer_id, member_spec.name.clone()),
8828 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8829 params: serde_json::to_value(&command).expect("serialize bridge command"),
8830 blocks: None,
8831 handling_mode: HandlingMode::Queue,
8832 stream: meerkat_core::comms::InputStreamMode::None,
8833 objective_id: None,
8834 })
8835 .await
8836 .expect("supervisor sends bootstrap BindMember over TCP");
8837 let meerkat_core::SendReceipt::PeerRequestSent { interaction_id, .. } = request_receipt
8838 else {
8839 panic!("expected PeerRequestSent receipt, got {request_receipt:?}");
8840 };
8841
8842 let candidate = drain_one_candidate(&member_runtime, "member BindMember request").await;
8843 assert_eq!(
8844 candidate.interaction.id, interaction_id,
8845 "member must receive the correlated bridge request"
8846 );
8847 assert!(
8848 try_handle_supervisor_bridge_command(
8849 &adapter,
8850 &session_id,
8851 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
8852 &candidate,
8853 )
8854 .await,
8855 "bridge handler must own BindMember"
8856 );
8857
8858 let binding = adapter.supervisor_binding(&session_id).await;
8859 let SupervisorBinding::Bound {
8860 address: bound_address,
8861 ..
8862 } = binding
8863 else {
8864 panic!("bootstrap bind should leave supervisor bound, got {binding:?}");
8865 };
8866 assert_eq!(
8867 bound_address,
8868 supervisor_spec.address.to_string(),
8869 "supervisor binding must store the supervisor route, not the member route"
8870 );
8871 assert_ne!(
8872 bound_address,
8873 member_spec.address.to_string(),
8874 "storing the member address under the supervisor pubkey makes later bridge replies loop back to the member"
8875 );
8876
8877 let response =
8878 drain_one_candidate(&supervisor_runtime, "supervisor BindMember response").await;
8879 let InteractionContent::Response { result, .. } = response.interaction.content else {
8880 panic!(
8881 "expected bridge response, got {:?}",
8882 response.interaction.content
8883 );
8884 };
8885 let BridgeReply::BindMember(bind_response) =
8886 serde_json::from_value(result).expect("typed bridge reply")
8887 else {
8888 panic!("expected BindMember reply");
8889 };
8890 assert_eq!(bind_response.peer_id, member_spec.peer_id.as_str());
8891 assert_eq!(
8892 bind_response.address,
8893 canonicalize_bridge_address(&member_spec.address.to_string()),
8894 "BindMember reply still reports the member's canonical advertised address"
8895 );
8896 }
8897
8898 #[cfg(not(target_arch = "wasm32"))]
8899 #[tokio::test]
8900 async fn bind_member_idempotent_ack_repairs_tcp_supervisor_trust_before_reply() {
8901 let suffix = Uuid::new_v4().simple().to_string();
8902 let member_name = format!("tcp-bind-member-{suffix}");
8903 let supervisor_name = format!("tcp-supervisor-{suffix}");
8904
8905 let mut member_runtime =
8906 meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member runtime");
8907 member_runtime
8908 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
8909 .expect("configure member TCP listener");
8910 member_runtime
8911 .start_listeners()
8912 .await
8913 .expect("start member TCP listener");
8914 let member_runtime = Arc::new(member_runtime);
8915 let member_peer_handle = Arc::new(CountingPeerInteractionHandle::default());
8916 install_test_peer_request_response_authority(&member_runtime, member_peer_handle.clone());
8917
8918 let mut supervisor_runtime =
8919 meerkat_comms::CommsRuntime::inproc_only(&supervisor_name).expect("supervisor runtime");
8920 supervisor_runtime
8921 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
8922 .expect("configure supervisor TCP listener");
8923 supervisor_runtime
8924 .start_listeners()
8925 .await
8926 .expect("start supervisor TCP listener");
8927 let supervisor_runtime = Arc::new(supervisor_runtime);
8928 install_test_peer_request_response_authority(
8929 &supervisor_runtime,
8930 Arc::new(CountingPeerInteractionHandle::default()),
8931 );
8932
8933 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
8934 let supervisor_dsl = install_running_test_peer_comms_handle(supervisor_runtime.as_ref());
8935 add_test_projection_trust_with_dsl(
8936 supervisor_runtime.as_ref(),
8937 Arc::clone(&supervisor_dsl),
8938 member_spec.clone(),
8939 1,
8940 "supervisor trusts member target",
8941 )
8942 .await;
8943 let supervisor_spec =
8944 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
8945
8946 let adapter = Arc::new(MeerkatMachine::ephemeral());
8947 let session_id = SessionId::new();
8948 adapter
8949 .register_session(session_id.clone())
8950 .await
8951 .expect("register session");
8952 adapter
8953 .stage_supervisor_bind(
8954 &session_id,
8955 supervisor_spec.name.to_string(),
8956 supervisor_spec.peer_id.to_string(),
8957 supervisor_spec.address.to_string(),
8958 signing_public_key_for_descriptor(&supervisor_spec),
8959 1,
8960 )
8961 .await
8962 .expect("pre-bind supervisor in DSL state");
8963 adapter
8964 .test_install_session_peer_comms_handle_on_runtime(&session_id, member_runtime.as_ref())
8965 .await
8966 .expect("install adapter session peer-comms handle on member runtime");
8967 assert!(
8968 member_runtime
8969 .peers()
8970 .await
8971 .iter()
8972 .all(|peer| peer.peer_id != supervisor_spec.peer_id),
8973 "fixture must start with DSL supervisor binding but no sendable runtime trust entry"
8974 );
8975
8976 let command = BridgeCommand::BindMember(
8977 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
8978 supervisor: BridgePeerSpec::from(supervisor_spec.clone()),
8979 epoch: 1,
8980 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
8981 expected_peer_id: member_spec.peer_id.as_str(),
8982 expected_address: member_spec.address.to_string(),
8983 bootstrap_token: member_runtime.bridge_bootstrap_token().to_owned().into(),
8984 },
8985 );
8986 let request_receipt = supervisor_runtime
8987 .send(CommsCommand::PeerRequest {
8988 content_taint: None,
8989 to: PeerRoute::with_display_name(member_spec.peer_id, member_spec.name.clone()),
8990 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8991 params: serde_json::to_value(&command).expect("serialize bridge command"),
8992 blocks: None,
8993 handling_mode: HandlingMode::Queue,
8994 stream: meerkat_core::comms::InputStreamMode::None,
8995 objective_id: None,
8996 })
8997 .await
8998 .expect("supervisor sends idempotent BindMember over TCP");
8999 let meerkat_core::SendReceipt::PeerRequestSent { interaction_id, .. } = request_receipt
9000 else {
9001 panic!("expected PeerRequestSent receipt, got {request_receipt:?}");
9002 };
9003
9004 let candidate = drain_one_candidate(&member_runtime, "member BindMember request").await;
9005 assert_eq!(
9006 candidate.interaction.id, interaction_id,
9007 "member must receive the correlated bridge request"
9008 );
9009 assert!(
9010 try_handle_supervisor_bridge_command(
9011 &adapter,
9012 &session_id,
9013 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
9014 &candidate,
9015 )
9016 .await,
9017 "bridge handler must own BindMember"
9018 );
9019
9020 let response =
9021 drain_one_candidate(&supervisor_runtime, "supervisor BindMember response").await;
9022 assert_eq!(
9023 member_peer_handle.response_replied_count(),
9024 1,
9025 "target must send the terminal bridge response after repairing trust"
9026 );
9027 assert!(
9028 member_runtime
9029 .router()
9030 .is_private(&meerkat_comms::PubKey::new(supervisor_spec.pubkey)),
9031 "idempotent BindMember must repair the PRIVATE supervisor admission edge"
9032 );
9033 assert!(
9034 member_runtime
9035 .peers()
9036 .await
9037 .iter()
9038 .all(|peer| peer.peer_id != supervisor_spec.peer_id),
9039 "repaired supervisor edge is a control-plane route; it must not surface in public peers()"
9040 );
9041 let InteractionContent::Response {
9042 in_reply_to,
9043 status,
9044 result,
9045 ..
9046 } = response.interaction.content
9047 else {
9048 panic!(
9049 "expected bridge response, got {:?}",
9050 response.interaction.content
9051 );
9052 };
9053 assert_eq!(in_reply_to, interaction_id);
9054 assert_eq!(status, meerkat_core::interaction::ResponseStatus::Completed);
9055 let reply: BridgeReply = serde_json::from_value(result).expect("typed bridge reply");
9056 let BridgeReply::BindMember(bind_response) = reply else {
9057 panic!("expected BindMember reply, got {reply:?}");
9058 };
9059 assert_eq!(bind_response.peer_id, member_spec.peer_id.as_str());
9060 assert_eq!(
9061 bind_response.address,
9062 canonicalize_bridge_address(&member_spec.address.to_string())
9063 );
9064 }
9065
9066 #[cfg(not(target_arch = "wasm32"))]
9067 #[tokio::test]
9068 async fn authorized_bridge_command_repairs_tcp_supervisor_response_route_before_reply() {
9069 let suffix = Uuid::new_v4().simple().to_string();
9070 let member_name = format!("tcp-observe-member-{suffix}");
9071 let supervisor_name = format!("tcp-observe-supervisor-{suffix}");
9072
9073 let mut member_runtime =
9074 meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member runtime");
9075 member_runtime
9076 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9077 .expect("configure member TCP listener");
9078 member_runtime
9079 .start_listeners()
9080 .await
9081 .expect("start member TCP listener");
9082 let member_runtime = Arc::new(member_runtime);
9083 let member_peer_handle = Arc::new(CountingPeerInteractionHandle::default());
9084 install_test_peer_request_response_authority(&member_runtime, member_peer_handle.clone());
9085
9086 let mut supervisor_runtime =
9087 meerkat_comms::CommsRuntime::inproc_only(&supervisor_name).expect("supervisor runtime");
9088 supervisor_runtime
9089 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9090 .expect("configure supervisor TCP listener");
9091 supervisor_runtime
9092 .start_listeners()
9093 .await
9094 .expect("start supervisor TCP listener");
9095 let supervisor_runtime = Arc::new(supervisor_runtime);
9096 install_test_peer_request_response_authority(
9097 &supervisor_runtime,
9098 Arc::new(CountingPeerInteractionHandle::default()),
9099 );
9100
9101 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
9102 let supervisor_dsl = install_running_test_peer_comms_handle(supervisor_runtime.as_ref());
9103 add_test_projection_trust_with_dsl(
9104 supervisor_runtime.as_ref(),
9105 Arc::clone(&supervisor_dsl),
9106 member_spec.clone(),
9107 1,
9108 "supervisor trusts member target",
9109 )
9110 .await;
9111 let supervisor_spec =
9112 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
9113
9114 let adapter = Arc::new(MeerkatMachine::ephemeral());
9115 let session_id = SessionId::new();
9116 adapter
9117 .register_session(session_id.clone())
9118 .await
9119 .expect("register session");
9120 adapter
9121 .stage_supervisor_bind(
9122 &session_id,
9123 supervisor_spec.name.to_string(),
9124 supervisor_spec.peer_id.as_str(),
9125 supervisor_spec.address.to_string(),
9126 signing_public_key_for_descriptor(&supervisor_spec),
9127 1,
9128 )
9129 .await
9130 .expect("pre-bind supervisor in DSL state");
9131 adapter
9132 .test_install_session_peer_comms_handle_on_runtime(&session_id, member_runtime.as_ref())
9133 .await
9134 .expect("install adapter session peer-comms handle on member runtime");
9135 assert!(
9136 member_runtime
9137 .peers()
9138 .await
9139 .iter()
9140 .all(|peer| peer.peer_id != supervisor_spec.peer_id),
9141 "fixture must start with authorized supervisor state but no public runtime route"
9142 );
9143
9144 let command = BridgeCommand::ObserveMember(BridgeSupervisorPayload {
9145 supervisor: BridgePeerSpec::from(supervisor_spec.clone()),
9146 epoch: 1,
9147 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
9148 });
9149 let interaction_id = InteractionId(Uuid::new_v4());
9150 let ingress = PeerIngressFact::peer(
9151 interaction_id,
9152 PeerInputClass::ActionableRequest,
9153 meerkat_core::PeerIngressKind::Request,
9154 Some(meerkat_core::PeerIngressAuthDecision::Required),
9155 PeerIngressIdentity::new(
9156 supervisor_spec.peer_id,
9157 supervisor_spec.name.as_str(),
9158 meerkat_core::PeerIngressConvention::Request {
9159 request_id: interaction_id.to_string(),
9160 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
9161 },
9162 )
9163 .with_signing_pubkey(supervisor_spec.pubkey),
9164 );
9165 let candidate =
9166 bridge_candidate_with_ingress(supervisor_spec.name.as_str(), &command, ingress);
9167
9168 assert!(
9169 try_handle_supervisor_bridge_command(
9170 &adapter,
9171 &session_id,
9172 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
9173 &candidate,
9174 )
9175 .await,
9176 "bridge handler must own ObserveMember"
9177 );
9178 assert_eq!(
9179 member_peer_handle.response_replied_count(),
9180 1,
9181 "target must send the terminal bridge response after repairing supervisor route"
9182 );
9183
9184 let supervisor_pubkey = meerkat_comms::PubKey::new(supervisor_spec.pubkey);
9185 assert!(
9186 member_runtime.router().is_private(&supervisor_pubkey),
9187 "repaired supervisor response route must remain private"
9188 );
9189 assert!(
9190 member_runtime
9191 .peers()
9192 .await
9193 .iter()
9194 .all(|peer| peer.peer_id != supervisor_spec.peer_id),
9195 "private supervisor response route must not leak through public peers()"
9196 );
9197
9198 let response =
9199 drain_one_candidate(&supervisor_runtime, "supervisor ObserveMember response").await;
9200 let InteractionContent::Response {
9201 in_reply_to,
9202 status,
9203 result,
9204 ..
9205 } = response.interaction.content
9206 else {
9207 panic!(
9208 "expected bridge response, got {:?}",
9209 response.interaction.content
9210 );
9211 };
9212 assert_eq!(in_reply_to, interaction_id);
9213 assert_eq!(status, meerkat_core::interaction::ResponseStatus::Completed);
9214 assert!(
9215 matches!(
9216 serde_json::from_value::<BridgeReply>(result).expect("typed bridge reply"),
9217 BridgeReply::Observation(_)
9218 ),
9219 "expected ObserveMember response"
9220 );
9221 }
9222
9223 #[cfg(not(target_arch = "wasm32"))]
9224 #[tokio::test]
9225 async fn supervisor_rotation_v4_is_one_way_then_observed_from_new_authority() {
9226 let suffix = Uuid::new_v4().simple().to_string();
9227 let member_name = format!("tcp-authorize-member-{suffix}");
9228 let old_supervisor_name = format!("tcp-authorize-old-supervisor-{suffix}");
9229 let new_supervisor_name = format!("tcp-authorize-new-supervisor-{suffix}");
9230
9231 let mut member_runtime =
9232 meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member runtime");
9233 member_runtime
9234 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9235 .expect("configure member TCP listener");
9236 member_runtime
9237 .start_listeners()
9238 .await
9239 .expect("start member TCP listener");
9240 let member_runtime = Arc::new(member_runtime);
9241 install_test_peer_request_response_authority(
9242 &member_runtime,
9243 Arc::new(CountingPeerInteractionHandle::default()),
9244 );
9245
9246 let mut old_supervisor_runtime =
9247 meerkat_comms::CommsRuntime::inproc_only(&old_supervisor_name)
9248 .expect("old supervisor runtime");
9249 old_supervisor_runtime
9250 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9251 .expect("configure old supervisor TCP listener");
9252 old_supervisor_runtime
9253 .start_listeners()
9254 .await
9255 .expect("start old supervisor TCP listener");
9256 let old_supervisor_runtime = Arc::new(old_supervisor_runtime);
9257 install_test_peer_request_response_authority(
9258 &old_supervisor_runtime,
9259 Arc::new(CountingPeerInteractionHandle::default()),
9260 );
9261
9262 let new_supervisor_keypair = meerkat_comms::Keypair::generate();
9263 let mut new_supervisor_runtime =
9264 meerkat_comms::CommsRuntime::inproc_only_with_keypair_and_silent_intents(
9265 &new_supervisor_name,
9266 None,
9267 new_supervisor_keypair.clone(),
9268 Arc::new(HashSet::new()),
9269 )
9270 .expect("new supervisor runtime");
9271 new_supervisor_runtime
9272 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9273 .expect("configure new supervisor TCP listener");
9274 new_supervisor_runtime
9275 .start_listeners()
9276 .await
9277 .expect("start new supervisor TCP listener");
9278 let new_supervisor_runtime = Arc::new(new_supervisor_runtime);
9279 install_test_peer_request_response_authority(
9280 &new_supervisor_runtime,
9281 Arc::new(CountingPeerInteractionHandle::default()),
9282 );
9283
9284 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
9285 let old_supervisor_dsl =
9286 install_running_test_peer_comms_handle(old_supervisor_runtime.as_ref());
9287 add_test_projection_trust_with_dsl(
9288 old_supervisor_runtime.as_ref(),
9289 Arc::clone(&old_supervisor_dsl),
9290 member_spec.clone(),
9291 1,
9292 "old supervisor trusts member target",
9293 )
9294 .await;
9295 let new_supervisor_dsl =
9296 install_running_test_peer_comms_handle(new_supervisor_runtime.as_ref());
9297 add_test_projection_trust_with_dsl(
9298 new_supervisor_runtime.as_ref(),
9299 Arc::clone(&new_supervisor_dsl),
9300 member_spec,
9301 1,
9302 "new supervisor trusts member target",
9303 )
9304 .await;
9305 let old_supervisor_spec =
9306 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &old_supervisor_runtime);
9307 let new_supervisor_spec =
9308 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &new_supervisor_runtime);
9309 let adapter = Arc::new(MeerkatMachine::ephemeral());
9315 let session_id = SessionId::new();
9316 adapter
9317 .register_session(session_id.clone())
9318 .await
9319 .expect("register session");
9320 adapter
9321 .stage_supervisor_bind(
9322 &session_id,
9323 old_supervisor_spec.name.to_string(),
9324 old_supervisor_spec.peer_id.as_str(),
9325 old_supervisor_spec.address.to_string(),
9326 signing_public_key_for_descriptor(&old_supervisor_spec),
9327 1,
9328 )
9329 .await
9330 .expect("pre-bind old supervisor in DSL state");
9331 adapter
9332 .test_install_session_peer_comms_handle_on_runtime(&session_id, member_runtime.as_ref())
9333 .await
9334 .expect("install adapter session peer-comms handle on member runtime");
9335 adapter
9336 .test_authorize_direct_comms_drain_runtime(
9337 &session_id,
9338 member_runtime.clone() as Arc<dyn CommsRuntime>,
9339 )
9340 .await
9341 .expect("install generated current member comms runtime fence");
9342 add_member_private_supervisor_trust_via_adapter(
9343 &adapter,
9344 &session_id,
9345 &member_runtime,
9346 &old_supervisor_spec,
9347 1,
9348 "member starts with old supervisor private route",
9349 )
9350 .await;
9351 assert!(
9352 member_runtime
9353 .router()
9354 .is_private(&meerkat_comms::PubKey::new(old_supervisor_spec.pubkey)),
9355 "member must start with a PRIVATE old supervisor route"
9356 );
9357
9358 let operation_id = SupervisorRotationOperationId::new();
9359 let delivery = serde_json::to_value(
9360 meerkat_contracts::wire::supervisor_bridge::BridgeSupervisorDelivery::SubmitSupervisorRotation(
9361 meerkat_contracts::wire::supervisor_bridge::BridgeSupervisorRotationSubmit {
9362 operation_id,
9363 target: BridgePeerSpec::from(new_supervisor_spec.clone()),
9364 target_epoch: 2,
9365 protocol_version: BridgeProtocolVersion::V4,
9366 },
9367 ),
9368 )
9369 .expect("serialize supervisor rotation delivery");
9370 let interaction_id = InteractionId(Uuid::new_v4());
9371 let ingress = PeerIngressFact::peer(
9372 interaction_id,
9373 PeerInputClass::ActionableMessage,
9374 meerkat_core::PeerIngressKind::Message,
9375 Some(meerkat_core::PeerIngressAuthDecision::Required),
9376 PeerIngressIdentity::new(
9377 old_supervisor_spec.peer_id,
9378 old_supervisor_spec.name.as_str(),
9379 meerkat_core::PeerIngressConvention::Message,
9380 )
9381 .with_signing_pubkey(old_supervisor_spec.pubkey),
9382 );
9383 let candidate = bridge_delivery_candidate_with_ingress(
9384 old_supervisor_spec.name.as_str(),
9385 &delivery,
9386 ingress,
9387 );
9388
9389 assert!(
9390 try_handle_supervisor_bridge_delivery(
9391 &adapter,
9392 &session_id,
9393 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
9394 &candidate,
9395 )
9396 .await,
9397 "delivery handler must own SubmitSupervisorRotation"
9398 );
9399 tokio::time::timeout(Duration::from_secs(3), async {
9400 loop {
9401 let receipt = adapter
9402 .active_supervisor_rotation(&session_id)
9403 .await
9404 .expect("read rotation receipt")
9405 .expect("rotation receipt exists");
9406 if receipt.phase == crate::meerkat_machine::dsl::SupervisorRotationPhase::Completed
9407 {
9408 break;
9409 }
9410 tokio::time::sleep(Duration::from_millis(10)).await;
9411 }
9412 })
9413 .await
9414 .expect("rotation worker reaches terminal completion");
9415
9416 let observe = BridgeCommand::ObserveSupervisorRotation(
9417 meerkat_contracts::wire::supervisor_bridge::BridgeSupervisorRotationObserve {
9418 operation_id,
9419 observer: BridgePeerSpec::from(new_supervisor_spec.clone()),
9420 observer_epoch: 2,
9421 protocol_version: BridgeProtocolVersion::V4,
9422 },
9423 );
9424 let probe_name = format!("{new_supervisor_name}/pending-supervisor-probe");
9430 let mut probe_runtime =
9431 meerkat_comms::CommsRuntime::inproc_only_with_keypair_and_silent_intents(
9432 &probe_name,
9433 None,
9434 new_supervisor_keypair,
9435 Arc::new(HashSet::new()),
9436 )
9437 .expect("alternate-authority probe runtime");
9438 probe_runtime
9439 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9440 .expect("configure alternate-authority probe TCP listener");
9441 probe_runtime
9442 .start_listeners()
9443 .await
9444 .expect("start alternate-authority probe TCP listener");
9445 let probe_runtime = Arc::new(probe_runtime);
9446 install_test_peer_request_response_authority(
9447 &probe_runtime,
9448 Arc::new(CountingPeerInteractionHandle::default()),
9449 );
9450 let probe_dsl = install_running_test_peer_comms_handle(probe_runtime.as_ref());
9451 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
9452 add_test_projection_trust_with_dsl(
9453 probe_runtime.as_ref(),
9454 Arc::clone(&probe_dsl),
9455 member_spec.clone(),
9456 1,
9457 "alternate-authority probe trusts member target",
9458 )
9459 .await;
9460 let reply_endpoint = PeerAddress::parse(probe_runtime.advertised_address())
9461 .expect("probe advertises a typed TCP reply endpoint");
9462 assert_ne!(
9463 reply_endpoint.to_string(),
9464 new_supervisor_spec.address.to_string(),
9465 "regression requires the durable authority and probe listeners to differ"
9466 );
9467 let receipt = probe_runtime
9468 .send_peer_request_at_endpoint(
9469 PeerRoute::with_display_name(member_spec.peer_id, member_spec.name.clone()),
9470 SUPERVISOR_BRIDGE_INTENT,
9471 serde_json::to_value(&observe).expect("serialize observe command"),
9472 reply_endpoint.clone(),
9473 )
9474 .await
9475 .expect("send signed observation from alternate-authority probe");
9476 let SendReceipt::PeerRequestSent { envelope_id, .. } = receipt else {
9477 panic!("expected peer request receipt, got {receipt:?}");
9478 };
9479 let observe_id = InteractionId(envelope_id);
9480 let observe_candidate =
9481 drain_one_candidate(&member_runtime, "alternate-authority observation request").await;
9482 assert_eq!(observe_candidate.interaction.id, observe_id);
9483 assert_eq!(
9484 observe_candidate.ingress.declared_reply_endpoint,
9485 Some(reply_endpoint),
9486 "the authenticated request reply endpoint must survive ingress classification"
9487 );
9488 assert!(
9489 try_handle_supervisor_bridge_command(
9490 &adapter,
9491 &session_id,
9492 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
9493 &observe_candidate,
9494 )
9495 .await,
9496 "bridge handler must own ObserveSupervisorRotation"
9497 );
9498
9499 let response = drain_one_candidate(
9500 &probe_runtime,
9501 "alternate-authority supervisor rotation observation",
9502 )
9503 .await;
9504 let InteractionContent::Response {
9505 in_reply_to,
9506 status,
9507 result,
9508 ..
9509 } = response.interaction.content
9510 else {
9511 panic!(
9512 "expected authorize bridge response, got {:?}",
9513 response.interaction.content
9514 );
9515 };
9516 assert_eq!(in_reply_to, observe_id);
9517 assert_eq!(status, meerkat_core::interaction::ResponseStatus::Completed);
9518 assert!(matches!(
9519 serde_json::from_value::<BridgeReply>(result).expect("typed bridge reply"),
9520 BridgeReply::SupervisorRotation(
9521 BridgeSupervisorRotationObservation::Found {
9522 state: BridgeSupervisorRotationState::Completed { receipt }
9523 }
9524 ) if receipt.operation_id == operation_id
9525 && receipt.target.target.peer_id == new_supervisor_spec.peer_id.as_str()
9526 ));
9527 let peers = member_runtime.peers().await;
9528 assert!(
9531 peers
9532 .iter()
9533 .all(|peer| peer.peer_id != old_supervisor_spec.peer_id),
9534 "old supervisor route must be removed after rotation"
9535 );
9536 assert!(
9537 peers
9538 .iter()
9539 .all(|peer| peer.peer_id != new_supervisor_spec.peer_id),
9540 "new supervisor route must stay private (not in public peers())"
9541 );
9542 assert!(
9543 member_runtime
9544 .router()
9545 .is_private(&meerkat_comms::PubKey::new(new_supervisor_spec.pubkey)),
9546 "new supervisor private route must remain installed after rotation"
9547 );
9548 assert!(
9549 !member_runtime
9550 .router()
9551 .is_private(&meerkat_comms::PubKey::new(old_supervisor_spec.pubkey)),
9552 "old supervisor private route must be fully removed after rotation"
9553 );
9554 }
9555
9556 #[cfg(not(target_arch = "wasm32"))]
9557 #[tokio::test]
9558 async fn revoke_supervisor_replies_after_removing_supervisor_route() {
9559 let suffix = Uuid::new_v4().simple().to_string();
9560 let member_name = format!("tcp-revoke-member-{suffix}");
9561 let supervisor_name = format!("tcp-revoke-supervisor-{suffix}");
9562
9563 let mut member_runtime =
9564 meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member runtime");
9565 member_runtime
9566 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9567 .expect("configure member TCP listener");
9568 member_runtime
9569 .start_listeners()
9570 .await
9571 .expect("start member TCP listener");
9572 let member_runtime = Arc::new(member_runtime);
9573 install_test_peer_request_response_authority(
9574 &member_runtime,
9575 Arc::new(CountingPeerInteractionHandle::default()),
9576 );
9577
9578 let mut supervisor_runtime =
9579 meerkat_comms::CommsRuntime::inproc_only(&supervisor_name).expect("supervisor runtime");
9580 supervisor_runtime
9581 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9582 .expect("configure supervisor TCP listener");
9583 supervisor_runtime
9584 .start_listeners()
9585 .await
9586 .expect("start supervisor TCP listener");
9587 let supervisor_runtime = Arc::new(supervisor_runtime);
9588 install_test_peer_request_response_authority(
9589 &supervisor_runtime,
9590 Arc::new(CountingPeerInteractionHandle::default()),
9591 );
9592
9593 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
9594 let supervisor_dsl = install_running_test_peer_comms_handle(supervisor_runtime.as_ref());
9595 add_test_projection_trust_with_dsl(
9596 supervisor_runtime.as_ref(),
9597 Arc::clone(&supervisor_dsl),
9598 member_spec,
9599 1,
9600 "supervisor trusts member target",
9601 )
9602 .await;
9603 let supervisor_spec =
9604 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
9605 let adapter = Arc::new(MeerkatMachine::ephemeral());
9610 let session_id = SessionId::new();
9611 adapter
9612 .register_session(session_id.clone())
9613 .await
9614 .expect("register session");
9615 adapter
9616 .stage_supervisor_bind(
9617 &session_id,
9618 supervisor_spec.name.to_string(),
9619 supervisor_spec.peer_id.as_str(),
9620 supervisor_spec.address.to_string(),
9621 signing_public_key_for_descriptor(&supervisor_spec),
9622 1,
9623 )
9624 .await
9625 .expect("pre-bind supervisor in DSL state");
9626 adapter
9627 .test_install_session_peer_comms_handle_on_runtime(&session_id, member_runtime.as_ref())
9628 .await
9629 .expect("install adapter session peer-comms handle on member runtime");
9630 add_member_private_supervisor_trust_via_adapter(
9631 &adapter,
9632 &session_id,
9633 &member_runtime,
9634 &supervisor_spec,
9635 1,
9636 "member starts with supervisor private route",
9637 )
9638 .await;
9639 assert!(
9640 member_runtime
9641 .router()
9642 .is_private(&meerkat_comms::PubKey::new(supervisor_spec.pubkey)),
9643 "member must start with a PRIVATE supervisor route"
9644 );
9645
9646 let command = BridgeCommand::RevokeSupervisor(BridgeSupervisorPayload {
9647 supervisor: BridgePeerSpec::from(supervisor_spec.clone()),
9648 epoch: 1,
9649 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
9650 });
9651 let interaction_id = InteractionId(Uuid::new_v4());
9652 let ingress = PeerIngressFact::peer(
9653 interaction_id,
9654 PeerInputClass::ActionableRequest,
9655 meerkat_core::PeerIngressKind::Request,
9656 Some(meerkat_core::PeerIngressAuthDecision::Required),
9657 PeerIngressIdentity::new(
9658 supervisor_spec.peer_id,
9659 supervisor_spec.name.as_str(),
9660 meerkat_core::PeerIngressConvention::Request {
9661 request_id: interaction_id.to_string(),
9662 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
9663 },
9664 )
9665 .with_signing_pubkey(supervisor_spec.pubkey),
9666 );
9667 let candidate =
9668 bridge_candidate_with_ingress(supervisor_spec.name.as_str(), &command, ingress);
9669
9670 assert!(
9671 try_handle_supervisor_bridge_command(
9672 &adapter,
9673 &session_id,
9674 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
9675 &candidate,
9676 )
9677 .await,
9678 "bridge handler must own RevokeSupervisor"
9679 );
9680
9681 let response =
9682 drain_one_candidate(&supervisor_runtime, "supervisor RevokeSupervisor response").await;
9683 let InteractionContent::Response {
9684 in_reply_to,
9685 status,
9686 result,
9687 ..
9688 } = response.interaction.content
9689 else {
9690 panic!(
9691 "expected revoke bridge response, got {:?}",
9692 response.interaction.content
9693 );
9694 };
9695 assert_eq!(in_reply_to, interaction_id);
9696 assert_eq!(status, meerkat_core::interaction::ResponseStatus::Completed);
9697 assert!(matches!(
9698 serde_json::from_value::<BridgeReply>(result).expect("typed bridge reply"),
9699 BridgeReply::Ack(BridgeAck { ok: true })
9700 ));
9701 assert!(
9702 member_runtime
9703 .peers()
9704 .await
9705 .iter()
9706 .all(|peer| peer.peer_id != supervisor_spec.peer_id),
9707 "supervisor route must be removed after revoke"
9708 );
9709 assert!(
9710 !member_runtime
9711 .router()
9712 .is_private(&meerkat_comms::PubKey::new(supervisor_spec.pubkey)),
9713 "supervisor private admission edge must be fully removed after revoke"
9714 );
9715 }
9716
9717 #[cfg(not(target_arch = "wasm32"))]
9718 async fn drain_one_candidate(
9719 runtime: &Arc<meerkat_comms::CommsRuntime>,
9720 label: &str,
9721 ) -> PeerInputCandidate {
9722 for _ in 0..40 {
9723 let candidates = runtime.drain_peer_input_candidates().await;
9724 if let Some(candidate) = candidates.into_iter().next() {
9725 return candidate;
9726 }
9727 let notify = runtime.inbox_notify();
9728 let notified = notify.notified();
9729 let _ = tokio::time::timeout(std::time::Duration::from_millis(25), notified).await;
9730 }
9731 panic!("timed out waiting for {label}");
9732 }
9733
9734 #[test]
9735 fn bridge_response_route_source_does_not_use_interaction_from_ladder() {
9736 let source = include_str!("comms_drain.rs");
9737 let route_start = source
9738 .find("async fn resolve_bridge_response_route")
9739 .expect("route function exists");
9740 let route_end = source[route_start..]
9741 .find("async fn resolve_peer_route")
9742 .map(|offset| route_start + offset)
9743 .expect("peer route function follows bridge response route");
9744 let route_source = &source[route_start..route_end];
9745 let forbidden_from = ["candidate", "interaction", "from"].join(".");
9746 assert!(
9747 !route_source.contains(&forbidden_from),
9748 "bridge response routing must use typed ingress facts, not candidate.interaction.from"
9749 );
9750
9751 for removed_helper in [
9752 ["bridge_response", "route_from_sender"].join("_"),
9753 ["resolve_bridge", "display_name_sender_route"].join("_"),
9754 ] {
9755 assert!(
9756 !source.contains(&format!("fn {removed_helper}")),
9757 "bridge response routing must not revive {removed_helper}"
9758 );
9759 }
9760 }
9761
9762 #[test]
9763 fn comms_drain_registers_inbox_waiter_before_draining() {
9764 let source = include_str!("comms_drain.rs");
9772 let enable = source
9773 .find("notified.as_mut().enable();")
9774 .expect("drain must enable() its pinned inbox waiter");
9775 let drain = source
9776 .find("comms_runtime.drain_classified_inbox_interactions().await")
9777 .expect("drain must consume the typed classified inbox path");
9778 assert!(
9779 enable < drain,
9780 "the inbox waiter must be registered (enable) BEFORE the drain, not after"
9781 );
9782 }
9783
9784 #[tokio::test]
9785 async fn notify_waiters_is_lost_without_prior_registration() {
9786 use std::sync::Arc;
9792 use tokio::sync::Notify;
9793
9794 let notify = Arc::new(Notify::new());
9795 notify.notify_waiters(); let late = notify.notified();
9797 assert!(
9798 tokio::time::timeout(std::time::Duration::from_millis(50), late)
9799 .await
9800 .is_err(),
9801 "a notify_waiters() with no prior registration must be lost"
9802 );
9803
9804 let mut early = std::pin::pin!(notify.notified());
9807 early.as_mut().enable();
9808 notify.notify_waiters();
9809 tokio::time::timeout(std::time::Duration::from_millis(100), early)
9810 .await
9811 .expect("a waiter enabled before notify_waiters() must observe it");
9812 }
9813
9814 struct BootstrapRuntime {
9815 peer_id: String,
9816 address: String,
9817 bootstrap_token: Option<String>,
9818 inbox_notify: Arc<tokio::sync::Notify>,
9819 add_trusted_peer_errors: HashMap<String, String>,
9820 remove_trusted_peer_errors: HashMap<String, String>,
9821 trusted_peer_ids: Arc<tokio::sync::Mutex<HashSet<String>>>,
9822 trusted_peers: Arc<tokio::sync::Mutex<HashMap<String, TrustedPeerDescriptor>>>,
9823 sent_commands: Arc<tokio::sync::Mutex<Vec<CommsCommand>>>,
9824 peer_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
9825 peer_request_response_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
9826 completed_count: Arc<std::sync::atomic::AtomicUsize>,
9827 add_mutates_then_hangs: Arc<std::sync::atomic::AtomicBool>,
9828 add_mutates_then_hangs_once_for: Arc<tokio::sync::Mutex<HashSet<String>>>,
9829 remove_hangs_remaining: Arc<std::sync::atomic::AtomicUsize>,
9830 remove_mutates_then_hangs_once_for: Arc<tokio::sync::Mutex<HashSet<String>>>,
9831 remove_failures_remaining: Arc<std::sync::atomic::AtomicUsize>,
9832 remove_attempts: Arc<std::sync::atomic::AtomicUsize>,
9833 staged_reply_endpoints: Arc<std::sync::Mutex<Vec<(PeerId, [u8; 32], String)>>>,
9834 staged_correlated_reply_endpoints:
9835 Arc<std::sync::Mutex<Vec<(PeerId, InteractionId, [u8; 32], PeerAddress)>>>,
9836 unstaged_correlated_reply_endpoints: Arc<std::sync::Mutex<Vec<(PeerId, InteractionId)>>>,
9837 }
9838
9839 #[async_trait::async_trait]
9840 impl CommsRuntime for BootstrapRuntime {
9841 fn peer_id(&self) -> Option<PeerId> {
9842 PeerId::parse(&self.peer_id).ok()
9843 }
9844
9845 fn public_key(&self) -> Option<String> {
9846 test_public_key_for_peer_id(&self.peer_id)
9847 }
9848
9849 fn public_key_bytes(&self) -> Option<[u8; 32]> {
9850 test_pubkey_bytes_for_peer_id(&self.peer_id)
9851 }
9852
9853 fn comms_name(&self) -> Option<String> {
9854 Some(test_comms_name_from_address(&self.address))
9855 }
9856
9857 fn advertised_address(&self) -> Option<String> {
9858 Some(self.address.clone())
9859 }
9860
9861 fn bridge_bootstrap_token(&self) -> Option<String> {
9862 self.bootstrap_token.clone()
9863 }
9864
9865 async fn drain_messages(&self) -> Vec<String> {
9866 Vec::new()
9867 }
9868
9869 fn inbox_notify(&self) -> Arc<tokio::sync::Notify> {
9870 self.inbox_notify.clone()
9871 }
9872
9873 fn peer_interaction_handle(
9874 &self,
9875 ) -> Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>> {
9876 self.peer_handle.clone()
9877 }
9878
9879 fn peer_request_response_authority_handle(
9880 &self,
9881 ) -> Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>> {
9882 self.peer_request_response_handle.clone()
9883 }
9884
9885 fn mark_interaction_complete(&self, _id: &InteractionId) {
9886 self.completed_count
9887 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
9888 }
9889
9890 async fn stage_declared_reply_endpoint(
9891 &self,
9892 dest: PeerId,
9893 signer_pubkey: [u8; 32],
9894 declared_address: String,
9895 ) -> Result<(), SendError> {
9896 self.staged_reply_endpoints
9897 .lock()
9898 .expect("staged reply endpoints mutex")
9899 .push((dest, signer_pubkey, declared_address));
9900 Ok(())
9901 }
9902
9903 async fn stage_correlated_reply_endpoint(
9904 &self,
9905 dest: PeerId,
9906 in_reply_to: InteractionId,
9907 signer_pubkey: [u8; 32],
9908 declared_endpoint: PeerAddress,
9909 ) -> Result<(), SendError> {
9910 self.staged_correlated_reply_endpoints
9911 .lock()
9912 .expect("staged correlated reply endpoints mutex")
9913 .push((dest, in_reply_to, signer_pubkey, declared_endpoint));
9914 Ok(())
9915 }
9916
9917 async fn unstage_correlated_reply_endpoint(
9918 &self,
9919 dest: PeerId,
9920 in_reply_to: InteractionId,
9921 ) -> Result<(), SendError> {
9922 self.unstaged_correlated_reply_endpoints
9923 .lock()
9924 .expect("unstaged correlated reply endpoints mutex")
9925 .push((dest, in_reply_to));
9926 Ok(())
9927 }
9928
9929 async fn apply_trust_mutation(
9930 &self,
9931 mutation: CommsTrustMutation,
9932 ) -> Result<CommsTrustMutationResult, SendError> {
9933 match mutation {
9934 CommsTrustMutation::AddPrivateTrustedPeer { peer, authority } => {
9935 authority
9936 .validate_private_add(self.peer_id(), &peer)
9937 .map_err(SendError::Validation)?;
9938 let peer_id_str = peer.peer_id.as_str();
9939 if let Some(message) = self.add_trusted_peer_errors.get(&peer_id_str) {
9940 return Err(SendError::Internal(message.clone()));
9941 }
9942 let created = self.trusted_peer_ids.lock().await.insert(peer_id_str);
9943 let mut mutate_then_hang = self.add_mutates_then_hangs_once_for.lock().await;
9944 let targeted_mutate_then_hang = mutate_then_hang.remove(&peer.peer_id.as_str());
9945 drop(mutate_then_hang);
9946 if self
9947 .add_mutates_then_hangs
9948 .load(std::sync::atomic::Ordering::SeqCst)
9949 || targeted_mutate_then_hang
9950 {
9951 std::future::pending::<()>().await;
9952 }
9953 Ok(CommsTrustMutationResult::Added { created })
9954 }
9955 CommsTrustMutation::RemovePrivateTrustedPeer { peer_id, authority } => {
9956 let parsed_peer_id = PeerId::parse(&peer_id)
9957 .map_err(|error| SendError::Validation(error.to_string()))?;
9958 authority
9959 .validate_private_remove(self.peer_id(), parsed_peer_id)
9960 .map_err(SendError::Validation)?;
9961 self.remove_attempts
9962 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
9963 let mut mutate_then_hang = self.remove_mutates_then_hangs_once_for.lock().await;
9964 let targeted_mutate_then_hang = mutate_then_hang.remove(&peer_id);
9965 drop(mutate_then_hang);
9966 if targeted_mutate_then_hang {
9967 self.trusted_peer_ids.lock().await.remove(&peer_id);
9968 std::future::pending::<()>().await;
9969 }
9970 if self
9971 .remove_hangs_remaining
9972 .fetch_update(
9973 std::sync::atomic::Ordering::SeqCst,
9974 std::sync::atomic::Ordering::SeqCst,
9975 |remaining| remaining.checked_sub(1),
9976 )
9977 .is_ok()
9978 {
9979 std::future::pending::<()>().await;
9980 }
9981 if self
9982 .remove_failures_remaining
9983 .fetch_update(
9984 std::sync::atomic::Ordering::SeqCst,
9985 std::sync::atomic::Ordering::SeqCst,
9986 |remaining| remaining.checked_sub(1),
9987 )
9988 .is_ok()
9989 {
9990 return Err(SendError::Internal(
9991 "synthetic one-shot remove failure".to_string(),
9992 ));
9993 }
9994 match self.remove_trusted_peer_errors.get(&peer_id) {
9995 Some(message) => Err(SendError::Internal(message.clone())),
9996 None => Ok(CommsTrustMutationResult::Removed {
9997 removed: self.trusted_peer_ids.lock().await.remove(&peer_id),
9998 }),
9999 }
10000 }
10001 _ => Err(SendError::Unsupported(
10002 "test runtime only supports generated private trust".into(),
10003 )),
10004 }
10005 }
10006
10007 async fn send(&self, cmd: CommsCommand) -> Result<meerkat_core::SendReceipt, SendError> {
10008 let receipt = match &cmd {
10009 CommsCommand::PeerResponse { in_reply_to, .. } => {
10010 meerkat_core::SendReceipt::PeerResponseSent {
10011 envelope_id: Uuid::new_v4(),
10012 in_reply_to: *in_reply_to,
10013 }
10014 }
10015 other => {
10016 return Err(SendError::Unsupported(format!(
10017 "BootstrapRuntime only records peer responses, got {}",
10018 other.command_kind()
10019 )));
10020 }
10021 };
10022 self.sent_commands.lock().await.push(cmd);
10023 Ok(receipt)
10024 }
10025
10026 async fn peer_ingress_runtime_snapshot(
10027 &self,
10028 ) -> Result<
10029 meerkat_core::interaction::PeerIngressRuntimeSnapshot,
10030 meerkat_core::CommsCapabilityError,
10031 > {
10032 Ok(meerkat_core::interaction::PeerIngressRuntimeSnapshot {
10033 self_peer_id: self
10034 .peer_id()
10035 .unwrap_or_else(|| PeerId::parse(PEER_ID_RECEIVER).expect("valid peer id")),
10036 auth_required: true,
10037 authority_phase: meerkat_core::interaction::PeerIngressAuthorityPhase::Received,
10038 trusted_peers: self.trusted_peers.lock().await.values().cloned().collect(),
10039 submission_queue_len: 0,
10040 queue: meerkat_core::interaction::PeerIngressQueueSnapshot::default(),
10041 })
10042 }
10043 }
10044
10045 fn bootstrap_runtime(
10046 peer_id: &str,
10047 address: &str,
10048 bootstrap_token: Option<&str>,
10049 ) -> BootstrapRuntime {
10050 let peer_handle = Arc::new(CountingPeerInteractionHandle::default());
10051 let peer_handle: Arc<dyn meerkat_core::handles::PeerInteractionHandle> = peer_handle;
10052 BootstrapRuntime {
10053 peer_id: peer_id.to_string(),
10054 address: address.to_string(),
10055 bootstrap_token: bootstrap_token.map(ToString::to_string),
10056 inbox_notify: Arc::new(tokio::sync::Notify::new()),
10057 add_trusted_peer_errors: HashMap::new(),
10058 remove_trusted_peer_errors: HashMap::new(),
10059 trusted_peer_ids: Arc::new(tokio::sync::Mutex::new(HashSet::new())),
10060 trusted_peers: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
10061 sent_commands: Arc::new(tokio::sync::Mutex::new(Vec::new())),
10062 peer_handle: Some(peer_handle.clone()),
10063 peer_request_response_handle: Some(peer_handle),
10064 completed_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
10065 add_mutates_then_hangs: Arc::new(std::sync::atomic::AtomicBool::new(false)),
10066 add_mutates_then_hangs_once_for: Arc::new(tokio::sync::Mutex::new(HashSet::new())),
10067 remove_hangs_remaining: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
10068 remove_mutates_then_hangs_once_for: Arc::new(tokio::sync::Mutex::new(HashSet::new())),
10069 remove_failures_remaining: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
10070 remove_attempts: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
10071 staged_reply_endpoints: Arc::new(std::sync::Mutex::new(Vec::new())),
10072 staged_correlated_reply_endpoints: Arc::new(std::sync::Mutex::new(Vec::new())),
10073 unstaged_correlated_reply_endpoints: Arc::new(std::sync::Mutex::new(Vec::new())),
10074 }
10075 }
10076
10077 #[tokio::test]
10078 async fn supervisor_rotation_task_slot_rejects_live_replacement_carrier() {
10079 struct DropSignal(Arc<std::sync::atomic::AtomicBool>);
10080
10081 impl Drop for DropSignal {
10082 fn drop(&mut self) {
10083 self.0.store(true, std::sync::atomic::Ordering::SeqCst);
10084 }
10085 }
10086
10087 let slot = crate::meerkat_machine::SupervisorRotationTaskSlot::new();
10088 let runtime_a: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
10089 PEER_ID_RECEIVER,
10090 "inproc://rotation-worker-a",
10091 Some("token"),
10092 ));
10093 let runtime_b: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
10094 PEER_ID_RECEIVER,
10095 "inproc://rotation-worker-b",
10096 Some("token"),
10097 ));
10098 let first_dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
10099 let first_started = Arc::new(tokio::sync::Notify::new());
10100 let first = tokio::spawn({
10101 let first_dropped = Arc::clone(&first_dropped);
10102 let first_started = Arc::clone(&first_started);
10103 async move {
10104 let _drop_signal = DropSignal(first_dropped);
10105 first_started.notify_one();
10106 std::future::pending::<()>().await;
10107 }
10108 });
10109 first_started.notified().await;
10110 assert!(
10111 slot.install("operation-a".to_string(), runtime_a, first)
10112 .await
10113 );
10114
10115 let replacement_dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
10116 let replacement = tokio::spawn({
10117 let drop_signal = DropSignal(Arc::clone(&replacement_dropped));
10118 async move {
10119 let _drop_signal = drop_signal;
10120 std::future::pending::<()>().await;
10121 }
10122 });
10123 assert!(
10124 !slot
10125 .install("operation-b".to_string(), runtime_b, replacement)
10126 .await,
10127 "a live carrier must never be displaced by last-writer-wins installation"
10128 );
10129 assert!(
10130 !first_dropped.load(std::sync::atomic::Ordering::SeqCst),
10131 "the incumbent carrier remains authoritative until its owning runtime is quiesced"
10132 );
10133 assert!(
10134 replacement_dropped.load(std::sync::atomic::Ordering::SeqCst),
10135 "the rejected candidate carrier must be aborted and joined"
10136 );
10137 slot.abort_and_wait().await;
10138 assert!(
10139 first_dropped.load(std::sync::atomic::Ordering::SeqCst),
10140 "explicit quiescence must abort and join the incumbent carrier"
10141 );
10142 }
10143
10144 fn lifecycle_candidate(
10145 class: PeerInputClass,
10146 intent: &str,
10147 params: serde_json::Value,
10148 ) -> PeerInputCandidate {
10149 let id = InteractionId(Uuid::new_v4());
10150 let lifecycle_kind = match class {
10151 PeerInputClass::PeerLifecycleRetired => {
10152 meerkat_core::comms::PeerLifecycleKind::PeerRetired
10153 }
10154 PeerInputClass::PeerLifecycleUnwired => {
10155 meerkat_core::comms::PeerLifecycleKind::PeerUnwired
10156 }
10157 _ => meerkat_core::comms::PeerLifecycleKind::PeerAdded,
10158 };
10159 PeerInputCandidate {
10160 interaction: InboxInteraction {
10161 objective_id: None,
10162 sender_taint: None,
10163 id,
10164 from_route: None,
10165 from: "test-mob/__mob_supervisor__".to_string(),
10166 content: InteractionContent::Request {
10167 intent: intent.to_string(),
10168 params,
10169 blocks: None,
10170 },
10171 rendered_text: String::new(),
10172 handling_mode: HandlingMode::Queue,
10173 render_metadata: None,
10174 },
10175 ingress: PeerIngressFact::peer(
10176 id,
10177 class,
10178 meerkat_core::PeerIngressKind::Request,
10179 Some(meerkat_core::PeerIngressAuthDecision::Required),
10180 PeerIngressIdentity::new(
10181 PeerId::new(),
10182 "test-mob/__mob_supervisor__",
10183 meerkat_core::PeerIngressConvention::Lifecycle {
10184 kind: lifecycle_kind,
10185 peer: "peer-1".to_string(),
10186 },
10187 ),
10188 ),
10189 lifecycle_peer: Some("peer-1".to_string()),
10190 response_terminality: None,
10191 }
10192 }
10193
10194 #[test]
10195 fn completed_outcome_maps_structured_output_to_interaction_complete() {
10196 let interaction_id = InteractionId(Uuid::new_v4());
10197 let event = interaction_terminal_event(
10198 interaction_id,
10199 CompletionOutcome::Completed(Box::new(meerkat_core::RunResult {
10200 text: "{\"answer\":42}".to_string(),
10201 session_id: meerkat_core::SessionId::new(),
10202 usage: meerkat_core::Usage::default(),
10203 turns: 1,
10204 tool_calls: 0,
10205 terminal_cause_kind: None,
10206 structured_output: Some(json!({"answer": 42})),
10207 extraction_error: None,
10208 schema_warnings: None,
10209 skill_diagnostics: None,
10210 })),
10211 );
10212
10213 match event {
10214 AgentEvent::InteractionComplete {
10215 interaction_id: actual_id,
10216 result,
10217 structured_output,
10218 } => {
10219 assert_eq!(actual_id, interaction_id);
10220 assert_eq!(result, "{\"answer\":42}");
10221 assert_eq!(structured_output, Some(json!({"answer": 42})));
10222 }
10223 other => panic!("expected InteractionComplete, got {other:?}"),
10224 }
10225 }
10226
10227 #[test]
10228 fn extraction_failure_maps_to_typed_interaction_failure() {
10229 let interaction_id = InteractionId(Uuid::new_v4());
10230 let event = interaction_terminal_event(
10231 interaction_id,
10232 CompletionOutcome::Completed(Box::new(meerkat_core::RunResult {
10233 text: "main output".to_string(),
10234 session_id: meerkat_core::SessionId::new(),
10235 usage: meerkat_core::Usage::default(),
10236 turns: 1,
10237 tool_calls: 0,
10238 terminal_cause_kind: None,
10239 structured_output: None,
10240 extraction_error: Some(meerkat_core::types::ExtractionError {
10241 last_output: "main output".to_string(),
10242 attempts: 3,
10243 reason: "invalid schema".to_string(),
10244 }),
10245 schema_warnings: None,
10246 skill_diagnostics: None,
10247 })),
10248 );
10249
10250 assert!(matches!(
10251 event,
10252 AgentEvent::InteractionFailed {
10253 interaction_id: actual_id,
10254 reason: meerkat_core::event::InteractionFailureReason::ExtractionFailed {
10255 ref last_output,
10256 attempts: 3,
10257 ref reason,
10258 },
10259 } if actual_id == interaction_id
10260 && last_output == "main output"
10261 && reason == "invalid schema"
10262 ));
10263 }
10264
10265 #[test]
10266 fn peer_turn_metadata_matches_live_interaction_complete_identity() {
10267 let interaction_id = InteractionId(Uuid::new_v4());
10268 let input = crate::input::Input::Peer(crate::input::PeerInput {
10269 directed_interaction_id: None,
10270 objective_id: None,
10271 injected_context: Vec::new(),
10272 header: crate::input::InputHeader {
10273 id: meerkat_core::lifecycle::InputId::new(),
10274 timestamp: chrono::Utc::now(),
10275 source: crate::input::InputOrigin::Peer {
10276 peer_id: PEER_ID_RECEIVER.to_string(),
10277 display_identity: Some("keepalive-peer".to_string()),
10278 runtime_id: None,
10279 },
10280 durability: crate::input::InputDurability::Durable,
10281 visibility: crate::input::InputVisibility::default(),
10282 idempotency_key: None,
10283 supersession_key: None,
10284 correlation_id: Some(crate::CorrelationId::from_uuid(interaction_id.0)),
10285 },
10286 convention: Some(crate::input::PeerConvention::Message),
10287 content: "poke".to_string().into(),
10288 payload: None,
10289 handling_mode: None,
10290 sender_taint: None,
10291 });
10292 let semantics =
10293 crate::ingress_types::RuntimeInputSemantics::try_from_generated_admission(&input, true)
10294 .expect("peer message should admit as a runtime turn");
10295 let metadata = crate::runtime_loop::for_input(&input, semantics);
10296
10297 let event =
10298 interaction_terminal_event(interaction_id, CompletionOutcome::CompletedWithoutResult);
10299 let AgentEvent::InteractionComplete {
10300 interaction_id: live_interaction_id,
10301 ..
10302 } = event
10303 else {
10304 panic!("expected InteractionComplete");
10305 };
10306
10307 assert_eq!(
10308 metadata.transcript_identity.interaction_id,
10309 Some(live_interaction_id),
10310 "runtime transcript identity must use the same peer interaction id as the live terminal event"
10311 );
10312 }
10313
10314 #[test]
10315 fn callback_pending_maps_to_interaction_callback_pending_terminal_event() {
10316 let interaction_id = InteractionId(Uuid::new_v4());
10317 let event = interaction_terminal_event(
10318 interaction_id,
10319 CompletionOutcome::CallbackPending {
10320 tool_name: "external_mock".to_string(),
10321 args: json!({ "value": "browser" }),
10322 },
10323 );
10324
10325 assert!(
10326 matches!(event, AgentEvent::InteractionCallbackPending { .. }),
10327 "expected callback-pending interaction event"
10328 );
10329 if let AgentEvent::InteractionCallbackPending {
10330 interaction_id: actual_id,
10331 tool_name,
10332 args,
10333 } = event
10334 {
10335 assert_eq!(actual_id, interaction_id);
10336 assert_eq!(tool_name, "external_mock");
10337 assert_eq!(args, json!({ "value": "browser" }));
10338 }
10339 }
10340
10341 #[tokio::test]
10342 async fn peer_lifecycle_added_does_not_change_comms_trust() {
10343 let runtime: Arc<dyn CommsRuntime> =
10344 Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-added").unwrap());
10345 let peer = meerkat_comms::CommsRuntime::inproc_only("peer-added").unwrap();
10346 let peer_spec = trusted_peer_from_runtime("peer-added", &peer);
10347 let candidate = lifecycle_candidate(
10348 PeerInputClass::PeerLifecycleAdded,
10349 "mob.peer_added",
10350 json!({
10351 "peer": "peer-1",
10352 "peer_spec": peer_spec,
10353 }),
10354 );
10355
10356 let peers_before = runtime.peers().await;
10357 assert!(
10358 peers_before.is_empty(),
10359 "test runtime should start without trust"
10360 );
10361 let input =
10362 classified_interaction_to_runtime_input(&candidate, &LogicalRuntimeId::new("s-1"))
10363 .expect("lifecycle candidate should project to runtime input");
10364 assert!(
10365 matches!(input, Input::Peer(_)),
10366 "lifecycle candidate should still route as peer input"
10367 );
10368 let peers = runtime.peers().await;
10369 assert!(
10370 peers.is_empty(),
10371 "peer lifecycle add must not materialize comms trust before topology validation"
10372 );
10373 }
10374
10375 #[tokio::test]
10376 async fn peer_lifecycle_unwired_and_retired_do_not_revoke_comms_trust() {
10377 let runtime =
10378 Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-removed").unwrap());
10379 let peer = meerkat_comms::CommsRuntime::inproc_only("peer-removed").unwrap();
10380 let peer_spec = trusted_peer_from_runtime("peer-removed", &peer);
10381 let dsl = test_projection_trust_dsl(runtime.as_ref());
10382 add_test_projection_trust_with_dsl(
10383 runtime.as_ref(),
10384 Arc::clone(&dsl),
10385 peer_spec.clone(),
10386 1,
10387 "trust peer",
10388 )
10389 .await;
10390
10391 let unwired = lifecycle_candidate(
10392 PeerInputClass::PeerLifecycleUnwired,
10393 "mob.peer_unwired",
10394 json!({
10395 "peer": "peer-1",
10396 "peer_spec": peer_spec.clone(),
10397 }),
10398 );
10399 let _ = classified_interaction_to_runtime_input(&unwired, &LogicalRuntimeId::new("s-1"))
10400 .expect("unwired lifecycle candidate should project");
10401 assert!(
10402 runtime
10403 .peers()
10404 .await
10405 .iter()
10406 .any(|entry| entry.name.as_str() == "peer-removed"),
10407 "peer lifecycle unwire must not revoke comms trust before topology validation"
10408 );
10409
10410 add_test_projection_trust_with_dsl(
10411 runtime.as_ref(),
10412 dsl,
10413 peer_spec.clone(),
10414 2,
10415 "refresh peer trust",
10416 )
10417 .await;
10418 let retired = lifecycle_candidate(
10419 PeerInputClass::PeerLifecycleRetired,
10420 "mob.peer_retired",
10421 json!({
10422 "peer": "peer-1",
10423 "peer_spec": peer_spec,
10424 }),
10425 );
10426 let _ = classified_interaction_to_runtime_input(&retired, &LogicalRuntimeId::new("s-1"))
10427 .expect("retired lifecycle candidate should project");
10428 assert!(
10429 runtime
10430 .peers()
10431 .await
10432 .iter()
10433 .any(|entry| entry.name.as_str() == "peer-removed"),
10434 "peer lifecycle retire must not revoke comms trust before topology validation"
10435 );
10436 }
10437
10438 #[test]
10439 fn bridge_capabilities_report_canonical_protocol_versions() {
10440 let capabilities = bridge_capabilities(
10441 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V4,
10442 );
10443 assert_eq!(
10444 capabilities.current_protocol_version,
10445 supervisor_bridge_current_protocol_version()
10446 );
10447 assert_eq!(
10448 capabilities.default_protocol_version,
10449 supervisor_bridge_default_protocol_version()
10450 );
10451 assert_eq!(
10452 capabilities.supported_protocol_versions,
10453 supervisor_bridge_supported_protocol_versions()
10454 );
10455 assert!(
10456 capabilities.hard_cancel_member,
10457 "phase 6 serves HardCancelMember: the bridge MUST advertise it — \
10458 the controlling side capability-gates dispatch on this fact \
10459 (ADJ-P6-5), so a false report would typed-reject every hard \
10460 cancel without bridge traffic"
10461 );
10462 assert!(
10463 capabilities.tracked_input_cancel,
10464 "V4 serves exact durable tracked-input cancellation and must advertise it"
10465 );
10466 }
10467
10468 #[test]
10469 fn v3_bind_capabilities_omit_v4_extensions_and_hard_cancel() {
10470 let capabilities = bridge_capabilities(
10471 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V3,
10472 );
10473 assert_eq!(
10474 capabilities.current_protocol_version,
10475 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V3
10476 );
10477 assert!(!capabilities.hard_cancel_member);
10478 assert!(!capabilities.tracked_input_cancel);
10479 assert_eq!(
10480 capabilities.supported_protocol_versions,
10481 vec![
10482 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V2,
10483 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V3,
10484 ]
10485 );
10486 let encoded = serde_json::to_value(&capabilities).expect("serialize V3 capabilities");
10487 for v4_field in [
10488 "durable_sessions",
10489 "autonomous_members",
10490 "tracked_input_cancel",
10491 "memory_store",
10492 "mcp",
10493 "engine_version",
10494 "approval_forwarding",
10495 "resolvable_providers",
10496 ] {
10497 assert!(
10498 encoded.get(v4_field).is_none(),
10499 "V3 bind response leaked V4 capability field {v4_field}: {encoded}"
10500 );
10501 }
10502 }
10503
10504 #[tokio::test]
10505 async fn validate_bind_request_rejects_missing_or_wrong_bootstrap_token() {
10506 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
10507 PEER_ID_RECEIVER,
10508 "inproc://receiver",
10509 Some("expected-token"),
10510 ));
10511 let (adapter, session_id) = bind_material_adapter().await;
10512 let supervisor = supervisor_bridge_spec();
10513 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
10514 supervisor: supervisor.clone(),
10515 epoch: 0,
10516 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10517 expected_peer_id: PEER_ID_RECEIVER.to_string(),
10518 expected_address: runtime.advertised_address().unwrap(),
10519 bootstrap_token: "wrong-token".into(),
10520 };
10521
10522 let (cause, error) = validate_bind_request(
10523 &adapter,
10524 &session_id,
10525 &runtime,
10526 &bridge_sender_fact(&supervisor.peer_id),
10527 &payload,
10528 )
10529 .await
10530 .expect_err("bind must reject incorrect bootstrap token");
10531 assert_eq!(cause, BridgeRejectionCause::InvalidBootstrapToken);
10532 assert!(
10533 error.contains("invalid bootstrap token"),
10534 "bind rejection should explain the bootstrap proof failure, got: {error}"
10535 );
10536 }
10537
10538 #[tokio::test]
10539 async fn validate_bind_request_accepts_matching_bootstrap_token() {
10540 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
10541 PEER_ID_RECEIVER,
10542 "inproc://receiver",
10543 Some("expected-token"),
10544 ));
10545 let (adapter, session_id) = bind_material_adapter().await;
10546 let supervisor = supervisor_bridge_spec();
10547 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
10548 supervisor: supervisor.clone(),
10549 epoch: 0,
10550 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10551 expected_peer_id: PEER_ID_RECEIVER.to_string(),
10552 expected_address: runtime.advertised_address().unwrap(),
10553 bootstrap_token: "expected-token".into(),
10554 };
10555
10556 let (authorized, advertised_address) = validate_bind_request(
10557 &adapter,
10558 &session_id,
10559 &runtime,
10560 &bridge_sender_fact(&supervisor.peer_id),
10561 &payload,
10562 )
10563 .await
10564 .expect("bind should accept the configured bootstrap token");
10565 assert_eq!(authorized.peer_id.as_str(), supervisor.peer_id);
10566 assert_eq!(advertised_address, runtime.advertised_address().unwrap());
10567 }
10568
10569 #[test]
10570 fn bridge_delivery_payload_preserves_explicit_queue_handling_mode() {
10571 let stable_input_id = Uuid::new_v4();
10572 let input = peer_input_from_delivery_payload(
10573 &SessionId::new(),
10574 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10575 BridgeDeliveryPayload {
10576 objective_id: None,
10577 injected_context: Vec::new(),
10578 supervisor: supervisor_bridge_spec(),
10579 epoch: 1,
10580 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10581 input_id: stable_input_id.to_string(),
10582 transcript_interaction_id: None,
10583 content: meerkat_core::types::ContentInput::Text("queued follow-up".to_string()),
10584 handling_mode: HandlingMode::Queue,
10585 expected_member: None,
10586 turn: None,
10587 outcome_tracking: None,
10588 },
10589 )
10590 .expect("UUID payload input id lowers");
10591
10592 let Input::Peer(peer) = &input else {
10593 panic!("bridge delivery must project to peer input");
10594 };
10595 assert_eq!(
10596 peer.handling_mode,
10597 Some(HandlingMode::Queue),
10598 "bridge delivery explicit queue must survive into MeerkatMachine admission"
10599 );
10600 assert_eq!(peer.header.id.0, stable_input_id);
10601 assert_eq!(
10602 peer.header.correlation_id.as_ref().map(|id| id.0),
10603 Some(stable_input_id),
10604 "payload input_id is the stable terminal correlation, not the transient envelope id"
10605 );
10606 assert_eq!(
10607 peer.directed_interaction_id, None,
10608 "peer-only legacy delivery must not mint terminal-publication authority"
10609 );
10610 assert_eq!(
10611 crate::input::validated_directed_interaction_id(&input)
10612 .expect("legacy peer shape validates"),
10613 None
10614 );
10615 let semantics =
10616 crate::ingress_types::RuntimeInputSemantics::try_from_generated_admission(&input, true)
10617 .expect("legacy peer admission semantics");
10618 assert!(
10619 crate::runtime_loop::for_input(&input, semantics)
10620 .directed_interaction_ids
10621 .is_empty(),
10622 "correlation alone must not opt legacy peer traffic into directed terminals"
10623 );
10624 }
10625
10626 #[test]
10627 fn explicitly_tracked_plain_bridge_delivery_mints_exact_directed_interaction() {
10628 let session_id = SessionId::new();
10629 let stable_input_id = Uuid::new_v4();
10630 let objective_id = meerkat_core::interaction::ObjectiveId::new();
10631 let payload = BridgeDeliveryPayload {
10632 objective_id: Some(objective_id),
10633 injected_context: vec![meerkat_core::types::ContentInput::Text(
10634 "tracked ambient context".to_string(),
10635 )],
10636 supervisor: supervisor_bridge_spec(),
10637 epoch: 1,
10638 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10639 input_id: stable_input_id.to_string(),
10640 transcript_interaction_id: Some(stable_input_id.to_string()),
10641 content: meerkat_core::types::ContentInput::Text("placed work".to_string()),
10642 handling_mode: HandlingMode::Queue,
10643 expected_member: Some(BridgeMemberIncarnation {
10644 mob_id: "mob".to_string(),
10645 agent_identity: "worker".to_string(),
10646 host_id: "host".to_string(),
10647 binding_generation: 1,
10648 member_session_id: session_id.to_string(),
10649 generation: 1,
10650 fence_token: 11,
10651 }),
10652 turn: None,
10653 outcome_tracking: Some(BridgeOutcomeTracking::Interaction),
10654 };
10655 assert!(delivery_requires_tracked_turn_custody(&payload));
10656 let input = peer_input_from_delivery_payload(
10657 &session_id,
10658 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10659 payload,
10660 )
10661 .expect("placed UUID payload lowers");
10662
10663 let expected = meerkat_core::interaction::InteractionId(stable_input_id);
10664 let Input::Peer(peer) = &input else {
10665 panic!("placed bridge delivery must project to peer input");
10666 };
10667 assert_eq!(peer.directed_interaction_id, Some(expected));
10668 assert_eq!(peer.objective_id, Some(objective_id));
10669 assert_eq!(
10670 peer.injected_context,
10671 vec![meerkat_core::types::ContentInput::Text(
10672 "tracked ambient context".to_string()
10673 )]
10674 );
10675 assert_eq!(
10676 crate::input::validated_directed_interaction_id(&input)
10677 .expect("placed peer shape validates"),
10678 Some(expected)
10679 );
10680 let semantics =
10681 crate::ingress_types::RuntimeInputSemantics::try_from_generated_admission(&input, true)
10682 .expect("placed peer admission semantics");
10683 assert_eq!(
10684 crate::runtime_loop::for_input(&input, semantics).directed_interaction_ids,
10685 vec![expected]
10686 );
10687 }
10688
10689 #[test]
10690 fn placed_delivery_without_explicit_tracking_remains_untracked() {
10691 let session_id = SessionId::new();
10692 let stable_input_id = Uuid::new_v4();
10693 let transcript_interaction_id = Uuid::new_v4();
10694 let payload = BridgeDeliveryPayload {
10695 objective_id: None,
10696 injected_context: Vec::new(),
10697 supervisor: supervisor_bridge_spec(),
10698 epoch: 1,
10699 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10700 input_id: stable_input_id.to_string(),
10701 transcript_interaction_id: Some(transcript_interaction_id.to_string()),
10702 content: meerkat_core::types::ContentInput::Text("ordinary placed work".to_string()),
10703 handling_mode: HandlingMode::Queue,
10704 expected_member: Some(BridgeMemberIncarnation {
10705 mob_id: "mob".to_string(),
10706 agent_identity: "worker".to_string(),
10707 host_id: "host".to_string(),
10708 binding_generation: 1,
10709 member_session_id: session_id.to_string(),
10710 generation: 1,
10711 fence_token: 11,
10712 }),
10713 turn: None,
10714 outcome_tracking: None,
10715 };
10716 assert!(!delivery_requires_tracked_turn_custody(&payload));
10717 validate_delivery_tracking_request(&payload)
10718 .expect("untracked placed admission accepts independent transcript identity");
10719 let mut unscoped = payload.clone();
10720 unscoped.expected_member = None;
10721 assert!(
10722 validate_delivery_tracking_request(&unscoped)
10723 .expect_err("transcript identity without placed residency must reject")
10724 .contains("exact placed-member incarnation")
10725 );
10726 let mut under_versioned = payload.clone();
10727 under_versioned.protocol_version = BridgeProtocolVersion::V3;
10728 assert!(
10729 validate_delivery_tracking_request(&under_versioned)
10730 .expect_err("placed transcript identity requires V4")
10731 .contains("V4")
10732 );
10733 let input = peer_input_from_delivery_payload(
10734 &session_id,
10735 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10736 payload,
10737 )
10738 .expect("ordinary placed payload lowers");
10739 let Input::Peer(peer) = &input else {
10740 panic!("ordinary placed bridge delivery must project to peer input");
10741 };
10742 assert_eq!(peer.directed_interaction_id, None);
10743 let expected = meerkat_core::interaction::InteractionId(transcript_interaction_id);
10744 let semantics =
10745 crate::ingress_types::RuntimeInputSemantics::try_from_generated_admission(&input, true)
10746 .expect("ordinary placed peer admission semantics");
10747 let metadata = crate::runtime_loop::for_input(&input, semantics);
10748 assert_eq!(
10749 metadata.transcript_identity.interaction_id,
10750 Some(expected),
10751 "the independent transcript carrier survives without terminal tracking"
10752 );
10753 assert!(
10754 metadata.directed_interaction_ids.is_empty(),
10755 "IngressAccepted must not acquire terminal-publication authority"
10756 );
10757 assert_eq!(
10758 crate::input::validated_directed_interaction_id(&input)
10759 .expect("ordinary placed peer shape validates"),
10760 None
10761 );
10762 }
10763
10764 #[test]
10765 fn placed_admission_without_caller_transcript_id_keeps_identity_absent() {
10766 let session_id = SessionId::new();
10767 let input = peer_input_from_delivery_payload(
10768 &session_id,
10769 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10770 BridgeDeliveryPayload {
10771 objective_id: None,
10772 injected_context: Vec::new(),
10773 supervisor: supervisor_bridge_spec(),
10774 epoch: 1,
10775 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10776 input_id: Uuid::new_v4().to_string(),
10777 transcript_interaction_id: None,
10778 content: meerkat_core::types::ContentInput::Text(
10779 "ordinary placed work".to_string(),
10780 ),
10781 handling_mode: HandlingMode::Queue,
10782 expected_member: Some(BridgeMemberIncarnation {
10783 mob_id: "mob".to_string(),
10784 agent_identity: "worker".to_string(),
10785 host_id: "host".to_string(),
10786 binding_generation: 1,
10787 member_session_id: session_id.to_string(),
10788 generation: 1,
10789 fence_token: 11,
10790 }),
10791 turn: None,
10792 outcome_tracking: None,
10793 },
10794 )
10795 .expect("ordinary placed payload lowers");
10796 assert!(input.header().correlation_id.is_none());
10797 let semantics =
10798 crate::ingress_types::RuntimeInputSemantics::try_from_generated_admission(&input, true)
10799 .expect("ordinary placed peer admission semantics");
10800 assert!(
10801 crate::runtime_loop::for_input(&input, semantics)
10802 .transcript_identity
10803 .interaction_id
10804 .is_none(),
10805 "absent caller identity must remain absent for runtime-side minting"
10806 );
10807 }
10808
10809 #[test]
10810 fn tracked_interaction_validation_is_v4_exact_member_and_mutually_exclusive() {
10811 let session_id = SessionId::new();
10812 let mut payload = BridgeDeliveryPayload {
10813 objective_id: None,
10814 injected_context: Vec::new(),
10815 supervisor: supervisor_bridge_spec(),
10816 epoch: 1,
10817 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10818 input_id: Uuid::new_v4().to_string(),
10819 transcript_interaction_id: None,
10820 content: meerkat_core::types::ContentInput::Text("tracked work".to_string()),
10821 handling_mode: HandlingMode::Queue,
10822 expected_member: Some(BridgeMemberIncarnation {
10823 mob_id: "mob".to_string(),
10824 agent_identity: "worker".to_string(),
10825 host_id: "host".to_string(),
10826 binding_generation: 1,
10827 member_session_id: session_id.to_string(),
10828 generation: 1,
10829 fence_token: 11,
10830 }),
10831 turn: None,
10832 outcome_tracking: Some(BridgeOutcomeTracking::Interaction),
10833 };
10834 payload.transcript_interaction_id = Some(payload.input_id.clone());
10835 validate_delivery_tracking_request(&payload).expect("valid tracked interaction");
10836
10837 payload.transcript_interaction_id = Some(Uuid::new_v4().to_string());
10838 assert!(
10839 validate_delivery_tracking_request(&payload)
10840 .expect_err("tracked terminal identity must not split from its sidecar key")
10841 .contains("equal input_id")
10842 );
10843 payload.transcript_interaction_id =
10844 Some("AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA".to_string());
10845 assert!(
10846 validate_delivery_tracking_request(&payload)
10847 .expect_err("non-canonical transcript UUID must reject")
10848 .contains("canonical non-nil UUID")
10849 );
10850 payload.transcript_interaction_id = Some(payload.input_id.clone());
10851
10852 let canonical_input_id = payload.input_id.clone();
10853 payload.input_id = Uuid::nil().to_string();
10854 payload.transcript_interaction_id = Some(payload.input_id.clone());
10855 assert!(
10856 validate_delivery_tracking_request(&payload)
10857 .expect_err("tracked peer nil UUID must reject")
10858 .contains("nil UUID")
10859 );
10860 assert!(
10861 peer_input_from_delivery_payload(
10862 &session_id,
10863 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10864 payload.clone(),
10865 )
10866 .expect_err("tracked peer lowering must independently reject nil UUID")
10867 .contains("nil UUID")
10868 );
10869 payload.input_id = canonical_input_id;
10870 payload.transcript_interaction_id = Some(payload.input_id.clone());
10871
10872 payload.protocol_version = BridgeProtocolVersion::V3;
10873 assert!(
10874 validate_delivery_tracking_request(&payload)
10875 .expect_err("pre-V4 tracking marker must reject")
10876 .contains("V4")
10877 );
10878 payload.protocol_version = SUPERVISOR_BRIDGE_PROTOCOL_VERSION;
10879 payload.expected_member = None;
10880 assert!(
10881 validate_delivery_tracking_request(&payload)
10882 .expect_err("tracking without exact residency must reject")
10883 .contains("exact placed-member incarnation")
10884 );
10885 payload.expected_member = Some(BridgeMemberIncarnation {
10886 mob_id: "mob".to_string(),
10887 agent_identity: "worker".to_string(),
10888 host_id: "host".to_string(),
10889 binding_generation: 1,
10890 member_session_id: session_id.to_string(),
10891 generation: 1,
10892 fence_token: 11,
10893 });
10894 payload.turn = Some(
10895 meerkat_contracts::wire::supervisor_bridge::BridgeTurnDirective {
10896 correlation: meerkat_contracts::wire::supervisor_bridge::BridgeTurnCorrelation {
10897 run_id: "run".to_string(),
10898 step_id: "step".to_string(),
10899 },
10900 tool_overlay: None,
10901 },
10902 );
10903 assert!(
10904 validate_delivery_tracking_request(&payload)
10905 .expect_err("turn and interaction marker must reject")
10906 .contains("mutually exclusive")
10907 );
10908
10909 payload.outcome_tracking = None;
10910 payload.transcript_interaction_id = None;
10911 payload.input_id = Uuid::nil().to_string();
10912 assert!(
10913 validate_delivery_tracking_request(&payload)
10914 .expect_err("tracked flow-step nil UUID must reject")
10915 .contains("nil UUID")
10916 );
10917 }
10918
10919 #[test]
10920 fn forged_tracked_peer_interaction_identity_is_rejected() {
10921 let session_id = SessionId::new();
10922 let tracked_id = Uuid::new_v4();
10923 let mut input = peer_input_from_delivery_payload(
10924 &session_id,
10925 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10926 BridgeDeliveryPayload {
10927 objective_id: None,
10928 injected_context: Vec::new(),
10929 supervisor: supervisor_bridge_spec(),
10930 epoch: 1,
10931 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10932 input_id: tracked_id.to_string(),
10933 transcript_interaction_id: Some(tracked_id.to_string()),
10934 content: meerkat_core::types::ContentInput::Text("placed work".to_string()),
10935 handling_mode: HandlingMode::Queue,
10936 expected_member: Some(BridgeMemberIncarnation {
10937 mob_id: "mob".to_string(),
10938 agent_identity: "worker".to_string(),
10939 host_id: "host".to_string(),
10940 binding_generation: 1,
10941 member_session_id: session_id.to_string(),
10942 generation: 1,
10943 fence_token: 11,
10944 }),
10945 turn: None,
10946 outcome_tracking: Some(BridgeOutcomeTracking::Interaction),
10947 },
10948 )
10949 .expect("placed UUID payload lowers");
10950 let Input::Peer(peer) = &mut input else {
10951 panic!("placed bridge delivery must project to peer input");
10952 };
10953 peer.directed_interaction_id =
10954 Some(meerkat_core::interaction::InteractionId(Uuid::new_v4()));
10955
10956 assert!(crate::input::validated_directed_interaction_id(&input).is_err());
10957 }
10958
10959 #[test]
10960 fn bridge_delivery_payload_rejects_non_uuid_stable_input_id() {
10961 let lowered = peer_input_from_delivery_payload(
10962 &SessionId::new(),
10963 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10964 BridgeDeliveryPayload {
10965 objective_id: None,
10966 injected_context: Vec::new(),
10967 supervisor: supervisor_bridge_spec(),
10968 epoch: 1,
10969 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10970 input_id: "transport-envelope-alias".to_string(),
10971 transcript_interaction_id: None,
10972 content: meerkat_core::types::ContentInput::Text("work".to_string()),
10973 handling_mode: HandlingMode::Queue,
10974 expected_member: None,
10975 turn: None,
10976 outcome_tracking: None,
10977 },
10978 );
10979 assert!(lowered.is_err());
10980 }
10981
10982 #[test]
10987 fn bridge_delivery_payload_injected_context_projects_before_peer_append() {
10988 let objective_id = meerkat_core::interaction::ObjectiveId::new();
10989 let input = peer_input_from_delivery_payload(
10990 &SessionId::new(),
10991 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10992 BridgeDeliveryPayload {
10993 objective_id: Some(objective_id),
10994 injected_context: vec![
10995 meerkat_core::types::ContentInput::Text("ambient alpha".to_string()),
10996 meerkat_core::types::ContentInput::Text("ambient beta".to_string()),
10997 ],
10998 supervisor: supervisor_bridge_spec(),
10999 epoch: 1,
11000 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11001 input_id: Uuid::new_v4().to_string(),
11002 transcript_interaction_id: None,
11003 content: meerkat_core::types::ContentInput::Text("work content".to_string()),
11004 handling_mode: HandlingMode::Queue,
11005 expected_member: None,
11006 turn: None,
11007 outcome_tracking: None,
11008 },
11009 )
11010 .expect("UUID payload input id lowers");
11011
11012 let Input::Peer(peer) = &input else {
11013 panic!("bridge delivery must project to peer input");
11014 };
11015 assert_eq!(
11016 peer.injected_context,
11017 vec![
11018 meerkat_core::types::ContentInput::Text("ambient alpha".to_string()),
11019 meerkat_core::types::ContentInput::Text("ambient beta".to_string()),
11020 ],
11021 );
11022 assert_eq!(peer.objective_id, Some(objective_id));
11023
11024 let projection = crate::input::runtime_input_projection_for_machine_batch(&input);
11025 assert_eq!(
11026 projection
11027 .injected_context_appends
11028 .iter()
11029 .map(|append| (append.role, append.content.render_text()))
11030 .collect::<Vec<_>>(),
11031 vec![
11032 (
11033 meerkat_core::lifecycle::run_primitive::ConversationAppendRole::InjectedContext,
11034 "ambient alpha".to_string()
11035 ),
11036 (
11037 meerkat_core::lifecycle::run_primitive::ConversationAppendRole::InjectedContext,
11038 "ambient beta".to_string()
11039 ),
11040 ],
11041 );
11042 assert!(
11043 projection.append.is_some(),
11044 "peer work append must survive alongside injected context"
11045 );
11046 }
11047
11048 #[test]
11049 fn bridge_delivery_handler_does_not_wait_for_completion_inline() {
11050 let source = include_str!("comms_drain.rs");
11051 let deliver_start = source
11052 .find("BridgeCommand::DeliverMemberInput")
11053 .expect("deliver command branch exists");
11054 let deliver_end = source[deliver_start..]
11055 .find("fn bridge_runtime_state")
11056 .map(|offset| deliver_start + offset)
11057 .expect("bridge runtime state helper follows command match");
11058 let deliver_source = &source[deliver_start..deliver_end];
11059 assert!(
11060 !deliver_source.contains(".wait().await"),
11061 "bridge delivery admission must not block the comms drain waiting for completion"
11062 );
11063 }
11064
11065 #[tokio::test]
11066 async fn validate_bind_request_rejects_same_display_name_different_canonical_sender() {
11067 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
11068 PEER_ID_RECEIVER,
11069 "inproc://receiver",
11070 Some("expected-token"),
11071 ));
11072 let (adapter, session_id) = bind_material_adapter().await;
11073 let supervisor = supervisor_bridge_spec();
11074 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11075 supervisor: supervisor.clone(),
11076 epoch: 0,
11077 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11078 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11079 expected_address: runtime.advertised_address().unwrap(),
11080 bootstrap_token: "expected-token".into(),
11081 };
11082 let attacker_peer_id =
11083 PeerId::parse(PEER_ID_OLD_SUPERVISOR).expect("valid attacker peer id");
11084 let sender = bridge_sender_fact_with_display(attacker_peer_id, &supervisor.name);
11085
11086 let (cause, error) =
11087 validate_bind_request(&adapter, &session_id, &runtime, &sender, &payload)
11088 .await
11089 .expect_err("same display name with a different canonical sender must reject");
11090 assert_eq!(cause, BridgeRejectionCause::SenderMismatch);
11091 assert!(
11092 error.contains("does not match supervisor"),
11093 "bind rejection should explain the sender mismatch, got: {error}"
11094 );
11095 }
11096
11097 #[tokio::test]
11098 async fn validate_bind_request_accepts_pubkey_sender_with_canonical_supervisor_peer_id() {
11099 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
11100 PEER_ID_RECEIVER,
11101 "inproc://receiver",
11102 Some("expected-token"),
11103 ));
11104 let (adapter, session_id) = bind_material_adapter().await;
11105 let supervisor_key = meerkat_comms::Keypair::generate();
11106 let supervisor_pubkey = supervisor_key.public_key();
11107 let supervisor = BridgePeerSpec {
11108 name: "mob/__mob_supervisor__".to_string(),
11109 peer_id: supervisor_pubkey.to_peer_id().as_str(),
11110 address: "inproc://mob/__mob_supervisor__".to_string(),
11111 pubkey: *supervisor_pubkey.as_bytes(),
11112 };
11113 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11114 supervisor: supervisor.clone(),
11115 epoch: 0,
11116 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11117 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11118 expected_address: runtime.advertised_address().unwrap(),
11119 bootstrap_token: "expected-token".into(),
11120 };
11121
11122 let (authorized, advertised_address) = validate_bind_request(
11123 &adapter,
11124 &session_id,
11125 &runtime,
11126 &bridge_sender_fact(&supervisor_pubkey.to_pubkey_string()),
11127 &payload,
11128 )
11129 .await
11130 .expect("bind should accept raw transport sender when payload carries pubkey");
11131 assert_eq!(authorized.peer_id.as_str(), supervisor.peer_id);
11132 assert_eq!(authorized.pubkey, supervisor.pubkey);
11133 assert_eq!(advertised_address, runtime.advertised_address().unwrap());
11134 }
11135
11136 #[tokio::test]
11137 async fn validate_bind_request_returns_runtime_advertised_address() {
11138 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
11139 PEER_ID_RECEIVER,
11140 &format!(
11141 "inproc://receiver-real?{SUPERVISOR_BRIDGE_BOOTSTRAP_TOKEN_PARAM}=expected-token"
11142 ),
11143 Some("expected-token"),
11144 ));
11145 let (adapter, session_id) = bind_material_adapter().await;
11146 let supervisor = supervisor_bridge_spec();
11147 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11148 supervisor: supervisor.clone(),
11149 epoch: 0,
11150 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11151 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11152 expected_address: "inproc://receiver-real".to_string(),
11153 bootstrap_token: "expected-token".into(),
11154 };
11155
11156 let (_, advertised_address) = validate_bind_request(
11157 &adapter,
11158 &session_id,
11159 &runtime,
11160 &bridge_sender_fact(&supervisor.peer_id),
11161 &payload,
11162 )
11163 .await
11164 .expect("bind should canonicalize to the callee's advertised address");
11165 assert_eq!(advertised_address, runtime.advertised_address().unwrap());
11166 }
11167
11168 #[tokio::test]
11169 async fn validate_bind_request_rejects_mismatched_expected_address() {
11170 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
11171 PEER_ID_RECEIVER,
11172 "inproc://receiver-real",
11173 Some("expected-token"),
11174 ));
11175 let (adapter, session_id) = bind_material_adapter().await;
11176 let supervisor = supervisor_bridge_spec();
11177 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11178 supervisor: supervisor.clone(),
11179 epoch: 0,
11180 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11181 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11182 expected_address: "inproc://receiver-stale".to_string(),
11183 bootstrap_token: "expected-token".into(),
11184 };
11185
11186 let (cause, error) = validate_bind_request(
11187 &adapter,
11188 &session_id,
11189 &runtime,
11190 &bridge_sender_fact(&supervisor.peer_id),
11191 &payload,
11192 )
11193 .await
11194 .expect_err("bind should reject mismatched expected addresses");
11195 assert_eq!(cause, BridgeRejectionCause::AddressMismatch);
11196 assert!(
11197 error.contains("bind address mismatch"),
11198 "bind rejection should explain the address mismatch, got: {error}"
11199 );
11200 }
11201
11202 #[tokio::test]
11203 async fn bridge_handler_rejects_unsupported_protocol_version_before_bind_validation() {
11204 let sent: Arc<tokio::sync::Mutex<Vec<CommsCommand>>> =
11205 Arc::new(tokio::sync::Mutex::new(Vec::new()));
11206 let runtime: Arc<dyn CommsRuntime> = Arc::new(CapturingRuntime {
11207 peer_id: PEER_ID_RECEIVER.to_string(),
11208 advertised_address: Some("inproc://receiver".to_string()),
11209 bootstrap_token: Some("expected-token".to_string()),
11210 inbox_notify: Arc::new(tokio::sync::Notify::new()),
11211 sent: sent.clone(),
11212 });
11213 let adapter = Arc::new(MeerkatMachine::ephemeral());
11214 let session_id = SessionId::new();
11215 adapter
11216 .register_session(session_id.clone())
11217 .await
11218 .expect("register session");
11219 let params = json!({
11220 "command": "bind_member",
11221 "supervisor": {
11222 "name": "mob/__mob_supervisor__",
11223 "peer_id": PEER_ID_SUPERVISOR,
11224 "address": "inproc://mob/__mob_supervisor__",
11225 "pubkey": vec![0u8; 32],
11226 },
11227 "epoch": 0,
11228 "protocol_version": 999,
11229 "expected_peer_id": PEER_ID_RECEIVER,
11230 "expected_address": "inproc://receiver",
11231 "bootstrap_token": "expected-token",
11232 });
11233 let interaction_id = InteractionId(Uuid::new_v4());
11234 let candidate = PeerInputCandidate {
11235 interaction: InboxInteraction {
11236 objective_id: None,
11237 sender_taint: None,
11238 id: interaction_id,
11239 from_route: None,
11240 from: PEER_ID_SUPERVISOR.to_string(),
11241 content: InteractionContent::Request {
11242 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
11243 params,
11244 blocks: None,
11245 },
11246 rendered_text: String::new(),
11247 handling_mode: HandlingMode::Queue,
11248 render_metadata: None,
11249 },
11250 ingress: bridge_sender_fact_with_id(interaction_id, PEER_ID_SUPERVISOR),
11251 lifecycle_peer: None,
11252 response_terminality: None,
11253 };
11254
11255 assert!(
11256 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await,
11257 "bridge handler must own malformed supervisor bridge commands"
11258 );
11259
11260 let (result, status) = sent
11261 .lock()
11262 .await
11263 .iter()
11264 .find_map(|cmd| match cmd {
11265 CommsCommand::PeerResponse { result, status, .. } => {
11266 Some((result.clone(), *status))
11267 }
11268 _ => None,
11269 })
11270 .expect("handler must send a typed rejection response");
11271 assert!(matches!(
11272 status,
11273 meerkat_core::interaction::ResponseStatus::Failed
11274 ));
11275 let reply: BridgeReply = serde_json::from_value(result).expect("typed bridge reply");
11276 match reply {
11277 BridgeReply::Rejected { cause, reason } => {
11278 assert_eq!(cause, BridgeRejectionCause::UnsupportedProtocolVersion);
11279 assert!(
11280 reason.contains("unsupported supervisor bridge protocol version"),
11281 "rejection should explain the typed version failure, got: {reason}"
11282 );
11283 }
11284 other => unreachable!("expected Rejected reply, got {other:?}"),
11285 }
11286 }
11287
11288 #[tokio::test]
11289 async fn validate_bind_request_rejects_invalid_supervisor_peer_name() {
11290 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
11291 PEER_ID_RECEIVER,
11292 "inproc://receiver",
11293 Some("expected-token"),
11294 ));
11295 let (adapter, session_id) = bind_material_adapter().await;
11296 let mut supervisor = supervisor_bridge_spec();
11297 supervisor.name = String::new();
11298 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11299 supervisor,
11300 epoch: 0,
11301 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11302 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11303 expected_address: runtime.advertised_address().unwrap(),
11304 bootstrap_token: "expected-token".into(),
11305 };
11306
11307 let (cause, error) = validate_bind_request(
11308 &adapter,
11309 &session_id,
11310 &runtime,
11311 &bridge_sender_fact(&payload.supervisor.peer_id),
11312 &payload,
11313 )
11314 .await
11315 .expect_err("bind should reject invalid supervisor peer names");
11316 assert_eq!(cause, BridgeRejectionCause::InvalidSupervisorSpec);
11317 assert!(
11318 error.contains("invalid supervisor peer spec"),
11319 "bind rejection should explain invalid supervisor identity, got: {error}"
11320 );
11321 }
11322
11323 fn sample_bind_payload() -> meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11324 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11325 supervisor: supervisor_bridge_spec(),
11326 epoch: 1,
11327 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11328 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11329 expected_address: "inproc://receiver".to_string(),
11330 bootstrap_token: "expected-token".into(),
11331 }
11332 }
11333
11334 async fn bind_material_adapter() -> (Arc<MeerkatMachine>, SessionId) {
11339 let adapter = Arc::new(MeerkatMachine::ephemeral());
11340 let session_id = SessionId::new();
11341 adapter
11342 .register_session(session_id.clone())
11343 .await
11344 .expect("register session");
11345 (adapter, session_id)
11346 }
11347
11348 #[tokio::test]
11354 async fn machine_owns_material_bind_admission_verdict_and_precedence() {
11355 use crate::meerkat_machine::dsl::SupervisorBindMaterialAdmissionKind as Verdict;
11356
11357 let (adapter, session_id) = bind_material_adapter().await;
11358
11359 let verdict = |address, sender, peer_id, token| {
11360 let adapter = adapter.clone();
11361 let session_id = session_id.clone();
11362 async move {
11363 adapter
11364 .resolve_supervisor_bind_material_admission(
11365 &session_id,
11366 address,
11367 sender,
11368 peer_id,
11369 token,
11370 )
11371 .await
11372 .expect("machine must emit a material bind-admission verdict")
11373 }
11374 };
11375
11376 assert_eq!(verdict(true, true, true, true).await, Verdict::Accept);
11378
11379 assert_eq!(
11381 verdict(false, true, true, true).await,
11382 Verdict::AddressMismatch
11383 );
11384 assert_eq!(
11385 verdict(true, false, true, true).await,
11386 Verdict::SenderMismatch
11387 );
11388 assert_eq!(
11389 verdict(true, true, false, true).await,
11390 Verdict::InvalidPeerSpec
11391 );
11392 assert_eq!(
11393 verdict(true, true, true, false).await,
11394 Verdict::InvalidBootstrapToken
11395 );
11396
11397 assert_eq!(
11399 verdict(false, false, false, false).await,
11400 Verdict::AddressMismatch
11401 );
11402 assert_eq!(
11404 verdict(true, false, false, false).await,
11405 Verdict::SenderMismatch
11406 );
11407 assert_eq!(
11409 verdict(true, true, false, false).await,
11410 Verdict::InvalidPeerSpec
11411 );
11412 }
11413
11414 #[tokio::test]
11415 async fn bridge_request_without_complete_peer_authority_fails_before_dispatch() {
11416 for install_peer_handle in [false, true] {
11417 let payload = sample_bind_payload();
11418 let sender = payload.supervisor.peer_id.clone();
11419 let mut runtime_impl = bootstrap_runtime(
11420 PEER_ID_RECEIVER,
11421 "inproc://receiver",
11422 Some("expected-token"),
11423 );
11424 let peer_handle = Arc::new(CountingPeerInteractionHandle::default());
11425 if install_peer_handle {
11426 let peer_handle: Arc<dyn meerkat_core::handles::PeerInteractionHandle> =
11427 peer_handle.clone();
11428 runtime_impl.peer_handle = Some(peer_handle);
11429 } else {
11430 runtime_impl.peer_handle = None;
11431 }
11432 runtime_impl.peer_request_response_handle = None;
11433 let completed_count = runtime_impl.completed_count.clone();
11434 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
11435 let adapter = Arc::new(MeerkatMachine::ephemeral());
11436 let session_id = SessionId::new();
11437 adapter
11438 .register_session(session_id.clone())
11439 .await
11440 .expect("register session");
11441 let candidate = bridge_candidate(&sender, &BridgeCommand::BindMember(payload));
11442
11443 assert!(
11444 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate)
11445 .await,
11446 "bridge handler must own bridge requests even when rejecting authority"
11447 );
11448 assert!(
11449 matches!(
11450 adapter.supervisor_binding(&session_id).await,
11451 SupervisorBinding::Unbound
11452 ),
11453 "bridge request must not mutate supervisor binding without complete authority"
11454 );
11455 assert_eq!(
11456 peer_handle.request_received_count(),
11457 0,
11458 "peer-only authority must not record PeerRequestReceived through bridge dispatch"
11459 );
11460 assert_eq!(
11461 completed_count.load(std::sync::atomic::Ordering::SeqCst),
11462 1,
11463 "bridge request rejected at the authority boundary should be marked complete"
11464 );
11465 }
11466 }
11467
11468 #[tokio::test]
11469 async fn bridge_request_rejected_by_peer_authority_fails_before_dispatch() {
11470 let payload = sample_bind_payload();
11471 let sender = payload.supervisor.peer_id.clone();
11472 let mut runtime_impl = bootstrap_runtime(
11473 PEER_ID_RECEIVER,
11474 "inproc://receiver",
11475 Some("expected-token"),
11476 );
11477 let peer_handle = Arc::new(CountingPeerInteractionHandle::rejecting_request_received());
11478 let peer_handle: Arc<dyn meerkat_core::handles::PeerInteractionHandle> = peer_handle;
11479 runtime_impl.peer_handle = Some(peer_handle.clone());
11480 runtime_impl.peer_request_response_handle = Some(peer_handle);
11481 let completed_count = runtime_impl.completed_count.clone();
11482 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
11483 let adapter = Arc::new(MeerkatMachine::ephemeral());
11484 let session_id = SessionId::new();
11485 adapter
11486 .register_session(session_id.clone())
11487 .await
11488 .expect("register session");
11489 let candidate = bridge_candidate(&sender, &BridgeCommand::BindMember(payload));
11490
11491 assert!(
11492 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await,
11493 "bridge handler must own bridge requests even when rejecting authority"
11494 );
11495 assert!(
11496 matches!(
11497 adapter.supervisor_binding(&session_id).await,
11498 SupervisorBinding::Unbound
11499 ),
11500 "bridge request must not mutate supervisor binding when PeerRequestReceived is rejected"
11501 );
11502 assert_eq!(
11503 completed_count.load(std::sync::atomic::Ordering::SeqCst),
11504 1,
11505 "bridge request rejected by peer authority should be marked complete"
11506 );
11507 }
11508
11509 async fn bind_sample_supervisor(
11510 adapter: &MeerkatMachine,
11511 session_id: &SessionId,
11512 payload: &meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload,
11513 ) {
11514 let spec = TrustedPeerDescriptor::try_from(payload.supervisor.clone())
11515 .expect("valid supervisor spec");
11516 adapter
11517 .stage_supervisor_bind(
11518 session_id,
11519 spec.name.to_string(),
11520 spec.peer_id.as_str(),
11521 spec.address.to_string(),
11522 signing_public_key_for_descriptor(&spec),
11523 payload.epoch,
11524 )
11525 .await
11526 .expect("supervisor bound");
11527 }
11528
11529 async fn stage_test_supervisor_rotation(
11530 adapter: &MeerkatMachine,
11531 session_id: &SessionId,
11532 next: crate::meerkat_machine::GeneratedSupervisorBinding,
11533 ) {
11534 let SupervisorBinding::Bound {
11535 peer_id: previous_peer_id,
11536 signing_public_key: previous_signing_public_key,
11537 epoch: previous_epoch,
11538 ..
11539 } = adapter.supervisor_binding(session_id).await
11540 else {
11541 panic!("test rotation requires a bound previous supervisor");
11542 };
11543 let operation_id = uuid::Uuid::new_v4().to_string();
11544 let submission = adapter
11545 .submit_supervisor_rotation(
11546 session_id,
11547 GeneratedSupervisorRotationSubmit {
11548 operation_id: operation_id.clone(),
11549 next: next.clone(),
11550 preflight_rejection: None,
11551 sender_peer_id: Some(previous_peer_id.clone()),
11552 sender_signing_public_key: Some(previous_signing_public_key),
11553 },
11554 )
11555 .await
11556 .expect("stage test rotation submit");
11557 assert!(matches!(submission, SupervisorRotationSubmission::New(_)));
11558 adapter
11559 .stage_supervisor_rotation_previous_revoked(
11560 session_id,
11561 operation_id.clone(),
11562 previous_peer_id,
11563 previous_epoch,
11564 )
11565 .await
11566 .expect("stage test rotation previous-revoked checkpoint");
11567 adapter
11568 .stage_supervisor_rotation_next_published(
11569 session_id,
11570 operation_id,
11571 next.peer_id,
11572 next.epoch,
11573 )
11574 .await
11575 .expect("stage test rotation bound checkpoint");
11576 }
11577
11578 #[tokio::test]
11579 async fn generated_bind_admission_allows_bootstrap_when_unbound() {
11580 let payload = sample_bind_payload();
11581 let supervisor = bridge_peer_identity(&payload.supervisor, "test").expect("valid peer");
11582 let adapter = MeerkatMachine::ephemeral();
11583 let session_id = SessionId::new();
11584 adapter
11585 .register_session(session_id.clone())
11586 .await
11587 .expect("register session");
11588 let sender = bridge_sender_fact(&payload.supervisor.peer_id);
11589 let admission = adapter
11590 .resolve_supervisor_bind_admission(
11591 &session_id,
11592 supervisor.peer_id.as_str().clone(),
11593 payload.epoch,
11594 generated_sender_peer_id(&sender),
11595 )
11596 .await
11597 .expect("generated bind admission");
11598 assert!(matches!(admission, SupervisorBindAdmission::Bootstrap));
11599 }
11600
11601 #[tokio::test]
11602 async fn generated_bind_admission_rejects_different_supervisor_takeover() {
11603 let current_payload = sample_bind_payload();
11607 let adapter = MeerkatMachine::ephemeral();
11608 let session_id = SessionId::new();
11609 adapter
11610 .register_session(session_id.clone())
11611 .await
11612 .expect("register session");
11613 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
11614 let mut takeover = sample_bind_payload();
11615 takeover.supervisor = old_supervisor_bridge_spec();
11616 let supervisor = bridge_peer_identity(&takeover.supervisor, "test").expect("valid peer");
11617 let sender = bridge_sender_fact(&takeover.supervisor.peer_id);
11618 let admission = adapter
11619 .resolve_supervisor_bind_admission(
11620 &session_id,
11621 supervisor.peer_id.as_str().clone(),
11622 takeover.epoch,
11623 generated_sender_peer_id(&sender),
11624 )
11625 .await
11626 .expect("generated bind admission");
11627 assert!(
11628 matches!(
11629 admission,
11630 SupervisorBindAdmission::Rejected(
11631 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::AlreadyBound
11632 )
11633 ),
11634 "different-supervisor bind must reject as already bound: {admission:?}"
11635 );
11636 }
11637
11638 #[tokio::test]
11639 async fn generated_bind_admission_rejects_lower_epoch_replay() {
11640 let current_payload = sample_bind_payload();
11644 let adapter = MeerkatMachine::ephemeral();
11645 let session_id = SessionId::new();
11646 adapter
11647 .register_session(session_id.clone())
11648 .await
11649 .expect("register session");
11650 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
11651 let mut replay = sample_bind_payload();
11652 replay.epoch = current_payload.epoch - 1;
11653 let supervisor = bridge_peer_identity(&replay.supervisor, "test").expect("valid peer");
11654 let sender = bridge_sender_fact(&replay.supervisor.peer_id);
11655 let admission = adapter
11656 .resolve_supervisor_bind_admission(
11657 &session_id,
11658 supervisor.peer_id.as_str().clone(),
11659 replay.epoch,
11660 generated_sender_peer_id(&sender),
11661 )
11662 .await
11663 .expect("generated bind admission");
11664 assert!(
11665 matches!(
11666 admission,
11667 SupervisorBindAdmission::Rejected(
11668 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::AlreadyBound
11669 )
11670 ),
11671 "lower-epoch bind replay must reject as already bound: {admission:?}"
11672 );
11673 }
11674
11675 #[tokio::test]
11676 async fn generated_bind_admission_rejects_higher_epoch_same_supervisor_rebind() {
11677 let current_payload = sample_bind_payload();
11681 let adapter = MeerkatMachine::ephemeral();
11682 let session_id = SessionId::new();
11683 adapter
11684 .register_session(session_id.clone())
11685 .await
11686 .expect("register session");
11687 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
11688 let mut advance = sample_bind_payload();
11689 advance.epoch = current_payload.epoch + 5;
11690 let supervisor = bridge_peer_identity(&advance.supervisor, "test").expect("valid peer");
11691 let sender = bridge_sender_fact(&advance.supervisor.peer_id);
11692 let admission = adapter
11693 .resolve_supervisor_bind_admission(
11694 &session_id,
11695 supervisor.peer_id.as_str().clone(),
11696 advance.epoch,
11697 generated_sender_peer_id(&sender),
11698 )
11699 .await
11700 .expect("generated bind admission");
11701 assert!(
11702 matches!(
11703 admission,
11704 SupervisorBindAdmission::Rejected(
11705 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::AlreadyBound
11706 )
11707 ),
11708 "higher-epoch rebind must reject as already bound: {admission:?}"
11709 );
11710 }
11711
11712 #[tokio::test]
11713 async fn generated_bind_admission_rejects_spoofed_sender() {
11714 let current_payload = sample_bind_payload();
11718 let adapter = MeerkatMachine::ephemeral();
11719 let session_id = SessionId::new();
11720 adapter
11721 .register_session(session_id.clone())
11722 .await
11723 .expect("register session");
11724 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
11725 let retry = sample_bind_payload();
11726 let supervisor = bridge_peer_identity(&retry.supervisor, "test").expect("valid peer");
11727 let attacker_peer_id = PeerId::new().as_str();
11728 let sender = bridge_sender_fact(&attacker_peer_id);
11729 let admission = adapter
11730 .resolve_supervisor_bind_admission(
11731 &session_id,
11732 supervisor.peer_id.as_str().clone(),
11733 retry.epoch,
11734 generated_sender_peer_id(&sender),
11735 )
11736 .await
11737 .expect("generated bind admission");
11738 assert!(
11739 matches!(
11740 admission,
11741 SupervisorBindAdmission::Rejected(
11742 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::SenderMismatch
11743 )
11744 ),
11745 "bind from unauthorized sender must reject as sender mismatch: {admission:?}"
11746 );
11747 }
11748
11749 #[tokio::test]
11750 async fn generated_bind_admission_rejects_same_display_name_different_canonical_sender() {
11751 let current_payload = sample_bind_payload();
11752 let adapter = MeerkatMachine::ephemeral();
11753 let session_id = SessionId::new();
11754 adapter
11755 .register_session(session_id.clone())
11756 .await
11757 .expect("register session");
11758 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
11759 let retry = sample_bind_payload();
11760 let supervisor = bridge_peer_identity(&retry.supervisor, "test").expect("valid peer");
11761 let attacker_peer_id =
11762 PeerId::parse(PEER_ID_OLD_SUPERVISOR).expect("valid attacker peer id");
11763 let sender = bridge_sender_fact_with_display(attacker_peer_id, &retry.supervisor.name);
11764
11765 let admission = adapter
11766 .resolve_supervisor_bind_admission(
11767 &session_id,
11768 supervisor.peer_id.as_str().clone(),
11769 retry.epoch,
11770 generated_sender_peer_id(&sender),
11771 )
11772 .await
11773 .expect("generated bind admission");
11774 assert!(
11775 matches!(
11776 admission,
11777 SupervisorBindAdmission::Rejected(
11778 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::SenderMismatch
11779 )
11780 ),
11781 "same display name with different canonical sender must reject: {admission:?}"
11782 );
11783 }
11784
11785 #[tokio::test]
11786 async fn generated_bind_admission_idempotently_acks_retry_from_current_supervisor() {
11787 let current_payload = sample_bind_payload();
11791 let adapter = MeerkatMachine::ephemeral();
11792 let session_id = SessionId::new();
11793 adapter
11794 .register_session(session_id.clone())
11795 .await
11796 .expect("register session");
11797 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
11798 let retry = sample_bind_payload();
11799 let supervisor = bridge_peer_identity(&retry.supervisor, "test").expect("valid peer");
11800 let sender = bridge_sender_fact(&retry.supervisor.peer_id);
11801 let admission = adapter
11802 .resolve_supervisor_bind_admission(
11803 &session_id,
11804 supervisor.peer_id.as_str().clone(),
11805 retry.epoch,
11806 generated_sender_peer_id(&sender),
11807 )
11808 .await
11809 .expect("generated bind admission");
11810 assert!(matches!(admission, SupervisorBindAdmission::IdempotentAck));
11811 }
11812
11813 #[tokio::test]
11814 async fn bind_member_handler_rejects_rebind_after_supervisor_bound() {
11815 let runtime: Arc<dyn CommsRuntime> = Arc::new(
11819 meerkat_comms::CommsRuntime::inproc_only("bind-rebind-receiver")
11820 .expect("receiver runtime"),
11821 );
11822 let supervisor_runtime = Arc::new(
11823 meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor__")
11824 .expect("supervisor runtime"),
11825 );
11826 let adapter = Arc::new(MeerkatMachine::ephemeral());
11827 let session_id = SessionId::new();
11828 adapter
11829 .register_session(session_id.clone())
11830 .await
11831 .expect("register session");
11832 let current_supervisor =
11833 trusted_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
11834 adapter
11835 .stage_supervisor_bind(
11836 &session_id,
11837 current_supervisor.name.to_string(),
11838 current_supervisor.peer_id.as_str(),
11839 current_supervisor.address.to_string(),
11840 signing_public_key_for_descriptor(¤t_supervisor),
11841 1,
11842 )
11843 .await
11844 .expect("initial bind must succeed");
11845 let adversary_supervisor = old_supervisor_bridge_spec();
11846 let adversary_peer_id = adversary_supervisor.peer_id.clone();
11847 let command = BridgeCommand::BindMember(
11848 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11849 supervisor: adversary_supervisor,
11850 epoch: 2,
11851 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11852 expected_peer_id: runtime
11853 .peer_id()
11854 .map(|peer_id| peer_id.as_str())
11855 .unwrap_or_else(|| PEER_ID_RECEIVER.to_string()),
11856 expected_address: runtime
11857 .advertised_address()
11858 .unwrap_or_else(|| "inproc://bind-rebind-receiver".to_string()),
11859 bootstrap_token: runtime.bridge_bootstrap_token().unwrap_or_default().into(),
11861 },
11862 );
11863 let candidate = bridge_candidate(&adversary_peer_id, &command);
11864
11865 assert!(
11866 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate,)
11867 .await,
11868 "bridge handler must own the BindMember command"
11869 );
11870 let binding = adapter.supervisor_binding(&session_id).await;
11871 let SupervisorBinding::Bound { peer_id, epoch, .. } = binding else {
11872 panic!("supervisor binding must be preserved as Bound");
11873 };
11874 assert_eq!(
11875 peer_id,
11876 current_supervisor.peer_id.as_str(),
11877 "rebind attempt must not replace the authorized supervisor"
11878 );
11879 assert_eq!(
11880 epoch, 1,
11881 "rebind attempt must not advance the authorized epoch"
11882 );
11883 }
11884
11885 #[tokio::test]
11894 async fn bind_member_handler_rebind_reply_is_typed_already_bound() {
11895 let sent: Arc<tokio::sync::Mutex<Vec<CommsCommand>>> =
11896 Arc::new(tokio::sync::Mutex::new(Vec::new()));
11897 let runtime: Arc<dyn CommsRuntime> = Arc::new(CapturingRuntime {
11898 peer_id: PEER_ID_RECEIVER.to_string(),
11899 advertised_address: Some("inproc://receiver".to_string()),
11900 bootstrap_token: Some("expected-token".to_string()),
11901 inbox_notify: Arc::new(tokio::sync::Notify::new()),
11902 sent: sent.clone(),
11903 });
11904 let adapter = Arc::new(MeerkatMachine::ephemeral());
11905 let session_id = SessionId::new();
11906 adapter
11907 .register_session(session_id.clone())
11908 .await
11909 .expect("register session");
11910 let current = trusted_supervisor_descriptor(0xbb);
11911 adapter
11912 .stage_supervisor_bind(
11913 &session_id,
11914 current.name.to_string(),
11915 current.peer_id.as_str(),
11916 current.address.to_string(),
11917 signing_public_key_for_descriptor(¤t),
11918 1,
11919 )
11920 .await
11921 .expect("initial bind must succeed");
11922
11923 let adversary = old_supervisor_bridge_spec();
11927 let adversary_peer_id = adversary.peer_id.clone();
11928 let command = BridgeCommand::BindMember(
11929 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11930 supervisor: adversary,
11931 epoch: 2,
11932 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11933 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11934 expected_address: "inproc://receiver".to_string(),
11935 bootstrap_token: "expected-token".into(),
11936 },
11937 );
11938 let candidate = bridge_candidate(&adversary_peer_id, &command);
11939
11940 assert!(
11941 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate,)
11942 .await,
11943 "bridge handler must own the BindMember command"
11944 );
11945
11946 let binding = adapter.supervisor_binding(&session_id).await;
11948 let SupervisorBinding::Bound { peer_id, .. } = binding else {
11949 panic!("binding preserved");
11950 };
11951 assert_eq!(peer_id, current.peer_id.as_str());
11952
11953 let (result, status) = sent
11955 .lock()
11956 .await
11957 .iter()
11958 .find_map(|cmd| match cmd {
11959 CommsCommand::PeerResponse { result, status, .. } => {
11960 Some((result.clone(), *status))
11961 }
11962 _ => None,
11963 })
11964 .expect("handler must send a PeerResponse for the rejection");
11965 assert!(
11966 matches!(status, meerkat_core::interaction::ResponseStatus::Failed),
11967 "rebind rejection must surface as Failed status"
11968 );
11969 let reply: BridgeReply = serde_json::from_value(result).expect("typed bridge reply");
11970 match reply {
11971 BridgeReply::Rejected { cause, .. } => {
11972 assert_eq!(
11973 cause,
11974 BridgeRejectionCause::AlreadyBound,
11975 "different-supervisor rebind must be rejected as AlreadyBound",
11976 );
11977 }
11978 other => unreachable!("expected Rejected reply, got {other:?}"),
11979 }
11980 }
11981
11982 struct CapturingRuntime {
11987 peer_id: String,
11988 advertised_address: Option<String>,
11989 bootstrap_token: Option<String>,
11990 inbox_notify: Arc<tokio::sync::Notify>,
11991 sent: Arc<tokio::sync::Mutex<Vec<CommsCommand>>>,
11992 }
11993
11994 #[async_trait::async_trait]
11995 impl CommsRuntime for CapturingRuntime {
11996 fn peer_id(&self) -> Option<PeerId> {
11997 PeerId::parse(&self.peer_id).ok()
11998 }
11999
12000 fn public_key(&self) -> Option<String> {
12001 test_public_key_for_peer_id(&self.peer_id)
12002 }
12003
12004 fn public_key_bytes(&self) -> Option<[u8; 32]> {
12005 test_pubkey_bytes_for_peer_id(&self.peer_id)
12006 }
12007
12008 fn comms_name(&self) -> Option<String> {
12009 self.advertised_address
12010 .as_deref()
12011 .map(test_comms_name_from_address)
12012 }
12013
12014 fn advertised_address(&self) -> Option<String> {
12015 self.advertised_address.clone()
12016 }
12017
12018 fn bridge_bootstrap_token(&self) -> Option<String> {
12019 self.bootstrap_token.clone()
12020 }
12021
12022 async fn drain_messages(&self) -> Vec<String> {
12023 Vec::new()
12024 }
12025
12026 fn inbox_notify(&self) -> Arc<tokio::sync::Notify> {
12027 self.inbox_notify.clone()
12028 }
12029
12030 fn peer_request_response_authority_handle(
12031 &self,
12032 ) -> Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>> {
12033 Some(Arc::new(CountingPeerInteractionHandle::default()))
12034 }
12035
12036 async fn apply_trust_mutation(
12037 &self,
12038 mutation: CommsTrustMutation,
12039 ) -> Result<CommsTrustMutationResult, SendError> {
12040 match mutation {
12041 CommsTrustMutation::AddPrivateTrustedPeer { peer, authority } => {
12042 authority
12043 .validate_private_add(self.peer_id(), &peer)
12044 .map_err(SendError::Validation)?;
12045 Ok(CommsTrustMutationResult::Added { created: true })
12046 }
12047 CommsTrustMutation::RemovePrivateTrustedPeer { peer_id, authority } => {
12048 let parsed_peer_id = PeerId::parse(&peer_id)
12049 .map_err(|error| SendError::Validation(error.to_string()))?;
12050 authority
12051 .validate_private_remove(self.peer_id(), parsed_peer_id)
12052 .map_err(SendError::Validation)?;
12053 Ok(CommsTrustMutationResult::Removed { removed: true })
12054 }
12055 _ => Err(SendError::Unsupported(
12056 "test runtime only supports generated private trust".into(),
12057 )),
12058 }
12059 }
12060
12061 async fn send(
12062 &self,
12063 cmd: CommsCommand,
12064 ) -> Result<meerkat_core::comms::SendReceipt, SendError> {
12065 let receipt = match &cmd {
12066 CommsCommand::PeerResponse { in_reply_to, .. } => {
12067 meerkat_core::comms::SendReceipt::PeerResponseSent {
12068 envelope_id: Uuid::new_v4(),
12069 in_reply_to: *in_reply_to,
12070 }
12071 }
12072 _ => meerkat_core::comms::SendReceipt::PeerMessageSent {
12073 envelope_id: Uuid::new_v4(),
12074 delivery: meerkat_core::comms::PeerDeliveryOutcome::Acked,
12075 },
12076 };
12077 self.sent.lock().await.push(cmd);
12078 Ok(receipt)
12079 }
12080 }
12081
12082 #[tokio::test]
12083 async fn bind_member_handler_response_reports_canonical_protocol_versions() {
12084 let sent: Arc<tokio::sync::Mutex<Vec<CommsCommand>>> =
12085 Arc::new(tokio::sync::Mutex::new(Vec::new()));
12086 let runtime: Arc<dyn CommsRuntime> = Arc::new(CapturingRuntime {
12087 peer_id: PEER_ID_RECEIVER.to_string(),
12088 advertised_address: Some("inproc://receiver".to_string()),
12089 bootstrap_token: Some("expected-token".to_string()),
12090 inbox_notify: Arc::new(tokio::sync::Notify::new()),
12091 sent: sent.clone(),
12092 });
12093 let adapter = Arc::new(MeerkatMachine::ephemeral());
12094 let session_id = SessionId::new();
12095 adapter
12096 .register_session(session_id.clone())
12097 .await
12098 .expect("register session");
12099 let supervisor = supervisor_bridge_spec();
12100 let command = BridgeCommand::BindMember(
12101 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
12102 supervisor: supervisor.clone(),
12103 epoch: 0,
12104 protocol_version: supervisor_bridge_current_protocol_version(),
12105 expected_peer_id: PEER_ID_RECEIVER.to_string(),
12106 expected_address: "inproc://receiver".to_string(),
12107 bootstrap_token: "expected-token".into(),
12108 },
12109 );
12110 let candidate = bridge_candidate(&supervisor.peer_id, &command);
12111
12112 assert!(
12113 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate,)
12114 .await,
12115 "bridge handler must own the BindMember command"
12116 );
12117
12118 let (result, status) = sent
12119 .lock()
12120 .await
12121 .iter()
12122 .find_map(|cmd| match cmd {
12123 CommsCommand::PeerResponse { result, status, .. } => {
12124 Some((result.clone(), *status))
12125 }
12126 _ => None,
12127 })
12128 .expect("handler must send a bind response");
12129 assert!(matches!(
12130 status,
12131 meerkat_core::interaction::ResponseStatus::Completed
12132 ));
12133 let reply: BridgeReply = serde_json::from_value(result).expect("typed bridge reply");
12134 let BridgeReply::BindMember(response) = reply else {
12135 panic!("expected bind response");
12136 };
12137 assert_eq!(
12138 response.capabilities.current_protocol_version,
12139 supervisor_bridge_current_protocol_version()
12140 );
12141 assert_eq!(
12142 response.capabilities.default_protocol_version,
12143 supervisor_bridge_default_protocol_version()
12144 );
12145 assert_eq!(
12146 response.capabilities.supported_protocol_versions,
12147 supervisor_bridge_supported_protocol_versions()
12148 );
12149 assert!(
12150 response.capabilities.hard_cancel_member,
12151 "bind response advertises the phase-6 hard-cancel arm"
12152 );
12153 assert!(
12154 response.capabilities.tracked_input_cancel,
12155 "bind response advertises exact durable tracked-input cancellation"
12156 );
12157 }
12158
12159 #[tokio::test]
12160 async fn idempotent_ack_invariant_rejects_without_echoing_attacker_fields() {
12161 let sent: Arc<tokio::sync::Mutex<Vec<CommsCommand>>> =
12168 Arc::new(tokio::sync::Mutex::new(Vec::new()));
12169 let runtime: Arc<dyn CommsRuntime> = Arc::new(CapturingRuntime {
12170 peer_id: PEER_ID_RECEIVER.to_string(),
12171 advertised_address: None,
12174 bootstrap_token: Some("expected-token".to_string()),
12175 inbox_notify: Arc::new(tokio::sync::Notify::new()),
12176 sent: sent.clone(),
12177 });
12178 let adapter = Arc::new(MeerkatMachine::ephemeral());
12179 let session_id = SessionId::new();
12180 adapter
12181 .register_session(session_id.clone())
12182 .await
12183 .expect("register session");
12184 let authorized = trusted_supervisor_descriptor(0xbb);
12185 adapter
12186 .stage_supervisor_bind(
12187 &session_id,
12188 authorized.name.to_string(),
12189 authorized.peer_id.as_str(),
12190 authorized.address.to_string(),
12191 signing_public_key_for_descriptor(&authorized),
12192 7,
12193 )
12194 .await
12195 .expect("pre-bind supervisor");
12196 let attacker_address = "inproc://ATTACKER-ADDRESS-DO-NOT-ECHO".to_string();
12199 let attacker_peer_id = format!("{}-ATTACKER-PEER-ID-DO-NOT-ECHO", PeerId::new().as_str());
12200 let command = BridgeCommand::BindMember(
12201 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
12202 supervisor: BridgePeerSpec::from(authorized.clone()),
12206 epoch: 7,
12207 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12208 expected_peer_id: attacker_peer_id.clone(),
12209 expected_address: attacker_address.clone(),
12210 bootstrap_token: "expected-token".into(),
12211 },
12212 );
12213 let candidate = bridge_candidate(&authorized.peer_id.as_str(), &command);
12214
12215 assert!(
12216 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate,)
12217 .await,
12218 "bridge handler must own the BindMember command"
12219 );
12220
12221 let binding = adapter.supervisor_binding(&session_id).await;
12224 let SupervisorBinding::Bound {
12225 peer_id: stored_peer_id,
12226 epoch: stored_epoch,
12227 ..
12228 } = binding
12229 else {
12230 panic!("binding must survive invariant failure");
12231 };
12232 assert_eq!(stored_peer_id, authorized.peer_id.as_str());
12233 assert_eq!(stored_epoch, 7);
12234
12235 let sent_commands = sent.lock().await.clone();
12238 let (result, status) = sent_commands
12239 .into_iter()
12240 .find_map(|cmd| match cmd {
12241 CommsCommand::PeerResponse { result, status, .. } => Some((result, status)),
12242 _ => None,
12243 })
12244 .expect("handler must send a PeerResponse for the invariant violation");
12245 assert!(
12246 matches!(status, meerkat_core::interaction::ResponseStatus::Failed),
12247 "invariant violation must surface as Failed status"
12248 );
12249 let reply: BridgeReply =
12250 serde_json::from_value(result.clone()).expect("typed bridge reply");
12251 assert!(
12252 matches!(reply, BridgeReply::Rejected { .. }),
12253 "expected Rejected reply for invariant violation, got: {reply:?}"
12254 );
12255 if let BridgeReply::Rejected { cause, reason } = reply {
12256 assert_eq!(cause, BridgeRejectionCause::Internal);
12257 assert!(
12258 !reason.contains(&attacker_address),
12259 "rejection reason must not echo attacker-supplied address: {reason}"
12260 );
12261 assert!(
12262 !reason.contains(&attacker_peer_id),
12263 "rejection reason must not echo attacker-supplied peer_id: {reason}"
12264 );
12265 }
12266 let raw = serde_json::to_string(&result).expect("serialize reply for attacker-field check");
12267 assert!(
12268 !raw.contains(&attacker_address),
12269 "reply payload must not contain attacker-supplied address: {raw}"
12270 );
12271 assert!(
12272 !raw.contains(&attacker_peer_id),
12273 "reply payload must not contain attacker-supplied peer_id: {raw}"
12274 );
12275 }
12276
12277 #[tokio::test]
12278 async fn generated_authorize_admission_rejects_initial_claim_without_bind() {
12279 let payload = BridgeSupervisorPayload {
12280 supervisor: supervisor_bridge_spec(),
12281 epoch: 0,
12282 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12283 };
12284 let supervisor = bridge_peer_identity(&payload.supervisor, "test").expect("valid peer");
12285 let adapter = MeerkatMachine::ephemeral();
12286 let session_id = SessionId::new();
12287 adapter
12288 .register_session(session_id.clone())
12289 .await
12290 .expect("register session");
12291 let sender = bridge_sender_fact(&payload.supervisor.peer_id);
12292
12293 assert!(
12294 matches!(
12295 adapter
12296 .resolve_supervisor_authorize_admission(
12297 &session_id,
12298 GeneratedSupervisorBinding {
12299 name: supervisor.name.to_string(),
12300 peer_id: supervisor.peer_id.as_str().clone(),
12301 address: supervisor.address.to_string(),
12302 signing_public_key: encode_supervisor_signing_public_key(
12303 *supervisor.pubkey.as_bytes(),
12304 ),
12305 epoch: payload.epoch,
12306 },
12307 generated_sender_peer_id(&sender),
12308 generated_sender_signing_public_key(&sender),
12309 )
12310 .await
12311 .expect("generated authorize admission"),
12312 SupervisorAuthorizeAdmission::Rejected(
12313 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::NotBound
12314 )
12315 ),
12316 "first supervisor claim must go through bind_member"
12317 );
12318 }
12319
12320 #[tokio::test]
12321 async fn generated_authorize_admission_rejects_same_display_name_different_canonical_sender() {
12322 let current_payload = sample_bind_payload();
12323 let payload = BridgeSupervisorPayload {
12324 supervisor: current_supervisor_bridge_spec(),
12325 epoch: current_payload.epoch + 1,
12326 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12327 };
12328 let supervisor = bridge_peer_identity(&payload.supervisor, "test").expect("valid peer");
12329 let adapter = MeerkatMachine::ephemeral();
12330 let session_id = SessionId::new();
12331 adapter
12332 .register_session(session_id.clone())
12333 .await
12334 .expect("register session");
12335 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
12336 let attacker_peer_id =
12337 PeerId::parse(PEER_ID_OLD_SUPERVISOR).expect("valid attacker peer id");
12338 let sender =
12339 bridge_sender_fact_with_display(attacker_peer_id, ¤t_payload.supervisor.name);
12340
12341 assert!(
12342 matches!(
12343 adapter
12344 .resolve_supervisor_authorize_admission(
12345 &session_id,
12346 GeneratedSupervisorBinding {
12347 name: supervisor.name.to_string(),
12348 peer_id: supervisor.peer_id.as_str().clone(),
12349 address: supervisor.address.to_string(),
12350 signing_public_key: encode_supervisor_signing_public_key(
12351 *supervisor.pubkey.as_bytes(),
12352 ),
12353 epoch: payload.epoch,
12354 },
12355 generated_sender_peer_id(&sender),
12356 generated_sender_signing_public_key(&sender),
12357 )
12358 .await
12359 .expect("generated authorize admission"),
12360 SupervisorAuthorizeAdmission::Rejected(
12361 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::SenderMismatch
12362 )
12363 ),
12364 "same display name with a different canonical sender must reject"
12365 );
12366 }
12367
12368 #[tokio::test]
12369 async fn supervisor_bridge_admission_rejects_same_display_name_different_canonical_sender() {
12370 let current_payload = sample_bind_payload();
12371 let payload = BridgeSupervisorPayload {
12372 supervisor: current_payload.supervisor.clone(),
12373 epoch: current_payload.epoch,
12374 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12375 };
12376 let attacker_peer_id =
12377 PeerId::parse(PEER_ID_OLD_SUPERVISOR).expect("valid attacker peer id");
12378 let sender =
12379 bridge_sender_fact_with_display(attacker_peer_id, ¤t_payload.supervisor.name);
12380
12381 let adapter = MeerkatMachine::ephemeral();
12382 let session_id = SessionId::new();
12383 adapter
12384 .register_session(session_id.clone())
12385 .await
12386 .expect("register session");
12387 let supervisor = TrustedPeerDescriptor::try_from(current_payload.supervisor.clone())
12388 .expect("valid supervisor spec");
12389 adapter
12390 .stage_supervisor_bind(
12391 &session_id,
12392 supervisor.name.to_string(),
12393 supervisor.peer_id.as_str(),
12394 supervisor.address.to_string(),
12395 signing_public_key_for_descriptor(&supervisor),
12396 current_payload.epoch,
12397 )
12398 .await
12399 .expect("supervisor bound");
12400
12401 let (cause, error) =
12402 resolve_authorized_supervisor(&adapter, &session_id, &sender, &payload)
12403 .await
12404 .expect_err("same display name with a different canonical sender must reject");
12405 assert_eq!(cause, BridgeRejectionCause::SenderMismatch);
12406 assert!(
12407 error.contains("MeerkatMachine"),
12408 "supervisor command rejection should explain the sender mismatch, got: {error}"
12409 );
12410 }
12411
12412 #[tokio::test]
12422 async fn validate_bind_request_rejects_empty_bootstrap_token_at_runtime() {
12423 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
12424 PEER_ID_RECEIVER,
12425 "inproc://receiver",
12426 Some(""),
12427 ));
12428 let (adapter, session_id) = bind_material_adapter().await;
12429 let supervisor = supervisor_bridge_spec();
12430 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
12431 supervisor: supervisor.clone(),
12432 epoch: 0,
12433 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12434 expected_peer_id: PEER_ID_RECEIVER.to_string(),
12435 expected_address: runtime.advertised_address().unwrap(),
12436 bootstrap_token: "whatever".into(),
12437 };
12438
12439 let (cause, _error) = validate_bind_request(
12440 &adapter,
12441 &session_id,
12442 &runtime,
12443 &bridge_sender_fact(&supervisor.peer_id),
12444 &payload,
12445 )
12446 .await
12447 .expect_err("runtime with empty token must refuse to validate");
12448 assert_eq!(cause, BridgeRejectionCause::InvalidBootstrapToken);
12449 }
12450
12451 #[tokio::test]
12452 async fn validate_bind_request_rejects_query_string_bootstrap_without_typed_runtime_token() {
12453 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
12454 PEER_ID_RECEIVER,
12455 &format!("inproc://receiver?{SUPERVISOR_BRIDGE_BOOTSTRAP_TOKEN_PARAM}=expected-token"),
12456 None,
12457 ));
12458 let (adapter, session_id) = bind_material_adapter().await;
12459 let supervisor = supervisor_bridge_spec();
12460 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
12461 supervisor: supervisor.clone(),
12462 epoch: 0,
12463 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12464 expected_peer_id: PEER_ID_RECEIVER.to_string(),
12465 expected_address: "inproc://receiver".to_string(),
12466 bootstrap_token: "expected-token".into(),
12467 };
12468
12469 let (cause, error) = validate_bind_request(
12470 &adapter,
12471 &session_id,
12472 &runtime,
12473 &bridge_sender_fact(&supervisor.peer_id),
12474 &payload,
12475 )
12476 .await
12477 .expect_err("query-string token must not satisfy typed bootstrap proof");
12478 assert_eq!(cause, BridgeRejectionCause::InvalidBootstrapToken);
12479 assert!(
12480 error.contains("typed bridge bootstrap token"),
12481 "runtime should explain that the typed token field is required, got: {error}"
12482 );
12483 }
12484
12485 fn bridge_candidate(sender: &str, command: &BridgeCommand) -> PeerInputCandidate {
12486 let id = InteractionId(Uuid::new_v4());
12487 let ingress = bridge_sender_fact_with_id(id, sender);
12488 bridge_candidate_with_ingress(sender, command, ingress)
12489 }
12490
12491 fn bridge_candidate_with_ingress(
12492 sender: &str,
12493 command: &BridgeCommand,
12494 ingress: PeerIngressFact,
12495 ) -> PeerInputCandidate {
12496 let id = ingress.interaction_id;
12497 PeerInputCandidate {
12498 interaction: InboxInteraction {
12499 objective_id: None,
12500 sender_taint: None,
12501 id,
12502 from_route: None,
12503 from: sender.to_string(),
12504 content: InteractionContent::Request {
12505 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
12506 params: serde_json::to_value(command).expect("serialize bridge command"),
12507 blocks: None,
12508 },
12509 rendered_text: String::new(),
12510 handling_mode: HandlingMode::Queue,
12511 render_metadata: None,
12512 },
12513 ingress,
12514 lifecycle_peer: None,
12515 response_terminality: None,
12516 }
12517 }
12518
12519 fn bridge_delivery_candidate_with_ingress(
12520 sender: &str,
12521 delivery: &serde_json::Value,
12522 ingress: PeerIngressFact,
12523 ) -> PeerInputCandidate {
12524 bridge_delivery_raw_candidate_with_ingress(
12525 sender,
12526 serde_json::to_string(delivery).expect("serialize bridge delivery body"),
12527 ingress,
12528 )
12529 }
12530
12531 fn bridge_delivery_raw_candidate_with_ingress(
12532 sender: &str,
12533 body: String,
12534 ingress: PeerIngressFact,
12535 ) -> PeerInputCandidate {
12536 let id = ingress.interaction_id;
12537 PeerInputCandidate {
12538 interaction: InboxInteraction {
12539 objective_id: None,
12540 sender_taint: None,
12541 id,
12542 from_route: None,
12543 from: sender.to_string(),
12544 content: InteractionContent::Message { body, blocks: None },
12545 rendered_text: String::new(),
12546 handling_mode: HandlingMode::Queue,
12547 render_metadata: None,
12548 },
12549 ingress,
12550 lifecycle_peer: None,
12551 response_terminality: None,
12552 }
12553 }
12554
12555 #[tokio::test]
12556 async fn raw_rotation_marker_versions_reject_durably_and_malformed_markers_fail_closed() {
12557 let current_spec = current_supervisor_bridge_spec();
12558 let current = TrustedPeerDescriptor::try_from(current_spec.clone())
12559 .expect("valid current supervisor");
12560 let target = supervisor_bridge_spec();
12561
12562 let runtime_impl = bootstrap_runtime(
12563 PEER_ID_RECEIVER,
12564 "inproc://raw-rotation-version",
12565 Some("token"),
12566 );
12567 let completed = Arc::clone(&runtime_impl.completed_count);
12568 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
12569 let adapter = Arc::new(MeerkatMachine::ephemeral());
12570 let session_id = SessionId::new();
12571 adapter
12572 .register_session(session_id.clone())
12573 .await
12574 .expect("register session");
12575 adapter
12576 .stage_supervisor_bind(
12577 &session_id,
12578 current.name.to_string(),
12579 current.peer_id.as_str(),
12580 current.address.to_string(),
12581 signing_public_key_for_descriptor(¤t),
12582 1,
12583 )
12584 .await
12585 .expect("bind current supervisor");
12586
12587 let operation_id = SupervisorRotationOperationId::new();
12588 let raw_unsupported = json!({
12589 "delivery": "submit_supervisor_rotation",
12590 "operation_id": operation_id,
12591 "target": target,
12592 "target_epoch": 2,
12593 "protocol_version": 99,
12594 });
12595 let interaction_id = InteractionId(Uuid::new_v4());
12596 let ingress = PeerIngressFact::peer(
12597 interaction_id,
12598 PeerInputClass::ActionableMessage,
12599 meerkat_core::PeerIngressKind::Message,
12600 Some(meerkat_core::PeerIngressAuthDecision::Required),
12601 PeerIngressIdentity::new(
12602 current.peer_id,
12603 current.name.as_str(),
12604 meerkat_core::PeerIngressConvention::Message,
12605 )
12606 .with_signing_pubkey(current.pubkey),
12607 );
12608 let candidate = bridge_delivery_candidate_with_ingress(
12609 current.name.as_str(),
12610 &raw_unsupported,
12611 ingress,
12612 );
12613 assert!(
12614 try_handle_supervisor_bridge_delivery(&adapter, &session_id, &runtime, &candidate,)
12615 .await,
12616 "exact delivery marker must be consumed before typed version decode"
12617 );
12618 let receipt = adapter
12619 .active_supervisor_rotation(&session_id)
12620 .await
12621 .expect("read rejected rotation")
12622 .expect("unsupported rotation is durably recorded");
12623 assert_eq!(
12624 receipt.phase,
12625 crate::meerkat_machine::dsl::SupervisorRotationPhase::Rejected
12626 );
12627 assert_eq!(
12628 receipt.rejection,
12629 Some(
12630 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::UnsupportedProtocolVersion
12631 )
12632 );
12633 assert_eq!(
12634 completed.load(std::sync::atomic::Ordering::SeqCst),
12635 1,
12636 "one-way unsupported submission must close local ingress"
12637 );
12638
12639 let malformed_runtime_impl = bootstrap_runtime(
12640 PEER_ID_RECEIVER,
12641 "inproc://raw-rotation-malformed",
12642 Some("token"),
12643 );
12644 let malformed_completed = Arc::clone(&malformed_runtime_impl.completed_count);
12645 let malformed_runtime: Arc<dyn CommsRuntime> = Arc::new(malformed_runtime_impl);
12646 let malformed_adapter = Arc::new(MeerkatMachine::ephemeral());
12647 let malformed_session = SessionId::new();
12648 malformed_adapter
12649 .register_session(malformed_session.clone())
12650 .await
12651 .expect("register malformed-marker session");
12652 let malformed_id = InteractionId(Uuid::new_v4());
12653 let malformed_ingress = PeerIngressFact::peer(
12654 malformed_id,
12655 PeerInputClass::ActionableMessage,
12656 meerkat_core::PeerIngressKind::Message,
12657 Some(meerkat_core::PeerIngressAuthDecision::Required),
12658 PeerIngressIdentity::new(
12659 current.peer_id,
12660 current.name.as_str(),
12661 meerkat_core::PeerIngressConvention::Message,
12662 )
12663 .with_signing_pubkey(current.pubkey),
12664 );
12665 let malformed = json!({
12666 "delivery": "submit_supervisor_rotation",
12667 "protocol_version": 4,
12668 });
12669 let malformed_candidate = bridge_delivery_candidate_with_ingress(
12670 current.name.as_str(),
12671 &malformed,
12672 malformed_ingress,
12673 );
12674 assert!(
12675 try_handle_supervisor_bridge_delivery(
12676 &malformed_adapter,
12677 &malformed_session,
12678 &malformed_runtime,
12679 &malformed_candidate,
12680 )
12681 .await,
12682 "malformed exact marker must be consumed fail-closed"
12683 );
12684 assert!(
12685 malformed_adapter
12686 .active_supervisor_rotation(&malformed_session)
12687 .await
12688 .expect("read malformed session")
12689 .is_none(),
12690 "malformed marker must not create partial durable operation state"
12691 );
12692 assert_eq!(
12693 malformed_completed.load(std::sync::atomic::Ordering::SeqCst),
12694 1,
12695 "malformed marker must be completed locally instead of reaching the user prompt"
12696 );
12697
12698 let duplicate_base = json!({
12699 "delivery": "submit_supervisor_rotation",
12700 "operation_id": SupervisorRotationOperationId::new(),
12701 "target": supervisor_bridge_spec(),
12702 "target_epoch": 2,
12703 "protocol_version": 4,
12704 })
12705 .to_string();
12706 let duplicate_body = format!(
12707 "{},\"delivery\":\"chat\"}}",
12708 duplicate_base
12709 .strip_suffix('}')
12710 .expect("delivery object has closing brace")
12711 );
12712 let duplicate_id = InteractionId(Uuid::new_v4());
12713 let duplicate_ingress = PeerIngressFact::peer(
12714 duplicate_id,
12715 PeerInputClass::ActionableMessage,
12716 meerkat_core::PeerIngressKind::Message,
12717 Some(meerkat_core::PeerIngressAuthDecision::Required),
12718 PeerIngressIdentity::new(
12719 current.peer_id,
12720 current.name.as_str(),
12721 meerkat_core::PeerIngressConvention::Message,
12722 )
12723 .with_signing_pubkey(current.pubkey),
12724 );
12725 let duplicate_candidate = bridge_delivery_raw_candidate_with_ingress(
12726 current.name.as_str(),
12727 duplicate_body,
12728 duplicate_ingress,
12729 );
12730 assert!(
12731 try_handle_supervisor_bridge_delivery(
12732 &malformed_adapter,
12733 &malformed_session,
12734 &malformed_runtime,
12735 &duplicate_candidate,
12736 )
12737 .await,
12738 "reserved-first duplicate delivery marker must remain reserved and fail closed"
12739 );
12740 assert_eq!(
12741 malformed_completed.load(std::sync::atomic::Ordering::SeqCst),
12742 2,
12743 "duplicate/conflicting delivery keys must be consumed instead of reaching the user prompt"
12744 );
12745 assert!(
12746 malformed_adapter
12747 .active_supervisor_rotation(&malformed_session)
12748 .await
12749 .expect("read duplicate-marker session")
12750 .is_none(),
12751 "duplicate marker must not create durable operation state"
12752 );
12753 }
12754
12755 #[tokio::test]
12756 async fn pending_rotation_observe_does_not_mint_temporary_supervisor_trust() {
12757 let previous_spec = current_supervisor_bridge_spec();
12758 let previous = TrustedPeerDescriptor::try_from(previous_spec.clone())
12759 .expect("valid previous supervisor");
12760 let next_spec = supervisor_bridge_spec();
12761 let runtime_impl = bootstrap_runtime(
12762 PEER_ID_RECEIVER,
12763 "inproc://pending-observe-no-retrust",
12764 Some("token"),
12765 );
12766 let trusted_peer_ids = Arc::clone(&runtime_impl.trusted_peer_ids);
12767 let sent_commands = Arc::clone(&runtime_impl.sent_commands);
12768 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
12769 let adapter = Arc::new(MeerkatMachine::ephemeral());
12770 let session_id = SessionId::new();
12771 adapter
12772 .register_session(session_id.clone())
12773 .await
12774 .expect("register session");
12775 adapter
12776 .stage_supervisor_bind(
12777 &session_id,
12778 previous.name.to_string(),
12779 previous.peer_id.as_str(),
12780 previous.address.to_string(),
12781 signing_public_key_for_descriptor(&previous),
12782 1,
12783 )
12784 .await
12785 .expect("bind previous supervisor");
12786
12787 let operation_id = SupervisorRotationOperationId::new();
12788 let next =
12789 TrustedPeerDescriptor::try_from(next_spec.clone()).expect("valid next supervisor");
12790 let submission = adapter
12791 .submit_supervisor_rotation(
12792 &session_id,
12793 GeneratedSupervisorRotationSubmit {
12794 operation_id: operation_id.to_string(),
12795 next: GeneratedSupervisorBinding {
12796 name: next.name.to_string(),
12797 peer_id: next.peer_id.as_str(),
12798 address: next.address.to_string(),
12799 signing_public_key: signing_public_key_for_descriptor(&next),
12800 epoch: 2,
12801 },
12802 preflight_rejection: None,
12803 sender_peer_id: Some(previous.peer_id.as_str()),
12804 sender_signing_public_key: Some(signing_public_key_for_descriptor(&previous)),
12805 },
12806 )
12807 .await
12808 .expect("submit pending rotation");
12809 assert!(matches!(submission, SupervisorRotationSubmission::New(_)));
12810
12811 let command = BridgeCommand::ObserveSupervisorRotation(
12812 meerkat_contracts::wire::supervisor_bridge::BridgeSupervisorRotationObserve {
12813 operation_id,
12814 observer: previous_spec,
12815 observer_epoch: 1,
12816 protocol_version: BridgeProtocolVersion::V4,
12817 },
12818 );
12819 let interaction_id = InteractionId(Uuid::new_v4());
12820 let ingress = PeerIngressFact::peer(
12821 interaction_id,
12822 PeerInputClass::ActionableRequest,
12823 meerkat_core::PeerIngressKind::Request,
12824 Some(meerkat_core::PeerIngressAuthDecision::Required),
12825 PeerIngressIdentity::new(
12826 previous.peer_id,
12827 previous.name.as_str(),
12828 meerkat_core::PeerIngressConvention::Request {
12829 request_id: interaction_id.to_string(),
12830 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
12831 },
12832 )
12833 .with_signing_pubkey(previous.pubkey),
12834 );
12835 let candidate = bridge_candidate_with_ingress(previous.name.as_str(), &command, ingress);
12836 assert!(
12837 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await
12838 );
12839 assert!(
12840 trusted_peer_ids.lock().await.is_empty(),
12841 "pending observation must not temporarily retrust either rotation participant"
12842 );
12843 let replies = recorded_bridge_replies(&sent_commands).await;
12844 assert!(matches!(
12845 replies.as_slice(),
12846 [(
12847 meerkat_core::interaction::ResponseStatus::Completed,
12848 BridgeReply::SupervisorRotation(BridgeSupervisorRotationObservation::Found {
12849 state: BridgeSupervisorRotationState::Pending { .. }
12850 })
12851 )]
12852 ));
12853 }
12854
12855 #[tokio::test]
12856 async fn replacement_runtime_hydrates_completed_and_rejected_rotation_authority() {
12857 let previous = trusted_supervisor_descriptor(0xcc);
12858 let next = trusted_supervisor_descriptor(0xbb);
12859
12860 let completed_runtime_name = format!("completed-rotation-hydration-{}", Uuid::new_v4());
12861 let completed_runtime = Arc::new(
12862 meerkat_comms::CommsRuntime::inproc_only(&completed_runtime_name)
12863 .expect("completed replacement runtime"),
12864 );
12865 let completed_runtime_dyn: Arc<dyn CommsRuntime> = completed_runtime.clone();
12866 let completed_adapter = Arc::new(MeerkatMachine::ephemeral());
12867 let completed_session = SessionId::new();
12868 completed_adapter
12869 .register_session(completed_session.clone())
12870 .await
12871 .expect("register completed session");
12872 completed_adapter
12873 .stage_supervisor_bind(
12874 &completed_session,
12875 previous.name.to_string(),
12876 previous.peer_id.as_str(),
12877 previous.address.to_string(),
12878 signing_public_key_for_descriptor(&previous),
12879 1,
12880 )
12881 .await
12882 .expect("bind previous completed supervisor");
12883 completed_adapter
12884 .test_install_session_peer_comms_handle_on_runtime(
12885 &completed_session,
12886 completed_runtime.as_ref(),
12887 )
12888 .await
12889 .expect("install completed runtime owner token");
12890 completed_adapter
12891 .stage_local_endpoint_for_comms_runtime(
12892 &completed_session,
12893 completed_runtime_dyn.as_ref(),
12894 )
12895 .await
12896 .expect("stage completed runtime local endpoint");
12897
12898 let completed_operation = SupervisorRotationOperationId::new();
12899 completed_adapter
12900 .submit_supervisor_rotation(
12901 &completed_session,
12902 GeneratedSupervisorRotationSubmit {
12903 operation_id: completed_operation.to_string(),
12904 next: GeneratedSupervisorBinding {
12905 name: next.name.to_string(),
12906 peer_id: next.peer_id.as_str(),
12907 address: next.address.to_string(),
12908 signing_public_key: signing_public_key_for_descriptor(&next),
12909 epoch: 2,
12910 },
12911 preflight_rejection: None,
12912 sender_peer_id: Some(previous.peer_id.as_str()),
12913 sender_signing_public_key: Some(signing_public_key_for_descriptor(&previous)),
12914 },
12915 )
12916 .await
12917 .expect("submit completed rotation");
12918 completed_adapter
12919 .stage_supervisor_rotation_resume(&completed_session, completed_operation.to_string())
12920 .await
12921 .expect("resume completed rotation");
12922 completed_adapter
12923 .stage_supervisor_rotation_previous_revoked(
12924 &completed_session,
12925 completed_operation.to_string(),
12926 previous.peer_id.as_str(),
12927 1,
12928 )
12929 .await
12930 .expect("checkpoint previous revoke");
12931 completed_adapter
12932 .stage_supervisor_rotation_next_published(
12933 &completed_session,
12934 completed_operation.to_string(),
12935 next.peer_id.as_str(),
12936 2,
12937 )
12938 .await
12939 .expect("checkpoint next publish");
12940 assert!(
12941 !completed_runtime
12942 .router()
12943 .is_private(&meerkat_comms::PubKey::new(next.pubkey))
12944 );
12945 assert!(
12946 hydrate_current_bound_supervisor_route(
12947 &completed_adapter,
12948 &completed_session,
12949 &completed_runtime_dyn,
12950 )
12951 .await
12952 .expect("hydrate completed authority")
12953 );
12954 assert!(
12955 completed_runtime
12956 .router()
12957 .is_private(&meerkat_comms::PubKey::new(next.pubkey)),
12958 "Completed replacement runtime must hydrate only the generated next authority"
12959 );
12960 assert!(
12961 !completed_runtime
12962 .router()
12963 .is_private(&meerkat_comms::PubKey::new(previous.pubkey)),
12964 "Completed replacement runtime must not hydrate the revoked authority"
12965 );
12966
12967 let rejected_runtime_name = format!("rejected-rotation-hydration-{}", Uuid::new_v4());
12968 let rejected_runtime = Arc::new(
12969 meerkat_comms::CommsRuntime::inproc_only(&rejected_runtime_name)
12970 .expect("rejected replacement runtime"),
12971 );
12972 let rejected_runtime_dyn: Arc<dyn CommsRuntime> = rejected_runtime.clone();
12973 let rejected_adapter = Arc::new(MeerkatMachine::ephemeral());
12974 let rejected_session = SessionId::new();
12975 rejected_adapter
12976 .register_session(rejected_session.clone())
12977 .await
12978 .expect("register rejected session");
12979 rejected_adapter
12980 .stage_supervisor_bind(
12981 &rejected_session,
12982 previous.name.to_string(),
12983 previous.peer_id.as_str(),
12984 previous.address.to_string(),
12985 signing_public_key_for_descriptor(&previous),
12986 1,
12987 )
12988 .await
12989 .expect("bind previous rejected supervisor");
12990 rejected_adapter
12991 .test_install_session_peer_comms_handle_on_runtime(
12992 &rejected_session,
12993 rejected_runtime.as_ref(),
12994 )
12995 .await
12996 .expect("install rejected runtime owner token");
12997 rejected_adapter
12998 .stage_local_endpoint_for_comms_runtime(
12999 &rejected_session,
13000 rejected_runtime_dyn.as_ref(),
13001 )
13002 .await
13003 .expect("stage rejected runtime local endpoint");
13004 rejected_adapter
13005 .submit_supervisor_rotation(
13006 &rejected_session,
13007 GeneratedSupervisorRotationSubmit {
13008 operation_id: SupervisorRotationOperationId::new().to_string(),
13009 next: GeneratedSupervisorBinding {
13010 name: next.name.to_string(),
13011 peer_id: next.peer_id.as_str(),
13012 address: next.address.to_string(),
13013 signing_public_key: signing_public_key_for_descriptor(&next),
13014 epoch: 2,
13015 },
13016 preflight_rejection: Some(
13017 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::UnsupportedProtocolVersion,
13018 ),
13019 sender_peer_id: Some(previous.peer_id.as_str()),
13020 sender_signing_public_key: Some(signing_public_key_for_descriptor(&previous)),
13021 },
13022 )
13023 .await
13024 .expect("durably reject rotation");
13025 assert!(
13026 hydrate_current_bound_supervisor_route(
13027 &rejected_adapter,
13028 &rejected_session,
13029 &rejected_runtime_dyn,
13030 )
13031 .await
13032 .expect("hydrate rejected authority")
13033 );
13034 assert!(
13035 rejected_runtime
13036 .router()
13037 .is_private(&meerkat_comms::PubKey::new(previous.pubkey)),
13038 "Rejected replacement runtime must hydrate only the retained previous authority"
13039 );
13040 assert!(
13041 !rejected_runtime
13042 .router()
13043 .is_private(&meerkat_comms::PubKey::new(next.pubkey)),
13044 "Rejected replacement runtime must not hydrate the rejected target"
13045 );
13046 }
13047
13048 #[tokio::test]
13049 async fn bind_member_rolls_back_binding_when_trust_publication_fails() {
13050 let payload = sample_bind_payload();
13053 let sender = payload.supervisor.peer_id.clone();
13054 let mut runtime_impl = bootstrap_runtime(
13055 PEER_ID_RECEIVER,
13056 "inproc://receiver",
13057 Some("expected-token"),
13058 );
13059 runtime_impl
13060 .add_trusted_peer_errors
13061 .insert(payload.supervisor.peer_id.clone(), "boom".to_string());
13062 let trusted_peer_ids = runtime_impl.trusted_peer_ids.clone();
13063 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
13064 let adapter = Arc::new(MeerkatMachine::ephemeral());
13065 let session_id = SessionId::new();
13066 adapter
13067 .register_session(session_id.clone())
13068 .await
13069 .expect("register session");
13070 let candidate = bridge_candidate(&sender, &BridgeCommand::BindMember(payload));
13071
13072 assert!(
13073 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await
13074 );
13075 assert!(
13076 matches!(
13077 adapter.supervisor_binding(&session_id).await,
13078 SupervisorBinding::Unbound
13079 ),
13080 "failed trust publication must roll BindMember back to Unbound"
13081 );
13082 assert!(
13083 trusted_peer_ids.lock().await.is_empty(),
13084 "failed trust publication must not leave the supervisor trusted"
13085 );
13086 }
13087
13088 #[tokio::test]
13089 async fn bind_member_invalid_bootstrap_does_not_publish_response_route() {
13090 let mut payload = sample_bind_payload();
13091 payload.bootstrap_token = "wrong-token".into();
13092 let sender = payload.supervisor.peer_id.clone();
13093 let runtime_impl = bootstrap_runtime(
13094 PEER_ID_RECEIVER,
13095 "inproc://receiver",
13096 Some("expected-token"),
13097 );
13098 let trusted_peer_ids = runtime_impl.trusted_peer_ids.clone();
13099 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
13100 let adapter = Arc::new(MeerkatMachine::ephemeral());
13101 let session_id = SessionId::new();
13102 adapter
13103 .register_session(session_id.clone())
13104 .await
13105 .expect("register session");
13106 let candidate = bridge_candidate(&sender, &BridgeCommand::BindMember(payload));
13107
13108 assert!(
13109 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await
13110 );
13111 assert!(
13112 matches!(
13113 adapter.supervisor_binding(&session_id).await,
13114 SupervisorBinding::Unbound
13115 ),
13116 "invalid bootstrap must not bind supervisor authority"
13117 );
13118 assert!(
13119 trusted_peer_ids.lock().await.is_empty(),
13120 "invalid bootstrap must not temporarily publish supervisor trust just to route rejection"
13121 );
13122 }
13123
13124 #[tokio::test]
13125 async fn revoke_supervisor_failure_leaves_pending_checkpoint_and_retry_completes() {
13126 let supervisor = supervisor_bridge_spec();
13127 let runtime = Arc::new(bootstrap_runtime(
13128 PEER_ID_RECEIVER,
13129 "inproc://receiver",
13130 Some("expected-token"),
13131 ));
13132 runtime
13133 .remove_failures_remaining
13134 .store(1, std::sync::atomic::Ordering::SeqCst);
13135 let runtime_dyn: Arc<dyn CommsRuntime> = Arc::clone(&runtime) as Arc<dyn CommsRuntime>;
13136 let adapter = Arc::new(MeerkatMachine::ephemeral());
13137 let session_id = SessionId::new();
13138 adapter
13139 .register_session(session_id.clone())
13140 .await
13141 .expect("register session");
13142 let payload = BridgeSupervisorPayload {
13143 supervisor: supervisor.clone(),
13144 epoch: 1,
13145 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
13146 };
13147 let candidate = bridge_candidate(
13148 &supervisor.peer_id,
13149 &BridgeCommand::RevokeSupervisor(payload.clone()),
13150 );
13151 let spec =
13152 TrustedPeerDescriptor::try_from(supervisor.clone()).expect("valid supervisor spec");
13153 adapter
13154 .stage_supervisor_bind(
13155 &session_id,
13156 spec.name.to_string(),
13157 spec.peer_id.as_str(),
13158 spec.address.to_string(),
13159 signing_public_key_for_descriptor(&spec),
13160 payload.epoch,
13161 )
13162 .await
13163 .expect("pre-bind supervisor");
13164
13165 assert!(
13166 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime_dyn, &candidate,)
13167 .await,
13168 "revoke command should be handled"
13169 );
13170 assert!(
13171 matches!(
13172 adapter.supervisor_binding(&session_id).await,
13173 SupervisorBinding::Unbound
13174 ),
13175 "failed removal must leave durable revoke pending, not restore a live binding"
13176 );
13177 let retry = bridge_candidate(
13178 &supervisor.peer_id,
13179 &BridgeCommand::RevokeSupervisor(payload),
13180 );
13181 assert!(
13182 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime_dyn, &retry).await,
13183 "exact revoke retry should complete the pending checkpoint"
13184 );
13185 assert!(
13186 runtime
13187 .remove_attempts
13188 .load(std::sync::atomic::Ordering::SeqCst)
13189 >= 2,
13190 "retry must re-drive generated trust removal with fresh authority"
13191 );
13192 }
13193
13194 #[tokio::test]
13201 async fn dsl_supervisor_guards_block_rebind_and_stale_revoke() {
13202 let adapter = Arc::new(MeerkatMachine::ephemeral());
13203 let session_id = SessionId::new();
13204 adapter
13205 .register_session(session_id.clone())
13206 .await
13207 .expect("register session");
13208
13209 adapter
13211 .stage_supervisor_bind(
13212 &session_id,
13213 "super-a".to_string(),
13214 "ed25519:super-a".to_string(),
13215 "inproc://super-a".to_string(),
13216 test_supervisor_signing_public_key(0xaa),
13217 1,
13218 )
13219 .await
13220 .expect("initial bind");
13221
13222 let rebind = adapter
13226 .stage_supervisor_bind(
13227 &session_id,
13228 "super-b".to_string(),
13229 "ed25519:super-b".to_string(),
13230 "inproc://super-b".to_string(),
13231 test_supervisor_signing_public_key(0xbb),
13232 2,
13233 )
13234 .await;
13235 assert!(
13236 matches!(rebind, Err(SupervisorBindingStageError::Dsl(_))),
13237 "double bind must be rejected by DSL guard, got: {rebind:?}"
13238 );
13239
13240 let stale_peer_revoke = adapter
13242 .stage_supervisor_revoke(&session_id, "ed25519:super-b".to_string(), 1)
13243 .await;
13244 assert!(
13245 matches!(stale_peer_revoke, Err(SupervisorBindingStageError::Dsl(_))),
13246 "revoke with mismatched peer_id must be DSL-rejected, got: {stale_peer_revoke:?}"
13247 );
13248
13249 let stale_epoch_revoke = adapter
13251 .stage_supervisor_revoke(&session_id, "ed25519:super-a".to_string(), 99)
13252 .await;
13253 assert!(
13254 matches!(stale_epoch_revoke, Err(SupervisorBindingStageError::Dsl(_))),
13255 "revoke with mismatched epoch must be DSL-rejected, got: {stale_epoch_revoke:?}"
13256 );
13257
13258 adapter
13260 .stage_supervisor_authorize(
13261 &session_id,
13262 "super-a-refreshed".to_string(),
13263 "ed25519:super-a".to_string(),
13264 "inproc://super-a-refreshed".to_string(),
13265 test_supervisor_signing_public_key(0xaa),
13266 2,
13267 )
13268 .await
13269 .expect("same-authority version advance must succeed");
13270
13271 match adapter.supervisor_binding(&session_id).await {
13273 SupervisorBinding::Bound { peer_id, epoch, .. } => {
13274 assert_eq!(peer_id, "ed25519:super-a");
13275 assert_eq!(epoch, 2);
13276 }
13277 SupervisorBinding::Unbound => panic!("expected Bound after rotation"),
13278 }
13279
13280 adapter
13283 .stage_supervisor_revoke(&session_id, "ed25519:super-a".to_string(), 2)
13284 .await
13285 .expect("matching revoke must succeed");
13286 assert!(
13287 matches!(
13288 adapter.supervisor_binding(&session_id).await,
13289 SupervisorBinding::Unbound
13290 ),
13291 "binding must be Unbound after matching revoke"
13292 );
13293 }
13294
13295 #[tokio::test]
13302 async fn dsl_supervisor_trust_publish_ack_stale_epoch_is_rejected() {
13303 let adapter = Arc::new(MeerkatMachine::ephemeral());
13304 let session_id = SessionId::new();
13305 adapter
13306 .register_session(session_id.clone())
13307 .await
13308 .expect("register session");
13309
13310 adapter
13312 .stage_supervisor_bind(
13313 &session_id,
13314 "super-a".to_string(),
13315 "ed25519:super-a".to_string(),
13316 "inproc://super-a".to_string(),
13317 test_supervisor_signing_public_key(0xaa),
13318 1,
13319 )
13320 .await
13321 .expect("initial bind");
13322
13323 adapter
13325 .stage_supervisor_authorize(
13326 &session_id,
13327 "super-a".to_string(),
13328 "ed25519:super-a".to_string(),
13329 "inproc://super-a".to_string(),
13330 test_supervisor_signing_public_key(0xaa),
13331 2,
13332 )
13333 .await
13334 .expect("rotation to epoch 2");
13335
13336 let stale = adapter
13340 .stage_supervisor_trust_published(&session_id, "ed25519:super-a".to_string(), 1)
13341 .await;
13342 assert!(
13343 matches!(stale, Err(SupervisorBindingStageError::Dsl(_))),
13344 "stale-epoch publish ack must be DSL-rejected, got: {stale:?}"
13345 );
13346 match adapter.supervisor_binding(&session_id).await {
13348 SupervisorBinding::Bound { epoch, .. } => {
13349 assert_eq!(epoch, 2, "binding must still be at epoch 2");
13350 }
13351 SupervisorBinding::Unbound => panic!("stale ack must not unbind"),
13352 }
13353
13354 adapter
13357 .stage_supervisor_trust_published(&session_id, "ed25519:super-a".to_string(), 2)
13358 .await
13359 .expect("matching-epoch ack must be accepted");
13360 }
13361
13362 #[tokio::test]
13363 async fn generated_supervisor_publish_authority_rejects_stale_descriptor_same_peer_epoch() {
13364 let adapter = Arc::new(MeerkatMachine::ephemeral());
13365 let session_id = SessionId::new();
13366 adapter
13367 .register_session(session_id.clone())
13368 .await
13369 .expect("register session");
13370 let runtime = bootstrap_runtime(PEER_ID_RECEIVER, "inproc://receiver", Some("token"));
13371 adapter
13372 .stage_local_endpoint_for_comms_runtime(&session_id, &runtime)
13373 .await
13374 .expect("stage local endpoint");
13375 let supervisor_peer_id = test_pubkey(0xaa).to_peer_id().as_str();
13376
13377 let transition = adapter
13378 .stage_supervisor_bind(
13379 &session_id,
13380 "super-a".to_string(),
13381 supervisor_peer_id.clone(),
13382 "inproc://super-old".to_string(),
13383 test_supervisor_signing_public_key(0xaa),
13384 7,
13385 )
13386 .await
13387 .expect("initial bind");
13388 let freshness = adapter
13389 .supervisor_trust_publish_freshness_authority(&session_id)
13390 .await
13391 .expect("publish freshness");
13392 let stale_obligation =
13393 crate::protocol_supervisor_trust_publish::extract_obligations_with_freshness(
13394 &transition,
13395 freshness,
13396 )
13397 .into_iter()
13398 .next()
13399 .expect("generated publish obligation");
13400
13401 adapter
13402 .stage_supervisor_binding_route_refresh(
13403 &session_id,
13404 GeneratedSupervisorBinding {
13405 name: "super-a-renamed".to_string(),
13406 peer_id: supervisor_peer_id.clone(),
13407 address: "inproc://super-new".to_string(),
13408 signing_public_key: test_supervisor_signing_public_key(0xaa),
13409 epoch: 7,
13410 },
13411 Some(supervisor_peer_id),
13412 )
13413 .await
13414 .expect("same-authority same-epoch route refresh");
13415
13416 let stale_authority = crate::protocol_supervisor_trust_publish::publish_authority_for_peer(
13417 &stale_obligation,
13418 stale_obligation.peer_id(),
13419 );
13420 assert!(
13421 matches!(stale_authority, Err(ref error) if error.contains("stale generated supervisor trust publish obligation")),
13422 "stale descriptor must not mint generated supervisor publish authority, got: {stale_authority:?}"
13423 );
13424 }
13425
13426 #[tokio::test]
13427 async fn dsl_supervisor_trust_revoke_ack_stale_epoch_is_rejected() {
13428 let adapter = Arc::new(MeerkatMachine::ephemeral());
13429 let session_id = SessionId::new();
13430 adapter
13431 .register_session(session_id.clone())
13432 .await
13433 .expect("register session");
13434
13435 adapter
13437 .stage_supervisor_bind(
13438 &session_id,
13439 "super-a".to_string(),
13440 "ed25519:super-a".to_string(),
13441 "inproc://super-a".to_string(),
13442 test_supervisor_signing_public_key(0xaa),
13443 1,
13444 )
13445 .await
13446 .expect("initial bind");
13447
13448 adapter
13450 .stage_supervisor_authorize(
13451 &session_id,
13452 "super-a".to_string(),
13453 "ed25519:super-a".to_string(),
13454 "inproc://super-a".to_string(),
13455 test_supervisor_signing_public_key(0xaa),
13456 2,
13457 )
13458 .await
13459 .expect("rotation to epoch 2");
13460
13461 adapter
13464 .stage_supervisor_revoke(&session_id, "ed25519:super-a".to_string(), 2)
13465 .await
13466 .expect("matching revoke must stage pending trust revoke");
13467
13468 let stale = adapter
13471 .stage_supervisor_trust_revoked(&session_id, "ed25519:super-a".to_string(), 1)
13472 .await;
13473 assert!(
13474 matches!(stale, Err(SupervisorBindingStageError::Dsl(_))),
13475 "stale-epoch revoke ack must be DSL-rejected, got: {stale:?}"
13476 );
13477 match adapter.supervisor_binding(&session_id).await {
13478 SupervisorBinding::Unbound => {}
13479 SupervisorBinding::Bound { .. } => panic!("revoke input must unbind before feedback"),
13480 }
13481
13482 adapter
13484 .stage_supervisor_trust_revoked(&session_id, "ed25519:super-a".to_string(), 2)
13485 .await
13486 .expect("matching-epoch revoke ack must be accepted");
13487 }
13488
13489 #[tokio::test]
13490 async fn dsl_supervisor_trust_ack_rejects_mismatched_peer_id() {
13491 let adapter = Arc::new(MeerkatMachine::ephemeral());
13492 let session_id = SessionId::new();
13493 adapter
13494 .register_session(session_id.clone())
13495 .await
13496 .expect("register session");
13497
13498 adapter
13499 .stage_supervisor_bind(
13500 &session_id,
13501 "super-a".to_string(),
13502 "ed25519:super-a".to_string(),
13503 "inproc://super-a".to_string(),
13504 test_supervisor_signing_public_key(0xaa),
13505 7,
13506 )
13507 .await
13508 .expect("initial bind");
13509
13510 let wrong_peer = adapter
13512 .stage_supervisor_trust_published(&session_id, "ed25519:unrelated".to_string(), 7)
13513 .await;
13514 assert!(
13515 matches!(wrong_peer, Err(SupervisorBindingStageError::Dsl(_))),
13516 "mismatched-peer ack must be DSL-rejected, got: {wrong_peer:?}"
13517 );
13518 }
13519
13520 #[tokio::test]
13521 async fn dsl_supervisor_trust_ack_rejected_when_unbound() {
13522 let adapter = Arc::new(MeerkatMachine::ephemeral());
13523 let session_id = SessionId::new();
13524 adapter
13525 .register_session(session_id.clone())
13526 .await
13527 .expect("register session");
13528
13529 let unbound = adapter
13531 .stage_supervisor_trust_published(&session_id, "ed25519:super-a".to_string(), 1)
13532 .await;
13533 assert!(
13534 matches!(unbound, Err(SupervisorBindingStageError::Dsl(_))),
13535 "publish ack must be rejected when Unbound, got: {unbound:?}"
13536 );
13537 }
13538
13539 #[tokio::test]
13540 async fn wire_member_rejects_invalid_peer_spec_before_trusting_it() {
13541 let runtime = Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-wire").unwrap());
13542 let supervisor_runtime =
13543 Arc::new(meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor__").unwrap());
13544 let adapter = Arc::new(MeerkatMachine::ephemeral());
13545 let session_id = SessionId::new();
13546 adapter
13547 .register_session(session_id.clone())
13548 .await
13549 .expect("register session");
13550 let supervisor = trusted_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
13551 add_test_projection_trust(runtime.as_ref(), supervisor.clone(), "trust supervisor").await;
13552 adapter
13553 .stage_supervisor_bind(
13554 &session_id,
13555 supervisor.name.to_string(),
13556 supervisor.peer_id.as_str(),
13557 supervisor.address.to_string(),
13558 signing_public_key_for_descriptor(&supervisor),
13559 1,
13560 )
13561 .await
13562 .expect("pre-bind supervisor");
13563 let invalid_peer_id = PeerId::new().as_str();
13564 let candidate = bridge_candidate(
13565 &supervisor.peer_id.as_str(),
13566 &BridgeCommand::WireMember(BridgePeerWiringPayload {
13567 supervisor: supervisor.clone().into(),
13568 epoch: 1,
13569 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
13570 peer_spec: BridgePeerSpec {
13571 name: "".to_string(),
13572 peer_id: invalid_peer_id.clone(),
13573 address: "inproc://peer".to_string(),
13574 pubkey: [0u8; 32],
13575 },
13576 mob_peer_overlay: Some(BridgeMobPeerOverlayHandoff {
13577 recipient_peer_id: runtime.peer_id().expect("runtime peer_id").as_str(),
13578 topology_epoch: 1,
13579 peer_specs: Vec::new(),
13580 }),
13581 }),
13582 );
13583
13584 assert!(
13585 try_handle_supervisor_bridge_command(
13586 &adapter,
13587 &session_id,
13588 &(runtime.clone() as Arc<dyn CommsRuntime>),
13589 &candidate,
13590 )
13591 .await,
13592 "wire bridge command should be handled"
13593 );
13594 let peers = runtime.peers().await;
13595 assert!(
13596 peers.iter().all(|entry| !entry.name.as_str().is_empty()),
13597 "invalid wire peer specs must not be materialized in comms trust"
13598 );
13599 }
13600
13601 #[tokio::test]
13602 async fn wire_member_without_overlay_rejects_v2_payload_without_wiring() {
13603 let runtime =
13609 Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-wire-noverlay").unwrap());
13610 let supervisor_runtime = Arc::new(
13611 meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor_noverlay__").unwrap(),
13612 );
13613 let peer_runtime =
13614 Arc::new(meerkat_comms::CommsRuntime::inproc_only("peer-wire-noverlay").unwrap());
13615 let adapter = Arc::new(MeerkatMachine::ephemeral());
13616 let session_id = SessionId::new();
13617 adapter
13618 .register_session(session_id.clone())
13619 .await
13620 .expect("register session");
13621 let supervisor =
13622 trusted_peer_from_runtime("mob/__mob_supervisor_noverlay__", &supervisor_runtime);
13623 add_test_projection_trust(runtime.as_ref(), supervisor.clone(), "trust supervisor").await;
13624 adapter
13625 .stage_supervisor_bind(
13626 &session_id,
13627 supervisor.name.to_string(),
13628 supervisor.peer_id.as_str(),
13629 supervisor.address.to_string(),
13630 signing_public_key_for_descriptor(&supervisor),
13631 1,
13632 )
13633 .await
13634 .expect("pre-bind supervisor");
13635 let peer_spec = trusted_peer_from_runtime("peer-wire-noverlay", &peer_runtime);
13636 let candidate = bridge_candidate(
13637 &supervisor.peer_id.as_str(),
13638 &BridgeCommand::WireMember(BridgePeerWiringPayload {
13639 supervisor: supervisor.clone().into(),
13640 epoch: 1,
13641 protocol_version: BridgeProtocolVersion::V2,
13642 peer_spec: peer_spec.clone().into(),
13643 mob_peer_overlay: None,
13644 }),
13645 );
13646
13647 assert!(
13648 try_handle_supervisor_bridge_command(
13649 &adapter,
13650 &session_id,
13651 &(runtime.clone() as Arc<dyn CommsRuntime>),
13652 &candidate,
13653 )
13654 .await,
13655 "wire bridge command should be handled (and cleanly rejected)"
13656 );
13657 let peers = runtime.peers().await;
13658 assert!(
13659 peers.iter().all(|entry| entry.peer_id != peer_spec.peer_id),
13660 "a V2 wiring payload without the overlay must not materialize peer trust"
13661 );
13662 }
13663
13664 #[tokio::test]
13665 async fn declare_member_outbound_taint_from_bound_supervisor_installs_declaration() {
13666 let runtime =
13667 Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-taint-install").unwrap());
13668 install_test_peer_request_response_authority(
13671 &runtime,
13672 Arc::new(CountingPeerInteractionHandle::default()),
13673 );
13674 let supervisor_runtime = Arc::new(
13675 meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor_taint__").unwrap(),
13676 );
13677 let adapter = Arc::new(MeerkatMachine::ephemeral());
13678 let session_id = SessionId::new();
13679 adapter
13680 .register_session(session_id.clone())
13681 .await
13682 .expect("register session");
13683 let supervisor =
13684 trusted_peer_from_runtime("mob/__mob_supervisor_taint__", &supervisor_runtime);
13685 adapter
13686 .stage_supervisor_bind(
13687 &session_id,
13688 supervisor.name.to_string(),
13689 supervisor.peer_id.as_str(),
13690 supervisor.address.to_string(),
13691 signing_public_key_for_descriptor(&supervisor),
13692 1,
13693 )
13694 .await
13695 .expect("pre-bind supervisor");
13696 adapter
13699 .test_install_session_peer_comms_handle_on_runtime(&session_id, runtime.as_ref())
13700 .await
13701 .expect("install adapter session peer-comms handle on member runtime");
13702
13703 let command = BridgeCommand::DeclareMemberOutboundTaint(BridgeOutboundTaintPayload {
13704 supervisor: supervisor.clone().into(),
13705 epoch: 1,
13706 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
13707 target: Some(BridgeOutboundTaintTarget::PeerOnly),
13708 taint: Some(meerkat_core::comms::SenderContentTaint::Tainted),
13709 });
13710 let interaction_id = InteractionId(Uuid::new_v4());
13711 let ingress = PeerIngressFact::peer(
13712 interaction_id,
13713 PeerInputClass::ActionableRequest,
13714 meerkat_core::PeerIngressKind::Request,
13715 Some(meerkat_core::PeerIngressAuthDecision::Required),
13716 PeerIngressIdentity::new(
13717 supervisor.peer_id,
13718 supervisor.name.as_str(),
13719 meerkat_core::PeerIngressConvention::Request {
13720 request_id: interaction_id.to_string(),
13721 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
13722 },
13723 )
13724 .with_signing_pubkey(supervisor.pubkey),
13725 );
13726 let candidate = bridge_candidate_with_ingress(supervisor.name.as_str(), &command, ingress);
13727
13728 assert!(
13729 try_handle_supervisor_bridge_command(
13730 &adapter,
13731 &session_id,
13732 &(runtime.clone() as Arc<dyn CommsRuntime>),
13733 &candidate,
13734 )
13735 .await,
13736 "declare-outbound-taint bridge command should be handled"
13737 );
13738 assert_eq!(
13739 runtime.outbound_content_taint(),
13740 Some(meerkat_core::comms::SenderContentTaint::Tainted),
13741 "the relayed host declaration must install on the member runtime"
13742 );
13743
13744 runtime.set_outbound_content_taint(None);
13745 let legacy_command =
13746 BridgeCommand::DeclareMemberOutboundTaint(BridgeOutboundTaintPayload {
13747 supervisor: supervisor.clone().into(),
13748 epoch: 1,
13749 protocol_version: BridgeProtocolVersion::V3,
13750 target: None,
13751 taint: Some(meerkat_core::comms::SenderContentTaint::Tainted),
13752 });
13753 let legacy_id = InteractionId(Uuid::new_v4());
13754 let legacy_ingress = PeerIngressFact::peer(
13755 legacy_id,
13756 PeerInputClass::ActionableRequest,
13757 meerkat_core::PeerIngressKind::Request,
13758 Some(meerkat_core::PeerIngressAuthDecision::Required),
13759 PeerIngressIdentity::new(
13760 supervisor.peer_id,
13761 supervisor.name.as_str(),
13762 meerkat_core::PeerIngressConvention::Request {
13763 request_id: legacy_id.to_string(),
13764 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
13765 },
13766 )
13767 .with_signing_pubkey(supervisor.pubkey),
13768 );
13769 let legacy_candidate = bridge_candidate_with_ingress(
13770 supervisor.name.as_str(),
13771 &legacy_command,
13772 legacy_ingress,
13773 );
13774 assert!(
13775 try_handle_supervisor_bridge_command(
13776 &adapter,
13777 &session_id,
13778 &(runtime.clone() as Arc<dyn CommsRuntime>),
13779 &legacy_candidate,
13780 )
13781 .await
13782 );
13783 assert_eq!(
13784 runtime.outbound_content_taint(),
13785 Some(meerkat_core::comms::SenderContentTaint::Tainted),
13786 "an omitted-target V3 declaration remains valid for a truly peer-only session"
13787 );
13788 }
13789
13790 #[tokio::test]
13791 async fn delayed_g1_outbound_taint_cannot_mutate_revived_g2_member() {
13792 let runtime =
13793 Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-taint-host-g2").unwrap());
13794 install_test_peer_request_response_authority(
13795 &runtime,
13796 Arc::new(CountingPeerInteractionHandle::default()),
13797 );
13798 let supervisor_runtime = Arc::new(
13799 meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor_taint_g2__").unwrap(),
13800 );
13801 let adapter = Arc::new(MeerkatMachine::ephemeral());
13802 let session_id = SessionId::new();
13803 adapter
13804 .register_session(session_id.clone())
13805 .await
13806 .expect("register session");
13807 let supervisor =
13808 trusted_peer_from_runtime("mob/__mob_supervisor_taint_g2__", &supervisor_runtime);
13809 adapter
13810 .stage_supervisor_bind(
13811 &session_id,
13812 supervisor.name.to_string(),
13813 supervisor.peer_id.as_str(),
13814 supervisor.address.to_string(),
13815 signing_public_key_for_descriptor(&supervisor),
13816 1,
13817 )
13818 .await
13819 .expect("pre-bind supervisor");
13820 adapter
13821 .test_install_session_peer_comms_handle_on_runtime(&session_id, runtime.as_ref())
13822 .await
13823 .expect("install adapter session peer-comms handle");
13824
13825 let current = BridgeMemberIncarnation {
13826 mob_id: "mob".to_string(),
13827 agent_identity: "worker".to_string(),
13828 host_id: "host".to_string(),
13829 binding_generation: 2,
13830 member_session_id: session_id.to_string(),
13831 generation: 1,
13832 fence_token: 7,
13833 };
13834 register_member_incarnation(&adapter, session_id.clone(), current.clone())
13835 .await
13836 .expect("register current member incarnation");
13837 let mut delayed = current;
13838 delayed.binding_generation = 1;
13839 let command = BridgeCommand::DeclareMemberOutboundTaint(BridgeOutboundTaintPayload {
13840 supervisor: supervisor.clone().into(),
13841 epoch: 1,
13842 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
13843 target: Some(BridgeOutboundTaintTarget::Placed(delayed)),
13844 taint: Some(meerkat_core::comms::SenderContentTaint::Tainted),
13845 });
13846 let interaction_id = InteractionId(Uuid::new_v4());
13847 let ingress = PeerIngressFact::peer(
13848 interaction_id,
13849 PeerInputClass::ActionableRequest,
13850 meerkat_core::PeerIngressKind::Request,
13851 Some(meerkat_core::PeerIngressAuthDecision::Required),
13852 PeerIngressIdentity::new(
13853 supervisor.peer_id,
13854 supervisor.name.as_str(),
13855 meerkat_core::PeerIngressConvention::Request {
13856 request_id: interaction_id.to_string(),
13857 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
13858 },
13859 )
13860 .with_signing_pubkey(supervisor.pubkey),
13861 );
13862 let candidate = bridge_candidate_with_ingress(supervisor.name.as_str(), &command, ingress);
13863 assert!(
13864 try_handle_supervisor_bridge_command(
13865 &adapter,
13866 &session_id,
13867 &(runtime.clone() as Arc<dyn CommsRuntime>),
13868 &candidate,
13869 )
13870 .await
13871 );
13872 assert_eq!(
13873 runtime.outbound_content_taint(),
13874 None,
13875 "a delayed G1 declaration must be rejected before mutating the revived G2 runtime"
13876 );
13877
13878 let legacy_command =
13879 BridgeCommand::DeclareMemberOutboundTaint(BridgeOutboundTaintPayload {
13880 supervisor: supervisor.clone().into(),
13881 epoch: 1,
13882 protocol_version: BridgeProtocolVersion::V3,
13883 target: None,
13884 taint: Some(meerkat_core::comms::SenderContentTaint::Tainted),
13885 });
13886 let legacy_id = InteractionId(Uuid::new_v4());
13887 let legacy_ingress = PeerIngressFact::peer(
13888 legacy_id,
13889 PeerInputClass::ActionableRequest,
13890 meerkat_core::PeerIngressKind::Request,
13891 Some(meerkat_core::PeerIngressAuthDecision::Required),
13892 PeerIngressIdentity::new(
13893 supervisor.peer_id,
13894 supervisor.name.as_str(),
13895 meerkat_core::PeerIngressConvention::Request {
13896 request_id: legacy_id.to_string(),
13897 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
13898 },
13899 )
13900 .with_signing_pubkey(supervisor.pubkey),
13901 );
13902 let legacy_candidate = bridge_candidate_with_ingress(
13903 supervisor.name.as_str(),
13904 &legacy_command,
13905 legacy_ingress,
13906 );
13907 assert!(
13908 try_handle_supervisor_bridge_command(
13909 &adapter,
13910 &session_id,
13911 &(runtime.clone() as Arc<dyn CommsRuntime>),
13912 &legacy_candidate,
13913 )
13914 .await
13915 );
13916 assert_eq!(
13917 runtime.outbound_content_taint(),
13918 None,
13919 "legacy omitted-target declarations must not address any registered placed incarnation"
13920 );
13921 }
13922
13923 #[test]
13924 fn delayed_g1_peer_message_fence_does_not_match_revived_g2_recipient() {
13925 let current = BridgeMemberIncarnation {
13926 mob_id: "mob".to_string(),
13927 agent_identity: "worker".to_string(),
13928 host_id: "host".to_string(),
13929 binding_generation: 2,
13930 member_session_id: "session".to_string(),
13931 generation: 1,
13932 fence_token: 7,
13933 };
13934 let mut delayed = meerkat_core::comms::PeerRecipientIncarnation {
13935 mob_id: current.mob_id.clone(),
13936 agent_identity: current.agent_identity.clone(),
13937 host_id: current.host_id.clone(),
13938 binding_generation: 1,
13939 member_session_id: current.member_session_id.clone(),
13940 generation: current.generation,
13941 fence_token: current.fence_token,
13942 };
13943 assert!(
13944 !peer_recipient_incarnation_matches(&delayed, ¤t),
13945 "a signed G1 envelope delayed across revoke/rebind cannot address G2"
13946 );
13947 delayed.binding_generation = 2;
13948 assert!(peer_recipient_incarnation_matches(&delayed, ¤t));
13949 }
13950
13951 #[tokio::test]
13952 async fn declare_member_outbound_taint_from_unauthorized_sender_installs_nothing() {
13953 let runtime = Arc::new(
13954 meerkat_comms::CommsRuntime::inproc_only("receiver-taint-unauthorized").unwrap(),
13955 );
13956 let supervisor_runtime = Arc::new(
13957 meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor_taint_real__").unwrap(),
13958 );
13959 let adapter = Arc::new(MeerkatMachine::ephemeral());
13960 let session_id = SessionId::new();
13961 adapter
13962 .register_session(session_id.clone())
13963 .await
13964 .expect("register session");
13965 let supervisor =
13966 trusted_peer_from_runtime("mob/__mob_supervisor_taint_real__", &supervisor_runtime);
13967 add_test_projection_trust(runtime.as_ref(), supervisor.clone(), "trust supervisor").await;
13968 adapter
13969 .stage_supervisor_bind(
13970 &session_id,
13971 supervisor.name.to_string(),
13972 supervisor.peer_id.as_str(),
13973 supervisor.address.to_string(),
13974 signing_public_key_for_descriptor(&supervisor),
13975 1,
13976 )
13977 .await
13978 .expect("pre-bind supervisor");
13979 let impostor_peer_id = PeerId::new();
13983 let candidate = bridge_candidate(
13984 &impostor_peer_id.as_str(),
13985 &BridgeCommand::DeclareMemberOutboundTaint(BridgeOutboundTaintPayload {
13986 supervisor: supervisor.clone().into(),
13987 epoch: 1,
13988 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
13989 target: Some(BridgeOutboundTaintTarget::PeerOnly),
13990 taint: Some(meerkat_core::comms::SenderContentTaint::Tainted),
13991 }),
13992 );
13993
13994 assert!(
13995 try_handle_supervisor_bridge_command(
13996 &adapter,
13997 &session_id,
13998 &(runtime.clone() as Arc<dyn CommsRuntime>),
13999 &candidate,
14000 )
14001 .await,
14002 "unauthorized declare must still be handled (and rejected)"
14003 );
14004 assert_eq!(
14005 runtime.outbound_content_taint(),
14006 None,
14007 "an unauthorized sender must not install an outbound taint declaration"
14008 );
14009 }
14010
14011 #[tokio::test]
14012 async fn wire_member_rejects_mob_overlay_for_different_recipient() {
14013 let runtime =
14014 Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-wire-mismatch").unwrap());
14015 let supervisor_runtime = Arc::new(
14016 meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor_mismatch__").unwrap(),
14017 );
14018 let peer_runtime =
14019 Arc::new(meerkat_comms::CommsRuntime::inproc_only("peer-wire-mismatch").unwrap());
14020 let adapter = Arc::new(MeerkatMachine::ephemeral());
14021 let session_id = SessionId::new();
14022 adapter
14023 .register_session(session_id.clone())
14024 .await
14025 .expect("register session");
14026 let supervisor =
14027 trusted_peer_from_runtime("mob/__mob_supervisor_mismatch__", &supervisor_runtime);
14028 add_test_projection_trust(runtime.as_ref(), supervisor.clone(), "trust supervisor").await;
14029 adapter
14030 .stage_supervisor_bind(
14031 &session_id,
14032 supervisor.name.to_string(),
14033 supervisor.peer_id.as_str(),
14034 supervisor.address.to_string(),
14035 signing_public_key_for_descriptor(&supervisor),
14036 1,
14037 )
14038 .await
14039 .expect("pre-bind supervisor");
14040 let peer_spec = trusted_peer_from_runtime("peer-wire-mismatch", &peer_runtime);
14041 let candidate = bridge_candidate(
14042 &supervisor.peer_id.as_str(),
14043 &BridgeCommand::WireMember(BridgePeerWiringPayload {
14044 supervisor: supervisor.clone().into(),
14045 epoch: 1,
14046 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
14047 peer_spec: peer_spec.clone().into(),
14048 mob_peer_overlay: Some(BridgeMobPeerOverlayHandoff {
14049 recipient_peer_id: PeerId::new().as_str(),
14050 topology_epoch: 1,
14051 peer_specs: vec![peer_spec.clone().into()],
14052 }),
14053 }),
14054 );
14055
14056 assert!(
14057 try_handle_supervisor_bridge_command(
14058 &adapter,
14059 &session_id,
14060 &(runtime.clone() as Arc<dyn CommsRuntime>),
14061 &candidate,
14062 )
14063 .await,
14064 "wire bridge command should be handled"
14065 );
14066 let peers = runtime.peers().await;
14067 assert!(
14068 peers.iter().all(|entry| entry.peer_id != peer_spec.peer_id),
14069 "MobMachine overlay handoff for another recipient must not update local peer trust"
14070 );
14071 }
14072
14073 enum TestWaiterEntry {
14076 Waiting(tokio::sync::oneshot::Sender<PeerInputCandidate>),
14077 TimedOut,
14078 }
14079
14080 struct WaiterReplyRuntime {
14084 notify: Arc<tokio::sync::Notify>,
14085 candidate: std::sync::Mutex<Option<PeerInputCandidate>>,
14086 peer_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
14087 completed_count: std::sync::atomic::AtomicUsize,
14088 waiters: std::sync::Mutex<HashMap<Uuid, TestWaiterEntry>>,
14089 }
14090
14091 impl WaiterReplyRuntime {
14092 fn new(
14093 candidate: PeerInputCandidate,
14094 peer_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
14095 in_reply_to: InteractionId,
14096 entry: TestWaiterEntry,
14097 ) -> Self {
14098 let mut waiters = HashMap::new();
14099 waiters.insert(in_reply_to.0, entry);
14100 Self {
14101 notify: Arc::new(tokio::sync::Notify::new()),
14102 candidate: std::sync::Mutex::new(Some(candidate)),
14103 peer_handle,
14104 completed_count: std::sync::atomic::AtomicUsize::new(0),
14105 waiters: std::sync::Mutex::new(waiters),
14106 }
14107 }
14108
14109 fn completed_count(&self) -> usize {
14110 self.completed_count
14111 .load(std::sync::atomic::Ordering::SeqCst)
14112 }
14113
14114 fn waiter_entry_count(&self) -> usize {
14115 self.waiters.lock().expect("waiters mutex").len()
14116 }
14117 }
14118
14119 #[async_trait::async_trait]
14120 impl CommsRuntime for WaiterReplyRuntime {
14121 async fn drain_messages(&self) -> Vec<String> {
14122 Vec::new()
14123 }
14124
14125 fn inbox_notify(&self) -> Arc<tokio::sync::Notify> {
14126 self.notify.clone()
14127 }
14128
14129 fn peer_interaction_handle(
14130 &self,
14131 ) -> Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>> {
14132 self.peer_handle.clone()
14133 }
14134
14135 async fn drain_classified_inbox_interactions(
14136 &self,
14137 ) -> Result<
14138 Vec<meerkat_core::interaction::ClassifiedInboxInteraction>,
14139 meerkat_core::agent::CommsCapabilityError,
14140 > {
14141 Ok(self
14142 .candidate
14143 .lock()
14144 .expect("candidate mutex")
14145 .take()
14146 .into_iter()
14147 .collect())
14148 }
14149
14150 fn mark_interaction_complete(&self, _id: &InteractionId) {
14151 self.completed_count
14152 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
14153 }
14154
14155 fn take_bridge_reply_waiter(
14156 &self,
14157 in_reply_to: &InteractionId,
14158 ) -> Option<tokio::sync::oneshot::Sender<PeerInputCandidate>> {
14159 match self
14160 .waiters
14161 .lock()
14162 .expect("waiters mutex")
14163 .remove(&in_reply_to.0)
14164 {
14165 Some(TestWaiterEntry::Waiting(sender)) => Some(sender),
14166 Some(TestWaiterEntry::TimedOut) | None => None,
14167 }
14168 }
14169
14170 fn has_bridge_reply_waiter(&self, in_reply_to: &InteractionId) -> bool {
14171 self.waiters
14172 .lock()
14173 .expect("waiters mutex")
14174 .contains_key(&in_reply_to.0)
14175 }
14176 }
14177
14178 fn response_candidate_replying_to(
14179 class: PeerInputClass,
14180 response_terminality: Option<meerkat_core::interaction::TerminalityClass>,
14181 in_reply_to: InteractionId,
14182 ) -> PeerInputCandidate {
14183 let mut candidate = response_candidate(class, response_terminality);
14184 if let InteractionContent::Response {
14185 in_reply_to: reply_slot,
14186 ..
14187 } = &mut candidate.interaction.content
14188 {
14189 *reply_slot = in_reply_to;
14190 }
14191 candidate
14192 }
14193
14194 #[tokio::test]
14199 async fn terminal_response_with_registered_waiter_is_delivered_not_injected() {
14200 let adapter = Arc::new(MeerkatMachine::ephemeral());
14201 let session_id = SessionId::new();
14202 adapter
14203 .register_session(session_id.clone())
14204 .await
14205 .expect("register session");
14206
14207 let in_reply_to = InteractionId(Uuid::new_v4());
14208 let candidate = response_candidate_replying_to(
14209 PeerInputClass::ResponseTerminal,
14210 Some(meerkat_core::interaction::TerminalityClass::Terminal {
14211 disposition: meerkat_core::interaction::TerminalDisposition::Completed,
14212 }),
14213 in_reply_to,
14214 );
14215 let (waiter_tx, waiter_rx) = tokio::sync::oneshot::channel();
14216 let runtime = Arc::new(WaiterReplyRuntime::new(
14217 candidate,
14218 None,
14219 in_reply_to,
14220 TestWaiterEntry::Waiting(waiter_tx),
14221 ));
14222
14223 let drain = spawn_authorized_test_comms_drain(
14224 adapter.clone(),
14225 session_id.clone(),
14226 runtime.clone(),
14227 Duration::from_millis(10),
14228 )
14229 .await;
14230 tokio::time::timeout(Duration::from_secs(1), drain)
14231 .await
14232 .expect("drain should exit after one candidate")
14233 .expect("drain task should not panic");
14234
14235 let delivered = waiter_rx.await.expect("waiter must receive the candidate");
14236 assert!(
14237 matches!(
14238 delivered.response_terminality,
14239 Some(meerkat_core::interaction::TerminalityClass::Terminal {
14240 disposition: meerkat_core::interaction::TerminalDisposition::Completed,
14241 })
14242 ),
14243 "typed terminality must reach the waiter intact"
14244 );
14245 assert!(
14246 matches!(
14247 &delivered.interaction.content,
14248 InteractionContent::Response { in_reply_to: got, .. } if *got == in_reply_to
14249 ),
14250 "the waiter must receive the correlated Response"
14251 );
14252 assert!(
14253 runtime.completed_count() >= 1,
14254 "waiter delivery must mark the interaction complete"
14255 );
14256 assert_eq!(
14257 runtime.waiter_entry_count(),
14258 0,
14259 "take must consume the waiter"
14260 );
14261 let snapshot = adapter
14262 .meerkat_machine_spine_snapshot(&session_id)
14263 .await
14264 .expect("registered session snapshot");
14265 assert_eq!(
14266 snapshot.ledger.input_count, 0,
14267 "a waiter-consumed reply must never become session input"
14268 );
14269 }
14270
14271 #[tokio::test]
14275 async fn tombstoned_waiter_consumes_late_terminal_reply_without_injection() {
14276 let adapter = Arc::new(MeerkatMachine::ephemeral());
14277 let session_id = SessionId::new();
14278 adapter
14279 .register_session(session_id.clone())
14280 .await
14281 .expect("register session");
14282
14283 let in_reply_to = InteractionId(Uuid::new_v4());
14284 let candidate = response_candidate_replying_to(
14285 PeerInputClass::ResponseTerminal,
14286 Some(meerkat_core::interaction::TerminalityClass::Terminal {
14287 disposition: meerkat_core::interaction::TerminalDisposition::Completed,
14288 }),
14289 in_reply_to,
14290 );
14291 let runtime = Arc::new(WaiterReplyRuntime::new(
14292 candidate,
14293 None,
14294 in_reply_to,
14295 TestWaiterEntry::TimedOut,
14296 ));
14297
14298 let drain = spawn_authorized_test_comms_drain(
14299 adapter.clone(),
14300 session_id.clone(),
14301 runtime.clone(),
14302 Duration::from_millis(10),
14303 )
14304 .await;
14305 tokio::time::timeout(Duration::from_secs(1), drain)
14306 .await
14307 .expect("drain should exit after one candidate")
14308 .expect("drain task should not panic");
14309
14310 assert_eq!(
14311 runtime.waiter_entry_count(),
14312 0,
14313 "the tombstone must be consumed by the late reply"
14314 );
14315 assert!(
14316 runtime.completed_count() >= 1,
14317 "late-reply discard must mark the interaction complete"
14318 );
14319 let snapshot = adapter
14320 .meerkat_machine_spine_snapshot(&session_id)
14321 .await
14322 .expect("registered session snapshot");
14323 assert_eq!(
14324 snapshot.ledger.input_count, 0,
14325 "a discarded late reply must never leak into the session"
14326 );
14327 }
14328
14329 #[tokio::test]
14333 async fn progress_response_with_registered_waiter_is_suppressed() {
14334 let adapter = Arc::new(MeerkatMachine::ephemeral());
14335 let session_id = SessionId::new();
14336 adapter
14337 .register_session(session_id.clone())
14338 .await
14339 .expect("register session");
14340
14341 let in_reply_to = InteractionId(Uuid::new_v4());
14342 let candidate = response_candidate_replying_to(
14343 PeerInputClass::ResponseProgress,
14344 Some(meerkat_core::interaction::TerminalityClass::Progress),
14345 in_reply_to,
14346 );
14347 let (waiter_tx, _waiter_rx) = tokio::sync::oneshot::channel();
14348 let runtime = Arc::new(WaiterReplyRuntime::new(
14349 candidate,
14350 None,
14351 in_reply_to,
14352 TestWaiterEntry::Waiting(waiter_tx),
14353 ));
14354
14355 let drain = spawn_authorized_test_comms_drain(
14356 adapter.clone(),
14357 session_id.clone(),
14358 runtime.clone(),
14359 Duration::from_millis(10),
14360 )
14361 .await;
14362 tokio::time::timeout(Duration::from_secs(1), drain)
14363 .await
14364 .expect("drain should exit after one candidate")
14365 .expect("drain task should not panic");
14366
14367 assert_eq!(
14368 runtime.waiter_entry_count(),
14369 1,
14370 "a Progress reply must not consume the waiter"
14371 );
14372 let snapshot = adapter
14373 .meerkat_machine_spine_snapshot(&session_id)
14374 .await
14375 .expect("registered session snapshot");
14376 assert_eq!(
14377 snapshot.ledger.input_count, 0,
14378 "a Progress reply for a waited interaction must not become session input"
14379 );
14380 }
14381
14382 #[tokio::test]
14386 async fn terminal_response_without_waiter_keeps_existing_injection_path() {
14387 let adapter = Arc::new(MeerkatMachine::ephemeral());
14388 let session_id = SessionId::new();
14389 adapter
14390 .register_session(session_id.clone())
14391 .await
14392 .expect("register session");
14393
14394 let runtime = Arc::new(OneShotPeerRequestRuntime::new(
14395 response_candidate(
14396 PeerInputClass::ResponseTerminal,
14397 Some(meerkat_core::interaction::TerminalityClass::Terminal {
14398 disposition: meerkat_core::interaction::TerminalDisposition::Completed,
14399 }),
14400 ),
14401 None,
14402 ));
14403 let drain = spawn_authorized_test_comms_drain(
14404 adapter.clone(),
14405 session_id.clone(),
14406 runtime.clone(),
14407 Duration::from_millis(10),
14408 )
14409 .await;
14410 tokio::time::timeout(Duration::from_secs(1), drain)
14411 .await
14412 .expect("drain should exit after one candidate")
14413 .expect("drain task should not panic");
14414
14415 let snapshot = adapter
14416 .meerkat_machine_spine_snapshot(&session_id)
14417 .await
14418 .expect("registered session snapshot");
14419 assert_eq!(
14420 snapshot.ledger.input_count, 1,
14421 "without a waiter the terminal response must still inject as session input"
14422 );
14423 }
14424
14425 #[tokio::test]
14432 async fn bind_member_invalid_bootstrap_never_stages_payload_reply_address() {
14433 let runtime_impl = bootstrap_runtime(
14434 PEER_ID_RECEIVER,
14435 "inproc://receiver",
14436 Some("expected-token"),
14437 );
14438 let staged = runtime_impl.staged_reply_endpoints.clone();
14439 let staged_correlated = runtime_impl.staged_correlated_reply_endpoints.clone();
14440 let sent = runtime_impl.sent_commands.clone();
14441 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14442 let adapter = Arc::new(MeerkatMachine::ephemeral());
14443 let session_id = SessionId::new();
14444 adapter
14445 .register_session(session_id.clone())
14446 .await
14447 .expect("register session");
14448
14449 for payload_address in [
14450 "tcp://127.0.0.1:9",
14451 "uds:///tmp/meerkat-payload-selected-reply.sock",
14452 ] {
14453 let mut payload = sample_bind_payload();
14454 payload.bootstrap_token = "wrong-token".into();
14455 payload.supervisor.address = payload_address.to_string();
14456 let sender = pubkey_sender_for_bridge_spec(&payload.supervisor);
14459 let candidate = bridge_candidate(&sender, &BridgeCommand::BindMember(payload));
14460
14461 assert!(
14462 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate,)
14463 .await
14464 );
14465 }
14466
14467 let mut open_payload = sample_bind_payload();
14471 open_payload.bootstrap_token = "wrong-token".into();
14472 open_payload.supervisor.address = "tcp://127.0.0.1:19".to_string();
14473 let open_sender = open_payload.supervisor.name.clone();
14474 let open_sender_peer_id =
14475 meerkat_comms::PubKey::new(open_payload.supervisor.pubkey).to_peer_id();
14476 let open_ingress =
14477 bridge_sender_fact_with_display(open_sender_peer_id, open_sender.as_str());
14478 let open_candidate = bridge_candidate_with_ingress(
14479 open_sender.as_str(),
14480 &BridgeCommand::BindMember(open_payload),
14481 open_ingress,
14482 );
14483 assert!(
14484 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &open_candidate,)
14485 .await
14486 );
14487
14488 let staged = staged.lock().expect("staged reply endpoints mutex").clone();
14489 assert!(
14490 staged.is_empty(),
14491 "payload-selected TCP/UDS addresses must never become callback routes: {staged:?}"
14492 );
14493 let staged_correlated = staged_correlated
14494 .lock()
14495 .expect("staged correlated reply endpoints mutex")
14496 .clone();
14497 assert!(
14498 staged_correlated.is_empty(),
14499 "payload-selected TCP/UDS/open-auth addresses must never become correlated callback routes: {staged_correlated:?}"
14500 );
14501 let replies = recorded_bridge_replies(&sent).await;
14502 assert_eq!(
14503 replies.len(),
14504 3,
14505 "one rejection attempt per malicious payload"
14506 );
14507 for (status, reply) in replies {
14508 assert_eq!(status, meerkat_core::interaction::ResponseStatus::Failed);
14509 assert!(
14510 matches!(&reply, BridgeReply::Rejected { .. }),
14511 "invalid bootstrap must produce a typed rejection attempt, got {reply:?}"
14512 );
14513 }
14514 }
14515
14516 #[tokio::test]
14522 async fn invalid_bootstrap_stages_exact_source_confined_callback_tuple() {
14523 let runtime_impl = bootstrap_runtime(
14524 PEER_ID_RECEIVER,
14525 "inproc://receiver",
14526 Some("expected-token"),
14527 );
14528 let staged_legacy = runtime_impl.staged_reply_endpoints.clone();
14529 let staged_correlated = runtime_impl.staged_correlated_reply_endpoints.clone();
14530 let unstaged_correlated = runtime_impl.unstaged_correlated_reply_endpoints.clone();
14531 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14532 let adapter = Arc::new(MeerkatMachine::ephemeral());
14533 let session_id = SessionId::new();
14534 adapter
14535 .register_session(session_id.clone())
14536 .await
14537 .expect("register session");
14538
14539 let mut payload = sample_bind_payload();
14540 payload.bootstrap_token = "wrong-token".into();
14541 payload.supervisor.address =
14542 "uds:///tmp/payload-address-is-not-callback-authority.sock".to_string();
14543 let sender_name = payload.supervisor.name.clone();
14544 let signer_pubkey = payload.supervisor.pubkey;
14545 let sender_peer_id = meerkat_comms::PubKey::new(signer_pubkey).to_peer_id();
14546 let interaction_id = InteractionId(Uuid::new_v4());
14547 let callback = PeerAddress::parse("tcp://192.0.2.44:4311")
14548 .expect("source-confined callback endpoint parses");
14549 let ingress = PeerIngressFact::peer(
14550 interaction_id,
14551 PeerInputClass::ActionableRequest,
14552 meerkat_core::PeerIngressKind::Request,
14553 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
14554 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
14555 )),
14556 PeerIngressIdentity::new(
14557 sender_peer_id,
14558 sender_name.as_str(),
14559 meerkat_core::PeerIngressConvention::Request {
14560 request_id: interaction_id.to_string(),
14561 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
14562 },
14563 )
14564 .with_signing_pubkey(signer_pubkey),
14565 )
14566 .with_declared_reply_endpoint(Some(callback.clone()));
14567 let candidate = bridge_candidate_with_ingress(
14568 sender_name.as_str(),
14569 &BridgeCommand::BindMember(payload),
14570 ingress,
14571 );
14572
14573 assert!(
14574 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate,)
14575 .await
14576 );
14577 assert!(
14578 staged_legacy
14579 .lock()
14580 .expect("staged legacy reply endpoints mutex")
14581 .is_empty(),
14582 "a correlated ingress callback must never use the legacy staging seam"
14583 );
14584 assert_eq!(
14585 staged_correlated
14586 .lock()
14587 .expect("staged correlated reply endpoints mutex")
14588 .as_slice(),
14589 &[(sender_peer_id, interaction_id, signer_pubkey, callback)],
14590 "bridge response staging must preserve peer, correlation, signer, and source-confined endpoint"
14591 );
14592 assert!(
14593 unstaged_correlated
14594 .lock()
14595 .expect("unstaged correlated reply endpoints mutex")
14596 .is_empty(),
14597 "a successfully recorded response must leave cleanup to the one-shot Router send"
14598 );
14599 }
14600
14601 #[tokio::test]
14604 async fn undecodable_bridge_command_stages_no_reply_endpoint() {
14605 let runtime_impl = bootstrap_runtime(
14606 PEER_ID_RECEIVER,
14607 "inproc://receiver",
14608 Some("expected-token"),
14609 );
14610 let staged = runtime_impl.staged_reply_endpoints.clone();
14611 let sent = runtime_impl.sent_commands.clone();
14612 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14613 let adapter = Arc::new(MeerkatMachine::ephemeral());
14614 let session_id = SessionId::new();
14615 adapter
14616 .register_session(session_id.clone())
14617 .await
14618 .expect("register session");
14619
14620 let id = InteractionId(Uuid::new_v4());
14621 let sender_key = meerkat_comms::PubKey::new([0xbb; 32]).to_pubkey_string();
14622 let ingress = bridge_sender_fact_with_id(id, &sender_key);
14623 let candidate = PeerInputCandidate {
14624 interaction: InboxInteraction {
14625 sender_taint: None,
14626 id,
14627 from_route: None,
14628 from: sender_key.clone(),
14629 content: InteractionContent::Request {
14630 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
14631 params: json!({ "not": "a bridge command" }),
14632 blocks: None,
14633 },
14634 rendered_text: String::new(),
14635 handling_mode: HandlingMode::Queue,
14636 render_metadata: None,
14637 objective_id: None,
14638 },
14639 ingress,
14640 lifecycle_peer: None,
14641 response_terminality: None,
14642 };
14643
14644 assert!(
14645 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await
14646 );
14647 assert!(
14648 staged
14649 .lock()
14650 .expect("staged reply endpoints mutex")
14651 .is_empty(),
14652 "pre-decode failures must stage nothing"
14653 );
14654 let replies = recorded_bridge_replies(&sent).await;
14655 assert_eq!(replies.len(), 1, "exactly one rejection reply");
14656 assert!(matches!(&replies[0].1, BridgeReply::Rejected { .. }));
14657 }
14658
14659 #[tokio::test]
14662 async fn bind_member_accept_path_stages_no_reply_endpoint() {
14663 let payload = sample_bind_payload();
14664 let sender = pubkey_sender_for_bridge_spec(&payload.supervisor);
14665 let runtime_impl = bootstrap_runtime(
14666 PEER_ID_RECEIVER,
14667 "inproc://receiver",
14668 Some("expected-token"),
14669 );
14670 let staged = runtime_impl.staged_reply_endpoints.clone();
14671 let sent = runtime_impl.sent_commands.clone();
14672 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14673 let adapter = Arc::new(MeerkatMachine::ephemeral());
14674 let session_id = SessionId::new();
14675 adapter
14676 .register_session(session_id.clone())
14677 .await
14678 .expect("register session");
14679 let candidate = bridge_candidate(&sender, &BridgeCommand::BindMember(payload));
14680
14681 assert!(
14682 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await
14683 );
14684 let replies = recorded_bridge_replies(&sent).await;
14685 assert_eq!(replies.len(), 1);
14686 assert!(
14687 matches!(&replies[0].1, BridgeReply::BindMember(_)),
14688 "expected a successful bind reply, got {:?}",
14689 replies[0].1
14690 );
14691 assert!(
14692 staged
14693 .lock()
14694 .expect("staged reply endpoints mutex")
14695 .is_empty(),
14696 "the accept path must not stage a declared reply endpoint"
14697 );
14698 }
14699
14700 #[tokio::test]
14706 async fn bind_supervisor_for_materialized_session_installs_binding_and_private_trust() {
14707 let runtime_impl = bootstrap_runtime(
14708 PEER_ID_RECEIVER,
14709 "inproc://receiver",
14710 Some("expected-token"),
14711 );
14712 let trusted_peer_ids = runtime_impl.trusted_peer_ids.clone();
14713 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14714 let adapter = Arc::new(MeerkatMachine::ephemeral());
14715 let session_id = SessionId::new();
14716 adapter
14717 .register_session(session_id.clone())
14718 .await
14719 .expect("register session");
14720 let supervisor = trusted_supervisor_descriptor(0xbb);
14721
14722 bind_supervisor_for_materialized_session(
14723 &adapter,
14724 &session_id,
14725 runtime.as_ref(),
14726 &supervisor,
14727 1,
14728 )
14729 .await
14730 .expect("host-seeded supervisor bind must succeed");
14731
14732 assert!(
14733 !matches!(
14734 adapter.supervisor_binding(&session_id).await,
14735 SupervisorBinding::Unbound
14736 ),
14737 "the supervisor binding must be installed"
14738 );
14739 assert!(
14740 trusted_peer_ids
14741 .lock()
14742 .await
14743 .contains(&supervisor.peer_id.to_string()),
14744 "the private supervisor admission edge must be installed"
14745 );
14746
14747 bind_supervisor_for_materialized_session(
14750 &adapter,
14751 &session_id,
14752 runtime.as_ref(),
14753 &supervisor,
14754 1,
14755 )
14756 .await
14757 .expect("idempotent host-seeded rebind must succeed");
14758 assert!(
14759 !matches!(
14760 adapter.supervisor_binding(&session_id).await,
14761 SupervisorBinding::Unbound
14762 ),
14763 "the binding must survive the idempotent rebind"
14764 );
14765 }
14766
14767 #[tokio::test]
14771 async fn authorize_supervisor_for_materialized_session_advances_same_peer_and_key() {
14772 let runtime_impl = bootstrap_runtime(
14773 PEER_ID_RECEIVER,
14774 "inproc://receiver",
14775 Some("expected-token"),
14776 );
14777 let trusted_peer_ids = runtime_impl.trusted_peer_ids.clone();
14778 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14779 let adapter = Arc::new(MeerkatMachine::ephemeral());
14780 let session_id = SessionId::new();
14781 adapter
14782 .register_session(session_id.clone())
14783 .await
14784 .expect("register session");
14785 let original = trusted_supervisor_descriptor(0xbb);
14786 let original_peer_id = original.peer_id.to_string();
14787 bind_supervisor_for_materialized_session(
14788 &adapter,
14789 &session_id,
14790 runtime.as_ref(),
14791 &original,
14792 7,
14793 )
14794 .await
14795 .expect("initial host-seeded supervisor bind");
14796 let refreshed_address = "inproc://mob/__mob_supervisor__/restarted";
14797 let refreshed = TrustedPeerDescriptor::unsigned_with_pubkey(
14798 original.name.clone(),
14799 original.peer_id.to_string(),
14800 original.pubkey,
14801 refreshed_address,
14802 )
14803 .expect("same-authority refreshed descriptor");
14804
14805 authorize_supervisor_for_materialized_session(
14806 &adapter,
14807 &session_id,
14808 runtime.as_ref(),
14809 &refreshed,
14810 8,
14811 )
14812 .await
14813 .expect("same-peer/key monotonic host refresh succeeds");
14814
14815 assert!(matches!(
14816 adapter.supervisor_binding(&session_id).await,
14817 SupervisorBinding::Bound {
14818 ref peer_id,
14819 ref address,
14820 epoch: 8,
14821 ..
14822 } if peer_id == &original_peer_id && address == refreshed_address
14823 ));
14824 assert!(
14825 trusted_peer_ids.lock().await.contains(&original_peer_id),
14826 "the refreshed private supervisor trust edge is published before success"
14827 );
14828 }
14829
14830 #[tokio::test]
14835 async fn authorize_supervisor_for_materialized_session_publication_failure_is_fail_closed_and_retryable()
14836 {
14837 let original = trusted_supervisor_descriptor(0xbb);
14838 let original_peer_id = original.peer_id.to_string();
14839 let refreshed_address = "inproc://mob/__mob_supervisor__/restarted";
14840 let refreshed = TrustedPeerDescriptor::unsigned_with_pubkey(
14841 original.name.clone(),
14842 original.peer_id.to_string(),
14843 original.pubkey,
14844 refreshed_address,
14845 )
14846 .expect("same-authority refreshed descriptor");
14847
14848 let mut failing_runtime_impl = bootstrap_runtime(
14849 PEER_ID_RECEIVER,
14850 "inproc://receiver",
14851 Some("expected-token"),
14852 );
14853 failing_runtime_impl
14854 .add_trusted_peer_errors
14855 .insert(original_peer_id.clone(), "boom".to_string());
14856 let failing_trusted_peer_ids = failing_runtime_impl.trusted_peer_ids.clone();
14857 let failing_runtime: Arc<dyn CommsRuntime> = Arc::new(failing_runtime_impl);
14858 let adapter = Arc::new(MeerkatMachine::ephemeral());
14859 let session_id = SessionId::new();
14860 adapter
14861 .register_session(session_id.clone())
14862 .await
14863 .expect("register session");
14864 adapter
14865 .stage_local_endpoint_for_comms_runtime(&session_id, failing_runtime.as_ref())
14866 .await
14867 .expect("stage local endpoint");
14868 adapter
14869 .stage_supervisor_bind(
14870 &session_id,
14871 original.name.to_string(),
14872 original.peer_id.as_str(),
14873 original.address.to_string(),
14874 signing_public_key_for_descriptor(&original),
14875 7,
14876 )
14877 .await
14878 .expect("seed current binding");
14879 adapter
14880 .stage_supervisor_trust_published(&session_id, original_peer_id.clone(), 7)
14881 .await
14882 .expect("close seeded publication");
14883 failing_trusted_peer_ids
14884 .lock()
14885 .await
14886 .insert(original_peer_id.clone());
14887
14888 let error = authorize_supervisor_for_materialized_session(
14889 &adapter,
14890 &session_id,
14891 failing_runtime.as_ref(),
14892 &refreshed,
14893 8,
14894 )
14895 .await
14896 .expect_err("synthetic trust publication failure must surface");
14897 assert!(matches!(
14898 error,
14899 SupervisorMaterializeAuthorizeError::Apply(_)
14900 ));
14901 assert!(matches!(
14902 adapter.supervisor_binding(&session_id).await,
14903 SupervisorBinding::Bound {
14904 ref peer_id,
14905 ref address,
14906 epoch: 8,
14907 ..
14908 } if peer_id == &original_peer_id && address == refreshed_address
14909 ));
14910 assert!(
14911 failing_trusted_peer_ids.lock().await.is_empty(),
14912 "failed replacement publication must remove the stale route"
14913 );
14914
14915 let retry_runtime_impl = bootstrap_runtime(
14916 PEER_ID_RECEIVER,
14917 "inproc://receiver",
14918 Some("expected-token"),
14919 );
14920 let retry_trusted_peer_ids = retry_runtime_impl.trusted_peer_ids.clone();
14921 let retry_runtime: Arc<dyn CommsRuntime> = Arc::new(retry_runtime_impl);
14922 authorize_supervisor_for_materialized_session(
14923 &adapter,
14924 &session_id,
14925 retry_runtime.as_ref(),
14926 &refreshed,
14927 8,
14928 )
14929 .await
14930 .expect("exact retry re-drives the pending private trust publication");
14931 assert!(
14932 retry_trusted_peer_ids
14933 .lock()
14934 .await
14935 .contains(&original_peer_id),
14936 "retry must publish the refreshed private route"
14937 );
14938 }
14939
14940 #[tokio::test]
14944 async fn bind_supervisor_for_materialized_session_rolls_back_on_publication_failure() {
14945 let supervisor = trusted_supervisor_descriptor(0xbb);
14946 let mut runtime_impl = bootstrap_runtime(
14947 PEER_ID_RECEIVER,
14948 "inproc://receiver",
14949 Some("expected-token"),
14950 );
14951 runtime_impl
14952 .add_trusted_peer_errors
14953 .insert(supervisor.peer_id.to_string(), "boom".to_string());
14954 let trusted_peer_ids = runtime_impl.trusted_peer_ids.clone();
14955 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14956 let adapter = Arc::new(MeerkatMachine::ephemeral());
14957 let session_id = SessionId::new();
14958 adapter
14959 .register_session(session_id.clone())
14960 .await
14961 .expect("register session");
14962
14963 let error = bind_supervisor_for_materialized_session(
14964 &adapter,
14965 &session_id,
14966 runtime.as_ref(),
14967 &supervisor,
14968 1,
14969 )
14970 .await
14971 .expect_err("trust publication failure must fail the bind");
14972 assert!(
14973 matches!(error, SupervisorMaterializeBindError::Commit(_)),
14974 "expected a Commit error, got {error:?}"
14975 );
14976 assert!(
14977 matches!(
14978 adapter.supervisor_binding(&session_id).await,
14979 SupervisorBinding::Unbound
14980 ),
14981 "a failed publication must roll the staged bind back to Unbound"
14982 );
14983 assert!(
14984 trusted_peer_ids.lock().await.is_empty(),
14985 "no trust edge may survive the rollback"
14986 );
14987 }
14988
14989 use crate::member_live::{MemberLiveError, MemberLiveHost, MemberLiveStatus};
14994 use meerkat_contracts::wire::supervisor_bridge::{
14995 BridgeLiveChannelPayload, BridgeLiveControlOutcome, BridgeLiveControlPayload,
14996 BridgeLiveControlVerb, BridgeLiveOpenPayload, BridgeLiveStatusPayload,
14997 };
14998 use meerkat_contracts::{
14999 LiveCloseStatus, LiveCommitInputStatus, LiveInterruptStatus, LiveOpenResult,
15000 LiveOpenTransport, LiveRefreshStatus, LiveTruncateStatus, RealtimeTurningMode,
15001 WireLiveAdapterStatus, WireLiveContinuityMode, WireLiveTransportBootstrap,
15002 };
15003
15004 #[test]
15009 fn member_bridge_capabilities_stay_live_free() {}
15010
15011 #[derive(Debug, Clone, PartialEq)]
15012 enum ScriptedLiveCall {
15013 Open {
15014 session: SessionId,
15015 turning_mode: Option<RealtimeTurningMode>,
15016 transport: Option<LiveOpenTransport>,
15017 },
15018 Close {
15019 session: SessionId,
15020 channel_id: String,
15021 },
15022 Status {
15023 session: SessionId,
15024 channel_id: Option<String>,
15025 },
15026 Control {
15027 session: SessionId,
15028 channel_id: String,
15029 verb: BridgeLiveControlVerb,
15030 },
15031 }
15032
15033 fn scripted_open_result() -> LiveOpenResult {
15034 LiveOpenResult {
15035 channel_id: "chan-scripted".to_string(),
15036 transport: WireLiveTransportBootstrap::Websocket {
15037 url: "wss://advertised.example:8443/live/ws?token=tok-1&channel=chan-scripted&format=pcm_24k_mono"
15038 .to_string(),
15039 token: "tok-1".to_string(),
15040 },
15041 capabilities: meerkat_core::live_adapter::LiveChannelCapabilities::default().into(),
15042 continuity: WireLiveContinuityMode::Fresh,
15043 }
15044 }
15045
15046 #[derive(Default)]
15050 struct ScriptedMemberLive {
15051 calls: std::sync::Mutex<Vec<ScriptedLiveCall>>,
15052 open_gate: Option<Arc<tokio::sync::Semaphore>>,
15053 }
15054
15055 impl ScriptedMemberLive {
15056 fn recorded(&self) -> Vec<ScriptedLiveCall> {
15057 self.calls.lock().expect("scripted live calls lock").clone()
15058 }
15059
15060 fn record(&self, call: ScriptedLiveCall) {
15061 self.calls
15062 .lock()
15063 .expect("scripted live calls lock")
15064 .push(call);
15065 }
15066 }
15067
15068 #[async_trait::async_trait]
15069 impl MemberLiveHost for ScriptedMemberLive {
15070 async fn open(
15071 &self,
15072 session: &SessionId,
15073 turning_mode: Option<RealtimeTurningMode>,
15074 transport: Option<LiveOpenTransport>,
15075 ) -> Result<LiveOpenResult, MemberLiveError> {
15076 self.record(ScriptedLiveCall::Open {
15077 session: session.clone(),
15078 turning_mode,
15079 transport,
15080 });
15081 if let Some(gate) = &self.open_gate {
15082 let _permit = gate.acquire().await.expect("open gate semaphore");
15083 }
15084 Ok(scripted_open_result())
15085 }
15086
15087 async fn close(
15088 &self,
15089 session: &SessionId,
15090 channel_id: &str,
15091 ) -> Result<LiveCloseStatus, MemberLiveError> {
15092 self.record(ScriptedLiveCall::Close {
15093 session: session.clone(),
15094 channel_id: channel_id.to_string(),
15095 });
15096 Ok(LiveCloseStatus::Closed)
15097 }
15098
15099 async fn status(
15100 &self,
15101 session: &SessionId,
15102 channel_id: Option<String>,
15103 ) -> Result<MemberLiveStatus, MemberLiveError> {
15104 self.record(ScriptedLiveCall::Status {
15105 session: session.clone(),
15106 channel_id: channel_id.clone(),
15107 });
15108 Ok(MemberLiveStatus {
15109 channel_id: channel_id.unwrap_or_else(|| "chan-scripted".to_string()),
15110 status: WireLiveAdapterStatus::Ready,
15111 })
15112 }
15113
15114 async fn control(
15115 &self,
15116 session: &SessionId,
15117 channel_id: &str,
15118 verb: BridgeLiveControlVerb,
15119 ) -> Result<BridgeLiveControlOutcome, MemberLiveError> {
15120 self.record(ScriptedLiveCall::Control {
15121 session: session.clone(),
15122 channel_id: channel_id.to_string(),
15123 verb: verb.clone(),
15124 });
15125 Ok(match verb {
15126 BridgeLiveControlVerb::CommitInput => BridgeLiveControlOutcome::CommitInput {
15127 status: LiveCommitInputStatus::Committed,
15128 },
15129 BridgeLiveControlVerb::Interrupt => BridgeLiveControlOutcome::Interrupt {
15130 status: LiveInterruptStatus::Interrupted,
15131 },
15132 BridgeLiveControlVerb::Truncate { .. } => BridgeLiveControlOutcome::Truncate {
15133 status: LiveTruncateStatus::Truncated,
15134 },
15135 BridgeLiveControlVerb::Refresh => BridgeLiveControlOutcome::Refresh {
15136 status: LiveRefreshStatus::Queued,
15137 },
15138 })
15139 }
15140 }
15141
15142 struct LiveArmFixture {
15143 adapter: Arc<MeerkatMachine>,
15144 session_id: SessionId,
15145 member_runtime: Arc<meerkat_comms::CommsRuntime>,
15146 supervisor_runtime: Arc<meerkat_comms::CommsRuntime>,
15147 supervisor_spec: TrustedPeerDescriptor,
15148 incarnation: BridgeMemberIncarnation,
15149 }
15150
15151 impl LiveArmFixture {
15152 async fn bound(label: &str) -> Self {
15156 let suffix = Uuid::new_v4().simple().to_string();
15157 let member_name = format!("live-{label}-member-{suffix}");
15158 let supervisor_name = format!("live-{label}-supervisor-{suffix}");
15159
15160 let mut member_runtime =
15161 meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member runtime");
15162 member_runtime
15163 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from((
15164 [127, 0, 0, 1],
15165 0,
15166 )))
15167 .expect("configure member TCP listener");
15168 member_runtime
15169 .start_listeners()
15170 .await
15171 .expect("start member TCP listener");
15172 let member_runtime = Arc::new(member_runtime);
15173 install_test_peer_request_response_authority(
15174 &member_runtime,
15175 Arc::new(CountingPeerInteractionHandle::default()),
15176 );
15177
15178 let mut supervisor_runtime = meerkat_comms::CommsRuntime::inproc_only(&supervisor_name)
15179 .expect("supervisor runtime");
15180 supervisor_runtime
15181 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from((
15182 [127, 0, 0, 1],
15183 0,
15184 )))
15185 .expect("configure supervisor TCP listener");
15186 supervisor_runtime
15187 .start_listeners()
15188 .await
15189 .expect("start supervisor TCP listener");
15190 let supervisor_runtime = Arc::new(supervisor_runtime);
15191 install_test_peer_request_response_authority(
15192 &supervisor_runtime,
15193 Arc::new(CountingPeerInteractionHandle::default()),
15194 );
15195
15196 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
15197 let supervisor_dsl =
15198 install_running_test_peer_comms_handle(supervisor_runtime.as_ref());
15199 add_test_projection_trust_with_dsl(
15200 supervisor_runtime.as_ref(),
15201 Arc::clone(&supervisor_dsl),
15202 member_spec.clone(),
15203 1,
15204 "supervisor trusts live member target",
15205 )
15206 .await;
15207 let supervisor_spec =
15208 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
15209
15210 let adapter = Arc::new(MeerkatMachine::ephemeral());
15211 let session_id = SessionId::new();
15212 adapter
15213 .register_session(session_id.clone())
15214 .await
15215 .expect("register session");
15216 adapter
15217 .stage_supervisor_bind(
15218 &session_id,
15219 supervisor_spec.name.to_string(),
15220 supervisor_spec.peer_id.as_str(),
15221 supervisor_spec.address.to_string(),
15222 signing_public_key_for_descriptor(&supervisor_spec),
15223 1,
15224 )
15225 .await
15226 .expect("pre-bind supervisor in DSL state");
15227 adapter
15228 .test_install_session_peer_comms_handle_on_runtime(
15229 &session_id,
15230 member_runtime.as_ref(),
15231 )
15232 .await
15233 .expect("install adapter session peer-comms handle on member runtime");
15234 add_member_private_supervisor_trust_via_adapter(
15238 &adapter,
15239 &session_id,
15240 &member_runtime,
15241 &supervisor_spec,
15242 1,
15243 "member starts with supervisor private route",
15244 )
15245 .await;
15246 let incarnation = BridgeMemberIncarnation {
15247 mob_id: "mob".to_string(),
15248 agent_identity: format!("live-{label}-member"),
15249 host_id: "host-a".to_string(),
15250 binding_generation: 1,
15251 member_session_id: session_id.to_string(),
15252 generation: 1,
15253 fence_token: 1,
15254 };
15255 register_member_incarnation(&adapter, session_id.clone(), incarnation.clone())
15256 .await
15257 .expect("register live fixture member incarnation");
15258
15259 Self {
15260 adapter,
15261 session_id,
15262 member_runtime,
15263 supervisor_runtime,
15264 supervisor_spec,
15265 incarnation,
15266 }
15267 }
15268
15269 fn candidate_for(&self, command: &BridgeCommand) -> PeerInputCandidate {
15270 let interaction_id = InteractionId(Uuid::new_v4());
15271 let ingress = PeerIngressFact::peer(
15272 interaction_id,
15273 PeerInputClass::ActionableRequest,
15274 meerkat_core::PeerIngressKind::Request,
15275 Some(meerkat_core::PeerIngressAuthDecision::Required),
15276 PeerIngressIdentity::new(
15277 self.supervisor_spec.peer_id,
15278 self.supervisor_spec.name.as_str(),
15279 meerkat_core::PeerIngressConvention::Request {
15280 request_id: interaction_id.to_string(),
15281 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
15282 },
15283 )
15284 .with_signing_pubkey(self.supervisor_spec.pubkey),
15285 );
15286 bridge_candidate_with_ingress(self.supervisor_spec.name.as_str(), command, ingress)
15287 }
15288
15289 fn supervisor_payload(&self) -> (BridgePeerSpec, u64, BridgeProtocolVersion) {
15290 (
15291 BridgePeerSpec::from(self.supervisor_spec.clone()),
15292 1,
15293 SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
15294 )
15295 }
15296
15297 async fn serve(&self, command: &BridgeCommand) {
15298 let candidate = self.candidate_for(command);
15299 assert!(
15300 try_handle_supervisor_bridge_command(
15301 &self.adapter,
15302 &self.session_id,
15303 &(Arc::clone(&self.member_runtime) as Arc<dyn CommsRuntime>),
15304 &candidate,
15305 )
15306 .await,
15307 "bridge handler must own the live command"
15308 );
15309 }
15310
15311 async fn next_reply(&self, label: &str) -> BridgeReply {
15312 let response = drain_one_candidate(&self.supervisor_runtime, label).await;
15313 let InteractionContent::Response { result, .. } = response.interaction.content else {
15314 panic!("expected bridge response for {label}");
15315 };
15316 serde_json::from_value::<BridgeReply>(result).expect("typed bridge reply")
15317 }
15318 }
15319
15320 #[tokio::test]
15325 async fn member_live_verbs_round_trip_payload_to_trait_to_reply() {
15326 let fixture = LiveArmFixture::bound("roundtrip").await;
15327 let live = Arc::new(ScriptedMemberLive::default());
15328 fixture
15329 .adapter
15330 .set_member_live_host(Arc::clone(&live) as Arc<dyn MemberLiveHost>);
15331 let (supervisor, epoch, protocol_version) = fixture.supervisor_payload();
15332
15333 fixture
15334 .serve(&BridgeCommand::OpenMemberLiveChannel(
15335 BridgeLiveOpenPayload {
15336 supervisor: supervisor.clone(),
15337 epoch,
15338 protocol_version,
15339 expected_member: fixture.incarnation.clone(),
15340 turning_mode: None,
15341 transport: None,
15342 },
15343 ))
15344 .await;
15345 match fixture.next_reply("open reply").await {
15346 BridgeReply::MemberLiveChannelOpened(response) => {
15347 assert_eq!(
15348 response.open,
15349 scripted_open_result(),
15350 "LiveOpenResult must embed VERBATIM (gotcha #14): URL + token untouched"
15351 );
15352 }
15353 other => panic!("expected MemberLiveChannelOpened, got {other:?}"),
15354 }
15355
15356 fixture
15357 .serve(&BridgeCommand::MemberLiveChannelStatus(
15358 BridgeLiveStatusPayload {
15359 supervisor: supervisor.clone(),
15360 epoch,
15361 protocol_version,
15362 expected_member: fixture.incarnation.clone(),
15363 channel_id: None,
15364 },
15365 ))
15366 .await;
15367 match fixture.next_reply("status reply").await {
15368 BridgeReply::MemberLiveChannelStatusReport { channel_id, status } => {
15369 assert_eq!(channel_id, "chan-scripted");
15370 assert_eq!(status, WireLiveAdapterStatus::Ready);
15371 }
15372 other => panic!("expected MemberLiveChannelStatusReport, got {other:?}"),
15373 }
15374
15375 fixture
15376 .serve(&BridgeCommand::ControlMemberLiveChannel(
15377 BridgeLiveControlPayload {
15378 supervisor: supervisor.clone(),
15379 epoch,
15380 protocol_version,
15381 expected_member: fixture.incarnation.clone(),
15382 channel_id: "chan-scripted".to_string(),
15383 verb: BridgeLiveControlVerb::CommitInput,
15384 },
15385 ))
15386 .await;
15387 match fixture.next_reply("control reply").await {
15388 BridgeReply::MemberLiveChannelControlled(response) => {
15389 assert_eq!(
15390 response.outcome,
15391 BridgeLiveControlOutcome::CommitInput {
15392 status: LiveCommitInputStatus::Committed,
15393 }
15394 );
15395 }
15396 other => panic!("expected MemberLiveChannelControlled, got {other:?}"),
15397 }
15398
15399 fixture
15400 .serve(&BridgeCommand::CloseMemberLiveChannel(
15401 BridgeLiveChannelPayload {
15402 supervisor,
15403 epoch,
15404 protocol_version,
15405 expected_member: fixture.incarnation.clone(),
15406 channel_id: "chan-scripted".to_string(),
15407 },
15408 ))
15409 .await;
15410 match fixture.next_reply("close reply").await {
15411 BridgeReply::MemberLiveChannelClosed { status } => {
15412 assert_eq!(status, LiveCloseStatus::Closed);
15413 }
15414 other => panic!("expected MemberLiveChannelClosed, got {other:?}"),
15415 }
15416
15417 let calls = live.recorded();
15418 assert_eq!(
15419 calls,
15420 vec![
15421 ScriptedLiveCall::Open {
15422 session: fixture.session_id.clone(),
15423 turning_mode: None,
15424 transport: None,
15425 },
15426 ScriptedLiveCall::Status {
15427 session: fixture.session_id.clone(),
15428 channel_id: None,
15429 },
15430 ScriptedLiveCall::Control {
15431 session: fixture.session_id.clone(),
15432 channel_id: "chan-scripted".to_string(),
15433 verb: BridgeLiveControlVerb::CommitInput,
15434 },
15435 ScriptedLiveCall::Close {
15436 session: fixture.session_id.clone(),
15437 channel_id: "chan-scripted".to_string(),
15438 },
15439 ],
15440 "every arm must reach the trait with the drain's bound member session"
15441 );
15442 }
15443
15444 #[tokio::test]
15447 async fn unauthorized_supervisor_never_reaches_member_live_host() {
15448 let fixture = LiveArmFixture::bound("unauthorized").await;
15449 let live = Arc::new(ScriptedMemberLive::default());
15450 fixture
15451 .adapter
15452 .set_member_live_host(Arc::clone(&live) as Arc<dyn MemberLiveHost>);
15453 let (supervisor, _epoch, protocol_version) = fixture.supervisor_payload();
15454
15455 fixture
15457 .serve(&BridgeCommand::CloseMemberLiveChannel(
15458 BridgeLiveChannelPayload {
15459 supervisor,
15460 epoch: 0,
15461 protocol_version,
15462 expected_member: fixture.incarnation.clone(),
15463 channel_id: "chan-any".to_string(),
15464 },
15465 ))
15466 .await;
15467 match fixture.next_reply("unauthorized close reply").await {
15468 BridgeReply::Rejected { .. } => {}
15469 other => panic!("expected typed rejection, got {other:?}"),
15470 }
15471 assert!(
15472 live.recorded().is_empty(),
15473 "an unauthorized command must never reach the member live host"
15474 );
15475 }
15476
15477 #[tokio::test]
15481 async fn absent_member_live_slot_rejects_live_transport_unavailable() {
15482 let fixture = LiveArmFixture::bound("absent-slot").await;
15483 let (supervisor, epoch, protocol_version) = fixture.supervisor_payload();
15484
15485 fixture
15486 .serve(&BridgeCommand::CloseMemberLiveChannel(
15487 BridgeLiveChannelPayload {
15488 supervisor,
15489 epoch,
15490 protocol_version,
15491 expected_member: fixture.incarnation.clone(),
15492 channel_id: "chan-any".to_string(),
15493 },
15494 ))
15495 .await;
15496 match fixture.next_reply("absent-slot close reply").await {
15497 BridgeReply::Rejected { cause, reason } => {
15498 assert_eq!(cause, BridgeRejectionCause::LiveTransportUnavailable);
15499 assert_eq!(reason, "member host serves no live substrate");
15500 }
15501 other => panic!("expected typed rejection, got {other:?}"),
15502 }
15503 }
15504
15505 #[test]
15508 fn resolve_member_live_host_absent_is_live_transport_unavailable() {
15509 let adapter = Arc::new(MeerkatMachine::ephemeral());
15510 let Err((cause, reason)) = resolve_member_live_host(&adapter) else {
15511 panic!("empty slot must reject");
15512 };
15513 assert_eq!(cause, BridgeRejectionCause::LiveTransportUnavailable);
15514 assert_eq!(reason, "member host serves no live substrate");
15515 }
15516
15517 #[test]
15522 fn member_live_error_rejection_uses_shared_conversion_and_display_reason() {
15523 let error = MemberLiveError::TransportUnsupported {
15524 requested: "webrtc".to_string(),
15525 };
15526 let (cause, reason) = member_live_error_to_bridge_rejection(&error);
15527 assert_eq!(cause, error.to_bridge_rejection());
15528 assert_eq!(reason, error.to_string());
15529
15530 let internal = MemberLiveError::Internal {
15531 reason: "invariant".to_string(),
15532 };
15533 let (cause, reason) = member_live_error_to_bridge_rejection(&internal);
15534 assert_eq!(cause, BridgeRejectionCause::Internal);
15535 assert_eq!(reason, "member live internal fault: invariant");
15536 }
15537
15538 #[tokio::test]
15542 async fn slow_member_live_open_does_not_block_inline_status() {
15543 let fixture = LiveArmFixture::bound("detached-open").await;
15544 let gate = Arc::new(tokio::sync::Semaphore::new(0));
15545 let live = Arc::new(ScriptedMemberLive {
15546 calls: std::sync::Mutex::new(Vec::new()),
15547 open_gate: Some(Arc::clone(&gate)),
15548 });
15549 fixture
15550 .adapter
15551 .set_member_live_host(Arc::clone(&live) as Arc<dyn MemberLiveHost>);
15552 let (supervisor, epoch, protocol_version) = fixture.supervisor_payload();
15553
15554 fixture
15557 .serve(&BridgeCommand::OpenMemberLiveChannel(
15558 BridgeLiveOpenPayload {
15559 supervisor: supervisor.clone(),
15560 epoch,
15561 protocol_version,
15562 expected_member: fixture.incarnation.clone(),
15563 turning_mode: None,
15564 transport: None,
15565 },
15566 ))
15567 .await;
15568
15569 fixture
15572 .serve(&BridgeCommand::MemberLiveChannelStatus(
15573 BridgeLiveStatusPayload {
15574 supervisor,
15575 epoch,
15576 protocol_version,
15577 expected_member: fixture.incarnation.clone(),
15578 channel_id: None,
15579 },
15580 ))
15581 .await;
15582 match fixture.next_reply("status while open parked").await {
15583 BridgeReply::MemberLiveChannelStatusReport { .. } => {}
15584 other => panic!("expected status report before the gated open reply, got {other:?}"),
15585 }
15586
15587 gate.add_permits(1);
15589 match fixture.next_reply("gated open reply").await {
15590 BridgeReply::MemberLiveChannelOpened(response) => {
15591 assert_eq!(response.open, scripted_open_result());
15592 }
15593 other => panic!("expected MemberLiveChannelOpened, got {other:?}"),
15594 }
15595 }
15596}
15597
15598fn spawn_completion_bridge(
15600 comms_runtime: Option<Arc<dyn CommsRuntime>>,
15601 interaction_id: meerkat_core::interaction::InteractionId,
15602 subscriber: Option<mpsc::Sender<AgentEvent>>,
15603 handle: Option<crate::completion::CompletionHandle>,
15604) {
15605 crate::tokio::spawn(async move {
15606 let delivered_terminal = if let Some(handle) = handle {
15607 match handle.try_wait().await {
15608 Ok(outcome) => {
15609 if let Some(tx) = subscriber {
15610 let event = interaction_terminal_event(interaction_id, outcome);
15611 match crate::tokio::time::timeout(
15612 std::time::Duration::from_secs(5),
15613 tx.send(event),
15614 )
15615 .await
15616 {
15617 Ok(Ok(())) => true,
15618 Ok(Err(_)) => {
15619 tracing::warn!(
15620 %interaction_id,
15621 "completion bridge could not deliver terminal event: subscriber closed"
15622 );
15623 false
15624 }
15625 Err(_) => {
15626 tracing::warn!(
15627 %interaction_id,
15628 "completion bridge could not deliver terminal event: subscriber send timed out after 5s"
15629 );
15630 false
15631 }
15632 }
15633 } else {
15634 tracing::warn!(
15635 %interaction_id,
15636 "completion bridge has no interaction subscriber for terminal event"
15637 );
15638 false
15639 }
15640 }
15641 Err(error) => {
15642 tracing::warn!(
15643 %interaction_id,
15644 error = %error,
15645 "completion bridge waiter failed without generated completion authority; not fabricating terminal event"
15646 );
15647 false
15648 }
15649 }
15650 } else {
15651 tracing::debug!(
15652 %interaction_id,
15653 "completion bridge has no completion handle; not fabricating terminal event"
15654 );
15655 false
15656 };
15657
15658 if !delivered_terminal && let Some(runtime) = comms_runtime {
15659 runtime.abandon_interaction_stream(
15660 &interaction_id,
15661 meerkat_core::InteractionStreamAbandonReason::TerminalDeliveryFailed,
15662 );
15663 }
15664 });
15665}