use super::types::{
FORKED_PARTICIPANT_CLEANUP_CLAIM_TTL, ForkedParticipantAttachmentId,
ForkedParticipantCapabilityId, ForkedParticipantCleanupAttemptId,
ForkedParticipantCleanupClaim, ForkedParticipantCleanupClaimOutcome,
ForkedParticipantCleanupDebt, ForkedParticipantCleanupId, ForkedParticipantCleanupLease,
ForkedParticipantCleanupPublish, ForkedParticipantCleanupReport,
ForkedParticipantExpirySweepReport, ForkedParticipantFingerprintError,
ForkedParticipantForkProtection, ForkedParticipantGrant, ForkedParticipantOwnerRoute,
ForkedParticipantPendingAttachment, ForkedParticipantPendingAttachmentReport,
ForkedParticipantPendingTerminal, ForkedParticipantProvenance, ForkedParticipantRef,
ForkedParticipantReleaseOutcome, ForkedParticipantRequest, ForkedParticipantRequestId,
ForkedParticipantReservation, ForkedParticipantRevocationId,
ForkedParticipantRevocationOutcome, ForkedParticipantSweepEntry, MAX_FORKED_PARTICIPANT_TTL,
MAX_FORKED_PARTICIPANT_USES,
};
use crate::ids::AgentIdentity;
use crate::machines::forked_participant_lifecycle as fp;
use crate::store::{
ForkedParticipantRecord, ForkedParticipantSidecar, ForkedParticipantStore, MobStoreError,
};
use async_trait::async_trait;
use chrono::{DateTime, Duration as ChronoDuration, Utc};
use meerkat_core::SessionId;
use meerkat_core::connection::RealmId;
use meerkat_core::service::SessionError;
use std::sync::Arc;
use thiserror::Error;
const FORKED_PARTICIPANT_CAS_ATTEMPTS: u32 = 5;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PlannedForkRequest {
pub source_identity: AgentIdentity,
pub owner_realm: RealmId,
pub source_session_id: SessionId,
pub planned_child_session_id: SessionId,
pub prefix_message_count: Option<usize>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PlannedForkOutcome {
pub child_session_id: SessionId,
pub prefix_message_count: usize,
pub prefix_digest: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SessionExecutionEvidence {
pub agent_identity: Option<AgentIdentity>,
pub realm_id: Option<RealmId>,
pub tool_access_policy: Option<meerkat_core::ops::ToolAccessPolicy>,
pub auth_binding: Option<meerkat_core::AuthBindingRef>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PlannedChildEvidence {
pub prefix_digest: String,
pub prefix_message_count: usize,
pub execution: SessionExecutionEvidence,
}
#[async_trait]
pub trait ForkedParticipantSourceRuntime: Send + Sync {
async fn session_execution_evidence(
&self,
session_id: &SessionId,
) -> Result<Option<SessionExecutionEvidence>, SessionError>;
async fn fork_planned_child(
&self,
request: PlannedForkRequest,
) -> Result<PlannedForkOutcome, SessionError>;
async fn planned_child_evidence(
&self,
child_session_id: &SessionId,
) -> Result<Option<PlannedChildEvidence>, SessionError>;
async fn archive_fork_session(&self, child_session_id: &SessionId) -> Result<(), SessionError>;
async fn bind_fork_session_to_member(
&self,
child_session_id: &SessionId,
mob_id: &str,
role: &str,
member: &str,
) -> Result<(), SessionError> {
let _ = (child_session_id, mob_id, role, member);
Err(SessionError::Unsupported(
"this source runtime cannot seat a fork session as a mob member".to_string(),
))
}
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum ForkedParticipantError {
#[error("forked participant store failure")]
Store(#[source] MobStoreError),
#[error("forked participant source session failure: {0}")]
Session(#[source] SessionError),
#[error("forked participant capability entropy failure")]
Entropy(#[source] meerkat_core::secret_entropy::SecretEntropyError),
#[error("forked participant request fingerprint failure")]
Fingerprint(#[source] ForkedParticipantFingerprintError),
#[error("invalid forked participant request: {detail}")]
InvalidRequest {
detail: String,
},
#[error("forked participant route is not owned by this service: {detail}")]
ForeignRoute {
detail: String,
},
#[error("forked participant source ownership rejected: {detail}")]
SourceOwnershipRejected {
detail: String,
},
#[error("forked participant reservation rejected: {reason:?}")]
ReservationRejected {
reason: fp::ForkedParticipantReservationRejection,
},
#[error("forked participant activation rejected: {reason:?}")]
ActivationRejected {
reason: fp::ForkedParticipantActivationRejection,
},
#[error("forked participant attach denied: {reason:?}")]
AttachDenied {
reason: fp::ForkedParticipantAttachDenial,
},
#[error("forked participant release rejected: {reason:?}")]
ReleaseRejected {
reason: fp::ForkedParticipantReleaseRejection,
},
#[error("forked participant revocation denied: {reason:?}")]
RevocationDenied {
reason: fp::ForkedParticipantRevocationDenial,
},
#[error("forked participant capability rejected: {detail}")]
CapabilityRejected {
detail: String,
},
#[error("planned fork child conflicts with durable evidence: {detail}")]
PlannedChildConflict {
detail: String,
},
#[error("forked participant record is under concurrent update: {detail}")]
ConcurrentUpdate {
detail: String,
},
#[error("forked participant lifecycle machine refused: {detail}")]
MachineRefused {
detail: String,
},
}
impl From<MobStoreError> for ForkedParticipantError {
fn from(error: MobStoreError) -> Self {
Self::Store(error)
}
}
impl From<SessionError> for ForkedParticipantError {
fn from(error: SessionError) -> Self {
Self::Session(error)
}
}
impl From<meerkat_core::secret_entropy::SecretEntropyError> for ForkedParticipantError {
fn from(error: meerkat_core::secret_entropy::SecretEntropyError) -> Self {
Self::Entropy(error)
}
}
impl From<ForkedParticipantFingerprintError> for ForkedParticipantError {
fn from(error: ForkedParticipantFingerprintError) -> Self {
Self::Fingerprint(error)
}
}
fn machine_refused(detail: impl std::fmt::Debug) -> ForkedParticipantError {
ForkedParticipantError::MachineRefused {
detail: format!("{detail:?}"),
}
}
#[derive(Clone, Copy)]
enum RecordLookup<'a> {
Exact(&'a ForkedParticipantRef),
CapabilityId(&'a ForkedParticipantCapabilityId),
}
struct TransitionPlan<T> {
outcome: T,
next: Option<ForkedParticipantRecord>,
}
pub struct ForkedParticipantService {
owner_route: ForkedParticipantOwnerRoute,
store: Arc<dyn ForkedParticipantStore>,
runtime: Arc<dyn ForkedParticipantSourceRuntime>,
}
impl std::fmt::Debug for ForkedParticipantService {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ForkedParticipantService")
.field("owner_route", &self.owner_route)
.field("store", &"<dyn ForkedParticipantStore>")
.field("runtime", &"<dyn ForkedParticipantSourceRuntime>")
.finish()
}
}
impl ForkedParticipantService {
pub fn new(
owner_route: ForkedParticipantOwnerRoute,
store: Arc<dyn ForkedParticipantStore>,
runtime: Arc<dyn ForkedParticipantSourceRuntime>,
) -> Result<Self, ForkedParticipantError> {
Ok(Self {
owner_route,
store,
runtime,
})
}
pub fn owner_route(&self) -> &ForkedParticipantOwnerRoute {
&self.owner_route
}
fn require_owned_route(
&self,
route: &ForkedParticipantOwnerRoute,
) -> Result<(), ForkedParticipantError> {
if route != &self.owner_route {
return Err(ForkedParticipantError::ForeignRoute {
detail: "the presented route is not the route this owner serves".to_string(),
});
}
Ok(())
}
async fn require_source_ownership(
&self,
source_session_id: &SessionId,
source_identity: &AgentIdentity,
) -> Result<SessionExecutionEvidence, ForkedParticipantError> {
let evidence = self
.runtime
.session_execution_evidence(source_session_id)
.await
.map_err(ForkedParticipantError::Session)?
.ok_or_else(|| ForkedParticipantError::SourceOwnershipRejected {
detail: "source session has no durable execution evidence".to_string(),
})?;
match evidence.agent_identity.as_ref() {
Some(identity) if identity == source_identity => {}
Some(_) => {
return Err(ForkedParticipantError::SourceOwnershipRejected {
detail: "source session belongs to a different member identity".to_string(),
});
}
None => {
return Err(ForkedParticipantError::SourceOwnershipRejected {
detail: "source session carries no member identity".to_string(),
});
}
}
match evidence.realm_id.as_ref() {
Some(realm) if realm == self.owner_route.realm_id() => {}
Some(_) => {
return Err(ForkedParticipantError::SourceOwnershipRejected {
detail: "source session belongs to a different realm".to_string(),
});
}
None => {
return Err(ForkedParticipantError::SourceOwnershipRejected {
detail: "source session carries no realm".to_string(),
});
}
}
Ok(evidence)
}
fn validate(
request: &ForkedParticipantRequest,
now: DateTime<Utc>,
) -> Result<(u64, DateTime<Utc>), ForkedParticipantError> {
if request.source_identity.as_str().trim().is_empty() {
return Err(ForkedParticipantError::InvalidRequest {
detail: "source identity must not be empty".to_string(),
});
}
let max_uses = request.reuse.max_uses();
if max_uses == 0 || max_uses > MAX_FORKED_PARTICIPANT_USES {
return Err(ForkedParticipantError::InvalidRequest {
detail: format!("reuse budget must be 1..={MAX_FORKED_PARTICIPANT_USES}"),
});
}
if request.ttl.is_zero() {
return Err(ForkedParticipantError::InvalidRequest {
detail: "time-to-live must be positive".to_string(),
});
}
if request.ttl > MAX_FORKED_PARTICIPANT_TTL {
return Err(ForkedParticipantError::InvalidRequest {
detail: format!(
"time-to-live must not exceed {} seconds",
MAX_FORKED_PARTICIPANT_TTL.as_secs()
),
});
}
let ttl = ChronoDuration::from_std(request.ttl).map_err(|error| {
ForkedParticipantError::InvalidRequest {
detail: format!("time-to-live is not representable: {error}"),
}
})?;
let expires_at =
now.checked_add_signed(ttl)
.ok_or_else(|| ForkedParticipantError::InvalidRequest {
detail: "expiry instant overflows the representable range".to_string(),
})?;
Ok((u64::from(max_uses), expires_at))
}
async fn load(
&self,
lookup: RecordLookup<'_>,
) -> Result<ForkedParticipantRecord, ForkedParticipantError> {
match lookup {
RecordLookup::Exact(capability) => {
let record = self.store.load_exact(capability).await?;
if &record.sidecar.owner_route != capability.owner_route() {
return Err(ForkedParticipantError::CapabilityRejected {
detail: "capability route does not match the owning record".to_string(),
});
}
if &record.sidecar.source_identity != capability.source_identity() {
return Err(ForkedParticipantError::CapabilityRejected {
detail: "capability source identity does not match the owning record"
.to_string(),
});
}
Ok(record)
}
RecordLookup::CapabilityId(capability_id) => self
.store
.load_by_capability_id(capability_id)
.await?
.ok_or_else(|| ForkedParticipantError::CapabilityRejected {
detail: format!("capability {} not found", capability_id.correlation_hint()),
}),
}
}
async fn transition<T>(
&self,
lookup: RecordLookup<'_>,
mut drive: impl FnMut(
&ForkedParticipantRecord,
) -> Result<TransitionPlan<T>, ForkedParticipantError>,
) -> Result<T, ForkedParticipantError> {
let mut last_conflict = String::new();
for _ in 0..FORKED_PARTICIPANT_CAS_ATTEMPTS {
let record = self.load(lookup).await?;
let plan = drive(&record)?;
let Some(next) = plan.next else {
return Ok(plan.outcome);
};
match self.store.commit(&next).await {
Ok(_) => return Ok(plan.outcome),
Err(MobStoreError::CasConflict(detail)) => {
last_conflict = detail;
}
Err(other) => return Err(other.into()),
}
}
Err(ForkedParticipantError::ConcurrentUpdate {
detail: last_conflict,
})
}
pub async fn reserve(
&self,
request: &ForkedParticipantRequest,
now: DateTime<Utc>,
) -> Result<ForkedParticipantReservation, ForkedParticipantError> {
self.require_owned_route(&request.owner_route)?;
let (max_uses, expires_at) = Self::validate(request, now)?;
self.require_source_ownership(&request.source_session_id, &request.source_identity)
.await?;
let fingerprint = request.fingerprint()?;
if let Some(existing) = self.store.load_by_request_id(&request.request_id).await? {
return self
.replay_reservation(existing, &fingerprint, max_uses)
.await;
}
let mut authority = fp::ForkedParticipantLifecycleMachineAuthority::new();
let transition = fp::ForkedParticipantLifecycleMachineMutator::apply(
&mut authority,
fp::ForkedParticipantLifecycleInput::Reserve {
request_fingerprint: fingerprint.clone(),
max_uses,
},
)
.map_err(machine_refused)?;
Self::expect_reserved(transition.effects())?;
let capability_id = ForkedParticipantCapabilityId::mint()?;
let record = ForkedParticipantRecord {
capability_id,
request_id: request.request_id.clone(),
request_fingerprint: fingerprint.clone(),
planned_child_session_id: SessionId::new(),
sidecar: ForkedParticipantSidecar {
source_identity: request.source_identity.clone(),
source_session_id: request.source_session_id.clone(),
owner_route: request.owner_route.clone(),
scope: request.scope,
reuse: request.reuse,
expires_at,
requested_prefix_message_count: request.prefix_message_count,
capability_ref: None,
},
machine_state: authority.state().clone(),
cleanup_debt: None,
cleanup_claim: None,
revision: 0,
created_at: now,
updated_at: now,
};
match self.store.insert_reserved(&record).await {
Ok(stored) => Ok(ForkedParticipantReservation {
capability_id: stored.capability_id.clone(),
request_id: stored.request_id.clone(),
request_fingerprint: stored.request_fingerprint.clone(),
planned_child_session_id: stored.planned_child_session_id,
replayed: false,
}),
Err(MobStoreError::CasConflict(_)) => {
let existing = self
.store
.load_by_request_id(&request.request_id)
.await?
.ok_or_else(|| ForkedParticipantError::InvalidRequest {
detail: "reservation conflict without a durable winner".to_string(),
})?;
self.replay_reservation(existing, &fingerprint, max_uses)
.await
}
Err(error) => Err(error.into()),
}
}
async fn replay_reservation(
&self,
existing: ForkedParticipantRecord,
fingerprint: &str,
max_uses: u64,
) -> Result<ForkedParticipantReservation, ForkedParticipantError> {
let capability_id = existing.capability_id.clone();
let fingerprint = fingerprint.to_owned();
self.transition(RecordLookup::CapabilityId(&capability_id), move |record| {
let mut authority = fp::ForkedParticipantLifecycleMachineAuthority::recover_from_state(
record.machine_state.clone(),
)
.map_err(machine_refused)?;
let transition = fp::ForkedParticipantLifecycleMachineMutator::apply(
&mut authority,
fp::ForkedParticipantLifecycleInput::Reserve {
request_fingerprint: fingerprint.clone(),
max_uses,
},
)
.map_err(machine_refused)?;
Self::expect_reserved(transition.effects())?;
let outcome = ForkedParticipantReservation {
capability_id: record.capability_id.clone(),
request_id: record.request_id.clone(),
request_fingerprint: record.request_fingerprint.clone(),
planned_child_session_id: record.planned_child_session_id.clone(),
replayed: true,
};
if authority.state() == &record.machine_state {
return Ok(TransitionPlan {
outcome,
next: None,
});
}
let mut next = record.clone();
next.machine_state = authority.state().clone();
Ok(TransitionPlan {
outcome,
next: Some(next),
})
})
.await
}
fn expect_reserved(
effects: &[fp::ForkedParticipantLifecycleEffect],
) -> Result<(), ForkedParticipantError> {
for effect in effects {
match effect {
fp::ForkedParticipantLifecycleEffect::CapabilityReserved { .. }
| fp::ForkedParticipantLifecycleEffect::ReservationReplayed { .. } => {}
fp::ForkedParticipantLifecycleEffect::ReservationRejected { reason } => {
return Err(ForkedParticipantError::ReservationRejected { reason: *reason });
}
other => return Err(machine_refused(other)),
}
}
Ok(())
}
pub async fn create(
&self,
request: &ForkedParticipantRequest,
now: DateTime<Utc>,
) -> Result<ForkedParticipantRef, ForkedParticipantError> {
self.require_owned_route(&request.owner_route)?;
let source_evidence = self
.require_source_ownership(&request.source_session_id, &request.source_identity)
.await?;
let fingerprint = request.fingerprint()?;
if let Some(existing) = self.store.load_by_request_id(&request.request_id).await?
&& existing.request_fingerprint == fingerprint
&& let Some(capability) = existing.sidecar.capability_ref.clone()
{
return self
.record_activation(&existing.capability_id, capability)
.await;
}
let reservation = self.reserve(request, now).await?;
let record = self
.store
.load_by_request_id(&request.request_id)
.await?
.ok_or_else(|| ForkedParticipantError::InvalidRequest {
detail: "reserved capability disappeared before activation".to_string(),
})?;
if let Some(capability) = record.sidecar.capability_ref.clone() {
return self
.record_activation(&record.capability_id, capability)
.await;
}
let planned_child = reservation.planned_child_session_id.clone();
let planned_evidence = match self.runtime.planned_child_evidence(&planned_child).await {
Ok(evidence) => evidence,
Err(error) => {
self.fail_activation_and_archive(&record.capability_id, &planned_child)
.await?;
return Err(ForkedParticipantError::Session(error));
}
};
let outcome = match planned_evidence {
Some(evidence) => {
if let Err(error) = Self::verify_planned_child(&record, &source_evidence, &evidence)
{
self.fail_activation_and_archive(&record.capability_id, &planned_child)
.await?;
return Err(error);
}
PlannedForkOutcome {
child_session_id: planned_child.clone(),
prefix_message_count: evidence.prefix_message_count,
prefix_digest: evidence.prefix_digest,
}
}
None => {
match self
.runtime
.fork_planned_child(PlannedForkRequest {
source_identity: request.source_identity.clone(),
owner_realm: self.owner_route.realm_id().clone(),
source_session_id: request.source_session_id.clone(),
planned_child_session_id: planned_child.clone(),
prefix_message_count: request.prefix_message_count,
})
.await
{
Ok(outcome) => outcome,
Err(error) => {
self.fail_activation_and_archive(&record.capability_id, &planned_child)
.await?;
return Err(ForkedParticipantError::Session(error));
}
}
}
};
if outcome.child_session_id != planned_child {
self.fail_activation_and_archive(&record.capability_id, &planned_child)
.await?;
return Err(ForkedParticipantError::PlannedChildConflict {
detail: "source runtime created a child other than the planned identity"
.to_string(),
});
}
let capability = ForkedParticipantRef::new_source_owned(
record.capability_id.clone(),
record.sidecar.source_identity.clone(),
planned_child.clone(),
record.sidecar.owner_route.clone(),
ForkedParticipantProvenance {
source_session_id: record.sidecar.source_session_id.clone(),
prefix_message_count: outcome.prefix_message_count,
prefix_digest: outcome.prefix_digest,
},
record.sidecar.scope,
record.sidecar.expires_at,
record.sidecar.reuse,
ForkedParticipantRevocationId::for_request(&record.request_id),
ForkedParticipantCleanupId::for_request(&record.request_id),
);
match self
.record_activation(&record.capability_id, capability)
.await
{
Ok(capability) => Ok(capability),
Err(error) => {
self.fail_activation_and_archive(&record.capability_id, &planned_child)
.await?;
Err(error)
}
}
}
fn verify_planned_child(
record: &ForkedParticipantRecord,
source: &SessionExecutionEvidence,
child: &PlannedChildEvidence,
) -> Result<(), ForkedParticipantError> {
if let Some(expected) = record.sidecar.requested_prefix_message_count
&& child.prefix_message_count != expected
{
return Err(ForkedParticipantError::PlannedChildConflict {
detail: format!(
"planned child holds {} messages, request selected {expected}",
child.prefix_message_count
),
});
}
if child.execution.tool_access_policy != source.tool_access_policy {
return Err(ForkedParticipantError::PlannedChildConflict {
detail: "planned child does not inherit the source tool access policy".to_string(),
});
}
if child.execution.auth_binding != source.auth_binding {
return Err(ForkedParticipantError::PlannedChildConflict {
detail: "planned child does not inherit the source auth binding".to_string(),
});
}
if child.execution.realm_id != source.realm_id {
return Err(ForkedParticipantError::PlannedChildConflict {
detail: "planned child does not inherit the source realm".to_string(),
});
}
Ok(())
}
async fn record_activation(
&self,
capability_id: &ForkedParticipantCapabilityId,
capability: ForkedParticipantRef,
) -> Result<ForkedParticipantRef, ForkedParticipantError> {
let capability_for_plan = capability.clone();
self.transition(RecordLookup::CapabilityId(capability_id), move |record| {
let mut authority = fp::ForkedParticipantLifecycleMachineAuthority::recover_from_state(
record.machine_state.clone(),
)
.map_err(machine_refused)?;
let transition = fp::ForkedParticipantLifecycleMachineMutator::apply(
&mut authority,
fp::ForkedParticipantLifecycleInput::RecordForkActivation {
request_fingerprint: record.request_fingerprint.clone(),
fork_activation_id: capability_for_plan.fork_session_id().to_string(),
},
)
.map_err(machine_refused)?;
for effect in transition.effects() {
match effect {
fp::ForkedParticipantLifecycleEffect::ForkActivated { .. }
| fp::ForkedParticipantLifecycleEffect::ForkActivationReplayed { .. } => {}
fp::ForkedParticipantLifecycleEffect::ActivationRejected { reason } => {
return Err(ForkedParticipantError::ActivationRejected { reason: *reason });
}
other => return Err(machine_refused(other)),
}
}
if authority.state() == &record.machine_state
&& let Some(existing) = record.sidecar.capability_ref.clone()
{
return Ok(TransitionPlan {
outcome: existing,
next: None,
});
}
let mut next = record.clone();
next.machine_state = authority.state().clone();
next.sidecar.capability_ref = Some(capability_for_plan.clone());
Ok(TransitionPlan {
outcome: capability_for_plan.clone(),
next: Some(next),
})
})
.await
}
async fn record_activation_failure(
&self,
capability_id: &ForkedParticipantCapabilityId,
) -> Result<(), ForkedParticipantError> {
self.transition(RecordLookup::CapabilityId(capability_id), |record| {
let mut authority = fp::ForkedParticipantLifecycleMachineAuthority::recover_from_state(
record.machine_state.clone(),
)
.map_err(machine_refused)?;
let transition = fp::ForkedParticipantLifecycleMachineMutator::apply(
&mut authority,
fp::ForkedParticipantLifecycleInput::RecordForkActivationFailure {
request_fingerprint: record.request_fingerprint.clone(),
},
)
.map_err(machine_refused)?;
for effect in transition.effects() {
match effect {
fp::ForkedParticipantLifecycleEffect::ForkActivationFailed { .. }
| fp::ForkedParticipantLifecycleEffect::ForkActivationFailureReplayed {
..
} => {}
fp::ForkedParticipantLifecycleEffect::ActivationRejected { reason } => {
return Err(ForkedParticipantError::ActivationRejected { reason: *reason });
}
other => return Err(machine_refused(other)),
}
}
if authority.state() == &record.machine_state {
return Ok(TransitionPlan {
outcome: (),
next: None,
});
}
let mut next = record.clone();
next.machine_state = authority.state().clone();
Ok(TransitionPlan {
outcome: (),
next: Some(next),
})
})
.await
}
async fn fail_activation_and_archive(
&self,
capability_id: &ForkedParticipantCapabilityId,
planned_child: &SessionId,
) -> Result<(), ForkedParticipantError> {
let activation_failure = self.record_activation_failure(capability_id).await;
let archive = self.archive_converging(planned_child).await;
match (activation_failure, archive) {
(Ok(()), Ok(())) => Ok(()),
(Err(error), Ok(())) => Err(error),
(Ok(()), Err(error)) => Err(error),
(Err(activation_error), Err(archive_error)) => {
tracing::error!(
capability = %capability_id.correlation_hint(),
child_session_id = %planned_child,
activation_error = %activation_error,
archive_error = %archive_error,
"fork activation failure and deterministic child archive both failed"
);
Err(archive_error)
}
}
}
pub async fn attach(
&self,
capability: &ForkedParticipantRef,
attachment_id: &ForkedParticipantAttachmentId,
caller_authorized: bool,
now: DateTime<Utc>,
) -> Result<ForkedParticipantGrant, ForkedParticipantError> {
self.require_owned_route(capability.owner_route())?;
let attachment = attachment_id.clone();
let fork_session_id = capability.fork_session_id().clone();
self.transition(RecordLookup::Exact(capability), move |record| {
let expired = now >= record.sidecar.expires_at;
let mut authority = fp::ForkedParticipantLifecycleMachineAuthority::recover_from_state(
record.machine_state.clone(),
)
.map_err(machine_refused)?;
let transition = fp::ForkedParticipantLifecycleMachineMutator::apply(
&mut authority,
fp::ForkedParticipantLifecycleInput::Attach {
attachment_id: attachment.as_str().to_string(),
authentication_valid: caller_authorized,
expired,
},
)
.map_err(machine_refused)?;
let mut grant = None;
let mut denial = None;
for effect in transition.effects() {
match effect {
fp::ForkedParticipantLifecycleEffect::AttachmentGranted {
use_index,
remaining_uses,
..
} => {
grant = Some(ForkedParticipantGrant {
attachment_id: attachment.clone(),
use_index: *use_index,
remaining_uses: *remaining_uses,
replayed: false,
scope: record.sidecar.scope,
fork_session_id: fork_session_id.clone(),
});
}
fp::ForkedParticipantLifecycleEffect::AttachmentGrantReplayed {
use_index,
..
} => {
let max_uses = u64::from(record.sidecar.reuse.max_uses());
grant = Some(ForkedParticipantGrant {
attachment_id: attachment.clone(),
use_index: *use_index,
remaining_uses: max_uses.saturating_sub(*use_index),
replayed: true,
scope: record.sidecar.scope,
fork_session_id: fork_session_id.clone(),
});
}
fp::ForkedParticipantLifecycleEffect::AttachDenied { reason, .. } => {
denial = Some(*reason);
}
fp::ForkedParticipantLifecycleEffect::CapabilityExpired { .. } => {}
other => return Err(machine_refused(other)),
}
}
let next = if authority.state() == &record.machine_state {
None
} else {
let mut next = record.clone();
next.machine_state = authority.state().clone();
Some(next)
};
let outcome = match (grant, denial) {
(Some(grant), _) => Ok(grant),
(None, Some(reason)) => Err(ForkedParticipantError::AttachDenied { reason }),
(None, None) => {
return Err(ForkedParticipantError::MachineRefused {
detail: "attach produced no typed verdict".to_string(),
});
}
};
Ok(TransitionPlan { outcome, next })
})
.await?
}
pub async fn release(
&self,
capability: &ForkedParticipantRef,
attachment_id: &ForkedParticipantAttachmentId,
) -> Result<ForkedParticipantReleaseOutcome, ForkedParticipantError> {
self.require_owned_route(capability.owner_route())?;
let attachment = attachment_id.clone();
self.transition(RecordLookup::Exact(capability), move |record| {
let mut authority = fp::ForkedParticipantLifecycleMachineAuthority::recover_from_state(
record.machine_state.clone(),
)
.map_err(machine_refused)?;
let transition = fp::ForkedParticipantLifecycleMachineMutator::apply(
&mut authority,
fp::ForkedParticipantLifecycleInput::Release {
attachment_id: attachment.as_str().to_string(),
},
)
.map_err(machine_refused)?;
let mut outcome = None;
let mut rejection = None;
for effect in transition.effects() {
match effect {
fp::ForkedParticipantLifecycleEffect::AttachmentReleased { .. } => {
outcome.get_or_insert(ForkedParticipantReleaseOutcome::Reusable);
}
fp::ForkedParticipantLifecycleEffect::CapabilityExhausted { .. } => {
outcome = Some(ForkedParticipantReleaseOutcome::Exhausted);
}
fp::ForkedParticipantLifecycleEffect::CapabilityRevoked { .. } => {
outcome = Some(ForkedParticipantReleaseOutcome::Revoked);
}
fp::ForkedParticipantLifecycleEffect::CapabilityExpired { .. } => {
outcome = Some(ForkedParticipantReleaseOutcome::Expired);
}
fp::ForkedParticipantLifecycleEffect::ReleaseReplayed { .. } => {
outcome = Some(ForkedParticipantReleaseOutcome::Replayed);
}
fp::ForkedParticipantLifecycleEffect::ReleaseRejected { reason, .. } => {
rejection = Some(*reason);
}
other => return Err(machine_refused(other)),
}
}
let next = if authority.state() == &record.machine_state {
None
} else {
let mut next = record.clone();
next.machine_state = authority.state().clone();
Some(next)
};
match (outcome, rejection) {
(Some(outcome), _) => Ok(TransitionPlan { outcome, next }),
(None, Some(reason)) => Err(ForkedParticipantError::ReleaseRejected { reason }),
(None, None) => Err(ForkedParticipantError::MachineRefused {
detail: "release produced no typed verdict".to_string(),
}),
}
})
.await
}
pub async fn revoke(
&self,
capability_id: &ForkedParticipantCapabilityId,
caller_authorized: bool,
) -> Result<ForkedParticipantRevocationOutcome, ForkedParticipantError> {
let owner_route = self.owner_route.clone();
self.transition(RecordLookup::CapabilityId(capability_id), move |record| {
if record.sidecar.owner_route != owner_route {
return Err(ForkedParticipantError::ForeignRoute {
detail: "the record's route is not the route this owner serves".to_string(),
});
}
let mut authority = fp::ForkedParticipantLifecycleMachineAuthority::recover_from_state(
record.machine_state.clone(),
)
.map_err(machine_refused)?;
let transition = fp::ForkedParticipantLifecycleMachineMutator::apply(
&mut authority,
fp::ForkedParticipantLifecycleInput::Revoke {
authentication_valid: caller_authorized,
},
)
.map_err(machine_refused)?;
let mut outcome = None;
let mut denial = None;
for effect in transition.effects() {
match effect {
fp::ForkedParticipantLifecycleEffect::CapabilityRevoked { cleanup_pending } => {
outcome = Some(ForkedParticipantRevocationOutcome::Revoked {
cleanup_pending: *cleanup_pending,
});
}
fp::ForkedParticipantLifecycleEffect::RevocationPendingRecorded => {
outcome = Some(ForkedParticipantRevocationOutcome::PendingAttachedRelease);
}
fp::ForkedParticipantLifecycleEffect::RevocationConverged => {
outcome = Some(ForkedParticipantRevocationOutcome::Converged);
}
fp::ForkedParticipantLifecycleEffect::RevocationDenied { reason } => {
denial = Some(*reason);
}
other => return Err(machine_refused(other)),
}
}
let next = if authority.state() == &record.machine_state {
None
} else {
let mut next = record.clone();
next.machine_state = authority.state().clone();
Some(next)
};
match (outcome, denial) {
(Some(outcome), _) => Ok(TransitionPlan { outcome, next }),
(None, Some(reason)) => Err(ForkedParticipantError::RevocationDenied { reason }),
(None, None) => Err(ForkedParticipantError::MachineRefused {
detail: "revoke produced no typed verdict".to_string(),
}),
}
})
.await
}
pub async fn sweep_expiry(
&self,
now: DateTime<Utc>,
) -> Result<ForkedParticipantExpirySweepReport, ForkedParticipantError> {
let mut report = ForkedParticipantExpirySweepReport::default();
for record in self.store.list_all().await? {
if record.sidecar.owner_route != self.owner_route {
continue;
}
let entry = ForkedParticipantSweepEntry {
capability_id: record.capability_id.clone(),
fork_session_id: record.fork_session_id().cloned(),
};
let mut observed = None;
let result = self
.transition(
RecordLookup::CapabilityId(&record.capability_id),
|record| {
let expired = now >= record.sidecar.expires_at;
let mut authority =
fp::ForkedParticipantLifecycleMachineAuthority::recover_from_state(
record.machine_state.clone(),
)
.map_err(machine_refused)?;
let transition = fp::ForkedParticipantLifecycleMachineMutator::apply(
&mut authority,
fp::ForkedParticipantLifecycleInput::ObserveExpiry { expired },
)
.map_err(machine_refused)?;
for effect in transition.effects() {
match effect {
fp::ForkedParticipantLifecycleEffect::CapabilityExpired {
..
} => observed = Some(ExpiryVerdict::Expired),
fp::ForkedParticipantLifecycleEffect::ExpiryPendingRecorded => {
observed = Some(ExpiryVerdict::PendingAttached);
}
fp::ForkedParticipantLifecycleEffect::ExpiryObservationIgnored {
..
} => {}
other => return Err(machine_refused(other)),
}
}
if authority.state() == &record.machine_state {
return Ok(TransitionPlan {
outcome: (),
next: None,
});
}
let mut next = record.clone();
next.machine_state = authority.state().clone();
Ok(TransitionPlan {
outcome: (),
next: Some(next),
})
},
)
.await;
match result {
Ok(()) => match observed {
Some(ExpiryVerdict::Expired) => report.expired.push(entry),
Some(ExpiryVerdict::PendingAttached) => {
report.expiry_pending_attached.push(entry);
}
None => {}
},
Err(error) => report.failed.push((entry, error.to_string())),
}
}
Ok(report)
}
pub async fn list_pending_attached(
&self,
) -> Result<ForkedParticipantPendingAttachmentReport, ForkedParticipantError> {
let mut report = ForkedParticipantPendingAttachmentReport::default();
for record in self.store.list_all().await? {
let terminal = match record.machine_state.lifecycle_phase {
fp::ForkedParticipantLifecycleState::ExpiryPendingAttached => {
ForkedParticipantPendingTerminal::Expiry
}
fp::ForkedParticipantLifecycleState::RevocationPendingAttached => {
ForkedParticipantPendingTerminal::Revocation
}
_ => continue,
};
if record.sidecar.owner_route != self.owner_route {
continue;
}
let Some(capability) = record.sidecar.capability_ref.clone() else {
report.unreadable.push((
record.capability_id.clone(),
"parked terminal on a record with no activated capability reference"
.to_string(),
));
continue;
};
let Some(raw_attachment_id) = record.machine_state.active_attachment_id.as_ref() else {
report.unreadable.push((
record.capability_id.clone(),
"parked-attached terminal with no active attachment id".to_string(),
));
continue;
};
match ForkedParticipantAttachmentId::new(raw_attachment_id) {
Ok(attachment_id) => report.pending.push(ForkedParticipantPendingAttachment {
capability,
attachment_id,
terminal,
}),
Err(error) => report.unreadable.push((
record.capability_id.clone(),
format!("active attachment id is not a valid identity: {error}"),
)),
}
}
Ok(report)
}
pub async fn protected_fork_session(
&self,
fork_session_id: &SessionId,
) -> Result<Option<ForkedParticipantForkProtection>, ForkedParticipantError> {
let Some(record) = self.store.load_by_fork_session_id(fork_session_id).await? else {
return Ok(None);
};
Ok(Some(ForkedParticipantForkProtection {
capability_hint: record.capability_id.correlation_hint(),
owner_route: record.sidecar.owner_route.clone(),
capability: record.sidecar.capability_ref.clone(),
}))
}
pub async fn sweep_cleanup(
&self,
now: DateTime<Utc>,
) -> Result<ForkedParticipantCleanupReport, ForkedParticipantError> {
let mut report = ForkedParticipantCleanupReport::default();
for record in self.store.list_all().await? {
if record.sidecar.owner_route != self.owner_route {
continue;
}
if record.machine_state.cleanup_state != fp::ForkedParticipantCleanupState::Pending {
continue;
}
let entry = ForkedParticipantSweepEntry {
capability_id: record.capability_id.clone(),
fork_session_id: record.fork_session_id().cloned(),
};
let lease = match self.claim_cleanup(&record.capability_id, now).await {
Ok(ForkedParticipantCleanupClaimOutcome::Claimed(lease)) => lease,
Ok(ForkedParticipantCleanupClaimOutcome::ClaimedElsewhere) => {
report.claimed_elsewhere.push(entry);
continue;
}
Ok(ForkedParticipantCleanupClaimOutcome::NotPending) => continue,
Err(error) => {
report.failed.push((entry, error.to_string()));
continue;
}
};
let archive = match entry.fork_session_id.as_ref() {
Some(fork_session_id) => self.archive_converging(fork_session_id).await,
None => Ok(()),
};
match archive {
Ok(()) => match self.publish_cleanup_success(&lease).await {
Ok(ForkedParticipantCleanupPublish::Published(())) => {
report.completed.push(entry);
}
Ok(ForkedParticipantCleanupPublish::ClaimLost) => {
report.claimed_elsewhere.push(entry);
}
Err(error) => report.failed.push((entry, error.to_string())),
},
Err(error) => {
let detail = error.to_string();
match self.publish_cleanup_failure(&lease, detail, now).await {
Ok(ForkedParticipantCleanupPublish::Published(debt)) => {
report.retained.push((entry, debt));
}
Ok(ForkedParticipantCleanupPublish::ClaimLost) => {
report.claimed_elsewhere.push(entry);
}
Err(error) => report.failed.push((entry, error.to_string())),
}
}
}
}
Ok(report)
}
async fn archive_converging(
&self,
fork_session_id: &SessionId,
) -> Result<(), ForkedParticipantError> {
match self.runtime.archive_fork_session(fork_session_id).await {
Ok(()) => Ok(()),
Err(SessionError::NotFound { .. }) => Ok(()),
Err(error) => Err(ForkedParticipantError::Session(error)),
}
}
pub async fn claim_cleanup(
&self,
capability_id: &ForkedParticipantCapabilityId,
now: DateTime<Utc>,
) -> Result<ForkedParticipantCleanupClaimOutcome, ForkedParticipantError> {
let attempt_id = ForkedParticipantCleanupAttemptId::mint()?;
let stale_after =
ChronoDuration::from_std(FORKED_PARTICIPANT_CLEANUP_CLAIM_TTL).map_err(|error| {
ForkedParticipantError::InvalidRequest {
detail: format!("cleanup claim ttl is not representable: {error}"),
}
})?;
self.transition(RecordLookup::CapabilityId(capability_id), move |record| {
if record.machine_state.cleanup_state != fp::ForkedParticipantCleanupState::Pending {
return Ok(TransitionPlan {
outcome: ForkedParticipantCleanupClaimOutcome::NotPending,
next: None,
});
}
if let Some(claim) = record.cleanup_claim.as_ref()
&& now.signed_duration_since(claim.claimed_at) < stale_after
{
return Ok(TransitionPlan {
outcome: ForkedParticipantCleanupClaimOutcome::ClaimedElsewhere,
next: None,
});
}
let claim_revision = record.revision.checked_add(1).ok_or_else(|| {
ForkedParticipantError::InvalidRequest {
detail: "cleanup claim revision counter is exhausted".to_string(),
}
})?;
let mut next = record.clone();
next.cleanup_claim = Some(ForkedParticipantCleanupClaim {
attempt_id: attempt_id.clone(),
claimed_at: now,
});
Ok(TransitionPlan {
outcome: ForkedParticipantCleanupClaimOutcome::Claimed(
ForkedParticipantCleanupLease::new_owned(
record.capability_id.clone(),
attempt_id.clone(),
now,
claim_revision,
),
),
next: Some(next),
})
})
.await
}
pub async fn publish_cleanup_failure(
&self,
lease: &ForkedParticipantCleanupLease,
detail: String,
now: DateTime<Utc>,
) -> Result<ForkedParticipantCleanupPublish<ForkedParticipantCleanupDebt>, ForkedParticipantError>
{
let attempt_id = lease.attempt_id().clone();
self.transition(
RecordLookup::CapabilityId(lease.capability_id()),
move |record| {
if !claim_is_held_by(record, &attempt_id) {
return Ok(TransitionPlan {
outcome: ForkedParticipantCleanupPublish::ClaimLost,
next: None,
});
}
let Some(fork_session_id) = record.fork_session_id().cloned() else {
return Err(ForkedParticipantError::MachineRefused {
detail: "cleanup failure recorded for a record without a fork".to_string(),
});
};
let attempts = record
.cleanup_debt
.as_ref()
.map_or(0, |debt| debt.attempts)
.saturating_add(1);
let debt = ForkedParticipantCleanupDebt {
fork_session_id,
attempts,
last_error: detail.clone(),
observed_at: now,
};
let mut next = record.clone();
next.cleanup_debt = Some(debt.clone());
next.cleanup_claim = None;
Ok(TransitionPlan {
outcome: ForkedParticipantCleanupPublish::Published(debt),
next: Some(next),
})
},
)
.await
}
pub async fn publish_cleanup_success(
&self,
lease: &ForkedParticipantCleanupLease,
) -> Result<ForkedParticipantCleanupPublish<()>, ForkedParticipantError> {
let attempt_id = lease.attempt_id().clone();
self.transition(
RecordLookup::CapabilityId(lease.capability_id()),
move |record| {
if !claim_is_held_by(record, &attempt_id) {
return Ok(TransitionPlan {
outcome: ForkedParticipantCleanupPublish::ClaimLost,
next: None,
});
}
let mut authority =
fp::ForkedParticipantLifecycleMachineAuthority::recover_from_state(
record.machine_state.clone(),
)
.map_err(machine_refused)?;
let transition = fp::ForkedParticipantLifecycleMachineMutator::apply(
&mut authority,
fp::ForkedParticipantLifecycleInput::CompleteCleanup {},
)
.map_err(machine_refused)?;
for effect in transition.effects() {
match effect {
fp::ForkedParticipantLifecycleEffect::CleanupCompleted
| fp::ForkedParticipantLifecycleEffect::CleanupCompletionReplayed => {}
fp::ForkedParticipantLifecycleEffect::CleanupCompletionRejected {
reason,
} => {
return Err(ForkedParticipantError::MachineRefused {
detail: format!("cleanup completion rejected: {reason:?}"),
});
}
other => return Err(machine_refused(other)),
}
}
let mut next = record.clone();
next.machine_state = authority.state().clone();
next.cleanup_debt = None;
next.cleanup_claim = None;
Ok(TransitionPlan {
outcome: ForkedParticipantCleanupPublish::Published(()),
next: Some(next),
})
},
)
.await
}
pub async fn load_record(
&self,
capability_id: &ForkedParticipantCapabilityId,
) -> Result<Option<ForkedParticipantRecord>, ForkedParticipantError> {
Ok(self.store.load_by_capability_id(capability_id).await?)
}
}
fn claim_is_held_by(
record: &ForkedParticipantRecord,
attempt_id: &ForkedParticipantCleanupAttemptId,
) -> bool {
record
.cleanup_claim
.as_ref()
.is_some_and(|claim| &claim.attempt_id == attempt_id)
}
#[derive(Clone, Copy)]
enum ExpiryVerdict {
Expired,
PendingAttached,
}