1mod approvals;
4mod checkpoints;
5mod host_events;
6mod runs;
7mod sessions;
8mod streams;
9mod traces;
10
11use std::{
12 collections::BTreeMap,
13 sync::{Arc, Mutex},
14};
15
16use async_trait::async_trait;
17use starweaver_context::{AgentCheckpoint, ResumableState};
18use starweaver_core::{RunId, RunLifecycle, SessionId};
19use starweaver_stream::{AgentStreamRecord, ReplayEvent, ReplayScope};
20
21use crate::{
22 AcquireBackgroundSubagentContinuation, AcquireRunAdmission, AdmitRunControl,
23 BackgroundSubagentArtifact, BackgroundSubagentArtifactLimits,
24 BackgroundSubagentContinuationReceipt, BackgroundSubagentRecord,
25 BackgroundSubagentTerminalCommit, DurableBackgroundSubagentDeliveryClaim,
26 DurableBackgroundSubagentDeliveryRelease, DurableBackgroundSubagentDeliveryStatus,
27 DurableBackgroundSubagentExecutionStatus, DurableBackgroundSubagentResultRef,
28 DurableControlReceipt, DurableRunControlIntent, DurableRunControlStatus, ManagedRunTarget,
29 ManagedSessionTarget, RunAdmissionLease, RunAdmissionReceipt, SessionContinuationFence,
30 SessionDeletionFence, UpdateManagedSession,
31 approval::{ApprovalRecord, ApprovalStatus, DeferredToolRecord},
32 claim::{
33 ContinuationEffectState, HitlResumeAbortOutcome, HitlResumeClaim, HitlResumeClaimState,
34 },
35 error::{SessionStoreError, SessionStoreResult},
36 evidence::RunEvidenceCommit,
37 host_events::{
38 DurableHostEventClass, DurableHostEventPage, DurableHostEventQuery, DurableHostEventRecord,
39 DurableHostEventScope, EventPublicationKey, PendingHostEventPublication,
40 append_authoritative_run_publications,
41 },
42 publication::{PendingStreamPublication, StreamPublicationTarget},
43 records::{
44 EnvironmentStateRef, ExecutionStatus, RunRecord, RunStatus, RunTerminalError,
45 RunTerminalProjection, SessionRecord, SessionStatus, StreamCursorRef,
46 },
47 resume::SessionResumeSnapshot,
48 trace::{CompactRunTrace, CompactSessionTrace},
49};
50
51use self::host_events::enqueue_host_event_publications_locked;
52use super::{SessionFilter, SessionPage, SessionPageQuery, SessionStore};
53
54#[derive(Clone, Debug, Default)]
56pub struct InMemorySessionStore {
57 inner: Arc<Mutex<StoreInner>>,
58}
59
60#[derive(Clone, Debug, Default)]
61struct StoreInner {
62 sessions: BTreeMap<SessionId, SessionRecord>,
63 runs: BTreeMap<(SessionId, RunId), RunRecord>,
64 checkpoints: BTreeMap<(SessionId, RunId), Vec<AgentCheckpoint>>,
65 streams: BTreeMap<(SessionId, RunId), Vec<AgentStreamRecord>>,
66 replay_events: BTreeMap<(ReplayScope, usize), ReplayEvent>,
67 approvals: BTreeMap<(SessionId, RunId), Vec<ApprovalRecord>>,
68 deferred_tools: BTreeMap<(SessionId, RunId), Vec<DeferredToolRecord>>,
69 evidence_commits: BTreeMap<(SessionId, RunId), RunEvidenceCommit>,
70 evidence_digests: BTreeMap<(SessionId, RunId), String>,
71 hitl_resume_claims: BTreeMap<(SessionId, RunId), HitlResumeClaim>,
72 stream_publication_outbox: BTreeMap<String, PendingStreamPublication>,
73 host_event_outbox: BTreeMap<EventPublicationKey, PendingHostEventPublication>,
74 host_event_outbox_order: BTreeMap<u64, EventPublicationKey>,
75 host_event_outbox_sequences: BTreeMap<EventPublicationKey, u64>,
76 last_host_event_outbox_sequence: u64,
77 host_event_records: BTreeMap<u64, DurableHostEventRecord>,
78 host_event_positions: BTreeMap<EventPublicationKey, u64>,
79 host_event_ids: BTreeMap<String, EventPublicationKey>,
80 last_host_event_position: u64,
81 session_idempotency: BTreeMap<(String, String), (String, SessionRecord)>,
82 run_admission_idempotency: BTreeMap<(String, String), (String, RunAdmissionReceipt)>,
83 run_admissions: BTreeMap<ManagedSessionTarget, RunAdmissionLease>,
84 admission_generations: BTreeMap<ManagedSessionTarget, u64>,
85 control_receipts: BTreeMap<String, DurableControlReceipt>,
86 control_idempotency: BTreeMap<(ManagedRunTarget, String), String>,
87 run_control_intents: BTreeMap<(ManagedRunTarget, String), DurableRunControlIntent>,
88 run_control_authority_keys: BTreeMap<(String, String), (ManagedRunTarget, String)>,
89 background_subagents: BTreeMap<starweaver_core::SubagentAttemptId, BackgroundSubagentRecord>,
90 background_artifacts: BTreeMap<String, BackgroundSubagentArtifact>,
91 background_terminal_fingerprints: BTreeMap<starweaver_core::SubagentAttemptId, String>,
92}
93
94impl InMemorySessionStore {
95 #[must_use]
97 pub fn new() -> Self {
98 Self::default()
99 }
100}
101
102fn run_key(session_id: &SessionId, run_id: &RunId) -> (SessionId, RunId) {
103 (session_id.clone(), run_id.clone())
104}
105
106fn run_key_label(session_id: &SessionId, run_id: &RunId) -> String {
107 format!("{}:{}", session_id.as_str(), run_id.as_str())
108}
109
110fn advance_run_revision(run: &mut RunRecord) -> SessionStoreResult<()> {
111 run.revision = run.revision.checked_add(1).ok_or_else(|| {
112 SessionStoreError::Failed(format!("run {} revision overflow", run.run_id.as_str()))
113 })?;
114 Ok(())
115}
116
117fn validate_approval_transition(
118 existing: &ApprovalRecord,
119 resolved: &ApprovalRecord,
120) -> SessionStoreResult<()> {
121 if existing == resolved {
122 return Ok(());
123 }
124 let same_request = existing.approval_id == resolved.approval_id
125 && existing.session_id == resolved.session_id
126 && existing.run_id == resolved.run_id
127 && existing.action_id == resolved.action_id
128 && existing.action_name == resolved.action_name
129 && existing.request == resolved.request
130 && existing.reviewed_arguments == resolved.reviewed_arguments
131 && existing.created_at == resolved.created_at
132 && existing.trace_context == resolved.trace_context
133 && existing.metadata == resolved.metadata;
134 if existing.status != ApprovalStatus::Pending
135 || resolved.status == ApprovalStatus::Pending
136 || resolved.decision.is_none()
137 || !same_request
138 {
139 return Err(SessionStoreError::Failed(format!(
140 "approval transition conflict for {}",
141 resolved.approval_id
142 )));
143 }
144 Ok(())
145}
146
147const fn checkpoint_run_status(status: RunLifecycle) -> RunStatus {
148 match status {
149 RunLifecycle::Starting | RunLifecycle::Running => RunStatus::Running,
150 RunLifecycle::Waiting => RunStatus::Waiting,
151 RunLifecycle::Completed => RunStatus::Completed,
152 RunLifecycle::Failed => RunStatus::Failed,
153 RunLifecycle::Cancelled => RunStatus::Cancelled,
154 }
155}
156
157fn checkpoint_terminal_error(status: RunLifecycle) -> Option<RunTerminalError> {
158 match status {
159 RunLifecycle::Failed => Some(RunTerminalError::new(
160 "checkpoint_run_failed",
161 "run failed while checkpointing",
162 )),
163 RunLifecycle::Cancelled => Some(RunTerminalError::new(
164 "checkpoint_run_cancelled",
165 "run was cancelled while checkpointing",
166 )),
167 RunLifecycle::Starting
168 | RunLifecycle::Running
169 | RunLifecycle::Waiting
170 | RunLifecycle::Completed => None,
171 }
172}
173
174fn validate_deferred_transition(
175 existing: &DeferredToolRecord,
176 resolved: &DeferredToolRecord,
177) -> SessionStoreResult<()> {
178 if existing == resolved {
179 return Ok(());
180 }
181 let same_request = existing.deferred_id == resolved.deferred_id
182 && existing.session_id == resolved.session_id
183 && existing.run_id == resolved.run_id
184 && existing.tool_call_id == resolved.tool_call_id
185 && existing.tool_name == resolved.tool_name
186 && existing.request == resolved.request
187 && existing.created_at == resolved.created_at
188 && existing.trace_context == resolved.trace_context;
189 if !matches!(
190 existing.status,
191 ExecutionStatus::Pending | ExecutionStatus::Waiting
192 ) || matches!(
193 resolved.status,
194 ExecutionStatus::Pending | ExecutionStatus::Running | ExecutionStatus::Waiting
195 ) || !same_request
196 {
197 return Err(SessionStoreError::Failed(format!(
198 "deferred tool transition conflict for {}",
199 resolved.deferred_id
200 )));
201 }
202 Ok(())
203}
204
205#[allow(clippy::needless_pass_by_value)]
206fn store_failed(
207 error: std::sync::PoisonError<std::sync::MutexGuard<'_, StoreInner>>,
208) -> SessionStoreError {
209 SessionStoreError::Failed(error.to_string())
210}
211
212fn session_mutation_time(
213 publications: &[PendingHostEventPublication],
214) -> SessionStoreResult<chrono::DateTime<chrono::Utc>> {
215 let Some(first) = publications.first() else {
216 return Ok(chrono::Utc::now());
217 };
218 if publications
219 .iter()
220 .any(|publication| publication.occurred_at != first.occurred_at)
221 {
222 return Err(SessionStoreError::Conflict(
223 "session mutation publications must share one occurred_at".to_string(),
224 ));
225 }
226 Ok(first.occurred_at)
227}
228
229fn ensure_active_admission_locked(
230 inner: &StoreInner,
231 lease: &RunAdmissionLease,
232 now: chrono::DateTime<chrono::Utc>,
233) -> SessionStoreResult<()> {
234 let key = ManagedSessionTarget::new(
235 lease.target.namespace_id.clone(),
236 lease.target.session_id.clone(),
237 );
238 let current = inner.run_admissions.get(&key).ok_or_else(|| {
239 SessionStoreError::StaleFence("run has no active owner lease".to_string())
240 })?;
241 if current.target != lease.target
242 || current.admission_id != lease.admission_id
243 || current.host_instance_id != lease.host_instance_id
244 || current.fencing_generation != lease.fencing_generation
245 {
246 return Err(SessionStoreError::StaleFence(
247 "stale admission owner".to_string(),
248 ));
249 }
250 if current.expired_at(now) {
251 return Err(SessionStoreError::StaleFence(
252 "run admission lease expired".to_string(),
253 ));
254 }
255 Ok(())
256}
257
258fn reconcile_run_control_intents_locked(
259 inner: &mut StoreInner,
260 lease: &RunAdmissionLease,
261 occurred_at: chrono::DateTime<chrono::Utc>,
262) -> SessionStoreResult<()> {
263 let keys = inner
264 .run_control_intents
265 .iter()
266 .filter(|(_, intent)| {
267 intent.target == lease.target
268 && intent.admission_id == lease.admission_id
269 && intent.fencing_generation == lease.fencing_generation
270 && matches!(
271 intent.status,
272 DurableRunControlStatus::Pending | DurableRunControlStatus::Delivered
273 )
274 })
275 .map(|(key, _)| key.clone())
276 .collect::<Vec<_>>();
277 for key in keys {
278 let mut intent = inner
279 .run_control_intents
280 .get(&key)
281 .cloned()
282 .ok_or_else(|| SessionStoreError::NotFound(key.1.clone()))?;
283 intent
284 .advance(DurableRunControlStatus::Reconciled, occurred_at)
285 .map_err(|error| SessionStoreError::Conflict(error.to_string()))?;
286 inner
287 .control_receipts
288 .insert(intent.receipt.receipt_id.clone(), intent.receipt.clone());
289 inner.run_control_intents.insert(key, intent);
290 }
291 Ok(())
292}
293
294fn apply_run_status_locked(
295 inner: &mut StoreInner,
296 session_id: &SessionId,
297 run_id: &RunId,
298 status: RunStatus,
299 output_preview: Option<String>,
300 terminal_error: Option<RunTerminalError>,
301 updated_at: chrono::DateTime<chrono::Utc>,
302) -> SessionStoreResult<RunRecord> {
303 let key = run_key(session_id, run_id);
304 let run = inner
305 .runs
306 .get_mut(&key)
307 .ok_or_else(|| SessionStoreError::NotFound(run_key_label(session_id, run_id)))?;
308 if (run.status, &run.output_preview, &run.terminal_error)
309 == (status, &output_preview, &terminal_error)
310 {
311 return Ok(run.clone());
312 }
313 run.status = status;
314 run.output_preview = output_preview;
315 run.terminal_error = terminal_error;
316 advance_run_revision(run)?;
317 run.updated_at = updated_at;
318 let result = run.clone();
319 if let Some(session) = inner.sessions.get_mut(session_id) {
320 session.head_run_id = Some(run_id.clone());
321 if status.is_active() {
322 session.active_run_id = Some(run_id.clone());
323 } else {
324 if status == RunStatus::Completed {
325 session.head_success_run_id = Some(run_id.clone());
326 }
327 if session.active_run_id.as_ref() == Some(run_id) {
328 session.active_run_id = None;
329 }
330 }
331 session.revision = session.revision.saturating_add(1);
332 session.updated_at = updated_at;
333 }
334 Ok(result)
335}
336
337fn terminalize_started_hitl_source_locked(
338 inner: &mut StoreInner,
339 replacement: &RunRecord,
340 now: chrono::DateTime<chrono::Utc>,
341) -> SessionStoreResult<bool> {
342 let Some(source_run_id) = replacement.restore_from_run_id.as_ref() else {
343 return Ok(false);
344 };
345 let source_key = run_key(&replacement.session_id, source_run_id);
346 let source = inner.runs.get(&source_key).ok_or_else(|| {
347 SessionStoreError::NotFound(run_key_label(&replacement.session_id, source_run_id))
348 })?;
349 if source.status != RunStatus::Waiting {
350 return Ok(false);
351 }
352 let claim = inner
353 .hitl_resume_claims
354 .get(&source_key)
355 .cloned()
356 .ok_or_else(|| {
357 SessionStoreError::Conflict(format!(
358 "waiting replacement source {} has no resume claim",
359 source_run_id.as_str()
360 ))
361 })?;
362 if claim.session_id != replacement.session_id || claim.run_id != *source_run_id {
363 return Err(SessionStoreError::Conflict(format!(
364 "waiting replacement source {} has a mismatched resume claim",
365 source_run_id.as_str()
366 )));
367 }
368 if claim.state == HitlResumeClaimState::Preflight {
369 return Err(SessionStoreError::Conflict(format!(
370 "waiting replacement source {} has an invalid preflight claim",
371 source_run_id.as_str()
372 )));
373 }
374 if claim.state == HitlResumeClaimState::Started {
375 let source = inner.runs.get_mut(&source_key).ok_or_else(|| {
376 SessionStoreError::NotFound(run_key_label(&replacement.session_id, source_run_id))
377 })?;
378 source.status = RunStatus::Cancelled;
379 source.output_preview = Some("interrupted after host lease expired".to_string());
380 source.terminal_error = Some(RunTerminalError::new(
381 "admission_lease_expired",
382 "interrupted after host lease expired",
383 ));
384 advance_run_revision(source)?;
385 source.updated_at = now;
386 ContinuationEffectState::indeterminate()
387 .insert_into(&mut source.metadata)
388 .map_err(|error| SessionStoreError::Failed(error.to_string()))?;
389 }
390 let consumed = inner.hitl_resume_claims.remove(&source_key);
391 if consumed.as_ref() != Some(&claim) {
392 return Err(SessionStoreError::Conflict(format!(
393 "resume claim changed while terminalizing run {}",
394 source_run_id.as_str()
395 )));
396 }
397 Ok(claim.state == HitlResumeClaimState::Started)
398}
399
400fn resolve_evidence_retry(
401 inner: &StoreInner,
402 key: &(SessionId, RunId),
403 commit: &RunEvidenceCommit,
404 digest: &str,
405) -> SessionStoreResult<Option<RunRecord>> {
406 let Some(existing_digest) = inner.evidence_digests.get(key) else {
407 return Ok(None);
408 };
409 if existing_digest == digest {
410 return inner.runs.get(key).cloned().map(Some).ok_or_else(|| {
411 SessionStoreError::NotFound(run_key_label(&commit.run.session_id, &commit.run.run_id))
412 });
413 }
414 Err(SessionStoreError::Failed(format!(
415 "run evidence conflict for session {} and run {}",
416 commit.run.session_id.as_str(),
417 commit.run.run_id.as_str()
418 )))
419}
420
421fn validate_related_evidence(
422 inner: &StoreInner,
423 commit: &RunEvidenceCommit,
424) -> SessionStoreResult<()> {
425 for update in &commit.related_run_updates {
426 let related_key = run_key(&commit.run.session_id, &update.run_id);
427 let claim_id = update.resume_claim_id.as_deref().ok_or_else(|| {
428 SessionStoreError::Failed(format!(
429 "related run {} requires an exclusive resume claim",
430 update.run_id.as_str()
431 ))
432 })?;
433 let claim = inner.hitl_resume_claims.get(&related_key).ok_or_else(|| {
434 SessionStoreError::Failed(format!(
435 "related run {} has no active resume claim",
436 update.run_id.as_str()
437 ))
438 })?;
439 if claim.claim_id != claim_id || claim.state != HitlResumeClaimState::Started {
440 return Err(SessionStoreError::Failed(format!(
441 "started resume claim conflict for related run {}",
442 update.run_id.as_str()
443 )));
444 }
445 for approval in &update.approvals {
446 let existing = inner
447 .approvals
448 .get(&related_key)
449 .into_iter()
450 .flatten()
451 .find(|existing| existing.approval_id == approval.approval_id)
452 .ok_or_else(|| SessionStoreError::NotFound(approval.approval_id.clone()))?;
453 validate_approval_transition(existing, approval)?;
454 }
455 for deferred in &update.deferred_tools {
456 let existing = inner
457 .deferred_tools
458 .get(&related_key)
459 .into_iter()
460 .flatten()
461 .find(|existing| existing.deferred_id == deferred.deferred_id)
462 .ok_or_else(|| SessionStoreError::NotFound(deferred.deferred_id.clone()))?;
463 validate_deferred_transition(existing, deferred)?;
464 }
465 }
466 Ok(())
467}
468
469fn validate_existing_evidence(
470 inner: &StoreInner,
471 key: &(SessionId, RunId),
472 commit: &RunEvidenceCommit,
473) -> SessionStoreResult<()> {
474 if let Some(existing_run) = inner.runs.get(key) {
475 for cursor in &commit.stream_cursors {
476 for existing in existing_run.stream_cursors.iter().chain(
477 inner
478 .sessions
479 .get(&commit.run.session_id)
480 .into_iter()
481 .flat_map(|session| session.stream_cursors.iter()),
482 ) {
483 cursor
484 .validate_progression(existing)
485 .map_err(|error| SessionStoreError::Failed(error.to_string()))?;
486 }
487 }
488 }
489 for approval in &commit.approvals {
490 if let Some(existing) = inner
491 .approvals
492 .get(key)
493 .into_iter()
494 .flatten()
495 .find(|existing| existing.approval_id == approval.approval_id)
496 && existing != approval
497 {
498 return Err(SessionStoreError::Failed(format!(
499 "approval conflict for id {}",
500 approval.approval_id
501 )));
502 }
503 }
504 for deferred in &commit.deferred_tools {
505 if let Some(existing) = inner
506 .deferred_tools
507 .get(key)
508 .into_iter()
509 .flatten()
510 .find(|existing| existing.deferred_id == deferred.deferred_id)
511 && existing != deferred
512 {
513 return Err(SessionStoreError::Failed(format!(
514 "deferred tool conflict for id {}",
515 deferred.deferred_id
516 )));
517 }
518 }
519 validate_related_evidence(inner, commit)
520}
521
522fn apply_related_evidence(
523 staged: &InMemorySessionStore,
524 commit: &RunEvidenceCommit,
525) -> SessionStoreResult<()> {
526 for update in &commit.related_run_updates {
527 let source = staged.load_run_record(&commit.run.session_id, &update.run_id)?;
528 if source.status != update.expected_status {
529 return Err(SessionStoreError::Failed(format!(
530 "related run {} status conflict: expected {}, found {}",
531 update.run_id.as_str(),
532 update.expected_status.as_str(),
533 source.status.as_str()
534 )));
535 }
536 staged.set_run_status(
537 &commit.run.session_id,
538 &update.run_id,
539 update.status,
540 update.output_preview.clone(),
541 update.terminal_error.clone(),
542 )?;
543 for approval in update.approvals.clone() {
544 staged.append_approval_record(approval)?;
545 }
546 for deferred in update.deferred_tools.clone() {
547 staged.append_deferred_tool_record(deferred)?;
548 }
549 staged
550 .inner
551 .lock()
552 .map_err(store_failed)?
553 .hitl_resume_claims
554 .remove(&run_key(&commit.run.session_id, &update.run_id));
555 }
556 Ok(())
557}
558
559fn apply_primary_evidence(
560 staged: &InMemorySessionStore,
561 commit: &RunEvidenceCommit,
562) -> SessionStoreResult<()> {
563 staged.append_run_record(commit.run.clone())?;
564 staged.save_context_state_snapshot(&commit.run.session_id, commit.context_state.clone())?;
565 for checkpoint in commit.checkpoints.clone() {
566 staged.append_checkpoint_record_with_revision(&commit.run.session_id, checkpoint, false)?;
567 }
568 staged.append_stream_record_batch_with_revision(
569 &commit.run.session_id,
570 &commit.run.run_id,
571 commit.stream_records.clone(),
572 false,
573 )?;
574 for cursor in commit.stream_cursors.clone() {
575 staged.save_stream_cursor_ref_with_revision(
576 &commit.run.session_id,
577 &commit.run.run_id,
578 cursor,
579 false,
580 )?;
581 }
582 for approval in commit.approvals.clone() {
583 staged.append_approval_record(approval)?;
584 }
585 for deferred in commit.deferred_tools.clone() {
586 staged.append_deferred_tool_record(deferred)?;
587 }
588 Ok(())
589}
590
591fn finalize_staged_evidence(
592 staged: &InMemorySessionStore,
593 mut commit: RunEvidenceCommit,
594 key: (SessionId, RunId),
595 digest: String,
596) -> SessionStoreResult<(StoreInner, RunRecord)> {
597 let mut inner = staged.inner.lock().map_err(store_failed)?;
598 let commit_timestamp = commit.run.updated_at;
599 let target_key = run_key(&commit.run.session_id, &commit.run.run_id);
600 inner
601 .runs
602 .get_mut(&target_key)
603 .ok_or_else(|| {
604 SessionStoreError::NotFound(run_key_label(&commit.run.session_id, &commit.run.run_id))
605 })?
606 .updated_at = commit_timestamp;
607 for update in &commit.related_run_updates {
608 if let Some(source) = inner
609 .runs
610 .get_mut(&run_key(&commit.run.session_id, &update.run_id))
611 {
612 source.updated_at = commit_timestamp;
613 }
614 }
615 if let Some(session) = inner.sessions.get_mut(&commit.run.session_id) {
616 session.updated_at = commit_timestamp;
617 }
618 let committed = inner.runs.get(&target_key).cloned().ok_or_else(|| {
619 SessionStoreError::NotFound(run_key_label(&commit.run.session_id, &commit.run.run_id))
620 })?;
621 commit.run = committed.clone();
622 let mut authoritative_runs = vec![committed.clone()];
623 for update in &commit.related_run_updates {
624 authoritative_runs.push(
625 inner
626 .runs
627 .get(&run_key(&commit.run.session_id, &update.run_id))
628 .cloned()
629 .ok_or_else(|| {
630 SessionStoreError::NotFound(run_key_label(
631 &commit.run.session_id,
632 &update.run_id,
633 ))
634 })?,
635 );
636 }
637 append_authoritative_run_publications(
638 &mut commit.host_event_publications,
639 &format!("run-evidence:{digest}"),
640 authoritative_runs.iter(),
641 )?;
642 if !commit.publication_targets.is_empty() {
643 let mut publication = PendingStreamPublication::new(
644 commit.run.session_id.clone(),
645 commit.run.run_id.clone(),
646 commit.publication_targets,
647 commit.run.updated_at,
648 );
649 publication
650 .stream_records
651 .clone_from(&commit.stream_records);
652 publication
653 .display_messages
654 .clone_from(&commit.display_messages);
655 publication.replay_events.clone_from(&commit.replay_events);
656 publication
657 .display_snapshot
658 .clone_from(&commit.display_snapshot);
659 inner
660 .stream_publication_outbox
661 .insert(publication.publication_id.clone(), publication);
662 }
663 enqueue_host_event_publications_locked(&mut inner, &commit.host_event_publications)?;
664 inner.evidence_commits.insert(key.clone(), commit);
665 inner.evidence_digests.insert(key, digest);
666 Ok((inner.clone(), committed))
667}
668
669#[allow(clippy::too_many_lines)]
670fn acquire_run_admission_locked(
671 inner: &mut StoreInner,
672 request: AcquireRunAdmission,
673) -> SessionStoreResult<RunAdmissionReceipt> {
674 let idempotency_key = (
675 request.namespace_id.clone(),
676 request.idempotency_key.clone(),
677 );
678 if let Some((fingerprint, receipt)) = inner.run_admission_idempotency.get(&idempotency_key) {
679 if fingerprint == &request.command_fingerprint {
680 let mut receipt = receipt.clone();
681 receipt.idempotent_replay = true;
682 return Ok(receipt);
683 }
684 return Err(SessionStoreError::IdempotencyConflict(
685 request.idempotency_key,
686 ));
687 }
688 if request.replaces_waiting_run_id.is_some() != request.hitl_resume_claim_id.is_some() {
689 return Err(SessionStoreError::Conflict(
690 "waiting-run replacement requires exactly one preflight HITL claim".to_string(),
691 ));
692 }
693 let session_target =
694 ManagedSessionTarget::new(request.namespace_id.clone(), request.run.session_id.clone());
695 let now = chrono::Utc::now();
696 if let Some(active) = inner.run_admissions.get(&session_target).cloned() {
697 if !active.expired_at(now) {
698 return Err(SessionStoreError::RunConflict(format!(
699 "session {} already has active run {}",
700 request.run.session_id.as_str(),
701 active.target.run_id.as_str()
702 )));
703 }
704 let replacement_key = run_key(&active.target.session_id, &active.target.run_id);
705 let replacement = inner.runs.get(&replacement_key).cloned().ok_or_else(|| {
706 SessionStoreError::NotFound(run_key_label(
707 &active.target.session_id,
708 &active.target.run_id,
709 ))
710 })?;
711 if replacement.status.is_active() {
712 let effect_started = terminalize_started_hitl_source_locked(inner, &replacement, now)?;
713 let run = inner.runs.get_mut(&replacement_key).ok_or_else(|| {
714 SessionStoreError::NotFound(run_key_label(
715 &active.target.session_id,
716 &active.target.run_id,
717 ))
718 })?;
719 run.status = RunStatus::Cancelled;
720 run.output_preview = Some("interrupted after host lease expired".to_string());
721 run.terminal_error = Some(RunTerminalError::new(
722 "admission_lease_expired",
723 "interrupted after host lease expired",
724 ));
725 advance_run_revision(run)?;
726 run.updated_at = now;
727 if effect_started {
728 ContinuationEffectState::indeterminate()
729 .insert_into(&mut run.metadata)
730 .map_err(|error| SessionStoreError::Failed(error.to_string()))?;
731 }
732 }
733 inner.run_admissions.remove(&session_target);
734 if let Some(session) = inner.sessions.get_mut(&request.run.session_id)
735 && session.active_run_id.as_ref() == Some(&active.target.run_id)
736 {
737 session.active_run_id = None;
738 }
739 }
740 {
741 let session = inner.sessions.get(&request.run.session_id).ok_or_else(|| {
742 SessionStoreError::NotFound(request.run.session_id.as_str().to_string())
743 })?;
744 if session.namespace_id != request.namespace_id {
745 return Err(SessionStoreError::NotFound(
746 request.run.session_id.as_str().to_string(),
747 ));
748 }
749 if session.status != SessionStatus::Active || session.deletion_fence.blocks_continuation() {
750 return Err(SessionStoreError::Conflict(
751 "session cannot admit new work".to_string(),
752 ));
753 }
754 if let Some(active_run_id) = session.active_run_id.as_ref() {
755 let valid_waiting_replacement = request.replaces_waiting_run_id.as_ref()
756 == Some(active_run_id)
757 && request.run.restore_from_run_id.as_ref() == Some(active_run_id)
758 && inner
759 .runs
760 .get(&run_key(&request.run.session_id, active_run_id))
761 .is_some_and(|source| source.status == RunStatus::Waiting);
762 if !valid_waiting_replacement {
763 return Err(SessionStoreError::RunConflict(format!(
764 "session {} already has active run {}",
765 request.run.session_id.as_str(),
766 active_run_id.as_str()
767 )));
768 }
769 } else if let Some(source_run_id) = request.replaces_waiting_run_id.as_ref() {
770 let valid_unparked_waiting_replacement = request.run.restore_from_run_id.as_ref()
775 == Some(source_run_id)
776 && inner
777 .runs
778 .get(&run_key(&request.run.session_id, source_run_id))
779 .is_some_and(|source| source.status == RunStatus::Waiting);
780 if !valid_unparked_waiting_replacement {
781 return Err(SessionStoreError::Conflict(
782 "waiting-run replacement has no retryable waiting source".to_string(),
783 ));
784 }
785 }
786 }
787 let hitl_claim_key = match (
788 request.replaces_waiting_run_id.as_ref(),
789 request.hitl_resume_claim_id.as_deref(),
790 ) {
791 (Some(source_run_id), Some(claim_id)) => {
792 let key = run_key(&request.run.session_id, source_run_id);
793 let claim = inner.hitl_resume_claims.get(&key).ok_or_else(|| {
794 SessionStoreError::NotFound(format!("resume claim for {}", source_run_id.as_str()))
795 })?;
796 if claim.claim_id != claim_id
797 || claim.session_id != request.run.session_id
798 || claim.run_id != *source_run_id
799 || claim.state != HitlResumeClaimState::Preflight
800 {
801 return Err(SessionStoreError::Conflict(format!(
802 "invalid preflight resume claim for run {}",
803 source_run_id.as_str()
804 )));
805 }
806 Some(key)
807 }
808 (None, None) => None,
809 _ => unreachable!("replacement and claim presence checked above"),
810 };
811 if let Some(key) = hitl_claim_key.as_ref() {
812 let claim = inner.hitl_resume_claims.get_mut(key).ok_or_else(|| {
813 SessionStoreError::Conflict("validated preflight resume claim disappeared".to_string())
814 })?;
815 claim.state = HitlResumeClaimState::Admitted;
816 }
817 let mut run = request.run;
818 run.normalize_for_admission();
819 run.revision = 1;
820 run.validate_new_write().map_err(|error| {
821 SessionStoreError::Failed(format!(
822 "invalid admitted run state for {}: {error}",
823 run.run_id.as_str()
824 ))
825 })?;
826 if run.sequence_no == 0 {
827 run.sequence_no = inner
828 .runs
829 .values()
830 .filter(|current| current.session_id == run.session_id)
831 .map(|current| current.sequence_no)
832 .max()
833 .unwrap_or(0)
834 .saturating_add(1);
835 }
836 run.updated_at = now;
837 let generation = inner
838 .admission_generations
839 .entry(session_target.clone())
840 .and_modify(|generation| *generation = generation.saturating_add(1))
841 .or_insert(1);
842 let lease = RunAdmissionLease {
843 target: ManagedRunTarget::new(
844 request.namespace_id,
845 run.session_id.clone(),
846 run.run_id.clone(),
847 ),
848 admission_id: request.admission_id,
849 host_instance_id: request.host_instance_id,
850 fencing_generation: *generation,
851 lease_expires_at: request.lease_expires_at,
852 heartbeat_at: now,
853 command_fingerprint: request.command_fingerprint.clone(),
854 idempotency_key: request.idempotency_key,
855 };
856 inner
857 .runs
858 .insert(run_key(&run.session_id, &run.run_id), run.clone());
859 let session = inner
860 .sessions
861 .get_mut(&run.session_id)
862 .ok_or_else(|| SessionStoreError::NotFound(run.session_id.as_str().to_string()))?;
863 session.head_run_id = Some(run.run_id.clone());
864 session.active_run_id = Some(run.run_id.clone());
865 session.revision = session.revision.saturating_add(1);
866 session.updated_at = now;
867 inner.run_admissions.insert(session_target, lease.clone());
868 let receipt = RunAdmissionReceipt {
869 run,
870 lease,
871 idempotent_replay: false,
872 };
873 inner.run_admission_idempotency.insert(
874 idempotency_key,
875 (request.command_fingerprint, receipt.clone()),
876 );
877 Ok(receipt)
878}
879
880fn same_background_identity(
881 current: &BackgroundSubagentRecord,
882 next: &BackgroundSubagentRecord,
883) -> bool {
884 current.schema_version == next.schema_version
885 && current.attempt_id == next.attempt_id
886 && current.agent_id == next.agent_id
887 && current.linked_task_id == next.linked_task_id
888 && current.subagent_name == next.subagent_name
889 && current.namespace_id == next.namespace_id
890 && current.parent_session_id == next.parent_session_id
891 && current.parent_run_id == next.parent_run_id
892 && current.profile == next.profile
893 && current.owner_lease.host_instance_id == next.owner_lease.host_instance_id
894 && current.owner_lease.fencing_generation == next.owner_lease.fencing_generation
895 && current.accepted_at == next.accepted_at
896}
897
898fn same_background_owner(
899 current: &BackgroundSubagentRecord,
900 next: &BackgroundSubagentRecord,
901) -> bool {
902 current.owner_lease.host_instance_id == next.owner_lease.host_instance_id
903 && current.owner_lease.fencing_generation == next.owner_lease.fencing_generation
904}
905
906fn valid_background_terminal_base(
907 current: &BackgroundSubagentRecord,
908 next: &BackgroundSubagentRecord,
909) -> bool {
910 current.delivery_status == DurableBackgroundSubagentDeliveryStatus::Undelivered
911 && current.delivery_claim.is_none()
912 && current.delivered_claim_id.is_none()
913 && current.continuation_run_id.is_none()
914 && current
915 .automatic_continuation_suppressed_by_run_id
916 .is_none()
917 && next.automatic_continuation_suppressed_by_run_id.is_none()
918 && current.retention_status == crate::DurableBackgroundSubagentRetentionStatus::Inline
919 && current.retention_expires_at.is_none()
920 && matches!(
921 next.retention_status,
922 crate::DurableBackgroundSubagentRetentionStatus::Inline
923 | crate::DurableBackgroundSubagentRetentionStatus::Artifact
924 )
925 && current.trace_context == next.trace_context
926 && next.updated_at >= current.updated_at
927}
928
929fn same_background_non_execution_state(
930 current: &BackgroundSubagentRecord,
931 next: &BackgroundSubagentRecord,
932) -> bool {
933 current.continuation_run_id == next.continuation_run_id
934 && current.result_ref == next.result_ref
935 && current.failure_category == next.failure_category
936 && current.cancellation_reason == next.cancellation_reason
937 && current.delivery_status == next.delivery_status
938 && current.delivery_claim == next.delivery_claim
939 && current.delivered_claim_id == next.delivered_claim_id
940 && current.automatic_continuation_suppressed_by_run_id
941 == next.automatic_continuation_suppressed_by_run_id
942 && current.retention_status == next.retention_status
943 && current.retention_expires_at == next.retention_expires_at
944 && current.trace_context == next.trace_context
945 && current.terminal_at == next.terminal_at
946 && next.updated_at >= current.updated_at
947}
948
949fn continuation_artifact_content(
950 inner: &StoreInner,
951 background: &BackgroundSubagentRecord,
952 now: chrono::DateTime<chrono::Utc>,
953) -> SessionStoreResult<Option<String>> {
954 if background.retention_status != crate::DurableBackgroundSubagentRetentionStatus::Artifact {
955 return Ok(None);
956 }
957 let result_ref = background.result_ref.as_ref().ok_or_else(|| {
958 SessionStoreError::Conflict("artifact result is missing terminal evidence".to_string())
959 })?;
960 let artifact_ref = result_ref.artifact_ref.as_deref().ok_or_else(|| {
961 SessionStoreError::Conflict("artifact result is missing its reference".to_string())
962 })?;
963 let artifact = inner
964 .background_artifacts
965 .get(artifact_ref)
966 .ok_or_else(|| SessionStoreError::NotFound(artifact_ref.to_string()))?;
967 if !artifact.is_available_at(now)
968 || artifact.namespace_id != background.namespace_id
969 || artifact.attempt_id != background.attempt_id
970 || artifact.digest != result_ref.digest.clone().unwrap_or_default()
971 || artifact.size_bytes != result_ref.size_bytes
972 || background.retention_expires_at != Some(artifact.expires_at)
973 {
974 return Err(SessionStoreError::Conflict(
975 "background-subagent artifact failed integrity or retention validation".to_string(),
976 ));
977 }
978 Ok(Some(artifact.content.clone()))
979}
980
981fn continuation_receipt_matches_request(
982 background: &BackgroundSubagentRecord,
983 request: &AcquireBackgroundSubagentContinuation,
984 admission: &RunAdmissionReceipt,
985) -> bool {
986 let proposed = &request.admission.run;
987 let admitted = &admission.run;
988 background.validates_continuation_cause_envelope(&request.cause, admitted)
989 && admitted.session_id == proposed.session_id
990 && admitted.run_id == proposed.run_id
991 && admitted.conversation_id == proposed.conversation_id
992 && admitted.input == proposed.input
993 && admitted.parent_run_id == proposed.parent_run_id
994 && admitted.parent_task_id == proposed.parent_task_id
995 && admitted.trigger_type == proposed.trigger_type
996 && admitted.profile == proposed.profile
997 && admitted.trace_context == proposed.trace_context
998 && admitted.metadata == proposed.metadata
999 && admission.lease.target.session_id == proposed.session_id
1000 && admission.lease.target.run_id == proposed.run_id
1001 && admission.lease.target.namespace_id == request.admission.namespace_id
1002 && admission.lease.command_fingerprint == request.admission.command_fingerprint
1003 && admission.lease.idempotency_key == request.admission.idempotency_key
1004}
1005
1006fn background_terminal_fingerprint(
1007 record: &BackgroundSubagentRecord,
1008 artifact: Option<&BackgroundSubagentArtifact>,
1009) -> SessionStoreResult<String> {
1010 BackgroundSubagentTerminalCommit {
1011 record: record.clone(),
1012 artifact: artifact.cloned(),
1013 artifact_limits: None,
1014 }
1015 .canonical_fingerprint()
1016 .map_err(|error| SessionStoreError::Failed(error.to_string()))
1017}
1018
1019fn persisted_background_terminal_fingerprint(
1020 inner: &StoreInner,
1021 current: &BackgroundSubagentRecord,
1022) -> SessionStoreResult<Option<String>> {
1023 if let Some(fingerprint) = inner
1024 .background_terminal_fingerprints
1025 .get(¤t.attempt_id)
1026 {
1027 return Ok(Some(fingerprint.clone()));
1028 }
1029 reconstruct_background_terminal_fingerprint(inner, current)
1030}
1031
1032fn reconstruct_background_terminal_fingerprint(
1033 inner: &StoreInner,
1034 current: &BackgroundSubagentRecord,
1035) -> SessionStoreResult<Option<String>> {
1036 if !current.execution_status.is_terminal()
1037 || current.retention_status == crate::DurableBackgroundSubagentRetentionStatus::Expired
1038 || current.result_ref.is_none()
1039 {
1040 return Ok(None);
1041 }
1042 let Some(terminal_at) = current.terminal_at else {
1043 return Ok(None);
1044 };
1045 if current
1046 .retention_expires_at
1047 .is_none_or(|deadline| deadline <= terminal_at)
1048 {
1049 return Ok(None);
1050 }
1051 let artifact = match current.retention_status {
1052 crate::DurableBackgroundSubagentRetentionStatus::Inline => None,
1053 crate::DurableBackgroundSubagentRetentionStatus::Artifact => {
1054 let Some(result_ref) = current.result_ref.as_ref() else {
1055 return Ok(None);
1056 };
1057 let Some(artifact_ref) = result_ref.artifact_ref.as_deref() else {
1058 return Ok(None);
1059 };
1060 let Some(artifact) = inner.background_artifacts.get(artifact_ref) else {
1061 return Ok(None);
1062 };
1063 if !artifact.is_valid()
1064 || artifact.namespace_id != current.namespace_id
1065 || artifact.attempt_id != current.attempt_id
1066 || artifact.digest != result_ref.digest.clone().unwrap_or_default()
1067 || artifact.size_bytes != result_ref.size_bytes
1068 || current.retention_expires_at != Some(artifact.expires_at)
1069 {
1070 return Ok(None);
1071 }
1072 Some(artifact)
1073 }
1074 crate::DurableBackgroundSubagentRetentionStatus::Expired => return Ok(None),
1075 };
1076 let mut terminal = current.clone();
1077 terminal.updated_at = terminal_at;
1078 background_terminal_fingerprint(&terminal, artifact).map(Some)
1079}
1080
1081#[derive(Clone, Debug, Eq, PartialEq)]
1082enum BackgroundClaimConsumerState {
1083 Reclaimable,
1084 Live,
1085 Completed(RunId),
1086 Terminated(RunId),
1087}
1088
1089fn background_claim_consumer_state(
1090 inner: &StoreInner,
1091 record: &BackgroundSubagentRecord,
1092 claim: &DurableBackgroundSubagentDeliveryClaim,
1093 now: chrono::DateTime<chrono::Utc>,
1094) -> BackgroundClaimConsumerState {
1095 let Some(run_id) = claim.continuation_run_id.as_ref() else {
1096 return BackgroundClaimConsumerState::Reclaimable;
1097 };
1098 let Some(run) = inner.runs.get(&run_key(&record.parent_session_id, run_id)) else {
1099 return BackgroundClaimConsumerState::Reclaimable;
1100 };
1101 match run.status {
1102 RunStatus::Completed => BackgroundClaimConsumerState::Completed(run_id.clone()),
1103 RunStatus::Failed | RunStatus::Cancelled => {
1104 BackgroundClaimConsumerState::Terminated(run_id.clone())
1105 }
1106 status
1107 if status.is_active()
1108 && inner
1109 .run_admissions
1110 .get(&ManagedSessionTarget::new(
1111 record.namespace_id.clone(),
1112 record.parent_session_id.clone(),
1113 ))
1114 .is_some_and(|lease| {
1115 lease.target.run_id == *run_id && !lease.expired_at(now)
1116 }) =>
1117 {
1118 BackgroundClaimConsumerState::Live
1119 }
1120 _ => BackgroundClaimConsumerState::Reclaimable,
1121 }
1122}
1123
1124fn validate_background_artifact_quota(
1125 inner: &StoreInner,
1126 artifact: &BackgroundSubagentArtifact,
1127 limits: BackgroundSubagentArtifactLimits,
1128 now: chrono::DateTime<chrono::Utc>,
1129) -> SessionStoreResult<()> {
1130 if !artifact.is_available_at(now) {
1131 return Err(SessionStoreError::Conflict(
1132 "background-subagent artifact is invalid or already expired".to_string(),
1133 ));
1134 }
1135 let retained_bytes = inner
1136 .background_artifacts
1137 .values()
1138 .filter(|current| {
1139 current.namespace_id == artifact.namespace_id
1140 && current.artifact_ref != artifact.artifact_ref
1141 && current.expires_at > now
1142 })
1143 .map(|current| current.size_bytes)
1144 .fold(0u64, u64::saturating_add);
1145 if artifact.size_bytes > limits.max_single_bytes
1146 || retained_bytes.saturating_add(artifact.size_bytes) > limits.max_retained_bytes
1147 {
1148 return Err(SessionStoreError::QuotaExceeded(
1149 "background-subagent artifact exceeds host retention quota".to_string(),
1150 ));
1151 }
1152 Ok(())
1153}
1154
1155fn ensure_background_parent_writable(
1156 inner: &StoreInner,
1157 record: &BackgroundSubagentRecord,
1158) -> SessionStoreResult<()> {
1159 let session = inner
1160 .sessions
1161 .get(&record.parent_session_id)
1162 .filter(|session| session.namespace_id == record.namespace_id)
1163 .ok_or_else(|| {
1164 SessionStoreError::NotFound(record.parent_session_id.as_str().to_string())
1165 })?;
1166 if session.status != SessionStatus::Active || session.deletion_fence.blocks_continuation() {
1167 return Err(SessionStoreError::Conflict(
1168 "background owner write rejected because parent session is deleting or deleted"
1169 .to_string(),
1170 ));
1171 }
1172 Ok(())
1173}
1174
1175fn commit_background_terminal(
1176 inner: &mut StoreInner,
1177 mut record: BackgroundSubagentRecord,
1178 artifact: Option<BackgroundSubagentArtifact>,
1179 artifact_limits: Option<BackgroundSubagentArtifactLimits>,
1180) -> SessionStoreResult<BackgroundSubagentRecord> {
1181 if !record.is_valid_terminal() {
1182 return Err(SessionStoreError::Conflict(
1183 "invalid background-subagent terminal record".to_string(),
1184 ));
1185 }
1186 if artifact.is_some() != artifact_limits.is_some()
1187 || artifact_limits.is_some_and(|limits| !limits.is_valid())
1188 {
1189 return Err(SessionStoreError::Conflict(
1190 "background-subagent artifact requires valid host quota limits".to_string(),
1191 ));
1192 }
1193 let current = inner
1194 .background_subagents
1195 .get(&record.attempt_id)
1196 .cloned()
1197 .ok_or_else(|| SessionStoreError::NotFound(record.attempt_id.as_str().to_string()))?;
1198 ensure_background_parent_writable(inner, ¤t)?;
1199 let terminal_fingerprint = background_terminal_fingerprint(&record, artifact.as_ref())?;
1200 if current.execution_status.is_terminal() {
1201 let persisted_fingerprint = persisted_background_terminal_fingerprint(inner, ¤t)?;
1202 return if persisted_fingerprint.as_deref() == Some(terminal_fingerprint.as_str()) {
1203 inner
1204 .background_terminal_fingerprints
1205 .insert(current.attempt_id.clone(), terminal_fingerprint);
1206 Ok(current)
1207 } else {
1208 Err(SessionStoreError::Conflict(format!(
1209 "terminal background attempt {} is immutable",
1210 record.attempt_id.as_str()
1211 )))
1212 };
1213 }
1214 let now = chrono::Utc::now();
1215 if current.owner_lease.expired_at(now) {
1216 return Err(SessionStoreError::Conflict(format!(
1217 "background owner lease expired for {}",
1218 record.attempt_id.as_str()
1219 )));
1220 }
1221 if let (Some(artifact), Some(limits)) = (artifact.as_ref(), artifact_limits) {
1222 validate_background_artifact_quota(inner, artifact, limits, now)?;
1223 }
1224 if !same_background_identity(¤t, &record)
1225 || !same_background_owner(¤t, &record)
1226 || !valid_background_terminal_base(¤t, &record)
1227 || !valid_background_transition(current.execution_status, record.execution_status)
1228 {
1229 return Err(SessionStoreError::Conflict(format!(
1230 "invalid terminal background transition for {}",
1231 record.attempt_id.as_str()
1232 )));
1233 }
1234 if let Some(artifact) = artifact {
1235 let artifact_matches = artifact.is_available_at(now)
1236 && artifact.attempt_id == record.attempt_id
1237 && artifact.namespace_id == record.namespace_id
1238 && record.retention_status == crate::DurableBackgroundSubagentRetentionStatus::Artifact
1239 && record.retention_expires_at == Some(artifact.expires_at)
1240 && record.result_ref.as_ref().is_some_and(|result| {
1241 result.artifact_ref.as_deref() == Some(artifact.artifact_ref.as_str())
1242 && result.digest.as_deref() == Some(artifact.digest.as_str())
1243 && result.size_bytes == artifact.size_bytes
1244 });
1245 if !artifact_matches {
1246 return Err(SessionStoreError::Conflict(
1247 "background-subagent artifact does not match terminal evidence".to_string(),
1248 ));
1249 }
1250 if let Some(existing) = inner.background_artifacts.get(&artifact.artifact_ref) {
1251 if existing != &artifact {
1252 return Err(SessionStoreError::Conflict(
1253 "background-subagent artifact identity conflict".to_string(),
1254 ));
1255 }
1256 } else {
1257 inner
1258 .background_artifacts
1259 .insert(artifact.artifact_ref.clone(), artifact);
1260 }
1261 } else if record.retention_status == crate::DurableBackgroundSubagentRetentionStatus::Artifact {
1262 return Err(SessionStoreError::Conflict(
1263 "artifact retention requires an atomic artifact payload".to_string(),
1264 ));
1265 }
1266 record.owner_lease = current.owner_lease;
1267 inner
1268 .background_terminal_fingerprints
1269 .insert(record.attempt_id.clone(), terminal_fingerprint);
1270 inner
1271 .background_subagents
1272 .insert(record.attempt_id.clone(), record.clone());
1273 Ok(record)
1274}
1275
1276fn valid_background_transition(
1277 current: DurableBackgroundSubagentExecutionStatus,
1278 next: DurableBackgroundSubagentExecutionStatus,
1279) -> bool {
1280 current == next
1281 || matches!(
1282 (current, next),
1283 (
1284 DurableBackgroundSubagentExecutionStatus::Accepted,
1285 DurableBackgroundSubagentExecutionStatus::Starting
1286 | DurableBackgroundSubagentExecutionStatus::Failed
1287 | DurableBackgroundSubagentExecutionStatus::Cancelled
1288 ) | (
1289 DurableBackgroundSubagentExecutionStatus::Starting,
1290 DurableBackgroundSubagentExecutionStatus::Running
1291 | DurableBackgroundSubagentExecutionStatus::Failed
1292 | DurableBackgroundSubagentExecutionStatus::Cancelled
1293 ) | (
1294 DurableBackgroundSubagentExecutionStatus::Running,
1295 DurableBackgroundSubagentExecutionStatus::Waiting
1296 | DurableBackgroundSubagentExecutionStatus::Completed
1297 | DurableBackgroundSubagentExecutionStatus::Failed
1298 | DurableBackgroundSubagentExecutionStatus::Cancelled
1299 ) | (
1300 DurableBackgroundSubagentExecutionStatus::Waiting,
1301 DurableBackgroundSubagentExecutionStatus::Running
1302 | DurableBackgroundSubagentExecutionStatus::Completed
1303 | DurableBackgroundSubagentExecutionStatus::Failed
1304 | DurableBackgroundSubagentExecutionStatus::Cancelled
1305 )
1306 )
1307}
1308
1309#[allow(clippy::too_many_lines)]
1310#[async_trait]
1311impl SessionStore for InMemorySessionStore {
1312 async fn commit_run_evidence(
1313 &self,
1314 mut commit: RunEvidenceCommit,
1315 ) -> SessionStoreResult<RunRecord> {
1316 commit.run.stream_cursors.clone_from(&commit.stream_cursors);
1317 commit.validate_structure()?;
1318 let digest = commit.digest()?;
1319 let key = run_key(&commit.run.session_id, &commit.run.run_id);
1320 let mut original = self.inner.lock().map_err(store_failed)?;
1321 if let Some(existing) = resolve_evidence_retry(&original, &key, &commit, &digest)? {
1322 return Ok(existing);
1323 }
1324 commit.validate_terminal_projections()?;
1325 validate_existing_evidence(&original, &key, &commit)?;
1326 let staged = Self {
1327 inner: Arc::new(Mutex::new(original.clone())),
1328 };
1329 apply_related_evidence(&staged, &commit)?;
1330 apply_primary_evidence(&staged, &commit)?;
1331 let (committed_inner, committed_run) =
1332 finalize_staged_evidence(&staged, commit, key, digest)?;
1333 *original = committed_inner;
1334 Ok(committed_run)
1335 }
1336
1337 async fn commit_run_evidence_fenced(
1338 &self,
1339 lease: &RunAdmissionLease,
1340 mut commit: RunEvidenceCommit,
1341 ) -> SessionStoreResult<RunRecord> {
1342 if lease.target.session_id != commit.run.session_id
1343 || lease.target.run_id != commit.run.run_id
1344 {
1345 return Err(SessionStoreError::Conflict(
1346 "run evidence does not match admission target".to_string(),
1347 ));
1348 }
1349 commit.run.stream_cursors.clone_from(&commit.stream_cursors);
1350 commit.validate_structure()?;
1351 let digest = commit.digest()?;
1352 let key = run_key(&commit.run.session_id, &commit.run.run_id);
1353 let mut original = self.inner.lock().map_err(store_failed)?;
1354 if let Some(existing) = resolve_evidence_retry(&original, &key, &commit, &digest)? {
1355 return Ok(existing);
1356 }
1357 commit.validate_terminal_projections()?;
1358 ensure_active_admission_locked(&original, lease, chrono::Utc::now())?;
1359 validate_existing_evidence(&original, &key, &commit)?;
1360 let staged = Self {
1361 inner: Arc::new(Mutex::new(original.clone())),
1362 };
1363 apply_related_evidence(&staged, &commit)?;
1364 apply_primary_evidence(&staged, &commit)?;
1365 let (committed_inner, committed_run) =
1366 finalize_staged_evidence(&staged, commit, key, digest)?;
1367 *original = committed_inner;
1368 Ok(committed_run)
1369 }
1370
1371 async fn append_replay_events_fenced(
1372 &self,
1373 lease: &RunAdmissionLease,
1374 events: Vec<ReplayEvent>,
1375 ) -> SessionStoreResult<()> {
1376 let expected_scope = ReplayScope::run(lease.target.run_id.as_str());
1377 for event in &events {
1378 if event.scope != expected_scope {
1379 return Err(SessionStoreError::Conflict(format!(
1380 "replay event scope {} does not match admission run {}",
1381 event.scope.as_str(),
1382 lease.target.run_id.as_str()
1383 )));
1384 }
1385 i64::try_from(event.sequence).map_err(|error| {
1386 SessionStoreError::Failed(format!("invalid replay event sequence: {error}"))
1387 })?;
1388 }
1389
1390 let mut inner = self.inner.lock().map_err(store_failed)?;
1391 ensure_active_admission_locked(&inner, lease, chrono::Utc::now())?;
1392 let mut staged = inner.replay_events.clone();
1393 for event in events {
1394 let key = (expected_scope.clone(), event.sequence);
1395 if let Some(persisted) = staged.get(&key) {
1396 if persisted != &event {
1397 return Err(SessionStoreError::Failed(format!(
1398 "replay event conflict for scope {} at sequence {}",
1399 expected_scope.as_str(),
1400 event.sequence
1401 )));
1402 }
1403 } else {
1404 staged.insert(key, event);
1405 }
1406 }
1407 inner.replay_events = staged;
1408 Ok(())
1409 }
1410
1411 async fn commit_checkpoint(
1412 &self,
1413 session_id: &SessionId,
1414 checkpoint: AgentCheckpoint,
1415 ) -> SessionStoreResult<()> {
1416 let mut original = self.inner.lock().map_err(store_failed)?;
1417 let staged = Self {
1418 inner: Arc::new(Mutex::new(original.clone())),
1419 };
1420 if staged.load_session_record(session_id).is_err() {
1421 staged.save_session_record(SessionRecord::new(session_id.clone()))?;
1422 }
1423 let created_run = staged
1424 .load_run_record(session_id, &checkpoint.run_id)
1425 .is_err();
1426 if created_run {
1427 let mut run = RunRecord::new(
1428 session_id.clone(),
1429 checkpoint.run_id.clone(),
1430 checkpoint.conversation_id.clone(),
1431 );
1432 run.status = checkpoint_run_status(checkpoint.resume.status);
1433 run.terminal_error = checkpoint_terminal_error(checkpoint.resume.status);
1434 run.trace_context = checkpoint.resume.trace_context.clone();
1435 run.parent_run_id
1436 .clone_from(&checkpoint.state.parent_run_id);
1437 run.parent_task_id
1438 .clone_from(&checkpoint.state.parent_task_id);
1439 staged.append_run_record(run)?;
1440 }
1441 staged.append_checkpoint_record_with_revision(session_id, checkpoint, !created_run)?;
1442 let staged_inner = staged.inner.lock().map_err(store_failed)?;
1443 *original = staged_inner.clone();
1444 Ok(())
1445 }
1446
1447 async fn commit_checkpoint_fenced(
1448 &self,
1449 lease: &RunAdmissionLease,
1450 checkpoint: AgentCheckpoint,
1451 ) -> SessionStoreResult<()> {
1452 if lease.target.run_id != checkpoint.run_id {
1453 return Err(SessionStoreError::Conflict(
1454 "checkpoint does not match admission target".to_string(),
1455 ));
1456 }
1457 let key = run_key(&lease.target.session_id, &checkpoint.run_id);
1458 let mut original = self.inner.lock().map_err(store_failed)?;
1459 if let Some(existing) = original
1460 .checkpoints
1461 .get(&key)
1462 .into_iter()
1463 .flatten()
1464 .find(|existing| existing.checkpoint_id == checkpoint.checkpoint_id)
1465 {
1466 if existing == &checkpoint {
1467 return Ok(());
1468 }
1469 return Err(SessionStoreError::Failed(format!(
1470 "checkpoint conflict for session {} run {} checkpoint {}",
1471 lease.target.session_id.as_str(),
1472 checkpoint.run_id.as_str(),
1473 checkpoint.checkpoint_id.as_str()
1474 )));
1475 }
1476 ensure_active_admission_locked(&original, lease, chrono::Utc::now())?;
1477 let staged = Self {
1478 inner: Arc::new(Mutex::new(original.clone())),
1479 };
1480 if staged
1481 .load_session_record(&lease.target.session_id)
1482 .is_err()
1483 {
1484 staged.save_session_record(SessionRecord::new(lease.target.session_id.clone()))?;
1485 }
1486 let created_run = staged
1487 .load_run_record(&lease.target.session_id, &checkpoint.run_id)
1488 .is_err();
1489 if created_run {
1490 let mut run = RunRecord::new(
1491 lease.target.session_id.clone(),
1492 checkpoint.run_id.clone(),
1493 checkpoint.conversation_id.clone(),
1494 );
1495 run.status = checkpoint_run_status(checkpoint.resume.status);
1496 run.terminal_error = checkpoint_terminal_error(checkpoint.resume.status);
1497 run.trace_context = checkpoint.resume.trace_context.clone();
1498 run.parent_run_id
1499 .clone_from(&checkpoint.state.parent_run_id);
1500 run.parent_task_id
1501 .clone_from(&checkpoint.state.parent_task_id);
1502 staged.append_run_record(run)?;
1503 }
1504 staged.append_checkpoint_record_with_revision(
1505 &lease.target.session_id,
1506 checkpoint,
1507 !created_run,
1508 )?;
1509 let staged_inner = staged.inner.lock().map_err(store_failed)?;
1510 *original = staged_inner.clone();
1511 Ok(())
1512 }
1513
1514 async fn claim_hitl_resume(&self, claim: HitlResumeClaim) -> SessionStoreResult<()> {
1515 if !claim.is_valid_preflight() {
1516 return Err(SessionStoreError::Failed(
1517 "invalid HITL preflight claim".to_string(),
1518 ));
1519 }
1520 let key = run_key(&claim.session_id, &claim.run_id);
1521 let mut inner = self.inner.lock().map_err(store_failed)?;
1522 let run = inner.runs.get(&key).ok_or_else(|| {
1523 SessionStoreError::NotFound(run_key_label(&claim.session_id, &claim.run_id))
1524 })?;
1525 if run.status != RunStatus::Waiting {
1526 return Err(SessionStoreError::Failed(format!(
1527 "run {} is not waiting",
1528 claim.run_id.as_str()
1529 )));
1530 }
1531 if let Some(existing) = inner.hitl_resume_claims.get(&key) {
1532 if existing.claim_id == claim.claim_id
1533 && existing.session_id == claim.session_id
1534 && existing.run_id == claim.run_id
1535 && existing.state == HitlResumeClaimState::Preflight
1536 {
1537 return Ok(());
1538 }
1539 return Err(SessionStoreError::Failed(format!(
1540 "run {} already has an active resume claim",
1541 claim.run_id.as_str()
1542 )));
1543 }
1544 inner.hitl_resume_claims.insert(key, claim);
1545 Ok(())
1546 }
1547
1548 async fn start_hitl_resume_effect(
1549 &self,
1550 lease: &RunAdmissionLease,
1551 source_run_id: &RunId,
1552 claim_id: &str,
1553 ) -> SessionStoreResult<()> {
1554 let mut inner = self.inner.lock().map_err(store_failed)?;
1555 ensure_active_admission_locked(&inner, lease, chrono::Utc::now())?;
1556 let target = inner
1557 .runs
1558 .get(&run_key(&lease.target.session_id, &lease.target.run_id))
1559 .ok_or_else(|| {
1560 SessionStoreError::NotFound(run_key_label(
1561 &lease.target.session_id,
1562 &lease.target.run_id,
1563 ))
1564 })?;
1565 if target.restore_from_run_id.as_ref() != Some(source_run_id) || !target.status.is_active()
1566 {
1567 return Err(SessionStoreError::Conflict(
1568 "active admission is not bound to the HITL source run".to_string(),
1569 ));
1570 }
1571 let source_key = run_key(&lease.target.session_id, source_run_id);
1572 let source = inner.runs.get(&source_key).ok_or_else(|| {
1573 SessionStoreError::NotFound(run_key_label(&lease.target.session_id, source_run_id))
1574 })?;
1575 if source.status != RunStatus::Waiting {
1576 return Err(SessionStoreError::Conflict(
1577 "HITL source run is not waiting".to_string(),
1578 ));
1579 }
1580 let claim = inner
1581 .hitl_resume_claims
1582 .get_mut(&source_key)
1583 .ok_or_else(|| {
1584 SessionStoreError::NotFound(format!("resume claim for {}", source_run_id.as_str()))
1585 })?;
1586 if claim.claim_id != claim_id
1587 || claim.session_id != lease.target.session_id
1588 || claim.run_id != *source_run_id
1589 || claim.state != HitlResumeClaimState::Admitted
1590 {
1591 return Err(SessionStoreError::Conflict(format!(
1592 "invalid admitted resume claim for run {}",
1593 source_run_id.as_str()
1594 )));
1595 }
1596 claim.state = HitlResumeClaimState::Started;
1597 Ok(())
1598 }
1599
1600 async fn abort_admitted_hitl_resume(
1601 &self,
1602 lease: &RunAdmissionLease,
1603 source_run_id: &RunId,
1604 claim_id: &str,
1605 output_preview: &str,
1606 ) -> SessionStoreResult<HitlResumeAbortOutcome> {
1607 let mut inner = self.inner.lock().map_err(store_failed)?;
1608 ensure_active_admission_locked(&inner, lease, chrono::Utc::now())?;
1609 let target_key = run_key(&lease.target.session_id, &lease.target.run_id);
1610 let target = inner.runs.get(&target_key).ok_or_else(|| {
1611 SessionStoreError::NotFound(run_key_label(
1612 &lease.target.session_id,
1613 &lease.target.run_id,
1614 ))
1615 })?;
1616 if target.restore_from_run_id.as_ref() != Some(source_run_id) || !target.status.is_active()
1617 {
1618 return Err(SessionStoreError::Conflict(
1619 "active admission is not bound to the HITL source run".to_string(),
1620 ));
1621 }
1622 let source_key = run_key(&lease.target.session_id, source_run_id);
1623 let source = inner.runs.get(&source_key).ok_or_else(|| {
1624 SessionStoreError::NotFound(run_key_label(&lease.target.session_id, source_run_id))
1625 })?;
1626 if source.status != RunStatus::Waiting {
1627 return Err(SessionStoreError::Conflict(
1628 "HITL source run is not waiting".to_string(),
1629 ));
1630 }
1631 let claim = inner
1632 .hitl_resume_claims
1633 .get(&source_key)
1634 .cloned()
1635 .ok_or_else(|| {
1636 SessionStoreError::NotFound(format!("resume claim for {}", source_run_id.as_str()))
1637 })?;
1638 if claim.claim_id != claim_id
1639 || claim.session_id != lease.target.session_id
1640 || claim.run_id != *source_run_id
1641 {
1642 return Err(SessionStoreError::Conflict(format!(
1643 "invalid resume claim for run {}",
1644 source_run_id.as_str()
1645 )));
1646 }
1647 if claim.state == HitlResumeClaimState::Started {
1648 return Ok(HitlResumeAbortOutcome::EffectStarted);
1649 }
1650 if claim.state != HitlResumeClaimState::Admitted {
1651 return Err(SessionStoreError::Conflict(format!(
1652 "invalid admitted resume claim for run {}",
1653 source_run_id.as_str()
1654 )));
1655 }
1656 apply_run_status_locked(
1657 &mut inner,
1658 &lease.target.session_id,
1659 &lease.target.run_id,
1660 RunStatus::Failed,
1661 Some(output_preview.to_string()),
1662 Some(RunTerminalError::new(
1663 "hitl_resume_preparation_failed",
1664 output_preview,
1665 )),
1666 chrono::Utc::now(),
1667 )?;
1668 let consumed = inner.hitl_resume_claims.remove(&source_key);
1669 if consumed.as_ref() != Some(&claim) {
1670 return Err(SessionStoreError::Conflict(format!(
1671 "admitted resume claim changed while aborting run {}",
1672 source_run_id.as_str()
1673 )));
1674 }
1675 Ok(HitlResumeAbortOutcome::AbortedBeforeEffect)
1676 }
1677
1678 async fn mark_hitl_resume_started(
1679 &self,
1680 session_id: &SessionId,
1681 run_id: &RunId,
1682 claim_id: &str,
1683 ) -> SessionStoreResult<()> {
1684 let key = run_key(session_id, run_id);
1685 let mut inner = self.inner.lock().map_err(store_failed)?;
1686 let claim = inner.hitl_resume_claims.get_mut(&key).ok_or_else(|| {
1687 SessionStoreError::NotFound(format!("resume claim for {}", run_id.as_str()))
1688 })?;
1689 if claim.claim_id != claim_id {
1690 return Err(SessionStoreError::Failed(format!(
1691 "resume claim conflict for run {}",
1692 run_id.as_str()
1693 )));
1694 }
1695 if claim.state == HitlResumeClaimState::Preflight {
1696 claim.state = HitlResumeClaimState::Started;
1697 }
1698 Ok(())
1699 }
1700
1701 async fn release_hitl_resume_claim(
1702 &self,
1703 session_id: &SessionId,
1704 run_id: &RunId,
1705 claim_id: &str,
1706 ) -> SessionStoreResult<()> {
1707 let key = run_key(session_id, run_id);
1708 let mut inner = self.inner.lock().map_err(store_failed)?;
1709 let Some(existing) = inner.hitl_resume_claims.get(&key) else {
1710 return Ok(());
1711 };
1712 if existing.claim_id != claim_id {
1713 return Err(SessionStoreError::Failed(format!(
1714 "resume claim conflict for run {}",
1715 run_id.as_str()
1716 )));
1717 }
1718 if existing.state != HitlResumeClaimState::Preflight {
1719 return Err(SessionStoreError::Failed(format!(
1720 "started resume claim for run {} cannot be released",
1721 run_id.as_str()
1722 )));
1723 }
1724 inner.hitl_resume_claims.remove(&key);
1725 Ok(())
1726 }
1727
1728 async fn pending_stream_publications(
1729 &self,
1730 session_id: &SessionId,
1731 ) -> SessionStoreResult<Vec<PendingStreamPublication>> {
1732 let inner = self.inner.lock().map_err(store_failed)?;
1733 Ok(inner
1734 .stream_publication_outbox
1735 .values()
1736 .filter(|publication| &publication.session_id == session_id)
1737 .cloned()
1738 .collect())
1739 }
1740
1741 async fn acknowledge_stream_publication(
1742 &self,
1743 publication_id: &str,
1744 target: StreamPublicationTarget,
1745 ) -> SessionStoreResult<()> {
1746 let mut inner = self.inner.lock().map_err(store_failed)?;
1747 let Some(publication) = inner.stream_publication_outbox.get_mut(publication_id) else {
1748 return Ok(());
1749 };
1750 match target {
1751 StreamPublicationTarget::Archive => publication.archive_pending = false,
1752 StreamPublicationTarget::Replay => publication.replay_pending = false,
1753 }
1754 if publication.is_complete() {
1755 inner.stream_publication_outbox.remove(publication_id);
1756 }
1757 Ok(())
1758 }
1759
1760 async fn enqueue_host_event_publications(
1761 &self,
1762 publications: Vec<PendingHostEventPublication>,
1763 ) -> SessionStoreResult<()> {
1764 self.enqueue_host_event_publication_batch(&publications)
1765 }
1766
1767 async fn pending_host_event_publications(
1768 &self,
1769 limit: usize,
1770 ) -> SessionStoreResult<Vec<PendingHostEventPublication>> {
1771 self.pending_host_event_publication_batch(limit)
1772 }
1773
1774 async fn materialize_host_event_publications(
1775 &self,
1776 limit: usize,
1777 ) -> SessionStoreResult<Vec<DurableHostEventRecord>> {
1778 self.materialize_host_event_publication_batch(limit)
1779 }
1780
1781 async fn replay_host_events(
1782 &self,
1783 query: DurableHostEventQuery,
1784 ) -> SessionStoreResult<DurableHostEventPage> {
1785 self.replay_host_event_page(query)
1786 }
1787
1788 async fn host_event_fence(
1789 &self,
1790 scope: &DurableHostEventScope,
1791 event_classes: &[DurableHostEventClass],
1792 ) -> SessionStoreResult<Option<u64>> {
1793 self.host_event_fence_position(scope, event_classes)
1794 }
1795
1796 async fn create_session_idempotent(
1797 &self,
1798 mut session: SessionRecord,
1799 idempotency_key: &str,
1800 command_fingerprint: &str,
1801 ) -> SessionStoreResult<SessionRecord> {
1802 let key = (session.namespace_id.clone(), idempotency_key.to_string());
1803 let mut inner = self.inner.lock().map_err(store_failed)?;
1804 if let Some((fingerprint, existing)) = inner.session_idempotency.get(&key) {
1805 if fingerprint == command_fingerprint {
1806 return Ok(existing.clone());
1807 }
1808 return Err(SessionStoreError::IdempotencyConflict(
1809 idempotency_key.to_string(),
1810 ));
1811 }
1812 if inner.sessions.contains_key(&session.session_id) {
1813 return Err(SessionStoreError::AlreadyExists(
1814 session.session_id.as_str().to_string(),
1815 ));
1816 }
1817 session.revision = session.revision.max(1);
1818 session.updated_at = chrono::Utc::now();
1819 inner
1820 .sessions
1821 .insert(session.session_id.clone(), session.clone());
1822 inner
1823 .session_idempotency
1824 .insert(key, (command_fingerprint.to_string(), session.clone()));
1825 Ok(session)
1826 }
1827
1828 async fn create_session_idempotent_with_host_events(
1829 &self,
1830 mut session: SessionRecord,
1831 idempotency_key: &str,
1832 command_fingerprint: &str,
1833 publications: Vec<PendingHostEventPublication>,
1834 ) -> SessionStoreResult<SessionRecord> {
1835 let key = (session.namespace_id.clone(), idempotency_key.to_string());
1836 let mut inner = self.inner.lock().map_err(store_failed)?;
1837 let mut staged = inner.clone();
1838 let result = if let Some((fingerprint, existing)) = staged.session_idempotency.get(&key) {
1839 if fingerprint != command_fingerprint {
1840 return Err(SessionStoreError::IdempotencyConflict(
1841 idempotency_key.to_string(),
1842 ));
1843 }
1844 existing.clone()
1845 } else {
1846 if staged.sessions.contains_key(&session.session_id) {
1847 return Err(SessionStoreError::AlreadyExists(
1848 session.session_id.as_str().to_string(),
1849 ));
1850 }
1851 session.revision = session.revision.max(1);
1852 session.updated_at = session_mutation_time(&publications)?;
1853 staged
1854 .sessions
1855 .insert(session.session_id.clone(), session.clone());
1856 staged
1857 .session_idempotency
1858 .insert(key, (command_fingerprint.to_string(), session.clone()));
1859 session
1860 };
1861 enqueue_host_event_publications_locked(&mut staged, &publications)?;
1862 *inner = staged;
1863 Ok(result)
1864 }
1865
1866 async fn load_session_mutation_receipt(
1867 &self,
1868 namespace_id: &str,
1869 idempotency_key: &str,
1870 command_fingerprint: &str,
1871 ) -> SessionStoreResult<Option<SessionRecord>> {
1872 let inner = self.inner.lock().map_err(store_failed)?;
1873 let key = (namespace_id.to_string(), idempotency_key.to_string());
1874 let Some((fingerprint, session)) = inner.session_idempotency.get(&key) else {
1875 return Ok(None);
1876 };
1877 if fingerprint != command_fingerprint {
1878 return Err(SessionStoreError::IdempotencyConflict(
1879 idempotency_key.to_string(),
1880 ));
1881 }
1882 Ok(Some(session.clone()))
1883 }
1884
1885 async fn update_managed_session(
1886 &self,
1887 command: UpdateManagedSession,
1888 command_fingerprint: &str,
1889 ) -> SessionStoreResult<SessionRecord> {
1890 let mut inner = self.inner.lock().map_err(store_failed)?;
1891 let namespace = inner
1892 .sessions
1893 .get(&command.session_id)
1894 .ok_or_else(|| SessionStoreError::NotFound(command.session_id.as_str().to_string()))?
1895 .namespace_id
1896 .clone();
1897 let idempotency_key = (namespace, command.idempotency_key.clone());
1898 if let Some((fingerprint, existing)) = inner.session_idempotency.get(&idempotency_key) {
1899 if fingerprint == command_fingerprint {
1900 return Ok(existing.clone());
1901 }
1902 return Err(SessionStoreError::IdempotencyConflict(
1903 command.idempotency_key,
1904 ));
1905 }
1906 let session = inner
1907 .sessions
1908 .get_mut(&command.session_id)
1909 .ok_or_else(|| SessionStoreError::NotFound(command.session_id.as_str().to_string()))?;
1910 if session.revision != command.expected_revision {
1911 return Err(SessionStoreError::Conflict(format!(
1912 "expected revision {}, current {}",
1913 command.expected_revision, session.revision
1914 )));
1915 }
1916 if session.deletion_fence.blocks_continuation() || session.status == SessionStatus::Deleted
1917 {
1918 return Err(SessionStoreError::Conflict(
1919 "session is deleting or deleted".to_string(),
1920 ));
1921 }
1922 if let Some(title) = command.patch.title {
1923 session.title = title.map(|value| value.chars().take(256).collect());
1924 }
1925 if let Some(profile) = command.patch.profile {
1926 session.profile = profile;
1927 }
1928 if let Some(archived) = command.patch.archived {
1929 session.status = if archived {
1930 SessionStatus::Archived
1931 } else {
1932 SessionStatus::Active
1933 };
1934 }
1935 for (key, value) in command.patch.metadata {
1936 if value.is_null() {
1937 session.metadata.remove(&key);
1938 } else {
1939 session.metadata.insert(key, value);
1940 }
1941 }
1942 session.revision = session.revision.saturating_add(1);
1943 session.updated_at = chrono::Utc::now();
1944 let result = session.clone();
1945 inner.session_idempotency.insert(
1946 idempotency_key,
1947 (command_fingerprint.to_string(), result.clone()),
1948 );
1949 Ok(result)
1950 }
1951
1952 async fn update_managed_session_with_host_events(
1953 &self,
1954 command: UpdateManagedSession,
1955 command_fingerprint: &str,
1956 publications: Vec<PendingHostEventPublication>,
1957 ) -> SessionStoreResult<SessionRecord> {
1958 let mut inner = self.inner.lock().map_err(store_failed)?;
1959 let mut staged = inner.clone();
1960 let namespace = staged
1961 .sessions
1962 .get(&command.session_id)
1963 .ok_or_else(|| SessionStoreError::NotFound(command.session_id.as_str().to_string()))?
1964 .namespace_id
1965 .clone();
1966 let idempotency_key = (namespace, command.idempotency_key.clone());
1967 let result = if let Some((fingerprint, existing)) =
1968 staged.session_idempotency.get(&idempotency_key)
1969 {
1970 if fingerprint != command_fingerprint {
1971 return Err(SessionStoreError::IdempotencyConflict(
1972 command.idempotency_key,
1973 ));
1974 }
1975 existing.clone()
1976 } else {
1977 let session = staged
1978 .sessions
1979 .get_mut(&command.session_id)
1980 .ok_or_else(|| {
1981 SessionStoreError::NotFound(command.session_id.as_str().to_string())
1982 })?;
1983 if session.revision != command.expected_revision {
1984 return Err(SessionStoreError::Conflict(format!(
1985 "expected revision {}, current {}",
1986 command.expected_revision, session.revision
1987 )));
1988 }
1989 if session.deletion_fence.blocks_continuation()
1990 || session.status == SessionStatus::Deleted
1991 {
1992 return Err(SessionStoreError::Conflict(
1993 "session is deleting or deleted".to_string(),
1994 ));
1995 }
1996 if let Some(title) = command.patch.title {
1997 session.title = title.map(|value| value.chars().take(256).collect());
1998 }
1999 if let Some(profile) = command.patch.profile {
2000 session.profile = profile;
2001 }
2002 if let Some(archived) = command.patch.archived {
2003 session.status = if archived {
2004 SessionStatus::Archived
2005 } else {
2006 SessionStatus::Active
2007 };
2008 }
2009 for (key, value) in command.patch.metadata {
2010 if value.is_null() {
2011 session.metadata.remove(&key);
2012 } else {
2013 session.metadata.insert(key, value);
2014 }
2015 }
2016 session.revision = session.revision.saturating_add(1);
2017 session.updated_at = session_mutation_time(&publications)?;
2018 let result = session.clone();
2019 staged.session_idempotency.insert(
2020 idempotency_key,
2021 (command_fingerprint.to_string(), result.clone()),
2022 );
2023 result
2024 };
2025 enqueue_host_event_publications_locked(&mut staged, &publications)?;
2026 *inner = staged;
2027 Ok(result)
2028 }
2029
2030 async fn acquire_session_deletion_fence(
2031 &self,
2032 session_id: &SessionId,
2033 expected_revision: u64,
2034 fence_id: &str,
2035 requested_by: &str,
2036 idempotency_key: &str,
2037 command_fingerprint: &str,
2038 ) -> SessionStoreResult<SessionRecord> {
2039 let mut inner = self.inner.lock().map_err(store_failed)?;
2040 let namespace = inner
2041 .sessions
2042 .get(session_id)
2043 .ok_or_else(|| SessionStoreError::NotFound(session_id.as_str().to_string()))?
2044 .namespace_id
2045 .clone();
2046 let key = (namespace, idempotency_key.to_string());
2047 if let Some((fingerprint, existing)) = inner.session_idempotency.get(&key) {
2048 if fingerprint == command_fingerprint {
2049 return Ok(existing.clone());
2050 }
2051 return Err(SessionStoreError::IdempotencyConflict(
2052 idempotency_key.to_string(),
2053 ));
2054 }
2055 if inner
2056 .sessions
2057 .get(session_id)
2058 .is_some_and(|session| session.active_run_id.is_some())
2059 || inner
2060 .run_admissions
2061 .contains_key(&ManagedSessionTarget::new(&key.0, session_id.clone()))
2062 {
2063 return Err(SessionStoreError::RunConflict(
2064 "session still has an admitted active run".to_string(),
2065 ));
2066 }
2067 let now = chrono::Utc::now();
2068 if inner.background_subagents.values().any(|record| {
2069 record.namespace_id == key.0
2070 && &record.parent_session_id == session_id
2071 && !record.execution_status.is_terminal()
2072 && !record.owner_lease.expired_at(now)
2073 }) {
2074 return Err(SessionStoreError::RunConflict(
2075 "session still has active background-subagent ownership".to_string(),
2076 ));
2077 }
2078 let session = inner
2079 .sessions
2080 .get_mut(session_id)
2081 .ok_or_else(|| SessionStoreError::NotFound(session_id.as_str().to_string()))?;
2082 if session.revision != expected_revision {
2083 return Err(SessionStoreError::Conflict(format!(
2084 "expected revision {expected_revision}, current {}",
2085 session.revision
2086 )));
2087 }
2088 if !matches!(session.deletion_fence, SessionDeletionFence::Stable) {
2089 return Err(SessionStoreError::Conflict(
2090 "session already has a deletion fence".to_string(),
2091 ));
2092 }
2093 session.deletion_fence = SessionDeletionFence::Deleting {
2094 fence_id: fence_id.to_string(),
2095 expected_revision,
2096 requested_by: requested_by.to_string(),
2097 started_at: chrono::Utc::now(),
2098 };
2099 session.revision = session.revision.saturating_add(1);
2100 session.updated_at = chrono::Utc::now();
2101 let result = session.clone();
2102 inner
2103 .session_idempotency
2104 .insert(key, (command_fingerprint.to_string(), result.clone()));
2105 Ok(result)
2106 }
2107
2108 async fn tombstone_session(
2109 &self,
2110 session_id: &SessionId,
2111 fence_id: &str,
2112 ) -> SessionStoreResult<SessionRecord> {
2113 let mut inner = self.inner.lock().map_err(store_failed)?;
2114 if inner
2115 .runs
2116 .values()
2117 .any(|run| &run.session_id == session_id && run.status.is_active())
2118 {
2119 return Err(SessionStoreError::RunConflict(
2120 "session still has an active run".to_string(),
2121 ));
2122 }
2123 let now = chrono::Utc::now();
2124 if inner.background_subagents.values().any(|record| {
2125 &record.parent_session_id == session_id
2126 && !record.execution_status.is_terminal()
2127 && !record.owner_lease.expired_at(now)
2128 }) {
2129 return Err(SessionStoreError::RunConflict(
2130 "session still has active background-subagent ownership".to_string(),
2131 ));
2132 }
2133 let session = inner
2134 .sessions
2135 .get_mut(session_id)
2136 .ok_or_else(|| SessionStoreError::NotFound(session_id.as_str().to_string()))?;
2137 match &session.deletion_fence {
2138 SessionDeletionFence::Deleted {
2139 fence_id: current, ..
2140 } if current == fence_id => {
2141 return Ok(session.clone());
2142 }
2143 SessionDeletionFence::Deleting {
2144 fence_id: current, ..
2145 } if current == fence_id => {}
2146 _ => {
2147 return Err(SessionStoreError::Conflict(
2148 "deletion fence mismatch".to_string(),
2149 ));
2150 }
2151 }
2152 session.status = SessionStatus::Deleted;
2153 session.active_run_id = None;
2154 session.deletion_fence = SessionDeletionFence::Deleted {
2155 fence_id: fence_id.to_string(),
2156 deleted_at: chrono::Utc::now(),
2157 };
2158 session.revision = session.revision.saturating_add(1);
2159 session.updated_at = chrono::Utc::now();
2160 Ok(session.clone())
2161 }
2162
2163 async fn tombstone_session_idempotent_with_host_events(
2164 &self,
2165 session_id: &SessionId,
2166 fence_id: &str,
2167 idempotency_key: &str,
2168 command_fingerprint: &str,
2169 publications: Vec<PendingHostEventPublication>,
2170 ) -> SessionStoreResult<SessionRecord> {
2171 let mut inner = self.inner.lock().map_err(store_failed)?;
2172 let mut staged = inner.clone();
2173 let namespace = staged
2174 .sessions
2175 .get(session_id)
2176 .ok_or_else(|| SessionStoreError::NotFound(session_id.as_str().to_string()))?
2177 .namespace_id
2178 .clone();
2179 let receipt_key = (namespace, idempotency_key.to_string());
2180 let (fingerprint, receipt_session) = staged
2181 .session_idempotency
2182 .get(&receipt_key)
2183 .ok_or_else(|| SessionStoreError::NotFound(idempotency_key.to_string()))?;
2184 if fingerprint != command_fingerprint {
2185 return Err(SessionStoreError::IdempotencyConflict(
2186 idempotency_key.to_string(),
2187 ));
2188 }
2189 if receipt_session.session_id != *session_id {
2190 return Err(SessionStoreError::Conflict(
2191 "session deletion receipt target mismatch".to_string(),
2192 ));
2193 }
2194 let already_deleted = staged.sessions.get(session_id).is_some_and(|session| {
2195 matches!(
2196 &session.deletion_fence,
2197 SessionDeletionFence::Deleted {
2198 fence_id: current,
2199 ..
2200 } if current == fence_id
2201 )
2202 });
2203 if !already_deleted {
2204 if staged
2205 .runs
2206 .values()
2207 .any(|run| &run.session_id == session_id && run.status.is_active())
2208 {
2209 return Err(SessionStoreError::RunConflict(
2210 "session still has an active run".to_string(),
2211 ));
2212 }
2213 let now = chrono::Utc::now();
2214 if staged.background_subagents.values().any(|record| {
2215 &record.parent_session_id == session_id
2216 && !record.execution_status.is_terminal()
2217 && !record.owner_lease.expired_at(now)
2218 }) {
2219 return Err(SessionStoreError::RunConflict(
2220 "session still has active background-subagent ownership".to_string(),
2221 ));
2222 }
2223 let session = staged
2224 .sessions
2225 .get_mut(session_id)
2226 .ok_or_else(|| SessionStoreError::NotFound(session_id.as_str().to_string()))?;
2227 match &session.deletion_fence {
2228 SessionDeletionFence::Deleting {
2229 fence_id: current, ..
2230 } if current == fence_id => {}
2231 _ => {
2232 return Err(SessionStoreError::Conflict(
2233 "deletion fence mismatch".to_string(),
2234 ));
2235 }
2236 }
2237 let deleted_at = session_mutation_time(&publications)?;
2238 session.status = SessionStatus::Deleted;
2239 session.active_run_id = None;
2240 session.deletion_fence = SessionDeletionFence::Deleted {
2241 fence_id: fence_id.to_string(),
2242 deleted_at,
2243 };
2244 session.revision = session.revision.saturating_add(1);
2245 session.updated_at = deleted_at;
2246 }
2247 let result = staged
2248 .sessions
2249 .get(session_id)
2250 .cloned()
2251 .ok_or_else(|| SessionStoreError::NotFound(session_id.as_str().to_string()))?;
2252 staged.session_idempotency.insert(
2253 receipt_key,
2254 (command_fingerprint.to_string(), result.clone()),
2255 );
2256 enqueue_host_event_publications_locked(&mut staged, &publications)?;
2257 *inner = staged;
2258 Ok(result)
2259 }
2260
2261 async fn session_continuation_fence(
2262 &self,
2263 namespace_id: &str,
2264 session_id: &SessionId,
2265 ) -> SessionStoreResult<SessionContinuationFence> {
2266 let inner = self.inner.lock().map_err(store_failed)?;
2267 let session = inner
2268 .sessions
2269 .get(session_id)
2270 .filter(|session| session.namespace_id == namespace_id)
2271 .ok_or_else(|| SessionStoreError::NotFound(session_id.as_str().to_string()))?;
2272 let fence_id = match &session.deletion_fence {
2273 SessionDeletionFence::Stable => None,
2274 SessionDeletionFence::Deleting { fence_id, .. }
2275 | SessionDeletionFence::Deleted { fence_id, .. } => Some(fence_id.clone()),
2276 };
2277 Ok(SessionContinuationFence {
2278 target: ManagedSessionTarget::new(namespace_id, session_id.clone()),
2279 revision: session.revision,
2280 continuation_allowed: !session.deletion_fence.blocks_continuation()
2281 && session.status != SessionStatus::Deleted,
2282 fence_id,
2283 })
2284 }
2285
2286 async fn acquire_run_admission(
2287 &self,
2288 request: AcquireRunAdmission,
2289 ) -> SessionStoreResult<RunAdmissionReceipt> {
2290 let mut inner = self.inner.lock().map_err(store_failed)?;
2291 let mut staged = inner.clone();
2292 let receipt = acquire_run_admission_locked(&mut staged, request)?;
2293 *inner = staged;
2294 Ok(receipt)
2295 }
2296
2297 async fn load_run_admission_receipt(
2298 &self,
2299 namespace_id: &str,
2300 idempotency_key: &str,
2301 command_fingerprint: &str,
2302 ) -> SessionStoreResult<Option<RunAdmissionReceipt>> {
2303 let inner = self.inner.lock().map_err(store_failed)?;
2304 let Some((fingerprint, receipt)) = inner
2305 .run_admission_idempotency
2306 .get(&(namespace_id.to_string(), idempotency_key.to_string()))
2307 else {
2308 return Ok(None);
2309 };
2310 if fingerprint != command_fingerprint {
2311 return Err(SessionStoreError::IdempotencyConflict(
2312 idempotency_key.to_string(),
2313 ));
2314 }
2315 let mut receipt = receipt.clone();
2316 receipt.idempotent_replay = true;
2317 Ok(Some(receipt))
2318 }
2319
2320 async fn heartbeat_run_admission(
2321 &self,
2322 lease: &RunAdmissionLease,
2323 lease_expires_at: chrono::DateTime<chrono::Utc>,
2324 ) -> SessionStoreResult<RunAdmissionLease> {
2325 let mut inner = self.inner.lock().map_err(store_failed)?;
2326 let key = ManagedSessionTarget::new(
2327 lease.target.namespace_id.clone(),
2328 lease.target.session_id.clone(),
2329 );
2330 let current = inner
2331 .run_admissions
2332 .get_mut(&key)
2333 .ok_or_else(|| SessionStoreError::NotFound(lease.admission_id.clone()))?;
2334 if current.admission_id != lease.admission_id
2335 || current.host_instance_id != lease.host_instance_id
2336 || current.fencing_generation != lease.fencing_generation
2337 || current.target != lease.target
2338 || current.expired_at(chrono::Utc::now())
2339 {
2340 return Err(SessionStoreError::Conflict(
2341 "stale admission owner".to_string(),
2342 ));
2343 }
2344 current.heartbeat_at = chrono::Utc::now();
2345 current.lease_expires_at = lease_expires_at;
2346 Ok(current.clone())
2347 }
2348
2349 async fn release_run_admission(&self, lease: &RunAdmissionLease) -> SessionStoreResult<()> {
2350 let mut inner = self.inner.lock().map_err(store_failed)?;
2351 let key = ManagedSessionTarget::new(
2352 lease.target.namespace_id.clone(),
2353 lease.target.session_id.clone(),
2354 );
2355 if let Some(current) = inner.run_admissions.get(&key) {
2356 if current.admission_id != lease.admission_id
2357 || current.host_instance_id != lease.host_instance_id
2358 || current.fencing_generation != lease.fencing_generation
2359 || current.target != lease.target
2360 || current.expired_at(chrono::Utc::now())
2361 {
2362 return Err(SessionStoreError::Conflict(
2363 "stale admission owner".to_string(),
2364 ));
2365 }
2366 inner.run_admissions.remove(&key);
2367 }
2368 Ok(())
2369 }
2370
2371 async fn update_run_status_fenced(
2372 &self,
2373 lease: &RunAdmissionLease,
2374 status: RunStatus,
2375 output_preview: Option<String>,
2376 ) -> SessionStoreResult<RunRecord> {
2377 if !status.is_active() {
2378 return Err(SessionStoreError::Conflict(
2379 "fenced status updates are non-terminal; use finalize_run_admission".to_string(),
2380 ));
2381 }
2382 let mut inner = self.inner.lock().map_err(store_failed)?;
2383 ensure_active_admission_locked(&inner, lease, chrono::Utc::now())?;
2384 apply_run_status_locked(
2385 &mut inner,
2386 &lease.target.session_id,
2387 &lease.target.run_id,
2388 status,
2389 output_preview,
2390 None,
2391 chrono::Utc::now(),
2392 )
2393 }
2394
2395 async fn finalize_run_admission(
2396 &self,
2397 lease: &RunAdmissionLease,
2398 terminal: RunTerminalProjection,
2399 ) -> SessionStoreResult<RunRecord> {
2400 if terminal.status.is_active() {
2401 return Err(SessionStoreError::Conflict(
2402 "run admission can only finalize to a non-active status".to_string(),
2403 ));
2404 }
2405 let mut inner = self.inner.lock().map_err(store_failed)?;
2406 let session_key = ManagedSessionTarget::new(
2407 lease.target.namespace_id.clone(),
2408 lease.target.session_id.clone(),
2409 );
2410 if !inner.run_admissions.contains_key(&session_key) {
2411 let run = inner
2412 .runs
2413 .get(&run_key(&lease.target.session_id, &lease.target.run_id))
2414 .cloned()
2415 .ok_or_else(|| {
2416 SessionStoreError::NotFound(run_key_label(
2417 &lease.target.session_id,
2418 &lease.target.run_id,
2419 ))
2420 })?;
2421 if terminal.matches(&run) {
2422 return Ok(run);
2423 }
2424 return Err(SessionStoreError::Conflict(
2425 "stale admission owner".to_string(),
2426 ));
2427 }
2428 ensure_active_admission_locked(&inner, lease, chrono::Utc::now())?;
2429 let mut staged = inner.clone();
2430 let target_key = run_key(&lease.target.session_id, &lease.target.run_id);
2431 let committed = staged.runs.get(&target_key).cloned().ok_or_else(|| {
2432 SessionStoreError::NotFound(run_key_label(
2433 &lease.target.session_id,
2434 &lease.target.run_id,
2435 ))
2436 })?;
2437 let (run, changed) = if committed.status.is_terminal() {
2438 (committed, false)
2442 } else {
2443 terminal
2444 .validate()
2445 .map_err(|error| SessionStoreError::Conflict(error.to_string()))?;
2446 (
2447 apply_run_status_locked(
2448 &mut staged,
2449 &lease.target.session_id,
2450 &lease.target.run_id,
2451 terminal.status,
2452 terminal.output_preview,
2453 terminal.error,
2454 chrono::Utc::now(),
2455 )?,
2456 true,
2457 )
2458 };
2459 if changed {
2460 let mut publications = Vec::new();
2461 append_authoritative_run_publications(
2462 &mut publications,
2463 &format!(
2464 "run-admission-finalize:{}:{}:{}",
2465 lease.admission_id, lease.fencing_generation, run.revision
2466 ),
2467 std::iter::once(&run),
2468 )?;
2469 enqueue_host_event_publications_locked(&mut staged, &publications)?;
2470 }
2471 reconcile_run_control_intents_locked(&mut staged, lease, chrono::Utc::now())?;
2472 staged.run_admissions.remove(&session_key);
2473 *inner = staged;
2474 Ok(run)
2475 }
2476
2477 async fn load_run_admission(
2478 &self,
2479 target: &ManagedRunTarget,
2480 ) -> SessionStoreResult<Option<RunAdmissionLease>> {
2481 let inner = self.inner.lock().map_err(store_failed)?;
2482 Ok(inner
2483 .run_admissions
2484 .get(&ManagedSessionTarget::new(
2485 target.namespace_id.clone(),
2486 target.session_id.clone(),
2487 ))
2488 .filter(|lease| &lease.target == target)
2489 .cloned())
2490 }
2491
2492 async fn reconcile_expired_run_admissions(
2493 &self,
2494 namespace_id: &str,
2495 now: chrono::DateTime<chrono::Utc>,
2496 ) -> SessionStoreResult<Vec<ManagedRunTarget>> {
2497 let mut inner = self.inner.lock().map_err(store_failed)?;
2498 let mut staged = inner.clone();
2499 let expired = staged
2500 .run_admissions
2501 .iter()
2502 .filter(|(session, lease)| {
2503 session.namespace_id == namespace_id && lease.expired_at(now)
2504 })
2505 .map(|(session, lease)| (session.clone(), lease.clone()))
2506 .collect::<Vec<_>>();
2507 for (session_target, lease) in &expired {
2508 let target = &lease.target;
2509 let replacement_key = run_key(&target.session_id, &target.run_id);
2510 let replacement = staged.runs.get(&replacement_key).cloned().ok_or_else(|| {
2511 SessionStoreError::NotFound(run_key_label(&target.session_id, &target.run_id))
2512 })?;
2513 if replacement.status.is_active() {
2514 let effect_started =
2515 terminalize_started_hitl_source_locked(&mut staged, &replacement, now)?;
2516 let mut authoritative_runs = Vec::with_capacity(2);
2517 if effect_started
2518 && let Some(source_run_id) = replacement.restore_from_run_id.as_ref()
2519 && let Some(source) = staged
2520 .runs
2521 .get(&run_key(&target.session_id, source_run_id))
2522 .cloned()
2523 {
2524 authoritative_runs.push(source);
2525 }
2526 let run = staged.runs.get_mut(&replacement_key).ok_or_else(|| {
2527 SessionStoreError::NotFound(run_key_label(&target.session_id, &target.run_id))
2528 })?;
2529 run.status = RunStatus::Cancelled;
2530 run.output_preview = Some("interrupted after host lease expired".to_string());
2531 run.terminal_error = Some(RunTerminalError::new(
2532 "admission_lease_expired",
2533 "interrupted after host lease expired",
2534 ));
2535 advance_run_revision(run)?;
2536 run.updated_at = now;
2537 if effect_started {
2538 ContinuationEffectState::indeterminate()
2539 .insert_into(&mut run.metadata)
2540 .map_err(|error| SessionStoreError::Failed(error.to_string()))?;
2541 }
2542 authoritative_runs.push(run.clone());
2543 let mut publications = Vec::new();
2544 append_authoritative_run_publications(
2545 &mut publications,
2546 &format!(
2547 "run-admission-expired:{}:{}",
2548 lease.admission_id, lease.fencing_generation
2549 ),
2550 authoritative_runs.iter(),
2551 )?;
2552 enqueue_host_event_publications_locked(&mut staged, &publications)?;
2553 }
2554 if let Some(session) = staged.sessions.get_mut(&target.session_id) {
2555 if session.active_run_id.as_ref() == Some(&target.run_id) {
2556 session.active_run_id = None;
2557 }
2558 session.revision = session.revision.saturating_add(1);
2559 session.updated_at = now;
2560 }
2561 reconcile_run_control_intents_locked(&mut staged, lease, now)?;
2562 staged.run_admissions.remove(session_target);
2563 }
2564 *inner = staged;
2565 Ok(expired.into_iter().map(|(_, lease)| lease.target).collect())
2566 }
2567
2568 async fn admit_run_control(
2569 &self,
2570 request: AdmitRunControl,
2571 ) -> SessionStoreResult<DurableRunControlIntent> {
2572 if request.authority_binding.is_empty()
2573 || request.operation_id.is_empty()
2574 || request.receipt_id.is_empty()
2575 || request.idempotency_key.is_empty()
2576 || request.command_fingerprint.is_empty()
2577 {
2578 return Err(SessionStoreError::Conflict(
2579 "durable run control identity fields cannot be empty".to_string(),
2580 ));
2581 }
2582 let mut inner = self.inner.lock().map_err(store_failed)?;
2583 let authority_key = (
2584 request.authority_binding.clone(),
2585 request.idempotency_key.clone(),
2586 );
2587 if let Some(intent_key) = inner.run_control_authority_keys.get(&authority_key)
2588 && let Some(existing) = inner.run_control_intents.get(intent_key)
2589 {
2590 return if existing.matches_admission(&request) {
2591 Ok(existing.clone())
2592 } else {
2593 Err(SessionStoreError::IdempotencyConflict(
2594 request.idempotency_key,
2595 ))
2596 };
2597 }
2598 let intent_key = (request.lease.target.clone(), request.operation_id.clone());
2599 if let Some(existing) = inner.run_control_intents.get(&intent_key) {
2600 return if existing.matches_admission(&request) {
2601 Ok(existing.clone())
2602 } else {
2603 Err(SessionStoreError::Conflict(format!(
2604 "run control operation {} already has different evidence",
2605 request.operation_id
2606 )))
2607 };
2608 }
2609 ensure_active_admission_locked(&inner, &request.lease, chrono::Utc::now())?;
2610 let receipt_key = (
2611 request.lease.target.clone(),
2612 request.idempotency_key.clone(),
2613 );
2614 if let Some(existing_id) = inner.control_idempotency.get(&receipt_key) {
2615 return Err(SessionStoreError::IdempotencyConflict(existing_id.clone()));
2616 }
2617 if inner.control_receipts.contains_key(&request.receipt_id) {
2618 return Err(SessionStoreError::Conflict(format!(
2619 "control receipt {} already exists",
2620 request.receipt_id
2621 )));
2622 }
2623 let intent = request.into_intent();
2624 inner
2625 .control_idempotency
2626 .insert(receipt_key, intent.receipt.receipt_id.clone());
2627 inner
2628 .control_receipts
2629 .insert(intent.receipt.receipt_id.clone(), intent.receipt.clone());
2630 inner
2631 .run_control_authority_keys
2632 .insert(authority_key, intent_key.clone());
2633 inner.run_control_intents.insert(intent_key, intent.clone());
2634 Ok(intent)
2635 }
2636
2637 async fn load_run_control_intent(
2638 &self,
2639 target: &ManagedRunTarget,
2640 operation_id: &str,
2641 ) -> SessionStoreResult<Option<DurableRunControlIntent>> {
2642 let inner = self.inner.lock().map_err(store_failed)?;
2643 Ok(inner
2644 .run_control_intents
2645 .get(&(target.clone(), operation_id.to_string()))
2646 .cloned())
2647 }
2648
2649 async fn list_run_control_intents(
2650 &self,
2651 target: &ManagedRunTarget,
2652 statuses: &[DurableRunControlStatus],
2653 limit: usize,
2654 ) -> SessionStoreResult<Vec<DurableRunControlIntent>> {
2655 if limit == 0 || limit > super::MAX_STABLE_PAGE_SIZE {
2656 return Err(SessionStoreError::Conflict(format!(
2657 "run control page limit must be between 1 and {}",
2658 super::MAX_STABLE_PAGE_SIZE
2659 )));
2660 }
2661 let inner = self.inner.lock().map_err(store_failed)?;
2662 let mut intents = inner
2663 .run_control_intents
2664 .values()
2665 .filter(|intent| {
2666 intent.target == *target
2667 && (statuses.is_empty() || statuses.contains(&intent.status))
2668 })
2669 .cloned()
2670 .collect::<Vec<_>>();
2671 intents.sort_by(|left, right| {
2672 left.created_at
2673 .cmp(&right.created_at)
2674 .then_with(|| left.operation_id.cmp(&right.operation_id))
2675 });
2676 intents.truncate(limit);
2677 Ok(intents)
2678 }
2679
2680 async fn advance_run_control_intent(
2681 &self,
2682 lease: &RunAdmissionLease,
2683 operation_id: &str,
2684 expected: DurableRunControlStatus,
2685 next: DurableRunControlStatus,
2686 occurred_at: chrono::DateTime<chrono::Utc>,
2687 ) -> SessionStoreResult<DurableRunControlIntent> {
2688 let mut inner = self.inner.lock().map_err(store_failed)?;
2689 ensure_active_admission_locked(&inner, lease, chrono::Utc::now())?;
2690 let key = (lease.target.clone(), operation_id.to_string());
2691 let mut intent = inner
2692 .run_control_intents
2693 .get(&key)
2694 .cloned()
2695 .ok_or_else(|| SessionStoreError::NotFound(operation_id.to_string()))?;
2696 if intent.admission_id != lease.admission_id
2697 || intent.host_instance_id != lease.host_instance_id
2698 || intent.fencing_generation != lease.fencing_generation
2699 {
2700 return Err(SessionStoreError::StaleFence(
2701 "run control intent belongs to a stale admission".to_string(),
2702 ));
2703 }
2704 if intent.status != expected && intent.status != next {
2705 return Err(SessionStoreError::Conflict(format!(
2706 "run control operation {operation_id} is {}, expected {}",
2707 intent.status.as_str(),
2708 expected.as_str()
2709 )));
2710 }
2711 intent
2712 .advance(next, occurred_at)
2713 .map_err(|error| SessionStoreError::Conflict(error.to_string()))?;
2714 inner
2715 .control_receipts
2716 .insert(intent.receipt.receipt_id.clone(), intent.receipt.clone());
2717 inner.run_control_intents.insert(key, intent.clone());
2718 Ok(intent)
2719 }
2720
2721 async fn reconcile_run_control_intent(
2722 &self,
2723 target: &ManagedRunTarget,
2724 operation_id: &str,
2725 occurred_at: chrono::DateTime<chrono::Utc>,
2726 ) -> SessionStoreResult<DurableRunControlIntent> {
2727 let mut inner = self.inner.lock().map_err(store_failed)?;
2728 let key = (target.clone(), operation_id.to_string());
2729 let mut intent = inner
2730 .run_control_intents
2731 .get(&key)
2732 .cloned()
2733 .ok_or_else(|| SessionStoreError::NotFound(operation_id.to_string()))?;
2734 intent
2735 .advance(DurableRunControlStatus::Reconciled, occurred_at)
2736 .map_err(|error| SessionStoreError::Conflict(error.to_string()))?;
2737 inner
2738 .control_receipts
2739 .insert(intent.receipt.receipt_id.clone(), intent.receipt.clone());
2740 inner.run_control_intents.insert(key, intent.clone());
2741 Ok(intent)
2742 }
2743
2744 async fn load_control_receipt(
2745 &self,
2746 target: &ManagedRunTarget,
2747 idempotency_key: &str,
2748 ) -> SessionStoreResult<Option<DurableControlReceipt>> {
2749 let inner = self.inner.lock().map_err(store_failed)?;
2750 let key = (target.clone(), idempotency_key.to_string());
2751 Ok(inner
2752 .control_idempotency
2753 .get(&key)
2754 .and_then(|receipt_id| inner.control_receipts.get(receipt_id))
2755 .cloned())
2756 }
2757
2758 async fn reserve_control_receipt(
2759 &self,
2760 receipt: DurableControlReceipt,
2761 ) -> SessionStoreResult<DurableControlReceipt> {
2762 let mut inner = self.inner.lock().map_err(store_failed)?;
2763 let key = (receipt.target.clone(), receipt.idempotency_key.clone());
2764 if let Some(receipt_id) = inner.control_idempotency.get(&key)
2765 && let Some(existing) = inner.control_receipts.get(receipt_id)
2766 {
2767 if existing.command_fingerprint == receipt.command_fingerprint {
2768 return Ok(existing.clone());
2769 }
2770 return Err(SessionStoreError::IdempotencyConflict(
2771 receipt.idempotency_key,
2772 ));
2773 }
2774 let session_key = ManagedSessionTarget::new(
2775 receipt.target.namespace_id.clone(),
2776 receipt.target.session_id.clone(),
2777 );
2778 let lease = inner.run_admissions.get(&session_key).ok_or_else(|| {
2779 SessionStoreError::Conflict("run has no active owner lease".to_string())
2780 })?;
2781 if lease.target != receipt.target || lease.fencing_generation != receipt.fencing_generation
2782 {
2783 return Err(SessionStoreError::Conflict(
2784 "stale control generation".to_string(),
2785 ));
2786 }
2787 inner
2788 .control_idempotency
2789 .insert(key, receipt.receipt_id.clone());
2790 inner
2791 .control_receipts
2792 .insert(receipt.receipt_id.clone(), receipt.clone());
2793 Ok(receipt)
2794 }
2795
2796 async fn update_control_receipt_state(
2797 &self,
2798 receipt_id: &str,
2799 state: &str,
2800 ) -> SessionStoreResult<DurableControlReceipt> {
2801 let mut inner = self.inner.lock().map_err(store_failed)?;
2802 let receipt = inner
2803 .control_receipts
2804 .get_mut(receipt_id)
2805 .ok_or_else(|| SessionStoreError::NotFound(receipt_id.to_string()))?;
2806 receipt.state = state.to_string();
2807 Ok(receipt.clone())
2808 }
2809
2810 async fn drain_background_subagent_operations(&self) -> SessionStoreResult<()> {
2811 Ok(())
2812 }
2813
2814 async fn record_background_subagent_acceptance(
2815 &self,
2816 record: BackgroundSubagentRecord,
2817 ) -> SessionStoreResult<BackgroundSubagentRecord> {
2818 if !record.is_valid_acceptance() {
2819 return Err(SessionStoreError::Conflict(
2820 "invalid background-subagent acceptance record".to_string(),
2821 ));
2822 }
2823 let mut inner = self.inner.lock().map_err(store_failed)?;
2824 if let Some(existing) = inner.background_subagents.get(&record.attempt_id) {
2825 return if same_background_identity(existing, &record) {
2826 Ok(existing.clone())
2827 } else {
2828 Err(SessionStoreError::Conflict(format!(
2829 "background attempt {} already exists with different identity",
2830 record.attempt_id.as_str()
2831 )))
2832 };
2833 }
2834 if record.owner_lease.expired_at(chrono::Utc::now()) {
2835 return Err(SessionStoreError::Conflict(
2836 "background-subagent acceptance owner lease is already expired".to_string(),
2837 ));
2838 }
2839 let session = inner
2840 .sessions
2841 .get(&record.parent_session_id)
2842 .filter(|session| session.namespace_id == record.namespace_id)
2843 .ok_or_else(|| {
2844 SessionStoreError::NotFound(record.parent_session_id.as_str().to_string())
2845 })?;
2846 if session.status != SessionStatus::Active || session.deletion_fence.blocks_continuation() {
2847 return Err(SessionStoreError::Conflict(
2848 "session cannot admit background delegation".to_string(),
2849 ));
2850 }
2851 if !inner
2852 .runs
2853 .contains_key(&run_key(&record.parent_session_id, &record.parent_run_id))
2854 {
2855 return Err(SessionStoreError::NotFound(format!(
2856 "{}:{}",
2857 record.parent_session_id.as_str(),
2858 record.parent_run_id.as_str()
2859 )));
2860 }
2861 if inner.background_subagents.values().any(|existing| {
2862 existing.namespace_id == record.namespace_id
2863 && existing.parent_session_id == record.parent_session_id
2864 && existing.agent_id == record.agent_id
2865 && !existing.execution_status.is_terminal()
2866 }) {
2867 return Err(SessionStoreError::Conflict(format!(
2868 "background agent {} already has an active durable attempt",
2869 record.agent_id
2870 )));
2871 }
2872 inner
2873 .background_subagents
2874 .insert(record.attempt_id.clone(), record.clone());
2875 Ok(record)
2876 }
2877
2878 async fn update_background_subagent_execution(
2879 &self,
2880 mut record: BackgroundSubagentRecord,
2881 ) -> SessionStoreResult<BackgroundSubagentRecord> {
2882 let mut inner = self.inner.lock().map_err(store_failed)?;
2883 let current = inner
2884 .background_subagents
2885 .get(&record.attempt_id)
2886 .ok_or_else(|| SessionStoreError::NotFound(record.attempt_id.as_str().to_string()))?;
2887 ensure_background_parent_writable(&inner, current)?;
2888 if current.owner_lease.expired_at(chrono::Utc::now())
2889 || !same_background_identity(current, &record)
2890 || !same_background_owner(current, &record)
2891 || current.execution_status.is_terminal()
2892 || record.execution_status.is_terminal()
2893 || !valid_background_transition(current.execution_status, record.execution_status)
2894 || !same_background_non_execution_state(current, &record)
2895 {
2896 return Err(SessionStoreError::Conflict(format!(
2897 "invalid background execution transition for {}",
2898 record.attempt_id.as_str()
2899 )));
2900 }
2901 record.owner_lease = current.owner_lease.clone();
2902 inner
2903 .background_subagents
2904 .insert(record.attempt_id.clone(), record.clone());
2905 Ok(record)
2906 }
2907
2908 async fn heartbeat_background_subagent(
2909 &self,
2910 attempt_id: &starweaver_core::SubagentAttemptId,
2911 host_instance_id: &str,
2912 fencing_generation: u64,
2913 lease_expires_at: chrono::DateTime<chrono::Utc>,
2914 ) -> SessionStoreResult<BackgroundSubagentRecord> {
2915 let now = chrono::Utc::now();
2916 let mut inner = self.inner.lock().map_err(store_failed)?;
2917 let identity = inner
2918 .background_subagents
2919 .get(attempt_id)
2920 .cloned()
2921 .ok_or_else(|| SessionStoreError::NotFound(attempt_id.as_str().to_string()))?;
2922 ensure_background_parent_writable(&inner, &identity)?;
2923 let record = inner
2924 .background_subagents
2925 .get_mut(attempt_id)
2926 .ok_or_else(|| SessionStoreError::NotFound(attempt_id.as_str().to_string()))?;
2927 if record.execution_status.is_terminal()
2928 || record.owner_lease.expired_at(now)
2929 || record.owner_lease.host_instance_id != host_instance_id
2930 || record.owner_lease.fencing_generation != fencing_generation
2931 || lease_expires_at <= now
2932 {
2933 return Err(SessionStoreError::Conflict(
2934 "stale or invalid background-subagent owner heartbeat".to_string(),
2935 ));
2936 }
2937 record.owner_lease.heartbeat_at = now;
2938 record.owner_lease.lease_expires_at = lease_expires_at;
2939 Ok(record.clone())
2940 }
2941
2942 async fn commit_background_subagent_terminal(
2943 &self,
2944 commit: BackgroundSubagentTerminalCommit,
2945 ) -> SessionStoreResult<BackgroundSubagentRecord> {
2946 let mut inner = self.inner.lock().map_err(store_failed)?;
2947 commit_background_terminal(
2948 &mut inner,
2949 commit.record,
2950 commit.artifact,
2951 commit.artifact_limits,
2952 )
2953 }
2954
2955 async fn load_background_subagent_artifact(
2956 &self,
2957 artifact_ref: &str,
2958 ) -> SessionStoreResult<BackgroundSubagentArtifact> {
2959 let artifact = self
2960 .inner
2961 .lock()
2962 .map_err(store_failed)?
2963 .background_artifacts
2964 .get(artifact_ref)
2965 .cloned()
2966 .ok_or_else(|| SessionStoreError::NotFound(artifact_ref.to_string()))?;
2967 if artifact.expires_at <= chrono::Utc::now() {
2968 return Err(SessionStoreError::NotFound(artifact_ref.to_string()));
2969 }
2970 if !artifact.is_valid() {
2971 return Err(SessionStoreError::Conflict(
2972 "background-subagent artifact failed integrity validation".to_string(),
2973 ));
2974 }
2975 Ok(artifact)
2976 }
2977
2978 async fn expire_background_subagent_retention(
2979 &self,
2980 namespace_id: &str,
2981 now: chrono::DateTime<chrono::Utc>,
2982 limit: usize,
2983 ) -> SessionStoreResult<Vec<BackgroundSubagentRecord>> {
2984 let mut inner = self.inner.lock().map_err(store_failed)?;
2985 let mut attempt_ids = inner
2986 .background_subagents
2987 .values()
2988 .filter(|record| {
2989 record.namespace_id == namespace_id
2990 && record.execution_status.is_terminal()
2991 && record.retention_status
2992 != crate::DurableBackgroundSubagentRetentionStatus::Expired
2993 && record
2994 .retention_expires_at
2995 .is_some_and(|deadline| deadline <= now)
2996 })
2997 .map(|record| (record.retention_expires_at, record.attempt_id.clone()))
2998 .collect::<Vec<_>>();
2999 attempt_ids.sort();
3000 attempt_ids.truncate(limit);
3001 let mut expired = Vec::with_capacity(attempt_ids.len());
3002 let mut artifact_refs = Vec::new();
3003 for (_, attempt_id) in attempt_ids {
3004 if !inner
3005 .background_terminal_fingerprints
3006 .contains_key(&attempt_id)
3007 && let Some(record) = inner.background_subagents.get(&attempt_id).cloned()
3008 && let Some(fingerprint) =
3009 reconstruct_background_terminal_fingerprint(&inner, &record)?
3010 {
3011 inner
3012 .background_terminal_fingerprints
3013 .insert(attempt_id.clone(), fingerprint);
3014 }
3015 let Some(record) = inner.background_subagents.get_mut(&attempt_id) else {
3016 continue;
3017 };
3018 if let Some(result_ref) = record.result_ref.as_mut() {
3019 if let Some(artifact_ref) = result_ref.artifact_ref.take() {
3020 artifact_refs.push(artifact_ref);
3021 }
3022 result_ref.content = None;
3023 result_ref.error = None;
3024 }
3025 record.retention_status = crate::DurableBackgroundSubagentRetentionStatus::Expired;
3026 record.retention_expires_at = None;
3027 expired.push(record.clone());
3028 }
3029 for artifact_ref in artifact_refs {
3030 inner.background_artifacts.remove(&artifact_ref);
3031 }
3032 Ok(expired)
3033 }
3034
3035 async fn record_background_subagent_terminal(
3036 &self,
3037 record: BackgroundSubagentRecord,
3038 ) -> SessionStoreResult<BackgroundSubagentRecord> {
3039 let mut inner = self.inner.lock().map_err(store_failed)?;
3040 commit_background_terminal(&mut inner, record, None, None)
3041 }
3042
3043 async fn load_background_subagent(
3044 &self,
3045 attempt_id: &starweaver_core::SubagentAttemptId,
3046 ) -> SessionStoreResult<BackgroundSubagentRecord> {
3047 self.inner
3048 .lock()
3049 .map_err(store_failed)?
3050 .background_subagents
3051 .get(attempt_id)
3052 .cloned()
3053 .ok_or_else(|| SessionStoreError::NotFound(attempt_id.as_str().to_string()))
3054 }
3055
3056 async fn list_background_subagents(
3057 &self,
3058 namespace_id: &str,
3059 session_id: Option<&SessionId>,
3060 limit: usize,
3061 ) -> SessionStoreResult<Vec<BackgroundSubagentRecord>> {
3062 let inner = self.inner.lock().map_err(store_failed)?;
3063 let mut records = inner
3064 .background_subagents
3065 .values()
3066 .filter(|record| {
3067 record.namespace_id == namespace_id
3068 && session_id.is_none_or(|session_id| &record.parent_session_id == session_id)
3069 })
3070 .cloned()
3071 .collect::<Vec<_>>();
3072 records.sort_by(|left, right| {
3073 right
3074 .updated_at
3075 .cmp(&left.updated_at)
3076 .then_with(|| left.attempt_id.cmp(&right.attempt_id))
3077 });
3078 records.truncate(limit);
3079 Ok(records)
3080 }
3081
3082 async fn list_pending_background_subagents(
3083 &self,
3084 namespace_id: &str,
3085 session_id: Option<&SessionId>,
3086 limit: usize,
3087 ) -> SessionStoreResult<Vec<BackgroundSubagentRecord>> {
3088 let inner = self.inner.lock().map_err(store_failed)?;
3089 let mut records = inner
3090 .background_subagents
3091 .values()
3092 .filter(|record| {
3093 record.namespace_id == namespace_id
3094 && session_id.is_none_or(|session_id| &record.parent_session_id == session_id)
3095 && record.execution_status.is_terminal()
3096 && record.delivery_status != DurableBackgroundSubagentDeliveryStatus::Delivered
3097 })
3098 .cloned()
3099 .collect::<Vec<_>>();
3100 records.sort_by(|left, right| {
3101 left.updated_at
3102 .cmp(&right.updated_at)
3103 .then_with(|| left.attempt_id.cmp(&right.attempt_id))
3104 });
3105 records.truncate(limit);
3106 Ok(records)
3107 }
3108
3109 async fn claim_background_subagent_delivery(
3110 &self,
3111 attempt_id: &starweaver_core::SubagentAttemptId,
3112 claim: DurableBackgroundSubagentDeliveryClaim,
3113 ) -> SessionStoreResult<BackgroundSubagentRecord> {
3114 let now = chrono::Utc::now();
3115 let mut inner = self.inner.lock().map_err(store_failed)?;
3116 let mut record = inner
3117 .background_subagents
3118 .get(attempt_id)
3119 .cloned()
3120 .ok_or_else(|| SessionStoreError::NotFound(attempt_id.as_str().to_string()))?;
3121 if record.delivery_status == DurableBackgroundSubagentDeliveryStatus::Claimed
3122 && record.delivery_claim.as_ref() == Some(&claim)
3123 {
3124 return Ok(record);
3125 }
3126 if let Some(current_claim) = record.delivery_claim.clone()
3127 && record.delivery_status == DurableBackgroundSubagentDeliveryStatus::Claimed
3128 && current_claim.deadline <= now
3129 {
3130 match background_claim_consumer_state(&inner, &record, ¤t_claim, now) {
3131 BackgroundClaimConsumerState::Live => {
3132 return Err(SessionStoreError::Conflict(
3133 "live admitted consumer still owns the background delivery claim"
3134 .to_string(),
3135 ));
3136 }
3137 BackgroundClaimConsumerState::Completed(run_id) => {
3138 record.delivery_status = DurableBackgroundSubagentDeliveryStatus::Delivered;
3139 record.delivery_claim = None;
3140 record.delivered_claim_id = Some(current_claim.claim_id);
3141 record.continuation_run_id = Some(run_id);
3142 record.automatic_continuation_suppressed_by_run_id = None;
3143 record.updated_at = now;
3144 inner
3145 .background_subagents
3146 .insert(attempt_id.clone(), record);
3147 return Err(SessionStoreError::Conflict(
3148 "completed consumer already delivered the background result".to_string(),
3149 ));
3150 }
3151 BackgroundClaimConsumerState::Terminated(run_id) => {
3152 record.delivery_status = DurableBackgroundSubagentDeliveryStatus::Undelivered;
3153 record.delivery_claim = None;
3154 record.automatic_continuation_suppressed_by_run_id = Some(run_id);
3155 record.updated_at = now;
3156 inner
3157 .background_subagents
3158 .insert(attempt_id.clone(), record);
3159 return Err(SessionStoreError::Conflict(
3160 "terminated consumer released the background delivery claim".to_string(),
3161 ));
3162 }
3163 BackgroundClaimConsumerState::Reclaimable => {}
3164 }
3165 }
3166 if !record.delivery_claimable_at(now) {
3167 return Err(SessionStoreError::Conflict(
3168 "background result delivery is not claimable".to_string(),
3169 ));
3170 }
3171 record.delivery_status = DurableBackgroundSubagentDeliveryStatus::Claimed;
3172 record.delivery_claim = Some(claim);
3173 record.updated_at = now;
3174 inner
3175 .background_subagents
3176 .insert(attempt_id.clone(), record.clone());
3177 Ok(record)
3178 }
3179
3180 async fn acknowledge_background_subagent_delivery(
3181 &self,
3182 attempt_id: &starweaver_core::SubagentAttemptId,
3183 claim_id: &str,
3184 ) -> SessionStoreResult<BackgroundSubagentRecord> {
3185 let mut inner = self.inner.lock().map_err(store_failed)?;
3186 let record = inner
3187 .background_subagents
3188 .get_mut(attempt_id)
3189 .ok_or_else(|| SessionStoreError::NotFound(attempt_id.as_str().to_string()))?;
3190 if record.delivery_status == DurableBackgroundSubagentDeliveryStatus::Delivered {
3191 return if record.delivered_claim_id.as_deref() == Some(claim_id) {
3192 Ok(record.clone())
3193 } else {
3194 Err(SessionStoreError::Conflict(
3195 "background result was delivered by another claim".to_string(),
3196 ))
3197 };
3198 }
3199 if record.delivery_status != DurableBackgroundSubagentDeliveryStatus::Claimed
3200 || record
3201 .delivery_claim
3202 .as_ref()
3203 .is_none_or(|claim| claim.claim_id != claim_id)
3204 {
3205 return Err(SessionStoreError::Conflict(
3206 "background delivery claim mismatch".to_string(),
3207 ));
3208 }
3209 record.continuation_run_id = record
3210 .delivery_claim
3211 .as_ref()
3212 .and_then(|claim| claim.continuation_run_id.clone());
3213 record.delivery_status = DurableBackgroundSubagentDeliveryStatus::Delivered;
3214 record.delivery_claim = None;
3215 record.delivered_claim_id = Some(claim_id.to_string());
3216 record.automatic_continuation_suppressed_by_run_id = None;
3217 record.updated_at = chrono::Utc::now();
3218 Ok(record.clone())
3219 }
3220
3221 async fn release_background_subagent_delivery(
3222 &self,
3223 attempt_id: &starweaver_core::SubagentAttemptId,
3224 claim_id: &str,
3225 release: DurableBackgroundSubagentDeliveryRelease,
3226 ) -> SessionStoreResult<BackgroundSubagentRecord> {
3227 let mut inner = self.inner.lock().map_err(store_failed)?;
3228 let mut record = inner
3229 .background_subagents
3230 .get(attempt_id)
3231 .cloned()
3232 .ok_or_else(|| SessionStoreError::NotFound(attempt_id.as_str().to_string()))?;
3233 if record.delivery_status == DurableBackgroundSubagentDeliveryStatus::Undelivered {
3234 return match release {
3235 DurableBackgroundSubagentDeliveryRelease::Retryable => Ok(record),
3236 DurableBackgroundSubagentDeliveryRelease::ConsumerTerminated { .. } => {
3237 Err(SessionStoreError::Conflict(
3238 "terminated consumer has no matching background delivery claim".to_string(),
3239 ))
3240 }
3241 };
3242 }
3243 if record.delivery_status != DurableBackgroundSubagentDeliveryStatus::Claimed
3244 || record
3245 .delivery_claim
3246 .as_ref()
3247 .is_none_or(|claim| claim.claim_id != claim_id)
3248 {
3249 return Err(SessionStoreError::Conflict(
3250 "background delivery claim mismatch".to_string(),
3251 ));
3252 }
3253 if let DurableBackgroundSubagentDeliveryRelease::ConsumerTerminated { run_id } = release {
3254 if record
3255 .delivery_claim
3256 .as_ref()
3257 .and_then(|claim| claim.continuation_run_id.as_ref())
3258 != Some(&run_id)
3259 || inner
3260 .runs
3261 .get(&run_key(&record.parent_session_id, &run_id))
3262 .is_none_or(|run| {
3263 !matches!(run.status, RunStatus::Failed | RunStatus::Cancelled)
3264 })
3265 {
3266 return Err(SessionStoreError::Conflict(
3267 "terminated consumer does not own the claim or is not terminal".to_string(),
3268 ));
3269 }
3270 record.automatic_continuation_suppressed_by_run_id = Some(run_id);
3271 }
3272 record.delivery_status = DurableBackgroundSubagentDeliveryStatus::Undelivered;
3273 record.delivery_claim = None;
3274 record.updated_at = chrono::Utc::now();
3275 inner
3276 .background_subagents
3277 .insert(attempt_id.clone(), record.clone());
3278 Ok(record)
3279 }
3280
3281 async fn acquire_background_subagent_continuation(
3282 &self,
3283 request: AcquireBackgroundSubagentContinuation,
3284 ) -> SessionStoreResult<BackgroundSubagentContinuationReceipt> {
3285 let mut inner = self.inner.lock().map_err(store_failed)?;
3286 let mut background = inner
3287 .background_subagents
3288 .get(&request.attempt_id)
3289 .cloned()
3290 .ok_or_else(|| SessionStoreError::NotFound(request.attempt_id.as_str().to_string()))?;
3291 let continuation_run_id = request.admission.run.run_id.clone();
3292 if request.attempt_id != request.cause.attempt_id
3293 || background
3294 .automatic_continuation_suppressed_by_run_id
3295 .is_some()
3296 || !background
3297 .validates_continuation_cause_envelope(&request.cause, &request.admission.run)
3298 {
3299 return Err(SessionStoreError::Conflict(
3300 "background continuation cause does not match its durable result".to_string(),
3301 ));
3302 }
3303 if background.delivery_status == DurableBackgroundSubagentDeliveryStatus::Delivered
3304 || background.delivery_status == DurableBackgroundSubagentDeliveryStatus::Claimed
3305 {
3306 let same_claim = background.continuation_run_id.as_ref() == Some(&continuation_run_id)
3307 && (background.delivered_claim_id.as_deref() == Some(&request.claim_id)
3308 || background
3309 .delivery_claim
3310 .as_ref()
3311 .is_some_and(|claim| claim.claim_id == request.claim_id));
3312 if !same_claim {
3313 return Err(SessionStoreError::Conflict(
3314 "background result was already claimed or delivered".to_string(),
3315 ));
3316 }
3317 let key = (
3318 request.admission.namespace_id.clone(),
3319 request.admission.idempotency_key.clone(),
3320 );
3321 let (fingerprint, admission) = inner
3322 .run_admission_idempotency
3323 .get(&key)
3324 .ok_or_else(|| SessionStoreError::NotFound(request.claim_id.clone()))?;
3325 if fingerprint != &request.admission.command_fingerprint {
3326 return Err(SessionStoreError::IdempotencyConflict(
3327 request.admission.idempotency_key.clone(),
3328 ));
3329 }
3330 let mut admission = admission.clone();
3331 admission.idempotent_replay = true;
3332 if !continuation_receipt_matches_request(&background, &request, &admission) {
3333 return Err(SessionStoreError::Conflict(
3334 "stored continuation receipt does not match the causal request".to_string(),
3335 ));
3336 }
3337 let cause =
3338 crate::BackgroundSubagentContinuationCause::new(&background, &admission.run.input)
3339 .map_err(|error| SessionStoreError::Failed(error.to_string()))?;
3340 return Ok(BackgroundSubagentContinuationReceipt {
3341 cause,
3342 background,
3343 admission,
3344 });
3345 }
3346 let now = chrono::Utc::now();
3347 let artifact_content = continuation_artifact_content(&inner, &background, now)?;
3348 if background.parent_session_id != request.admission.run.session_id
3349 || background.namespace_id != request.admission.namespace_id
3350 || !background.validates_continuation_cause(
3351 &request.cause,
3352 &request.admission.run,
3353 artifact_content.as_deref(),
3354 )
3355 || request.claim_id.is_empty()
3356 || request.claim_deadline <= now
3357 || !background.delivery_claimable_at(now)
3358 {
3359 return Err(SessionStoreError::Conflict(
3360 "background result cannot admit this continuation".to_string(),
3361 ));
3362 }
3363 let session = inner
3364 .sessions
3365 .get(&background.parent_session_id)
3366 .ok_or_else(|| {
3367 SessionStoreError::NotFound(background.parent_session_id.as_str().to_string())
3368 })?;
3369 if request.admission.run.restore_from_run_id != session.head_run_id {
3370 return Err(SessionStoreError::Conflict(
3371 "background continuation source no longer matches the session head".to_string(),
3372 ));
3373 }
3374 let admission_request = request.clone();
3375 let mut staged = inner.clone();
3376 let admission = acquire_run_admission_locked(&mut staged, request.admission.clone())?;
3377 if !continuation_receipt_matches_request(&background, &admission_request, &admission) {
3378 return Err(SessionStoreError::Conflict(
3379 "admitted continuation receipt lost causal input binding".to_string(),
3380 ));
3381 }
3382 background.delivery_status = DurableBackgroundSubagentDeliveryStatus::Delivered;
3383 background.delivery_claim = None;
3384 background.delivered_claim_id = Some(request.claim_id);
3385 background.continuation_run_id = Some(admission.run.run_id.clone());
3386 background.updated_at = chrono::Utc::now();
3387 staged
3388 .background_subagents
3389 .insert(background.attempt_id.clone(), background.clone());
3390 let cause =
3391 crate::BackgroundSubagentContinuationCause::new(&background, &admission.run.input)
3392 .map_err(|error| SessionStoreError::Failed(error.to_string()))?;
3393 *inner = staged;
3394 Ok(BackgroundSubagentContinuationReceipt {
3395 cause,
3396 background,
3397 admission,
3398 })
3399 }
3400
3401 async fn reconcile_background_subagents(
3402 &self,
3403 namespace_id: &str,
3404 now: chrono::DateTime<chrono::Utc>,
3405 ) -> SessionStoreResult<Vec<BackgroundSubagentRecord>> {
3406 let mut inner = self.inner.lock().map_err(store_failed)?;
3407 let attempt_ids = inner
3408 .background_subagents
3409 .values()
3410 .filter(|record| record.namespace_id == namespace_id)
3411 .map(|record| record.attempt_id.clone())
3412 .collect::<Vec<_>>();
3413 let mut changed = Vec::new();
3414 for attempt_id in attempt_ids {
3415 let Some(mut record) = inner.background_subagents.get(&attempt_id).cloned() else {
3416 continue;
3417 };
3418 let mut terminal_fingerprint = None;
3419 if !record.execution_status.is_terminal() && record.owner_lease.expired_at(now) {
3420 let error = "in-process background execution was interrupted by host restart";
3421 record.execution_status = DurableBackgroundSubagentExecutionStatus::Failed;
3422 record.failure_category = Some("host_process_lost".to_string());
3423 record.result_ref = Some(DurableBackgroundSubagentResultRef {
3424 error: Some(error.to_string()),
3425 size_bytes: u64::try_from(error.len()).unwrap_or(u64::MAX),
3426 ..DurableBackgroundSubagentResultRef::default()
3427 });
3428 record.delivery_status = DurableBackgroundSubagentDeliveryStatus::Undelivered;
3429 record.delivery_claim = None;
3430 record.retention_expires_at = Some(
3431 now + chrono::Duration::seconds(
3432 crate::DEFAULT_BACKGROUND_RESULT_RETENTION_SECS,
3433 ),
3434 );
3435 record.terminal_at = Some(now);
3436 record.updated_at = now;
3437 terminal_fingerprint = Some(background_terminal_fingerprint(&record, None)?);
3438 } else if record.delivery_status == DurableBackgroundSubagentDeliveryStatus::Claimed
3439 && record
3440 .delivery_claim
3441 .as_ref()
3442 .is_some_and(|claim| claim.deadline <= now)
3443 {
3444 let claim = record.delivery_claim.clone().ok_or_else(|| {
3445 SessionStoreError::Conflict(
3446 "claimed background result is missing its claim".to_string(),
3447 )
3448 })?;
3449 let consumer = claim.continuation_run_id.as_ref().and_then(|run_id| {
3450 inner
3451 .runs
3452 .get(&run_key(&record.parent_session_id, run_id))
3453 .map(|run| (run_id, run.status))
3454 });
3455 let consumer_has_live_admission = consumer.is_some_and(|(run_id, status)| {
3456 status.is_active()
3457 && inner
3458 .run_admissions
3459 .get(&ManagedSessionTarget::new(
3460 record.namespace_id.clone(),
3461 record.parent_session_id.clone(),
3462 ))
3463 .is_some_and(|lease| {
3464 lease.target.run_id == *run_id && !lease.expired_at(now)
3465 })
3466 });
3467 if consumer_has_live_admission {
3468 continue;
3469 }
3470 if consumer.is_some_and(|(_, status)| status == RunStatus::Completed) {
3471 record.delivery_status = DurableBackgroundSubagentDeliveryStatus::Delivered;
3472 record
3473 .continuation_run_id
3474 .clone_from(&claim.continuation_run_id);
3475 record.delivered_claim_id = Some(claim.claim_id);
3476 record.automatic_continuation_suppressed_by_run_id = None;
3477 } else {
3478 record.delivery_status = DurableBackgroundSubagentDeliveryStatus::Undelivered;
3479 if let Some((run_id, status)) = consumer
3480 && matches!(status, RunStatus::Failed | RunStatus::Cancelled)
3481 {
3482 record.automatic_continuation_suppressed_by_run_id = Some(run_id.clone());
3483 }
3484 }
3485 record.delivery_claim = None;
3486 record.updated_at = now;
3487 } else {
3488 continue;
3489 }
3490 if let Some(fingerprint) = terminal_fingerprint {
3491 inner
3492 .background_terminal_fingerprints
3493 .insert(attempt_id.clone(), fingerprint);
3494 }
3495 inner
3496 .background_subagents
3497 .insert(attempt_id, record.clone());
3498 changed.push(record);
3499 }
3500 Ok(changed)
3501 }
3502
3503 async fn save_session(&self, session: SessionRecord) -> SessionStoreResult<()> {
3504 self.save_session_record(session)
3505 }
3506
3507 async fn load_session(&self, session_id: &SessionId) -> SessionStoreResult<SessionRecord> {
3508 self.load_session_record(session_id)
3509 }
3510
3511 async fn list_sessions(&self, filter: SessionFilter) -> SessionStoreResult<Vec<SessionRecord>> {
3512 self.list_session_records(filter)
3513 }
3514
3515 async fn list_session_page(&self, query: SessionPageQuery) -> SessionStoreResult<SessionPage> {
3516 self.list_session_record_page(&query)
3517 }
3518
3519 async fn update_session_status(
3520 &self,
3521 session_id: &SessionId,
3522 status: SessionStatus,
3523 ) -> SessionStoreResult<()> {
3524 self.set_session_status(session_id, status)
3525 }
3526
3527 async fn save_context_state(
3528 &self,
3529 session_id: &SessionId,
3530 state: ResumableState,
3531 ) -> SessionStoreResult<()> {
3532 self.save_context_state_snapshot(session_id, state)
3533 }
3534
3535 async fn save_environment_state(
3536 &self,
3537 session_id: &SessionId,
3538 environment_state: EnvironmentStateRef,
3539 ) -> SessionStoreResult<()> {
3540 self.save_environment_state_ref(session_id, environment_state)
3541 }
3542
3543 async fn append_run(&self, run: RunRecord) -> SessionStoreResult<()> {
3544 self.append_run_record(run)
3545 }
3546
3547 async fn load_run(
3548 &self,
3549 session_id: &SessionId,
3550 run_id: &RunId,
3551 ) -> SessionStoreResult<RunRecord> {
3552 self.load_run_record(session_id, run_id)
3553 }
3554
3555 async fn list_runs(&self, session_id: &SessionId) -> SessionStoreResult<Vec<RunRecord>> {
3556 self.list_run_records(session_id)
3557 }
3558
3559 async fn update_run_status(
3560 &self,
3561 session_id: &SessionId,
3562 run_id: &RunId,
3563 status: RunStatus,
3564 output_preview: Option<String>,
3565 ) -> SessionStoreResult<()> {
3566 self.set_legacy_run_status(session_id, run_id, status, output_preview)
3567 }
3568
3569 async fn append_checkpoint(
3570 &self,
3571 session_id: &SessionId,
3572 checkpoint: AgentCheckpoint,
3573 ) -> SessionStoreResult<()> {
3574 self.append_checkpoint_record(session_id, checkpoint)
3575 }
3576
3577 async fn load_checkpoints(
3578 &self,
3579 session_id: &SessionId,
3580 run_id: &RunId,
3581 ) -> SessionStoreResult<Vec<AgentCheckpoint>> {
3582 self.load_checkpoint_records(session_id, run_id)
3583 }
3584
3585 async fn append_stream_records(
3586 &self,
3587 session_id: &SessionId,
3588 run_id: &RunId,
3589 records: Vec<AgentStreamRecord>,
3590 ) -> SessionStoreResult<()> {
3591 self.append_stream_record_batch(session_id, run_id, records)
3592 }
3593
3594 async fn replay_stream_records(
3595 &self,
3596 session_id: &SessionId,
3597 run_id: &RunId,
3598 ) -> SessionStoreResult<Vec<AgentStreamRecord>> {
3599 self.replay_stream_record_batch(session_id, run_id)
3600 }
3601
3602 async fn save_stream_cursor(
3603 &self,
3604 session_id: &SessionId,
3605 run_id: &RunId,
3606 cursor: StreamCursorRef,
3607 ) -> SessionStoreResult<()> {
3608 self.save_stream_cursor_ref(session_id, run_id, cursor)
3609 }
3610
3611 async fn append_approval(&self, approval: ApprovalRecord) -> SessionStoreResult<()> {
3612 self.append_approval_record(approval)
3613 }
3614
3615 async fn load_approvals(
3616 &self,
3617 session_id: &SessionId,
3618 run_id: &RunId,
3619 ) -> SessionStoreResult<Vec<ApprovalRecord>> {
3620 self.load_approval_records(session_id, run_id)
3621 }
3622
3623 async fn append_deferred_tool(&self, record: DeferredToolRecord) -> SessionStoreResult<()> {
3624 self.append_deferred_tool_record(record)
3625 }
3626
3627 async fn load_deferred_tools(
3628 &self,
3629 session_id: &SessionId,
3630 run_id: &RunId,
3631 ) -> SessionStoreResult<Vec<DeferredToolRecord>> {
3632 self.load_deferred_tool_records(session_id, run_id)
3633 }
3634
3635 async fn resume_snapshot(
3636 &self,
3637 session_id: &SessionId,
3638 run_id: &RunId,
3639 ) -> SessionStoreResult<SessionResumeSnapshot> {
3640 let session = self.load_session(session_id).await?;
3641 let run = self.load_run(session_id, run_id).await?;
3642 let state = {
3643 let inner = self.inner.lock().map_err(store_failed)?;
3644 inner
3645 .evidence_commits
3646 .get(&run_key(session_id, run_id))
3647 .map_or_else(
3648 || session.state.clone(),
3649 |commit| commit.context_state.clone(),
3650 )
3651 };
3652 let latest_checkpoint = self.latest_checkpoint(session_id, run_id).await?;
3653 let after_sequence = latest_checkpoint
3654 .as_ref()
3655 .and_then(|checkpoint| checkpoint.resume.cursor.stream_cursor);
3656 let stream_records = self
3657 .replay_stream_records_after(session_id, run_id, after_sequence)
3658 .await?;
3659 let approvals = self.load_approvals(session_id, run_id).await?;
3660 let deferred_tools = self.load_deferred_tools(session_id, run_id).await?;
3661 let environment_state = run
3662 .environment_state
3663 .clone()
3664 .or_else(|| session.environment_state.clone());
3665 let mut stream_cursors = session.stream_cursors.clone();
3666 stream_cursors.extend(run.stream_cursors.clone());
3667 Ok(SessionResumeSnapshot {
3668 session,
3669 run,
3670 state,
3671 environment_state,
3672 latest_checkpoint,
3673 stream_records,
3674 approvals,
3675 deferred_tools,
3676 stream_cursors,
3677 })
3678 }
3679
3680 async fn compact_run_trace(
3681 &self,
3682 session_id: &SessionId,
3683 run_id: &RunId,
3684 ) -> SessionStoreResult<CompactRunTrace> {
3685 self.compact_run_trace_projection(session_id, run_id)
3686 }
3687
3688 async fn compact_session_trace(
3689 &self,
3690 session_id: &SessionId,
3691 ) -> SessionStoreResult<CompactSessionTrace> {
3692 self.compact_session_trace_projection(session_id)
3693 }
3694}