use car_engine::{Runtime, ToolExecutor};
use car_eventlog::EventLog;
use car_proto::{ToolCancelRequest, ToolExecuteRequest, ToolExecuteResponse};
use futures::Sink;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot, Mutex};
use tokio_tungstenite::tungstenite::{Error as WsError, Message};
pub type WsSink = Pin<Box<dyn Sink<Message, Error = WsError> + Send + Unpin + 'static>>;
pub const RUN_COMPLETE_GRACE: std::time::Duration = std::time::Duration::from_millis(250);
pub const RUN_RESUME_LEASE: std::time::Duration = std::time::Duration::from_secs(10);
const CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT: std::time::Duration =
std::time::Duration::from_secs(5);
const STARTUP_RECONCILIATION_ACKNOWLEDGEMENT_TIMEOUT: std::time::Duration =
CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT;
pub const RUN_DISCONNECT_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(22);
const RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT: usize = 3;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", tag = "type")]
pub enum A2aRouteAuth {
None,
Bearer { token: String },
Header { name: String, value: String },
}
#[cfg(test)]
mod run_reservation_liveness_tests {
use super::*;
use crate::run_store::RunStoreLookupGate;
use std::time::Duration;
fn pristine_run(run_id: &str, client_id: &str) -> RunMeta {
RunMeta {
run_id: run_id.to_string(),
agent_id: "agent-release".to_string(),
client_id: client_id.to_string(),
active_client_id: client_id.to_string(),
resume_predecessor_client_id: None,
resume_lease: None,
intent: "release without blocking the registry".to_string(),
outcome_description: None,
started_at: chrono::Utc::now(),
termination: None,
ended_at: None,
turns: Vec::new(),
start_committed: false,
pending_terminal: None,
cancellation_pending: None,
cancellation_receipt: None,
trace_corruption: None,
durability_generation: 0,
}
}
#[tokio::test]
async fn blocked_release_lookup_does_not_hold_global_runs_lock() {
let tmp = tempfile::TempDir::new().unwrap();
let gate = RunStoreLookupGate::default();
let state = Arc::new(ServerState::with_config(
ServerStateConfig::new(tmp.path().join("journals"))
.with_run_store_lookup_gate(gate.clone()),
));
let session = state
.create_session("release-client", Arc::new(WsChannel::test_stub()))
.await
.unwrap();
let expected = pristine_run("release-candidate", &session.client_id);
assert!(matches!(
state.reserve_run(expected.clone()).await,
Ok(RunReservation::New)
));
gate.block_next();
let release_state = state.clone();
let release_session = session.clone();
let release_expected = expected.clone();
let release = tokio::spawn(async move {
release_state
.release_unpersisted_run_reservation(&release_session, &release_expected)
.await
});
let wait_gate = gate.clone();
assert!(
tokio::task::spawn_blocking(move || {
wait_gate.wait_until_entered(Duration::from_secs(3))
})
.await
.unwrap(),
"release must reach the blocked durable lookup"
);
let registry = tokio::time::timeout(Duration::from_secs(1), state.runs.lock())
.await
.expect("blocked release lookup must not hold the global runs lock");
drop(registry);
gate.release();
assert_eq!(release.await.unwrap(), Ok(()));
assert!(state.run_meta(&expected.run_id).await.is_none());
}
}
pub struct WsChannel {
pub write: Mutex<WsSink>,
pub pending: Mutex<HashMap<String, oneshot::Sender<ToolExecuteResponse>>>,
pub active_actions: Mutex<HashMap<String, String>>,
pub next_id: AtomicU64,
}
#[async_trait::async_trait]
impl crate::host::EventSubscriber for WsChannel {
async fn send_text(&self, json: String) {
use futures::SinkExt;
let _ = tokio::time::timeout(std::time::Duration::from_secs(10), async {
self.write
.lock()
.await
.send(Message::Text(json.into()))
.await
})
.await;
}
}
impl WsChannel {
pub fn next_request_id(&self) -> String {
let id = self.next_id.fetch_add(1, Ordering::SeqCst);
format!("cb-{}", id)
}
#[cfg(test)]
pub fn test_stub() -> Self {
use futures::sink::SinkExt;
let sink: WsSink = Box::pin(
futures::sink::drain()
.sink_map_err(|_| tokio_tungstenite::tungstenite::Error::ConnectionClosed),
);
WsChannel {
write: Mutex::new(sink),
pending: Mutex::new(HashMap::new()),
active_actions: Mutex::new(HashMap::new()),
next_id: AtomicU64::new(0),
}
}
#[cfg(test)]
pub fn test_capture() -> (Self, std::sync::Arc<std::sync::Mutex<Vec<String>>>) {
let frames = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
let sink: WsSink = Box::pin(futures::sink::unfold(
frames.clone(),
|frames: std::sync::Arc<std::sync::Mutex<Vec<String>>>, msg: Message| {
if let Message::Text(text) = &msg {
if let Ok(mut held) = frames.lock() {
held.push(text.as_str().to_string());
}
}
futures::future::ready(Ok::<_, WsError>(frames))
},
));
(
WsChannel {
write: Mutex::new(sink),
pending: Mutex::new(HashMap::new()),
active_actions: Mutex::new(HashMap::new()),
next_id: AtomicU64::new(0),
},
frames,
)
}
}
#[derive(Debug, Clone)]
pub struct ChatSession {
pub agent_id: String,
pub host_client_id: String,
pub created_at: u64,
pub local_cancel: Option<Arc<AtomicBool>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChatGoalState {
pub session_id: String,
pub check: String,
pub max_iterations: u32,
pub status: String,
pub last_iteration: Option<u32>,
pub last_met: Option<bool>,
pub last_grounded: Option<bool>,
pub last_reason: Option<String>,
pub terminal_kind: Option<String>,
pub terminal_message: Option<String>,
pub updated_at: u64,
}
fn chat_goals_path_from_journal_dir(journal_dir: &Path) -> PathBuf {
let car_dir = journal_dir
.parent()
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("."));
car_dir.join("chat-goals.json")
}
fn load_chat_goals_from_disk(journal_dir: &Path) -> HashMap<String, ChatGoalState> {
let path = chat_goals_path_from_journal_dir(journal_dir);
let Ok(text) = std::fs::read_to_string(&path) else {
return HashMap::new();
};
match serde_json::from_str::<HashMap<String, ChatGoalState>>(&text) {
Ok(mut goals) => {
for goal in goals.values_mut() {
if goal.status == "running" {
goal.status = "active".to_string();
}
}
goals
}
Err(e) => {
tracing::warn!(
path = %path.display(),
error = %e,
"chat-goals store was unreadable; starting with an empty in-memory goal registry"
);
HashMap::new()
}
}
}
fn write_chat_goals_atomic(
path: &std::path::Path,
goals: &HashMap<String, ChatGoalState>,
) -> std::io::Result<()> {
use std::sync::atomic::{AtomicU64, Ordering};
static SEQ: AtomicU64 = AtomicU64::new(0);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let json = serde_json::to_vec_pretty(goals)?;
let seq = SEQ.fetch_add(1, Ordering::Relaxed);
let mut tmp_os = path.as_os_str().to_owned();
tmp_os.push(format!(".tmp.{}.{}", std::process::id(), seq));
let tmp = PathBuf::from(tmp_os);
std::fs::write(&tmp, json)?;
std::fs::rename(&tmp, path)
}
#[derive(Debug, Clone)]
pub struct ChatStreamChunk {
pub kind: String,
pub delta: Option<String>,
pub error: Option<String>,
}
pub struct ChatCollector {
pub tx: tokio::sync::mpsc::UnboundedSender<ChatStreamChunk>,
pub host_client_id: String,
}
#[derive(Debug, Clone)]
#[doc(hidden)]
pub struct RunResumeLease {
pub disconnected_client_id: String,
pub expires_at: tokio::time::Instant,
}
#[derive(Debug, Clone, Default)]
#[doc(hidden)]
pub struct RunCompletionFenceGate {
armed: Arc<std::sync::atomic::AtomicBool>,
entered: Arc<tokio::sync::Notify>,
release: Arc<tokio::sync::Notify>,
}
impl RunCompletionFenceGate {
pub fn block_next(&self) {
self.armed.store(true, std::sync::atomic::Ordering::Release);
}
pub async fn wait_until_entered(&self, timeout: std::time::Duration) -> bool {
tokio::time::timeout(timeout, self.entered.notified())
.await
.is_ok()
}
pub fn release(&self) {
self.armed
.store(false, std::sync::atomic::Ordering::Release);
self.release.notify_one();
}
async fn wait_if_armed(&self) {
if !self.armed.load(std::sync::atomic::Ordering::Acquire) {
return;
}
self.entered.notify_one();
if self.armed.load(std::sync::atomic::Ordering::Acquire) {
self.release.notified().await;
}
}
}
#[derive(Debug, Clone)]
pub struct RunMeta {
pub run_id: String,
pub agent_id: String,
pub client_id: String,
pub active_client_id: String,
pub resume_predecessor_client_id: Option<String>,
#[doc(hidden)]
pub resume_lease: Option<RunResumeLease>,
pub intent: String,
pub outcome_description: Option<String>,
pub started_at: chrono::DateTime<chrono::Utc>,
pub termination: Option<car_proto::RunTermination>,
pub ended_at: Option<chrono::DateTime<chrono::Utc>>,
pub turns: Vec<car_proto::RunRecord>,
pub start_committed: bool,
pub pending_terminal: Option<car_proto::RunEnded>,
pub cancellation_pending: Option<car_proto::RunCancellationRequested>,
pub cancellation_receipt: Option<car_proto::RunCancelResponse>,
pub trace_corruption: Option<String>,
#[doc(hidden)]
pub durability_generation: u64,
}
#[derive(Debug, Clone)]
pub enum RunReservation {
New,
Existing(RunMeta),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct RunResumeBinding {
pub run_id: String,
pub agent_id: String,
pub active_client_id: String,
pub resumed_from_client_id: String,
}
pub(crate) fn run_completion_digest(
termination: &car_proto::RunTermination,
) -> Result<String, String> {
let canonical = car_inference::catalog_identity::canonical_json(termination)?;
Ok(format!("{:x}", Sha256::digest(canonical.as_bytes())))
}
fn run_trace_corruption_message(run_id: &str, detail: impl std::fmt::Display) -> String {
format!(
"{} run `{run_id}`: {detail}",
car_proto::RUN_TRACE_CORRUPTION_MESSAGE_PREFIX
)
}
fn same_run_termination(
left: &car_proto::RunTermination,
right: &car_proto::RunTermination,
) -> bool {
match (run_completion_digest(left), run_completion_digest(right)) {
(Ok(left), Ok(right)) => left == right,
_ => false,
}
}
fn same_run_ended(left: &car_proto::RunEnded, right: &car_proto::RunEnded) -> bool {
left.run_id == right.run_id
&& left.client_id == right.client_id
&& left.agent_id == right.agent_id
&& left.completion_digest == right.completion_digest
&& left.ended_at == right.ended_at
&& same_run_termination(&left.termination, &right.termination)
}
struct StartupJournalCache {
journal_dir: PathBuf,
failures: Option<car_eventlog::JournalFailureInjector>,
logs: HashMap<String, EventLog>,
}
impl StartupJournalCache {
fn new(journal_dir: &Path, failures: Option<&car_eventlog::JournalFailureInjector>) -> Self {
Self {
journal_dir: journal_dir.to_path_buf(),
failures: failures.cloned(),
logs: HashMap::new(),
}
}
fn get(&mut self, client_id: &str) -> Result<&mut EventLog, String> {
if !self.logs.contains_key(client_id) {
let path = self.journal_dir.join(format!("{client_id}.jsonl"));
let log = match (path.exists(), self.failures.as_ref()) {
(true, Some(failures)) => {
EventLog::load_with_journal_failure_injector(&path, failures.clone())
.map_err(|error| error.to_string())
}
(true, None) => EventLog::load(&path).map_err(|error| error.to_string()),
(false, Some(failures)) => Ok(EventLog::with_journal_failure_injector(
path,
failures.clone(),
)),
(false, None) => Ok(EventLog::with_journal(path)),
}?;
self.logs.insert(client_id.to_string(), log);
}
self.logs
.get_mut(client_id)
.ok_or_else(|| "startup journal cache insertion failed".to_string())
}
}
fn append_recovered_cancellation_result_journal(
journal_dir: PathBuf,
failures: Option<car_eventlog::JournalFailureInjector>,
client_id: String,
result: car_proto::RunCancelResponse,
) -> Result<bool, String> {
let mut journals = StartupJournalCache::new(&journal_dir, failures.as_ref());
let log = journals.get(&client_id)?;
log.bind_run(&result.run_id, &client_id)?;
let Value::Object(data) = serde_json::to_value(&result).map_err(|error| error.to_string())?
else {
return Err("recovered cancellation result did not serialize as an object".into());
};
let data: HashMap<String, Value> = data.into_iter().collect();
let already_durable = log.events().iter().any(|event| {
event.kind == car_eventlog::EventKind::RunCancellationResult
&& event.run_id.as_deref() == Some(result.run_id.as_str())
&& event.client_id.as_deref() == Some(client_id.as_str())
&& event.action_id.as_deref() == result.action_id.as_deref()
&& event.proposal_id.is_none()
&& event.data == data
});
log.append_critical_bounded(
car_eventlog::EventKind::RunCancellationResult,
result.action_id.as_deref(),
None,
data,
CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
)
.map_err(|error| format!("recovered cancellation journal append failed: {error}"))?;
Ok(!already_durable)
}
fn reconcile_durable_run_journals(
journals: &mut StartupJournalCache,
store: &crate::run_store::RunStore,
acknowledgement_timeout: std::time::Duration,
) -> Result<(), String> {
type DurableRunBoundaries = (
car_proto::RunStarted,
Option<car_proto::RunEnded>,
Option<car_proto::RunCancellationRequested>,
Option<car_proto::RunCancelResponse>,
);
let mut by_client: HashMap<String, Vec<DurableRunBoundaries>> = HashMap::new();
store.visit_run_boundaries(|started, ended, requested, result| {
let Some(client_id) = started.client_id.as_deref() else {
return;
};
if client_id.is_empty()
|| client_id.contains('/')
|| client_id.contains('\\')
|| client_id == "."
|| client_id == ".."
{
tracing::error!(run_id = %started.run_id, "run-journal outbox has unsafe client_id");
return;
}
by_client
.entry(client_id.to_string())
.or_default()
.push((started, ended, requested, result));
});
for (client_id, mut runs) in by_client {
runs.sort_by_key(|(started, _, _, _)| started.started_at);
let log = match journals.get(&client_id) {
Ok(log) => log,
Err(error) => {
tracing::error!(%client_id, %error, "cannot load run journal for outbox reconciliation");
continue;
}
};
for (started, ended, requested, result) in runs {
if let Err(error) = log.bind_run(&started.run_id, &client_id) {
tracing::error!(run_id = %started.run_id, %error, "cannot bind run journal during outbox reconciliation");
continue;
}
let mut start_data = HashMap::from([
(
"agent_id".to_string(),
Value::from(started.agent_id.clone()),
),
("intent".to_string(), Value::from(started.intent.clone())),
(
"started_at".to_string(),
serde_json::to_value(started.started_at).unwrap_or(Value::Null),
),
]);
if let Some(description) = &started.outcome_description {
start_data.insert(
"outcome_description".to_string(),
Value::from(description.clone()),
);
}
log.append_critical_bounded(
car_eventlog::EventKind::RunStarted,
None,
None,
start_data,
acknowledgement_timeout,
)
.map_err(|error| {
format!(
"startup reconciliation failed for run {} run_started: {error}",
started.run_id
)
})?;
if let Some(requested) = requested {
let Value::Object(data) = serde_json::to_value(&requested).unwrap_or(Value::Null)
else {
continue;
};
log.append_critical_bounded(
car_eventlog::EventKind::RunCancellationRequested,
requested.action_id.as_deref(),
None,
data.into_iter().collect(),
acknowledgement_timeout,
)
.map_err(|error| {
format!(
"startup reconciliation failed for run {} cancellation request: {error}",
started.run_id
)
})?;
}
if ended.is_none() {
if let Some(result) = result {
let Value::Object(data) = serde_json::to_value(&result).unwrap_or(Value::Null)
else {
continue;
};
log.append_critical_bounded(
car_eventlog::EventKind::RunCancellationResult,
result.action_id.as_deref(),
None,
data.into_iter().collect(),
acknowledgement_timeout,
)
.map_err(|error| {
format!(
"startup reconciliation failed for run {} cancellation result: {error}",
started.run_id
)
})?;
}
}
let Some(ended) = ended else {
continue;
};
let Some(completion_digest) = ended.completion_digest.as_deref() else {
log.clear_run_binding(&started.run_id, &client_id)?;
continue;
};
if ended.client_id.as_deref() != Some(client_id.as_str()) {
tracing::error!(run_id = %started.run_id, "RunEnded owner does not match RunStarted during outbox reconciliation");
log.clear_run_binding(&started.run_id, &client_id)?;
continue;
}
let termination_kind = match &ended.termination {
car_proto::RunTermination::Outcome { .. } => "outcome",
car_proto::RunTermination::Incomplete => "incomplete",
car_proto::RunTermination::Cancelled { .. } => "cancelled",
};
let data = HashMap::from([
(
"termination_kind".to_string(),
Value::from(termination_kind),
),
(
"completion_digest".to_string(),
Value::from(completion_digest),
),
(
"termination".to_string(),
serde_json::to_value(&ended.termination).unwrap_or(Value::Null),
),
]);
log.append_critical_bounded(
car_eventlog::EventKind::RunCompleted,
None,
None,
data,
acknowledgement_timeout,
)
.map_err(|error| {
format!(
"startup reconciliation failed for run {} run_completed: {error}",
started.run_id
)
})?;
log.clear_run_binding(&started.run_id, &client_id)?;
}
}
Ok(())
}
fn reconcile_pending_proposal_journals(
journals: &mut StartupJournalCache,
store: &crate::run_store::RunStore,
acknowledgement_timeout: std::time::Duration,
) -> Result<(), String> {
let pending_proposals = match store.all_pending_proposals() {
Ok(pending) => pending,
Err(error) => {
tracing::error!(%error, "cannot enumerate proposal-finalization outbox; all unresolved runs remain quarantined");
return Ok(());
}
};
for pending in pending_proposals {
match store.completed_proposal(
&pending.run_id,
&pending.client_id,
pending.requested_policy_session_id.as_deref(),
&pending.original_submission,
) {
Ok(Some(receipt)) => {
if let Err(error) = store.cleanup_completed_proposal_guards(&receipt) {
tracing::error!(run_id = %pending.run_id, %error, "completed proposal response is durable; startup guard cleanup remains pending");
}
continue;
}
Ok(None) => {}
Err(error) => {
tracing::error!(run_id = %pending.run_id, %error, "cannot validate exact completed proposal response during startup cleanup");
continue;
}
}
let (started, marker) = match store.pending_provenance(&pending) {
Ok(provenance) => provenance,
Err(error) => {
tracing::error!(run_id = %pending.run_id, %error, "proposal-finalization provenance is invalid; preserving outcome-unknown quarantine");
continue;
}
};
let Some(client_id) = started.client_id.as_deref() else {
tracing::error!(run_id = %pending.run_id, "durable RunStarted is missing client_id");
continue;
};
if client_id.is_empty()
|| client_id.contains('/')
|| client_id.contains('\\')
|| client_id == "."
|| client_id == ".."
{
tracing::error!(run_id = %pending.run_id, "proposal-finalization outbox has unsafe client_id");
continue;
}
let canonical = match car_inference::catalog_identity::canonical_json(
&pending.proposal_result,
) {
Ok(canonical) => canonical,
Err(error) => {
tracing::error!(run_id = %pending.run_id, %error, "proposal-finalization result is not RFC 8785 canonicalizable");
continue;
}
};
let digest = format!("{:x}", Sha256::digest(canonical.as_bytes()));
if digest != pending.result_digest {
tracing::error!(run_id = %pending.run_id, "proposal-finalization digest does not match its durable result preimage");
continue;
}
let log = match journals.get(client_id) {
Ok(log) => log,
Err(error) => {
tracing::error!(run_id = %pending.run_id, %error, "cannot load proposal journal for outbox reconciliation");
continue;
}
};
if let Err(error) = log.bind_run(&started.run_id, client_id) {
tracing::error!(run_id = %pending.run_id, %error, "cannot bind proposal journal during outbox reconciliation");
continue;
}
if let Some(policy_session_id) = marker.policy_session_id.as_deref() {
if let Err(error) = log.bind_policy_session(policy_session_id) {
tracing::error!(run_id = %pending.run_id, %error, "cannot restore proposal policy binding during outbox reconciliation");
continue;
}
}
store.ensure_proposal_turns(&pending).map_err(|error| {
format!(
"startup reconciliation failed for run {} proposal trace: {error}",
pending.run_id
)
})?;
log.append_critical_bounded(
car_eventlog::EventKind::ProposalCompleted,
None,
Some(&pending.final_proposal_id),
pending.event_data(),
acknowledgement_timeout,
)
.map_err(|error| {
format!(
"startup reconciliation failed for run {} proposal_completed: {error}",
pending.run_id
)
})?;
if let Some(policy_session_id) = marker.policy_session_id.as_deref() {
log.clear_policy_session(policy_session_id)?;
}
let receipt = match store.write_completed_proposal(&pending) {
Ok(receipt) => receipt,
Err(error) => {
tracing::error!(run_id = %pending.run_id, %error, "cannot persist reconciled completed proposal response");
continue;
}
};
if let Err(error) = store.cleanup_completed_proposal_guards(&receipt) {
tracing::error!(run_id = %pending.run_id, %error, "completed proposal response is durable; startup guard cleanup remains pending");
continue;
}
}
Ok(())
}
fn reconcile_completed_proposal_guards(store: &crate::run_store::RunStore) {
if let Err(error) = store.reconcile_completed_proposal_migration() {
tracing::error!(%error, "cannot migrate completed proposal response index; guards remain quarantined");
}
}
pub const RECORD_TURNS_RUN_CEILING: usize = 2000;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RecordRunTurnsOutcome {
Appended { new_total: usize },
RefusedCeiling,
UnknownOrTerminal,
PersistenceFailed(String),
}
pub(crate) enum RunSubscribePageResult {
Ready(car_proto::RunSubscribeResponse),
Durable {
agent_id: String,
status: car_proto::RunLiveStatus,
},
}
impl RunMeta {
pub fn is_terminal(&self) -> bool {
self.termination.is_some()
}
pub fn accepts_proposals(&self) -> bool {
self.start_committed
&& self.pending_terminal.is_none()
&& self.cancellation_pending.is_none()
&& self.trace_corruption.is_none()
&& !self.is_terminal()
}
pub fn live_status(&self) -> car_proto::RunLiveStatus {
match &self.termination {
None if self.cancellation_pending.is_some() => {
car_proto::RunLiveStatus::CancellationPending
}
None => car_proto::RunLiveStatus::InProgress,
Some(car_proto::RunTermination::Outcome { .. }) => car_proto::RunLiveStatus::Completed,
Some(car_proto::RunTermination::Incomplete) => car_proto::RunLiveStatus::Incomplete,
Some(car_proto::RunTermination::Cancelled { .. }) => {
car_proto::RunLiveStatus::Cancelled
}
}
}
pub fn turn_cursor(&self) -> usize {
self.turns.len()
}
}
pub const DEFAULT_TOOL_TIMEOUT_MS: u64 = 300_000;
const TOOL_TIMEOUT_GRACE_MS: u64 = 5_000;
fn tool_callback_timeout(action_timeout_ms: Option<u64>) -> std::time::Duration {
let ms = match action_timeout_ms {
Some(budget) => budget.saturating_add(TOOL_TIMEOUT_GRACE_MS),
None => std::env::var("CAR_TOOL_TIMEOUT")
.ok()
.and_then(|s| s.trim().parse::<u64>().ok())
.map(|secs| secs.saturating_mul(1000))
.unwrap_or(DEFAULT_TOOL_TIMEOUT_MS),
};
std::time::Duration::from_millis(ms)
}
pub(crate) async fn write_tool_cancel(
channel: &WsChannel,
request_id: String,
action_id: String,
reason: String,
) {
use futures::SinkExt;
let cancel = ToolCancelRequest {
request_id,
action_id,
reason,
};
let notification = serde_json::json!({
"jsonrpc": "2.0",
"method": "tools.cancel",
"params": cancel,
});
let Ok(text) = serde_json::to_string(¬ification) else {
return;
};
let _ = channel
.write
.lock()
.await
.send(Message::Text(text.into()))
.await;
}
struct PendingToolCall {
channel: Arc<WsChannel>,
request_id: String,
action_id: String,
reason: String,
armed: bool,
}
impl PendingToolCall {
fn new(channel: Arc<WsChannel>, request_id: String, action_id: String, reason: String) -> Self {
Self {
channel,
request_id,
action_id,
reason,
armed: true,
}
}
fn disarm(&mut self) {
self.armed = false;
}
}
impl Drop for PendingToolCall {
fn drop(&mut self) {
if !self.armed {
return;
}
let Ok(handle) = tokio::runtime::Handle::try_current() else {
return;
};
let channel = self.channel.clone();
let request_id = std::mem::take(&mut self.request_id);
let action_id = std::mem::take(&mut self.action_id);
let reason = std::mem::take(&mut self.reason);
handle.spawn(async move {
let claimed = channel.pending.lock().await.remove(&request_id).is_some();
channel.active_actions.lock().await.remove(&request_id);
if claimed {
write_tool_cancel(&channel, request_id, action_id, reason).await;
}
});
}
}
pub struct WsToolExecutor {
pub channel: Arc<WsChannel>,
negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
}
#[async_trait::async_trait]
impl ToolExecutor for WsToolExecutor {
async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
self.execute_with_action(tool, params, "", None).await
}
async fn execute_with_action(
&self,
tool: &str,
params: &Value,
action_id: &str,
timeout_ms: Option<u64>,
) -> Result<Value, String> {
self.execute_with_action_in_session(tool, params, action_id, timeout_ms, None, 1)
.await
}
async fn execute_with_action_in_session(
&self,
tool: &str,
params: &Value,
action_id: &str,
timeout_ms: Option<u64>,
session_id: Option<&str>,
attempt: u32,
) -> Result<Value, String> {
self.execute_callback(
tool,
params,
action_id,
timeout_ms,
session_id,
attempt,
&HashMap::new(),
None,
false,
)
.await
.map(|execution| execution.output)
}
async fn execute_with_action_state_in_session(
&self,
tool: &str,
params: &Value,
action_id: &str,
timeout_ms: Option<u64>,
session_id: Option<&str>,
attempt: u32,
expected_effects: &HashMap<String, Value>,
return_schema: Option<&Value>,
) -> Result<car_engine::ToolExecution, String> {
self.execute_callback(
tool,
params,
action_id,
timeout_ms,
session_id,
attempt,
expected_effects,
return_schema,
self.callback_state_negotiated(),
)
.await
}
}
impl WsToolExecutor {
pub fn new(
channel: Arc<WsChannel>,
negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
) -> Self {
Self {
channel,
negotiated_capabilities,
}
}
#[cfg(test)]
fn legacy(channel: Arc<WsChannel>) -> Self {
Self::new(
channel,
Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new())),
)
}
fn callback_state_negotiated(&self) -> bool {
self.negotiated_capabilities
.read()
.map(|capabilities| capabilities.contains(car_proto::TOOLS_CALLBACK_STATE_CAPABILITY))
.unwrap_or(false)
}
#[allow(clippy::too_many_arguments)]
async fn execute_callback(
&self,
tool: &str,
params: &Value,
action_id: &str,
timeout_ms: Option<u64>,
session_id: Option<&str>,
attempt: u32,
expected_effects: &HashMap<String, Value>,
return_schema: Option<&Value>,
callback_state_negotiated: bool,
) -> Result<car_engine::ToolExecution, String> {
use futures::SinkExt;
let request_id = self.channel.next_request_id();
let callback = ToolExecuteRequest {
action_id: action_id.to_string(),
tool: tool.to_string(),
parameters: params.clone(),
timeout_ms,
attempt,
request_id: request_id.clone(),
session_id: session_id.map(str::to_string),
};
let (tx, rx) = oneshot::channel();
self.channel
.pending
.lock()
.await
.insert(request_id.clone(), tx);
self.channel
.active_actions
.lock()
.await
.insert(request_id.clone(), action_id.to_string());
let mut pending_call = PendingToolCall::new(
self.channel.clone(),
request_id.clone(),
action_id.to_string(),
format!("tool '{tool}' call cancelled before completion (request {request_id})"),
);
let rpc_request = serde_json::json!({
"jsonrpc": "2.0",
"method": "tools.execute",
"params": callback,
"id": request_id,
});
let msg = Message::Text(
serde_json::to_string(&rpc_request)
.map_err(|e| e.to_string())?
.into(),
);
self.channel
.write
.lock()
.await
.send(msg)
.await
.map_err(|e| format!("failed to send tool callback: {}", e))?;
let wait = tool_callback_timeout(timeout_ms);
let response = match tokio::time::timeout(wait, rx).await {
Ok(inner) => inner.map_err(|_| format!("tool '{}' callback channel closed", tool))?,
Err(_) => {
let reason = format!("tool '{}' callback timed out ({}s)", tool, wait.as_secs());
pending_call.reason = reason.clone();
return Err(reason);
}
};
self.channel.active_actions.lock().await.remove(&request_id);
pending_call.disarm();
if let Some(err) = response.error {
return Err(err);
}
decode_callback_execution(
tool,
action_id,
response.output.unwrap_or(Value::Null),
expected_effects,
return_schema,
callback_state_negotiated,
)
}
}
fn decode_callback_execution(
tool: &str,
action_id: &str,
result: Value,
expected_effects: &HashMap<String, Value>,
return_schema: Option<&Value>,
callback_state_negotiated: bool,
) -> Result<car_engine::ToolExecution, String> {
if !callback_state_negotiated {
return Ok(car_engine::ToolExecution::output_only(result));
}
let envelope = result.as_object().filter(|object| {
object.len() == 2 && object.contains_key("output") && object.contains_key("state_changes")
});
if expected_effects.is_empty() && envelope.is_none() {
return Ok(car_engine::ToolExecution::output_only(result));
}
let envelope = envelope.ok_or_else(|| {
format!(
"tool '{tool}' callback for action '{action_id}' must return exact envelope {{output,state_changes}}"
)
})?;
let output = envelope
.get("output")
.cloned()
.expect("exact envelope contains output");
let state_changes: HashMap<String, Value> = serde_json::from_value(
envelope
.get("state_changes")
.cloned()
.expect("exact envelope contains state_changes"),
)
.map_err(|_| {
format!("tool '{tool}' callback for action '{action_id}' state_changes must be an object")
})?;
let expected_keys: std::collections::BTreeSet<&str> =
expected_effects.keys().map(String::as_str).collect();
let actual_keys: std::collections::BTreeSet<&str> =
state_changes.keys().map(String::as_str).collect();
if expected_keys != actual_keys {
return Err(format!(
"tool '{tool}' callback state_changes keys do not match action '{action_id}': expected {:?}, got {:?}",
expected_keys, actual_keys
));
}
if let Some(schema) = return_schema {
car_engine::validate_tool_output(tool, schema, &output)?;
}
let changes_value = serde_json::to_value(&state_changes).map_err(|error| {
format!("tool '{tool}' callback state_changes serialization failed: {error}")
})?;
car_inference::catalog_identity::canonical_json(&changes_value).map_err(|error| {
format!("tool '{tool}' callback state_changes failed JCS/I-JSON validation: {error}")
})?;
Ok(car_engine::ToolExecution {
output,
state_changes,
})
}
#[cfg(test)]
mod tool_cancel_tests {
use super::*;
use serde_json::json;
use std::time::Duration;
fn written(frames: &std::sync::Arc<std::sync::Mutex<Vec<String>>>) -> Vec<Value> {
frames
.lock()
.unwrap()
.iter()
.map(|t| serde_json::from_str::<Value>(t).expect("frame is JSON"))
.collect()
}
fn frame_with_method<'a>(frames: &'a [Value], method: &str) -> Option<&'a Value> {
frames.iter().find(|f| f["method"] == json!(method))
}
async fn settle() {
for _ in 0..8 {
tokio::task::yield_now().await;
}
tokio::time::sleep(Duration::from_millis(100)).await;
for _ in 0..8 {
tokio::task::yield_now().await;
}
}
#[tokio::test(start_paused = true)]
async fn callback_wait_expiry_cancels() {
let (channel, frames) = WsChannel::test_capture();
let channel = Arc::new(channel);
let executor = WsToolExecutor::legacy(channel.clone());
let err = executor
.execute_with_action_in_session("drive_cli", &json!({}), "a0", Some(1), None, 1)
.await
.expect_err("the callback wait must expire");
assert!(
err.contains("tool 'drive_cli' callback timed out"),
"unexpected error text: {err}"
);
settle().await;
let frames = written(&frames);
let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
let cancel = frame_with_method(&frames, "tools.cancel")
.expect("a tools.cancel notification must reach the wire");
assert_eq!(
cancel["params"]["request_id"], execute["params"]["request_id"],
"cancel must correlate to the originating request_id"
);
assert_eq!(cancel["params"]["action_id"], json!("a0"));
assert!(
cancel["id"].is_null(),
"cancel is a notification, not a request"
);
assert!(
channel.pending.lock().await.is_empty(),
"the pending entry must be released"
);
}
#[tokio::test(start_paused = true)]
async fn tools_execute_carries_the_execution_session() {
let (channel, frames) = WsChannel::test_capture();
let channel = Arc::new(channel);
let executor = WsToolExecutor::legacy(channel.clone());
let _ = tokio::time::timeout(
Duration::from_millis(50),
executor.execute_with_action_in_session(
"search",
&json!({}),
"a0",
Some(10_000),
Some("sess-42"),
1,
),
)
.await;
settle().await;
let frames = written(&frames);
let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
assert_eq!(
execute["params"]["session_id"],
json!("sess-42"),
"the daemon must stamp the session it already knows, not leave \
attribution to the host"
);
}
#[tokio::test(start_paused = true)]
async fn tools_execute_carries_the_real_attempt_number() {
for attempt in [1u32, 2, 7] {
let (channel, frames) = WsChannel::test_capture();
let channel = Arc::new(channel);
let executor = WsToolExecutor::legacy(channel.clone());
let _ = tokio::time::timeout(
Duration::from_millis(50),
executor.execute_with_action_in_session(
"search",
&json!({}),
"a0",
Some(10_000),
Some("sess-1"),
attempt,
),
)
.await;
settle().await;
let frames = written(&frames);
let execute =
frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
assert_eq!(
execute["params"]["attempt"],
json!(attempt),
"the payload must carry the engine's attempt, not a constant"
);
}
}
#[tokio::test(start_paused = true)]
async fn a_sessionless_call_omits_the_session_key() {
let (channel, frames) = WsChannel::test_capture();
let channel = Arc::new(channel);
let executor = WsToolExecutor::legacy(channel.clone());
let _ = tokio::time::timeout(
Duration::from_millis(50),
executor.execute_with_action_in_session(
"search",
&json!({}),
"a0",
Some(10_000),
None,
1,
),
)
.await;
settle().await;
let frames = written(&frames);
let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
assert!(
execute["params"].get("session_id").is_none(),
"a sessionless call must omit the key entirely, got: {}",
execute["params"]
);
}
#[tokio::test(start_paused = true)]
async fn engine_deadline_drop_cancels() {
let (channel, frames) = WsChannel::test_capture();
let channel = Arc::new(channel);
let executor = WsToolExecutor::legacy(channel.clone());
let outcome = tokio::time::timeout(
Duration::from_millis(50),
executor.execute_with_action_in_session(
"drive_cli",
&json!({}),
"a0",
Some(10_000),
None,
1,
),
)
.await;
assert!(
outcome.is_err(),
"the outer deadline must fire first and drop the dispatch future"
);
settle().await;
let frames = written(&frames);
let execute = frame_with_method(&frames, "tools.execute").expect("tools.execute was sent");
let cancel = frame_with_method(&frames, "tools.cancel").expect(
"dropping the dispatch future must still emit tools.cancel so the host child isn't orphaned",
);
assert_eq!(
cancel["params"]["request_id"], execute["params"]["request_id"],
"cancel must correlate to the originating request_id"
);
assert!(
channel.pending.lock().await.is_empty(),
"the pending entry must not leak"
);
}
#[tokio::test]
async fn success_emits_no_cancel() {
let (channel, frames) = WsChannel::test_capture();
let channel = Arc::new(channel);
let executor = WsToolExecutor::legacy(channel.clone());
let responder = channel.clone();
let host = tokio::spawn(async move {
loop {
let claimed = {
let mut pending = responder.pending.lock().await;
let key = pending.keys().next().cloned();
key.and_then(|k| pending.remove(&k).map(|tx| (k, tx)))
};
if let Some((request_id, tx)) = claimed {
let _ = tx.send(ToolExecuteResponse {
action_id: request_id,
output: Some(json!({"ok": true})),
error: None,
});
return;
}
tokio::task::yield_now().await;
}
});
let out = executor
.execute_with_action_in_session("drive_cli", &json!({}), "a0", Some(10_000), None, 1)
.await
.expect("the host responded, so the call succeeds");
assert_eq!(out, json!({"ok": true}));
host.await.expect("responder task finishes");
settle().await;
let frames = written(&frames);
assert!(
frame_with_method(&frames, "tools.execute").is_some(),
"tools.execute was sent"
);
assert!(
frame_with_method(&frames, "tools.cancel").is_none(),
"a completed call must not emit tools.cancel"
);
assert!(channel.pending.lock().await.is_empty());
}
}
pub(crate) const SUBSTRATE_OWNED_TOOLS: &[&str] = &[
"read_file",
"write_file",
"edit_file",
"list_dir",
"find_files",
"grep_files",
];
pub struct SubstrateShadowExecutor {
inner: Arc<dyn ToolExecutor>,
}
impl SubstrateShadowExecutor {
pub fn new(inner: Arc<dyn ToolExecutor>) -> Self {
Self { inner }
}
}
#[async_trait::async_trait]
impl ToolExecutor for SubstrateShadowExecutor {
async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
self.execute_with_action(tool, params, "", None).await
}
async fn execute_with_action(
&self,
tool: &str,
params: &Value,
action_id: &str,
timeout_ms: Option<u64>,
) -> Result<Value, String> {
self.execute_with_action_in_session(tool, params, action_id, timeout_ms, None, 1)
.await
}
async fn execute_with_action_in_session(
&self,
tool: &str,
params: &Value,
action_id: &str,
timeout_ms: Option<u64>,
session_id: Option<&str>,
attempt: u32,
) -> Result<Value, String> {
if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
return Err(format!("unknown tool: {tool}"));
}
self.inner
.execute_with_action_in_session(
tool, params, action_id, timeout_ms, session_id, attempt,
)
.await
}
async fn execute_with_action_state_in_session(
&self,
tool: &str,
params: &Value,
action_id: &str,
timeout_ms: Option<u64>,
session_id: Option<&str>,
attempt: u32,
expected_effects: &HashMap<String, Value>,
return_schema: Option<&Value>,
) -> Result<car_engine::ToolExecution, String> {
if SUBSTRATE_OWNED_TOOLS.contains(&tool) {
return Err(format!("unknown tool: {tool}"));
}
self.inner
.execute_with_action_state_in_session(
tool,
params,
action_id,
timeout_ms,
session_id,
attempt,
expected_effects,
return_schema,
)
.await
}
}
pub struct WsVoiceEventSink {
pub channel: Arc<WsChannel>,
}
impl car_voice::VoiceEventSink for WsVoiceEventSink {
fn send(&self, session_id: &str, event_json: String) {
use futures::SinkExt;
let channel = self.channel.clone();
let session_id = session_id.to_string();
tokio::spawn(async move {
let payload: Value = serde_json::from_str(&event_json)
.unwrap_or_else(|_| Value::String(event_json.clone()));
let notification = serde_json::json!({
"jsonrpc": "2.0",
"method": "voice.event",
"params": {
"session_id": session_id,
"event": payload,
},
});
let Ok(text) = serde_json::to_string(¬ification) else {
return;
};
let _ = channel
.write
.lock()
.await
.send(Message::Text(text.into()))
.await;
});
}
fn send_binary(&self, frame: Vec<u8>) {
use futures::SinkExt;
let channel = self.channel.clone();
tokio::spawn(async move {
let _ = channel
.write
.lock()
.await
.send(Message::Binary(frame.into()))
.await;
});
}
}
pub struct WsMemgineIngestSink {
pub meeting_id: String,
pub engine: Arc<Mutex<car_memgine::MemgineEngine>>,
pub upstream: Arc<dyn car_voice::VoiceEventSink>,
}
impl car_voice::VoiceEventSink for WsMemgineIngestSink {
fn send(&self, voice_session_id: &str, event_json: String) {
if let Ok(value) = serde_json::from_str::<Value>(&event_json) {
if let Some((speaker, text)) = car_meeting::extract_transcript_for_ingest(
&value,
&self.meeting_id,
voice_session_id,
) {
let engine = self.engine.clone();
tokio::spawn(async move {
let mut guard = engine.lock().await;
guard.ingest_conversation(&speaker, &text, chrono::Utc::now());
});
}
}
self.upstream.send(voice_session_id, event_json);
}
}
#[derive(Debug, Clone)]
pub struct LastChatTurn {
pub user_text: String,
pub assistant_text: String,
pub trace_id: String,
pub model_id: String,
}
pub struct ClientSession {
pub client_id: String,
pub runtime: Arc<Runtime>,
pub channel: Arc<WsChannel>,
pub host: Arc<crate::host::HostState>,
pub memgine: Arc<Mutex<car_memgine::MemgineEngine>>,
pub browser: car_ffi_common::browser::BrowserSessionSlot,
pub authenticated: Arc<std::sync::atomic::AtomicBool>,
pub negotiated_protocol_version: std::sync::atomic::AtomicU32,
pub negotiated_capabilities: Arc<std::sync::RwLock<std::collections::BTreeSet<String>>>,
pub inference_control: Arc<crate::inference_control::InferenceRegistry>,
pub is_host: std::sync::atomic::AtomicBool,
pub agent_id: Arc<tokio::sync::Mutex<Option<String>>>,
pub callback_tool_schema_digests:
Arc<tokio::sync::RwLock<std::collections::HashMap<String, String>>>,
pub memory_namespace: tokio::sync::Mutex<Option<String>>,
pub bound_memgine: tokio::sync::Mutex<Option<Arc<Mutex<car_memgine::MemgineEngine>>>>,
pub current_run_id: tokio::sync::Mutex<Option<String>>,
pub run_lifecycle_guard: Arc<tokio::sync::Mutex<()>>,
pub permission_gate: Arc<tokio::sync::RwLock<car_policy::PermissionGate>>,
pub evolution_guard: crate::evolution::CycleGuard,
pub last_chat_turn: tokio::sync::Mutex<Option<LastChatTurn>>,
pub chat_inflight: std::sync::atomic::AtomicUsize,
pub tenant: tokio::sync::Mutex<Option<String>>,
pub tool_stream_subscribed: std::sync::Arc<std::sync::atomic::AtomicBool>,
}
impl ClientSession {
pub async fn effective_memgine(&self) -> Arc<Mutex<car_memgine::MemgineEngine>> {
if let Some(eng) = self.bound_memgine.lock().await.as_ref() {
return eng.clone();
}
self.memgine.clone()
}
pub async fn bind_run_journal(&self, run_id: &str) -> Result<(), String> {
self.runtime
.event_log_handle()
.lock()
.await
.bind_run(run_id, &self.client_id)
}
pub async fn require_run_journal_binding(&self, run_id: &str) -> Result<(), String> {
let log = self.runtime.event_log_handle();
let log = log.lock().await;
match log.active_run_binding() {
Some((bound_run, bound_client, _))
if bound_run == run_id && bound_client == self.client_id =>
{
Ok(())
}
Some((bound_run, bound_client, _)) => Err(format!(
"active journal binding mismatch: expected run_id `{run_id}` / client_id `{}`, got `{bound_run}` / `{bound_client}`",
self.client_id
)),
None => Err(format!(
"active run `{run_id}` has no authenticated journal binding"
)),
}
}
pub async fn clear_run_journal_binding(&self, run_id: &str) -> Result<(), String> {
self.runtime
.event_log_handle()
.lock()
.await
.clear_run_binding(run_id, &self.client_id)
}
pub async fn append_run_terminal_event(
&self,
ended: &car_proto::RunEnded,
) -> Result<(), String> {
self.append_run_terminal_event_once(ended, &self.client_id)
.await
.map_err(|error| error.to_string())
}
pub(crate) async fn append_resumed_run_terminal_event(
&self,
ended: &car_proto::RunEnded,
durable_client_id: &str,
) -> Result<(), String> {
self.append_run_terminal_event_once(ended, durable_client_id)
.await
.map_err(|error| error.to_string())
}
pub async fn append_run_cancellation_requested_event(
&self,
requested: &car_proto::RunCancellationRequested,
) -> Result<(), String> {
self.require_run_journal_binding(&requested.run_id).await?;
let value = serde_json::to_value(requested).map_err(|error| error.to_string())?;
let Value::Object(data) = value else {
return Err("cancellation request did not serialize as an object".into());
};
self.runtime
.event_log_handle()
.lock()
.await
.append_critical_async(
car_eventlog::EventKind::RunCancellationRequested,
requested.action_id.as_deref(),
None,
data.into_iter().collect(),
CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
)
.await
.map_err(|error| error.to_string())?;
Ok(())
}
pub async fn append_run_cancellation_result_event(
&self,
result: &car_proto::RunCancelResponse,
) -> Result<(), String> {
self.require_run_journal_binding(&result.run_id).await?;
let value = serde_json::to_value(result).map_err(|error| error.to_string())?;
let Value::Object(data) = value else {
return Err("cancellation result did not serialize as an object".into());
};
self.runtime
.event_log_handle()
.lock()
.await
.append_critical_async(
car_eventlog::EventKind::RunCancellationResult,
result.action_id.as_deref(),
None,
data.into_iter().collect(),
CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
)
.await
.map_err(|error| error.to_string())?;
Ok(())
}
async fn append_run_terminal_event_once(
&self,
ended: &car_proto::RunEnded,
durable_client_id: &str,
) -> Result<(), car_eventlog::CriticalAppendError> {
let rejected = |reason| car_eventlog::CriticalAppendError::Rejected { reason };
self.require_run_journal_binding(&ended.run_id)
.await
.map_err(rejected)?;
if ended.client_id.as_deref() != Some(durable_client_id) {
return Err(rejected(
"RunEnded client_id does not match the durable run owner".into(),
));
}
let completion_digest = ended
.completion_digest
.as_deref()
.ok_or_else(|| rejected("RunEnded is missing completion_digest".into()))?;
let termination_kind = match &ended.termination {
car_proto::RunTermination::Outcome { .. } => "outcome",
car_proto::RunTermination::Incomplete => "incomplete",
car_proto::RunTermination::Cancelled { .. } => "cancelled",
};
let log = self.runtime.event_log_handle();
let mut log = log.lock().await;
log.append_critical_async(
car_eventlog::EventKind::RunCompleted,
None,
None,
[
(
"termination_kind".to_string(),
Value::from(termination_kind),
),
(
"completion_digest".to_string(),
Value::from(completion_digest),
),
(
"termination".to_string(),
serde_json::to_value(&ended.termination)
.map_err(|error| rejected(error.to_string()))?,
),
]
.into(),
CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
)
.await?;
Ok(())
}
pub async fn append_run_started_event(
&self,
started: &car_proto::RunStarted,
) -> Result<(), String> {
self.require_run_journal_binding(&started.run_id).await?;
if started.client_id.as_deref() != Some(self.client_id.as_str()) {
return Err("RunStarted client_id does not match the live ClientSession".into());
}
let mut data = HashMap::from([
(
"agent_id".to_string(),
Value::from(started.agent_id.clone()),
),
("intent".to_string(), Value::from(started.intent.clone())),
(
"started_at".to_string(),
serde_json::to_value(started.started_at).unwrap_or(Value::Null),
),
]);
if let Some(description) = &started.outcome_description {
data.insert(
"outcome_description".to_string(),
Value::from(description.clone()),
);
}
self.runtime
.event_log_handle()
.lock()
.await
.append_critical_async(
car_eventlog::EventKind::RunStarted,
None,
None,
data,
CRITICAL_JOURNAL_ACKNOWLEDGEMENT_TIMEOUT,
)
.await
.map_err(|error| error.to_string())?;
Ok(())
}
}
#[derive(Debug, Clone)]
pub struct ApprovalGate {
pub enabled: bool,
pub methods: std::collections::HashSet<String>,
pub timeout: std::time::Duration,
}
impl Default for ApprovalGate {
fn default() -> Self {
let methods = [
"automation.run_applescript",
"automation.run_powershell",
"automation.shortcuts.run",
"messages.send",
"mail.send",
"vision.ocr",
]
.iter()
.map(|s| s.to_string())
.collect();
Self {
enabled: true,
methods,
timeout: std::time::Duration::from_secs(60),
}
}
}
impl ApprovalGate {
pub fn disabled() -> Self {
Self {
enabled: false,
methods: std::collections::HashSet::new(),
timeout: std::time::Duration::from_secs(60),
}
}
pub fn requires_approval(&self, method: &str) -> bool {
self.enabled && self.methods.contains(method)
}
}
pub struct ServerStateConfig {
pub journal_dir: PathBuf,
pub shared_memgine: Option<Arc<Mutex<car_memgine::MemgineEngine>>>,
pub inference: Option<Arc<car_inference::InferenceEngine>>,
pub a2a_runtime: Option<Arc<car_engine::Runtime>>,
pub a2a_store: Option<Arc<dyn car_a2a::TaskStore>>,
pub a2a_card_source: Option<Arc<car_a2a::AgentCardSource>>,
pub approval_gate: Option<ApprovalGate>,
pub approval_journal: Option<PathBuf>,
pub trajectory_dir: Option<PathBuf>,
pub run_store_failures: Option<crate::run_store::RunStoreFailureInjector>,
#[doc(hidden)]
pub run_store_summary_read_gate: Option<crate::run_store::RunStoreSummaryReadGate>,
#[doc(hidden)]
pub run_store_summary_write_gate: Option<crate::run_store::RunStoreSummaryWriteGate>,
#[doc(hidden)]
pub run_store_append_gate: Option<crate::run_store::RunStoreAppendGate>,
#[doc(hidden)]
pub run_completion_fence_gate: Option<RunCompletionFenceGate>,
#[doc(hidden)]
pub run_store_lookup_gate: Option<crate::run_store::RunStoreLookupGate>,
pub run_store_private_path_failures: Option<car_secrets::PrivatePathDurabilityFailureInjector>,
pub journal_failures: Option<car_eventlog::JournalFailureInjector>,
pub selfheal_ledger: Option<PathBuf>,
pub selfheal_interval_secs: u64,
#[doc(hidden)]
pub selfheal_evidence: Option<crate::selfheal::SelfhealEvidence>,
pub selfheal_source_probe: crate::selfheal::SelfhealSourceProbe,
#[doc(hidden)]
pub startup_reconciliation_acknowledgement_timeout: std::time::Duration,
#[doc(hidden)]
pub run_resume_lease: std::time::Duration,
}
impl ServerStateConfig {
pub fn new(journal_dir: PathBuf) -> Self {
Self {
journal_dir,
shared_memgine: None,
inference: None,
a2a_runtime: None,
a2a_store: None,
a2a_card_source: None,
approval_gate: None,
approval_journal: None,
trajectory_dir: None,
run_store_failures: None,
run_store_summary_read_gate: None,
run_store_summary_write_gate: None,
run_store_append_gate: None,
run_completion_fence_gate: None,
run_store_lookup_gate: None,
run_store_private_path_failures: None,
journal_failures: None,
selfheal_ledger: None,
selfheal_interval_secs: crate::selfheal::DEFAULT_SELFHEAL_INTERVAL_SECS,
selfheal_evidence: None,
selfheal_source_probe: crate::selfheal::SelfhealSourceProbe::empty(),
startup_reconciliation_acknowledgement_timeout:
STARTUP_RECONCILIATION_ACKNOWLEDGEMENT_TIMEOUT,
run_resume_lease: RUN_RESUME_LEASE,
}
}
pub fn with_shared_memgine(mut self, engine: Arc<Mutex<car_memgine::MemgineEngine>>) -> Self {
self.shared_memgine = Some(engine);
self
}
pub fn with_approval_journal(mut self, path: PathBuf) -> Self {
self.approval_journal = Some(path);
self
}
pub fn with_trajectory_dir(mut self, dir: PathBuf) -> Self {
self.trajectory_dir = Some(dir);
self
}
pub fn with_run_store_failures(
mut self,
failures: crate::run_store::RunStoreFailureInjector,
) -> Self {
self.run_store_failures = Some(failures);
self
}
#[doc(hidden)]
pub fn with_run_store_summary_read_gate(
mut self,
gate: crate::run_store::RunStoreSummaryReadGate,
) -> Self {
self.run_store_summary_read_gate = Some(gate);
self
}
#[doc(hidden)]
pub fn with_run_store_summary_write_gate(
mut self,
gate: crate::run_store::RunStoreSummaryWriteGate,
) -> Self {
self.run_store_summary_write_gate = Some(gate);
self
}
#[doc(hidden)]
pub fn with_run_store_append_gate(
mut self,
gate: crate::run_store::RunStoreAppendGate,
) -> Self {
self.run_store_append_gate = Some(gate);
self
}
#[doc(hidden)]
pub fn with_run_completion_fence_gate(mut self, gate: RunCompletionFenceGate) -> Self {
self.run_completion_fence_gate = Some(gate);
self
}
#[doc(hidden)]
pub fn with_run_resume_lease(mut self, lease: std::time::Duration) -> Self {
self.run_resume_lease = lease;
self
}
#[doc(hidden)]
pub fn with_run_store_lookup_gate(
mut self,
gate: crate::run_store::RunStoreLookupGate,
) -> Self {
self.run_store_lookup_gate = Some(gate);
self
}
pub fn with_run_store_private_path_failures(
mut self,
failures: car_secrets::PrivatePathDurabilityFailureInjector,
) -> Self {
self.run_store_private_path_failures = Some(failures);
self
}
pub fn with_journal_failures(mut self, failures: car_eventlog::JournalFailureInjector) -> Self {
self.journal_failures = Some(failures);
self
}
pub fn with_selfheal_ledger(mut self, path: PathBuf) -> Self {
self.selfheal_ledger = Some(path);
self
}
pub fn with_selfheal_interval_secs(mut self, interval_secs: u64) -> Self {
self.selfheal_interval_secs = interval_secs.max(1);
self
}
#[doc(hidden)]
pub fn with_selfheal_evidence(mut self, evidence: crate::selfheal::SelfhealEvidence) -> Self {
self.selfheal_evidence = Some(evidence);
self
}
pub fn with_selfheal_source_probe(
mut self,
probe: crate::selfheal::SelfhealSourceProbe,
) -> Self {
self.selfheal_source_probe = probe;
self
}
#[doc(hidden)]
pub fn with_startup_reconciliation_acknowledgement_timeout(
mut self,
timeout: std::time::Duration,
) -> Self {
self.startup_reconciliation_acknowledgement_timeout = timeout;
self
}
pub fn with_inference(mut self, engine: Arc<car_inference::InferenceEngine>) -> Self {
self.inference = Some(engine);
self
}
pub fn with_a2a_runtime(mut self, runtime: Arc<car_engine::Runtime>) -> Self {
self.a2a_runtime = Some(runtime);
self
}
pub fn with_a2a_store(mut self, store: Arc<dyn car_a2a::TaskStore>) -> Self {
self.a2a_store = Some(store);
self
}
pub fn with_a2a_card_source(mut self, source: Arc<car_a2a::AgentCardSource>) -> Self {
self.a2a_card_source = Some(source);
self
}
pub fn with_approval_gate(mut self, gate: ApprovalGate) -> Self {
self.approval_gate = Some(gate);
self
}
}
pub struct OrgSyncHolder {
pub org: String,
pub subsystem: Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>,
}
#[derive(Debug, PartialEq, Eq)]
enum SyncRoute {
User,
Org,
}
fn route_for_scope(scope: &car_sync::Scope, org_holder: Option<&str>) -> SyncRoute {
if let car_sync::Scope::Shared { org } = scope {
if org_holder == Some(org.as_str()) {
return SyncRoute::Org;
}
}
SyncRoute::User
}
pub struct ServerState {
pub journal_dir: PathBuf,
pub sessions: Mutex<HashMap<String, Arc<ClientSession>>>,
run_resume_liveness: Mutex<()>,
run_resume_lease: std::time::Duration,
run_completion_fence_gate: Option<RunCompletionFenceGate>,
pub inference: std::sync::OnceLock<Arc<car_inference::InferenceEngine>>,
pub host: Arc<crate::host::HostState>,
pub shared_memgine: Option<Arc<Mutex<car_memgine::MemgineEngine>>>,
pub trajectory_store: Arc<car_memgine::TrajectoryStore>,
pub selfheal: Arc<crate::selfheal::SelfhealService>,
pub voice_sessions: Arc<car_voice::VoiceSessionRegistry>,
pub meetings: Arc<car_meeting::MeetingRegistry>,
pub a2ui: car_a2ui::A2uiSurfaceStore,
pub ui_agent: Arc<car_ui_agent::UIImprovementAgent>,
pub ui_agent_oscillation: Arc<crate::ui_agent_loop::OscillationDetector>,
pub ui_agent_budget: Arc<crate::ui_agent_loop::IterationBudget>,
pub admission: Arc<crate::admission::InferenceAdmission>,
pub a2ui_route_auth: Mutex<HashMap<String, A2aRouteAuth>>,
pub supervisor: std::sync::OnceLock<Arc<car_registry::supervisor::Supervisor>>,
pub declagents: std::sync::OnceLock<Arc<car_registry::declarative::DeclRegistry>>,
pub routing: std::sync::OnceLock<Arc<car_registry::routing::RoutingStore>>,
pub sync: std::sync::Mutex<Option<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>>>,
pub org_sync: std::sync::Mutex<Option<OrgSyncHolder>>,
pub observer_manifest_path: std::sync::OnceLock<PathBuf>,
pub a2a_dispatcher: std::sync::OnceLock<Arc<car_a2a::A2aDispatcher>>,
pub a2ui_subscribers: Mutex<HashMap<String, Arc<WsChannel>>>,
pub mcp_executor: Arc<car_engine::McpToolExecutor>,
pub connectors: std::sync::OnceLock<Arc<car_connectors::ConnectorManager>>,
connectors_loaded: std::sync::atomic::AtomicBool,
pub channel_supervisor: std::sync::OnceLock<Arc<crate::channel_supervisor::ChannelSupervisor>>,
pub durable_tasks: Mutex<tokio::task::JoinSet<String>>,
pub auth_completion_owner_id: String,
pub auth_token: std::sync::OnceLock<String>,
pub host_token: std::sync::OnceLock<String>,
pub mobile_runtime_url: std::sync::OnceLock<String>,
pub mobile_registration_url: std::sync::OnceLock<String>,
pub parslee_session: std::sync::OnceLock<crate::parslee_auth::ParsleeSession>,
pub attached_agents: Mutex<HashMap<String, String>>,
pub agent_memgines: Mutex<HashMap<String, Arc<Mutex<car_memgine::MemgineEngine>>>>,
pub namespace_memgines: Mutex<HashMap<String, Arc<Mutex<car_memgine::MemgineEngine>>>>,
pub coder_sessions: Mutex<crate::coder::rpc::CoderSessionMap>,
pub coder_subscribers: Mutex<HashMap<(String, String), Arc<WsChannel>>>,
pub coder_watchers: Mutex<HashMap<String, (u64, Arc<WsChannel>)>>,
pub(crate) coder_watch_notify: std::sync::OnceLock<mpsc::UnboundedSender<String>>,
pub coder_discussions: Mutex<crate::coder::discuss::DiscussionMap>,
pub(crate) coder_discussion_slots: Arc<tokio::sync::Semaphore>,
pub chat_sessions: Mutex<HashMap<String, ChatSession>>,
pub peer_guards: Mutex<HashMap<String, car_peers::DeliveryGuard>>,
pub held_peer_messages: Mutex<std::collections::VecDeque<crate::peers::HeldPeerMessage>>,
pub lan_discovery: std::sync::Mutex<Option<car_a2a::lan::LanDirectory>>,
pub peer_identity: std::sync::Mutex<Option<Arc<car_a2a::peer_auth::PeerIdentity>>>,
pub peer_trust: car_a2a::peer_auth::PeerTrust,
pub chat_collectors: Mutex<HashMap<String, ChatCollector>>,
pub chat_goals: Mutex<HashMap<String, ChatGoalState>>,
pub runs: Mutex<HashMap<String, RunMeta>>,
run_durability_locks: Mutex<HashMap<String, std::sync::Weak<Mutex<()>>>>,
pub run_subscribers: Mutex<HashMap<(String, String), crate::host::RunTraceSubscriber>>,
pub browser_views: Arc<crate::browser_view::BrowserViewRegistry>,
pub run_store: crate::run_store::RunStore,
journal_failures: Option<car_eventlog::JournalFailureInjector>,
pub mcp_url: std::sync::OnceLock<String>,
pub mcp_sessions: std::sync::OnceLock<Arc<crate::mcp::SessionMap>>,
pub approval_gate: ApprovalGate,
pub supervision: Arc<crate::supervision::SupervisionRegistry>,
pub approval_ledger: Arc<tokio::sync::RwLock<car_policy::ApprovalLedger>>,
pub harness_measurer: std::sync::RwLock<Option<Arc<dyn crate::evolution::HarnessMeasurer>>>,
pub(crate) a2a_runtime: std::sync::Mutex<Option<Arc<car_engine::Runtime>>>,
pub(crate) a2a_store: std::sync::Mutex<Option<Arc<dyn car_a2a::TaskStore>>>,
pub(crate) a2a_card_source: std::sync::Mutex<Option<Arc<car_a2a::AgentCardSource>>>,
}
fn default_approval_journal_path() -> Option<PathBuf> {
let dir = car_home::root()?;
let _ = std::fs::create_dir_all(&dir);
Some(dir.join("approvals.jsonl"))
}
fn car_home_dir() -> Option<PathBuf> {
car_home::root()
}
pub(crate) async fn apply_project_policies(
runtime: &Runtime,
car_dir: &std::path::Path,
) -> Result<(), String> {
match runtime.load_project_policies(car_dir).await {
Ok(_) => Ok(()),
Err(e) => Err(format!(
"refusing to start with unreadable project policy rules: {e}. \
Fix or remove the file — a policy rule that fails to parse is a \
security control that would silently not exist."
)),
}
}
impl ServerState {
async fn run_durability_lock(&self, run_id: &str) -> Arc<Mutex<()>> {
let mut locks = self.run_durability_locks.lock().await;
if let Some(lock) = locks.get(run_id).and_then(std::sync::Weak::upgrade) {
return lock;
}
locks.retain(|_, lock| lock.strong_count() > 0);
let lock = Arc::new(Mutex::new(()));
locks.insert(run_id.to_string(), Arc::downgrade(&lock));
lock
}
async fn quarantine_run_trace_locked(
&self,
runs: &mut HashMap<String, RunMeta>,
run_id: &str,
detail: String,
) -> String {
let message = run_trace_corruption_message(run_id, detail);
if let Some(meta) = runs.get_mut(run_id) {
if meta.trace_corruption.as_deref() != Some(message.as_str()) {
meta.durability_generation = meta.durability_generation.wrapping_add(1);
}
meta.trace_corruption = Some(message.clone());
}
let mut subs = self.run_subscribers.lock().await;
subs.retain(|(subscribed_run, _), _| subscribed_run != run_id);
message
}
pub(crate) async fn quarantine_run_trace_from_read(
&self,
run_id: &str,
detail: String,
) -> String {
let durability_lock = self.run_durability_lock(run_id).await;
let _durability_guard = durability_lock.lock().await;
let mut runs = self.runs.lock().await;
self.quarantine_run_trace_locked(&mut runs, run_id, detail)
.await
}
pub fn standalone(journal_dir: PathBuf) -> Self {
Self::with_config(ServerStateConfig::new(journal_dir))
}
pub fn sync_subsystem(
&self,
) -> Result<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>, String> {
let mut guard = self
.sync
.lock()
.map_err(|_| "sync subsystem init lock poisoned".to_string())?;
if let Some(existing) = guard.as_ref() {
return Ok(existing.clone());
}
let root = self.journal_dir.join("sync");
let subsystem = self.open_sync_subsystem(&root)?;
let arc = Arc::new(tokio::sync::Mutex::new(subsystem));
*guard = Some(arc.clone());
Ok(arc)
}
pub fn subsystem_for_scope(
&self,
scope: &car_sync::Scope,
) -> Result<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>, String> {
let user = self.sync_subsystem()?;
let guard = self
.org_sync
.lock()
.map_err(|_| "org sync lock poisoned".to_string())?;
match route_for_scope(scope, guard.as_ref().map(|h| h.org.as_str())) {
SyncRoute::Org => Ok(guard
.as_ref()
.expect("route_for_scope returns Org only when a holder exists")
.subsystem
.clone()),
SyncRoute::User => Ok(user),
}
}
pub fn org_subsystem(&self) -> Option<Arc<tokio::sync::Mutex<crate::sync::SyncSubsystem>>> {
match self.org_sync.lock() {
Ok(g) => g.as_ref().map(|h| h.subsystem.clone()),
Err(_) => {
tracing::error!(
"org_sync lock poisoned — skipping the org pump (org-scope delivery degraded)"
);
None
}
}
}
fn open_sync_subsystem(
&self,
root: &std::path::Path,
) -> Result<crate::sync::SyncSubsystem, String> {
let backend = car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_BACKEND")
.unwrap_or_default()
.to_lowercase();
if backend != "parslee" {
return crate::sync::SyncSubsystem::open(root);
}
let Some(account_id) = self
.parslee_session
.get()
.map(|s| s.identity.account_id.clone())
.filter(|a| !a.is_empty())
else {
tracing::warn!(
"sync backend=parslee but no Parslee login is present; using local sync until login"
);
return crate::sync::SyncSubsystem::open(root);
};
let Some(passphrase) = car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_PASSPHRASE")
.filter(|p| !p.is_empty())
else {
return Err(
"sync backend=parslee requires PARSLEE_SYNC_PASSPHRASE (the zero-knowledge \
cross-device sync key) — set it in the keychain; refusing to sync unencrypted"
.to_string(),
);
};
let base_url = car_secrets::resolve_env_or_keychain("PARSLEE_CAR_SYNC_BASE_URL")
.filter(|u| !u.is_empty())
.unwrap_or_else(|| {
let api_base = car_auth::api_base(None);
format!("{}/sync", api_base.trim_end_matches('/'))
});
let bearer: car_parslee::sync_transport::BearerFn = Arc::new(car_auth::access_token);
let transport = Arc::new(
car_parslee::sync_transport::ParsleeSyncTransport::new(base_url, bearer)
.map_err(|e| format!("sync: build Parslee transport: {e}"))?,
);
let master = car_sync::StretchedMaster::from_passphrase(passphrase.as_bytes(), &account_id);
let base: Arc<dyn car_sync::SyncKeyProvider> =
Arc::new(car_sync::DerivedKeyProvider::from_master(&master));
let mut pending_org: Option<OrgSyncHolder> = None;
let provider: Arc<dyn car_sync::SyncKeyProvider> = match Self::parse_org_scope_config(
car_secrets::resolve_env_or_keychain("PARSLEE_SYNC_ORG_SCOPE"),
)? {
None => base,
Some((org_id, granters)) => {
let active_org = self
.parslee_session
.get()
.and_then(|s| s.identity.active_organization.clone())
.filter(|o| !o.is_empty());
if active_org.as_deref() != Some(org_id.as_str()) {
return Err(format!(
"PARSLEE_SYNC_ORG_SCOPE org {org_id:?} does not match the signed-in active \
organization {active_org:?} — refusing to activate org scope"
));
}
use car_sync::OrgKeyDirectory as _;
let my_secret = car_sync::derive_x25519_identity(&master, &account_id);
let mut directory = car_sync::NetworkOrgKeyDirectory::new(
transport.clone(),
format!("org:{org_id}"),
);
let my_pub = car_sync::x25519_public(&my_secret);
let my_pub_hex: String = my_pub
.as_bytes()
.iter()
.map(|b| format!("{b:02x}"))
.collect();
if let Err(e) = directory.publish_pubkey(&account_id, &my_pub_hex) {
tracing::warn!(org = %org_id, error = %e, "org-scope: failed to publish member pubkey — future grants may not reach this member");
}
let mut roots = std::collections::HashMap::new();
match car_sync::resolve_all_org_roots(
&directory,
&org_id,
&my_secret,
&account_id,
&granters,
) {
Ok(map) if !map.is_empty() => {
tracing::info!(org = %org_id, epochs = ?map.keys().collect::<Vec<_>>(), "org-scope: resolved K_org generations");
roots.insert(org_id.clone(), map);
}
Ok(_) => tracing::warn!(
org = %org_id,
"org-scope: no trusted grant for this member — org scope fails closed (DenyCipher)"
),
Err(e) => tracing::warn!(
org = %org_id, error = %e,
"org-scope: org-key directory unreachable — org scope fails closed (DenyCipher), NOT demoted to the personal key"
),
}
let provider: Arc<dyn car_sync::SyncKeyProvider> =
Arc::new(car_sync::OrgAwareKeyProvider::new(base, roots));
let org_root = root.join(format!("org-{org_id}"));
let org_subsystem = crate::sync::SyncSubsystem::open_remote(
&org_root,
transport.clone(),
format!("org:{org_id}"),
provider.clone(),
)?;
pending_org = Some(OrgSyncHolder {
org: org_id.clone(),
subsystem: Arc::new(tokio::sync::Mutex::new(org_subsystem)),
});
tracing::info!(org = %org_id, "org-scope: opened shared org delivery subsystem (relay org:{org_id})");
provider
}
};
let user = crate::sync::SyncSubsystem::open_remote(
root,
transport,
format!("user:{account_id}"),
provider,
)?;
if let Some(holder) = pending_org {
*self
.org_sync
.lock()
.map_err(|_| "org sync init lock poisoned".to_string())? = Some(holder);
}
Ok(user)
}
fn parse_org_scope_config(
raw: Option<String>,
) -> Result<Option<(String, Vec<car_sync::OrgVerifyingKey>)>, String> {
let Some(raw) = raw.filter(|s| !s.is_empty()) else {
return Ok(None);
};
let (org_id, granters_csv) = raw.split_once(':').ok_or_else(|| {
"PARSLEE_SYNC_ORG_SCOPE must be <orgId>:<granter_ed25519_pubkey_hex>[,...]".to_string()
})?;
car_sync::require_canonical_org(org_id)
.map_err(|e| format!("PARSLEE_SYNC_ORG_SCOPE: bad orgId: {e}"))?;
let granters = granters_csv
.split(',')
.map(str::trim)
.filter(|h| !h.is_empty())
.map(|hex| {
car_sync::parse_ed25519_verifying(hex)
.map_err(|e| format!("PARSLEE_SYNC_ORG_SCOPE: bad granter key {hex:?}: {e}"))
})
.collect::<Result<Vec<_>, _>>()?;
if granters.is_empty() {
return Err(
"PARSLEE_SYNC_ORG_SCOPE: at least one granter ed25519 pubkey is required".into(),
);
}
Ok(Some((org_id.to_string(), granters)))
}
pub async fn persist_chat_goals(&self) -> Result<(), String> {
let path = chat_goals_path_from_journal_dir(&self.journal_dir);
let goals = self.chat_goals.lock().await.clone();
tokio::task::spawn_blocking(move || {
write_chat_goals_atomic(&path, &goals)
.map_err(|e| format!("write {}: {e}", path.display()))
})
.await
.map_err(|e| format!("persist chat goals join: {e}"))?
}
pub fn embedded(
journal_dir: PathBuf,
shared_memgine: Arc<Mutex<car_memgine::MemgineEngine>>,
) -> Self {
Self::with_config(ServerStateConfig::new(journal_dir).with_shared_memgine(shared_memgine))
}
pub fn with_config(cfg: ServerStateConfig) -> Self {
Self::try_with_config(cfg)
.unwrap_or_else(|error| panic!("server state startup failed: {error}"))
}
pub fn try_with_config(cfg: ServerStateConfig) -> Result<Self, String> {
let startup_acknowledgement_timeout = cfg.startup_reconciliation_acknowledgement_timeout;
if startup_acknowledgement_timeout.is_zero()
|| startup_acknowledgement_timeout > car_eventlog::MAX_CRITICAL_ACKNOWLEDGEMENT_TIMEOUT
{
return Err(format!(
"startup reconciliation acknowledgement timeout must be between 1ns and {}ms, got {}ms",
car_eventlog::MAX_CRITICAL_ACKNOWLEDGEMENT_TIMEOUT.as_millis(),
startup_acknowledgement_timeout.as_millis()
));
}
let inference = std::sync::OnceLock::new();
if let Some(eng) = cfg.inference {
let _ = inference.set(eng);
}
let voice_sessions = Arc::new(car_voice::VoiceSessionRegistry::new());
voice_sessions.start_sweeper();
let ui_agent = Arc::new(car_ui_agent::UIImprovementAgent::with_default_strategies());
let ui_agent_oscillation = Arc::new(crate::ui_agent_loop::OscillationDetector::new());
let ui_agent_budget = Arc::new(crate::ui_agent_loop::IterationBudget::new());
let mut run_store = crate::run_store::RunStore::from_journal_dir(&cfg.journal_dir);
if let Some(failures) = cfg.run_store_failures.clone() {
run_store = run_store.with_failure_injector(failures);
}
if let Some(gate) = cfg.run_store_summary_read_gate.clone() {
run_store = run_store.with_summary_read_gate(gate);
}
if let Some(gate) = cfg.run_store_summary_write_gate.clone() {
run_store = run_store.with_summary_write_gate(gate);
}
if let Some(gate) = cfg.run_store_append_gate.clone() {
run_store = run_store.with_append_gate(gate);
}
if let Some(gate) = cfg.run_store_lookup_gate.clone() {
run_store = run_store.with_lookup_gate(gate);
}
if let Some(failures) = cfg.run_store_private_path_failures.clone() {
run_store = run_store.with_private_path_failure_injector(failures);
}
run_store
.reconcile_proposal_retry_rollbacks()
.map_err(|error| format!("proposal retry rollback reconciliation failed: {error}"))?;
let chat_goals = load_chat_goals_from_disk(&cfg.journal_dir);
let selfheal_ledger = cfg.selfheal_ledger.clone().unwrap_or_else(|| {
cfg.journal_dir
.parent()
.unwrap_or(&cfg.journal_dir)
.join("selfheal")
.join("detections.jsonl")
});
let selfheal = Arc::new(crate::selfheal::SelfhealService::open(
selfheal_ledger,
cfg.selfheal_interval_secs,
cfg.selfheal_evidence.clone(),
cfg.selfheal_source_probe.clone(),
)?);
let _evicted_before_replay = run_store.gc();
let mut startup_journals =
StartupJournalCache::new(&cfg.journal_dir, cfg.journal_failures.as_ref());
reconcile_durable_run_journals(
&mut startup_journals,
&run_store,
startup_acknowledgement_timeout,
)?;
reconcile_pending_proposal_journals(
&mut startup_journals,
&run_store,
startup_acknowledgement_timeout,
)?;
reconcile_completed_proposal_guards(&run_store);
let _adopted = run_store.adopt_orphans();
reconcile_durable_run_journals(
&mut startup_journals,
&run_store,
startup_acknowledgement_timeout,
)?;
let _evicted = run_store.gc();
if let Ok(coder_dir) = crate::coder::rpc::coder_state_dir() {
let _coder_adopted = crate::coder::session::adopt_orphaned_sessions(&coder_dir);
}
let approval_ledger = {
let path = cfg.approval_journal.or_else(default_approval_journal_path);
match path {
Some(p) => {
if let Some(parent) = p.parent() {
let _ = std::fs::create_dir_all(parent);
}
match car_policy::ApprovalLedger::with_journal(&p) {
Ok(l) => {
if l.skipped_on_load() > 0 {
tracing::warn!(
path = %p.display(),
skipped = l.skipped_on_load(),
"approval-ledger journal had unparseable lines; partial ledger loaded"
);
}
l
}
Err(e) => {
tracing::warn!(
path = %p.display(), error = %e,
"approval-ledger journal unopenable; falling back to IN-MEMORY \
ledger — approvals will NOT survive restart"
);
car_policy::ApprovalLedger::new()
}
}
}
None => {
tracing::warn!(
"no CAR_HOME/HOME/USERPROFILE to place approvals.jsonl under the CAR \
state root; approval ledger is IN-MEMORY — approvals will NOT survive \
restart"
);
car_policy::ApprovalLedger::new()
}
}
};
let host = Arc::new(crate::host::HostState::new());
let browser_views = Arc::new(crate::browser_view::BrowserViewRegistry::default());
browser_views.set_signin_attention(Arc::new(
crate::browser_attention::HostSignInAttention::new(Arc::clone(&host)),
));
Ok(Self {
journal_dir: cfg.journal_dir,
sessions: Mutex::new(HashMap::new()),
inference,
host,
shared_memgine: cfg.shared_memgine,
trajectory_store: Arc::new(car_memgine::TrajectoryStore::new(
&cfg.trajectory_dir
.unwrap_or_else(car_memgine::TrajectoryStore::default_path),
)),
selfheal,
voice_sessions,
meetings: Arc::new(car_meeting::MeetingRegistry::new()),
a2ui: car_a2ui::A2uiSurfaceStore::new(),
ui_agent,
ui_agent_oscillation,
ui_agent_budget,
admission: Arc::new(crate::admission::InferenceAdmission::new()),
a2ui_route_auth: Mutex::new(HashMap::new()),
supervisor: std::sync::OnceLock::new(),
declagents: std::sync::OnceLock::new(),
routing: std::sync::OnceLock::new(),
sync: std::sync::Mutex::new(None),
org_sync: std::sync::Mutex::new(None),
observer_manifest_path: std::sync::OnceLock::new(),
a2a_dispatcher: std::sync::OnceLock::new(),
a2a_runtime: std::sync::Mutex::new(cfg.a2a_runtime),
a2a_store: std::sync::Mutex::new(cfg.a2a_store),
a2a_card_source: std::sync::Mutex::new(cfg.a2a_card_source),
a2ui_subscribers: Mutex::new(HashMap::new()),
mcp_executor: Arc::new(car_engine::McpToolExecutor::new()),
connectors: std::sync::OnceLock::new(),
connectors_loaded: std::sync::atomic::AtomicBool::new(false),
channel_supervisor: std::sync::OnceLock::new(),
durable_tasks: Mutex::new(tokio::task::JoinSet::new()),
auth_completion_owner_id: uuid::Uuid::new_v4().simple().to_string(),
auth_token: std::sync::OnceLock::new(),
host_token: std::sync::OnceLock::new(),
mobile_runtime_url: std::sync::OnceLock::new(),
mobile_registration_url: std::sync::OnceLock::new(),
parslee_session: std::sync::OnceLock::new(),
attached_agents: Mutex::new(HashMap::new()),
agent_memgines: Mutex::new(HashMap::new()),
namespace_memgines: Mutex::new(HashMap::new()),
coder_sessions: Mutex::new(HashMap::new()),
coder_subscribers: Mutex::new(HashMap::new()),
coder_watchers: Mutex::new(HashMap::new()),
coder_watch_notify: std::sync::OnceLock::new(),
coder_discussions: Mutex::new(HashMap::new()),
coder_discussion_slots: Arc::new(tokio::sync::Semaphore::new(
crate::coder::discuss::MAX_OPEN_DISCUSSIONS,
)),
chat_sessions: Mutex::new(HashMap::new()),
peer_guards: Mutex::new(HashMap::new()),
held_peer_messages: Mutex::new(std::collections::VecDeque::new()),
lan_discovery: std::sync::Mutex::new(None),
peer_identity: std::sync::Mutex::new(None),
peer_trust: car_a2a::peer_auth::PeerTrust::new(),
chat_collectors: Mutex::new(HashMap::new()),
chat_goals: Mutex::new(chat_goals),
runs: Mutex::new(HashMap::new()),
run_resume_liveness: Mutex::new(()),
run_resume_lease: cfg.run_resume_lease,
run_completion_fence_gate: cfg.run_completion_fence_gate,
run_durability_locks: Mutex::new(HashMap::new()),
run_subscribers: Mutex::new(HashMap::new()),
browser_views,
run_store,
journal_failures: cfg.journal_failures,
mcp_url: std::sync::OnceLock::new(),
mcp_sessions: std::sync::OnceLock::new(),
approval_gate: cfg.approval_gate.unwrap_or_default(),
supervision: Arc::new(crate::supervision::SupervisionRegistry::default()),
approval_ledger: Arc::new(tokio::sync::RwLock::new(approval_ledger)),
harness_measurer: std::sync::RwLock::new(None),
})
}
pub fn set_harness_measurer(&self, m: Arc<dyn crate::evolution::HarnessMeasurer>) {
match self.harness_measurer.write() {
Ok(mut guard) => *guard = Some(m),
Err(e) => tracing::error!(
error = %e,
"harness_measurer lock poisoned; the in-process harness evaluator was NOT installed"
),
}
}
pub fn harness_measurer(&self) -> Option<Arc<dyn crate::evolution::HarnessMeasurer>> {
match self.harness_measurer.read() {
Ok(guard) => guard.clone(),
Err(e) => {
tracing::error!(error = %e, "harness_measurer lock poisoned; treating as absent");
None
}
}
}
pub async fn spawn_durable_operation<F, E>(
&self,
operation_name: impl Into<String>,
operation: F,
) -> tokio::sync::oneshot::Receiver<Result<serde_json::Value, E>>
where
F: std::future::Future<Output = Result<serde_json::Value, E>> + Send + 'static,
E: Send + 'static,
{
let (result_tx, result_rx) = tokio::sync::oneshot::channel();
let operation_name = operation_name.into();
let mut tasks = self.durable_tasks.lock().await;
while let Some(result) = tasks.try_join_next() {
if let Err(error) = result {
tracing::warn!(error = %error, "daemon-owned task failed to join");
}
}
tasks.spawn(async move {
let result = operation.await;
if result_tx.send(result).is_err() {
tracing::debug!(
operation = %operation_name,
"daemon-owned operation finished after its response waiter closed"
);
}
operation_name
});
result_rx
}
pub async fn spawn_durable_task<F>(&self, operation_name: impl Into<String>, operation: F)
where
F: std::future::Future<Output = ()> + Send + 'static,
{
let operation_name = operation_name.into();
let mut tasks = self.durable_tasks.lock().await;
while let Some(result) = tasks.try_join_next() {
if let Err(error) = result {
tracing::warn!(error = %error, "daemon-owned task failed to join");
}
}
tasks.spawn(async move {
operation.await;
operation_name
});
}
pub fn install_auth_token(&self, token: String) -> Result<(), String> {
self.auth_token.set(token)
}
pub fn install_host_token(&self, token: String) -> Result<(), String> {
self.host_token.set(token)
}
pub fn install_mobile_runtime_url(&self, url: String) -> Result<(), String> {
self.mobile_runtime_url.set(url)
}
pub fn install_mobile_registration_url(&self, url: String) -> Result<(), String> {
self.mobile_registration_url.set(url)
}
pub fn install_parslee_session(
&self,
session: crate::parslee_auth::ParsleeSession,
) -> Result<(), crate::parslee_auth::ParsleeSession> {
self.parslee_session.set(session)
}
pub fn install_channel_supervisor(
&self,
supervisor: Arc<crate::channel_supervisor::ChannelSupervisor>,
) -> Result<(), Arc<crate::channel_supervisor::ChannelSupervisor>> {
self.channel_supervisor.set(supervisor)
}
pub fn install_mcp_url(&self, url: String) -> Result<(), String> {
self.mcp_url.set(url)
}
pub fn install_mcp_sessions(
&self,
sessions: Arc<crate::mcp::SessionMap>,
) -> Result<(), Arc<crate::mcp::SessionMap>> {
self.mcp_sessions.set(sessions)
}
pub fn connectors(&self) -> Arc<car_connectors::ConnectorManager> {
self.connectors
.get_or_init(|| {
let mgr = car_connectors::ConnectorManager::new(self.mcp_executor.clone())
.unwrap_or_else(|_| {
car_connectors::ConnectorManager::with_path(
self.mcp_executor.clone(),
std::path::PathBuf::from("connectors.json"),
)
});
Arc::new(mgr)
})
.clone()
}
pub async fn ensure_connectors_loaded(&self) {
use std::sync::atomic::Ordering;
if self.connectors_loaded.swap(true, Ordering::SeqCst) {
return;
}
let mgr = self.connectors();
match mgr.load_and_connect().await {
Ok(entries) => self.register_connector_entries(&entries).await,
Err(e) => tracing::warn!("connector load failed: {e}"),
}
}
pub async fn register_connector_entries(&self, entries: &[car_engine::ToolEntry]) {
if entries.is_empty() {
return;
}
let sessions: Vec<Arc<ClientSession>> =
self.sessions.lock().await.values().cloned().collect();
for session in sessions {
for entry in entries {
session.runtime.register_tool_entry(entry.clone()).await;
}
}
}
pub async fn unregister_connector_tools(&self, canonical_names: &[String]) {
if canonical_names.is_empty() {
return;
}
let sessions: Vec<Arc<ClientSession>> =
self.sessions.lock().await.values().cloned().collect();
for session in sessions {
for name in canonical_names {
session.runtime.unregister_tool(name).await;
}
}
}
pub async fn any_host_connected(&self) -> bool {
self.sessions
.lock()
.await
.values()
.any(|s| s.is_host.load(std::sync::atomic::Ordering::Acquire))
}
pub fn declagents(&self) -> Result<Arc<car_registry::declarative::DeclRegistry>, String> {
if let Some(r) = self.declagents.get() {
return Ok(r.clone());
}
let r = Arc::new(car_registry::declarative::DeclRegistry::user_default()?);
let _ = self.declagents.set(r);
Ok(self.declagents.get().expect("set or pre-existing").clone())
}
pub fn routing(&self) -> Result<Arc<car_registry::routing::RoutingStore>, String> {
if let Some(r) = self.routing.get() {
return Ok(r.clone());
}
let r = Arc::new(car_registry::routing::RoutingStore::user_default()?);
let _ = self.routing.set(r);
Ok(self.routing.get().expect("set or pre-existing").clone())
}
pub fn supervisor(&self) -> Result<Arc<car_registry::supervisor::Supervisor>, String> {
if let Some(s) = self.supervisor.get() {
return Ok(s.clone());
}
if let Some(p) = self.observer_manifest_path.get() {
return Err(format!(
"this car-server is observe-only — another car-server process \
holds the supervisor lock for {}. Mutations refuse here; route \
them to the primary daemon, or stop the other car-server first.",
p.display()
));
}
let s = car_registry::supervisor::Supervisor::user_default()
.map(Arc::new)
.map_err(|e| e.to_string())?;
let _ = self.supervisor.set(s);
Ok(self.supervisor.get().expect("set or pre-existing").clone())
}
pub fn install_supervisor(
&self,
supervisor: Arc<car_registry::supervisor::Supervisor>,
) -> Result<(), Arc<car_registry::supervisor::Supervisor>> {
self.supervisor.set(supervisor)
}
pub fn supervisor_if_installed(&self) -> Option<Arc<car_registry::supervisor::Supervisor>> {
self.supervisor.get().cloned()
}
pub fn install_observer_manifest(&self, path: PathBuf) -> Result<(), PathBuf> {
self.observer_manifest_path.set(path)
}
pub fn observer_manifest_path(&self) -> Option<&PathBuf> {
self.observer_manifest_path.get()
}
pub async fn a2a_dispatcher(&self) -> Arc<car_a2a::A2aDispatcher> {
if let Some(d) = self.a2a_dispatcher.get() {
return d.clone();
}
let runtime = self
.a2a_runtime
.lock()
.expect("a2a_runtime mutex poisoned")
.take();
let runtime = match runtime {
Some(r) => r,
None => {
let r = Arc::new(car_engine::Runtime::new());
r.register_agent_basics().await;
r
}
};
let store = self
.a2a_store
.lock()
.expect("a2a_store mutex poisoned")
.take()
.unwrap_or_else(|| Arc::new(car_a2a::InMemoryTaskStore::new()));
let card_source = self
.a2a_card_source
.lock()
.expect("a2a_card_source mutex poisoned")
.take();
let card_source = match card_source {
Some(c) => c,
None => {
let card = car_a2a::build_default_agent_card(
&runtime,
car_a2a::AgentCardConfig::minimal(
"Common Agent Runtime",
"Embedded CAR daemon — A2A v1.0 reachable over WebSocket JSON-RPC.",
"ws://127.0.0.1:9100/",
car_a2a::AgentProvider {
organization: "Parslee".into(),
url: Some("https://github.com/Parslee-ai/car".into()),
},
),
)
.await;
Arc::new(move || card.clone()) as Arc<car_a2a::AgentCardSource>
}
};
let dispatcher = Arc::new(car_a2a::A2aDispatcher::new(runtime, store, card_source));
let _ = self.a2a_dispatcher.set(dispatcher);
self.a2a_dispatcher
.get()
.expect("a2a_dispatcher set or pre-existing")
.clone()
}
fn fanout_locked(
subscribers: &HashMap<(String, String), crate::host::RunTraceSubscriber>,
run_id: &str,
agent_id: &str,
record: car_proto::RunRecord,
cursor: usize,
status: car_proto::RunLiveStatus,
) {
for ((sub_run, _client), sub) in subscribers.iter() {
if sub_run != run_id {
continue;
}
let event = car_proto::RunTraceEvent {
run_id: run_id.to_string(),
agent_id: agent_id.to_string(),
record: record.clone(),
cursor,
status,
};
if !sub.push(event) {
tracing::debug!(
run_id,
"run-trace: dropped event for slow subscriber (channel full)"
);
}
}
}
pub async fn reserve_run(&self, meta: RunMeta) -> Result<RunReservation, String> {
fn existing_reservation(
existing: &RunMeta,
requested: &RunMeta,
) -> Result<RunReservation, String> {
if existing.client_id != requested.client_id {
return Err(format!(
"{} run `{}` belongs to client_id `{}`",
car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
requested.run_id,
existing.client_id
));
}
if existing.agent_id != requested.agent_id {
return Err(format!(
"{} run `{}` was retried under a different agent",
car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
requested.run_id
));
}
let changed_fields = [
(existing.intent != requested.intent).then_some("intent"),
(existing.outcome_description != requested.outcome_description)
.then_some("outcome_description"),
]
.into_iter()
.flatten()
.collect::<Vec<_>>();
if !changed_fields.is_empty() {
return Err(format!(
"{} idempotency conflict for run `{}`: retry changed occurrence-defining {}",
car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
requested.run_id,
changed_fields.join(" and ")
));
}
Ok(RunReservation::Existing(existing.clone()))
}
let durability_lock = self.run_durability_lock(&meta.run_id).await;
let _durability_guard = durability_lock.lock().await;
{
let runs = self.runs.lock().await;
if let Some(existing) = runs.get(&meta.run_id) {
return existing_reservation(existing, &meta);
}
}
let store = self.run_store.clone();
let durable_run_id = meta.run_id.clone();
let durable = tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
.await
.map_err(|error| format!("RunStarted reservation read task failed: {error}"))?
.map_err(|error| format!("RunStarted reservation read failed: {error}"))?;
if let Some(started) = durable {
return Err(format!(
"{} persisted run `{}` belongs to client_id `{}` and cannot be adopted",
car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
meta.run_id,
started.client_id.as_deref().unwrap_or("historical/unknown")
));
}
let mut runs = self.runs.lock().await;
if let Some(existing) = runs.get(&meta.run_id) {
return existing_reservation(existing, &meta);
}
runs.insert(meta.run_id.clone(), meta);
Ok(RunReservation::New)
}
pub(crate) async fn release_unpersisted_run_reservation(
&self,
session: &ClientSession,
expected: &RunMeta,
) -> Result<(), String> {
if expected.client_id != session.client_id
|| expected.start_committed
|| expected.termination.is_some()
|| expected.ended_at.is_some()
|| !expected.turns.is_empty()
|| expected.pending_terminal.is_some()
{
return Err(format!(
"run `{}` is not a pristine reservation owned by this session",
expected.run_id
));
}
let still_expected = |existing: &RunMeta| {
existing.run_id == expected.run_id
&& existing.client_id == expected.client_id
&& existing.agent_id == expected.agent_id
&& existing.intent == expected.intent
&& existing.outcome_description == expected.outcome_description
&& existing.started_at == expected.started_at
&& existing.durability_generation == expected.durability_generation
&& !existing.start_committed
&& existing.termination.is_none()
&& existing.ended_at.is_none()
&& existing.turns.is_empty()
&& existing.pending_terminal.is_none()
};
let durability_lock = self.run_durability_lock(&expected.run_id).await;
let _durability_guard = durability_lock.lock().await;
{
let runs = self.runs.lock().await;
let existing = runs
.get(&expected.run_id)
.ok_or_else(|| format!("reserved run `{}` is absent", expected.run_id))?;
if !still_expected(existing) {
return Err(format!(
"reserved run `{}` changed before pre-start release",
expected.run_id
));
}
}
let store = self.run_store.clone();
let durable_run_id = expected.run_id.clone();
if tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
.await
.map_err(|error| format!("RunStarted release read task failed: {error}"))?
.map_err(|error| format!("RunStarted release read failed: {error}"))?
.is_some()
{
return Err(format!(
"reserved run `{}` reached durable storage and cannot be released",
expected.run_id
));
}
let mut runs = self.runs.lock().await;
let existing = runs
.get(&expected.run_id)
.ok_or_else(|| format!("reserved run `{}` is absent", expected.run_id))?;
if !still_expected(existing) {
return Err(format!(
"reserved run `{}` changed during pre-start release",
expected.run_id
));
}
runs.remove(&expected.run_id);
Ok(())
}
pub async fn persist_run_start(&self, run_id: &str) -> Result<car_proto::RunStarted, String> {
let durability_lock = self.run_durability_lock(run_id).await;
let _durability_guard = durability_lock.lock().await;
let started = {
let runs = self.runs.lock().await;
let meta = runs
.get(run_id)
.ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
car_proto::RunStarted {
run_id: meta.run_id.clone(),
client_id: Some(meta.client_id.clone()),
agent_id: meta.agent_id.clone(),
intent: meta.intent.clone(),
outcome_description: meta.outcome_description.clone(),
started_at: meta.started_at,
}
};
let store = self.run_store.clone();
let durable_started = started.clone();
tokio::task::spawn_blocking(move || store.write_started(&durable_started))
.await
.map_err(|error| format!("RunStarted durability task failed: {error}"))?
.map_err(|error| format!("RunStarted durability failed: {error}"))?;
Ok(started)
}
pub async fn commit_run_start(&self, run_id: &str) -> Result<(), String> {
let mut runs = self.runs.lock().await;
let meta = runs
.get_mut(run_id)
.ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
if meta.start_committed {
return Ok(());
}
meta.start_committed = true;
meta.durability_generation = meta.durability_generation.wrapping_add(1);
let started = car_proto::RunStarted {
run_id: meta.run_id.clone(),
client_id: Some(meta.client_id.clone()),
agent_id: meta.agent_id.clone(),
intent: meta.intent.clone(),
outcome_description: meta.outcome_description.clone(),
started_at: meta.started_at,
};
let subs = self.run_subscribers.lock().await;
Self::fanout_locked(
&subs,
&started.run_id,
&started.agent_id,
car_proto::RunRecord::Started(started.clone()),
0,
car_proto::RunLiveStatus::InProgress,
);
Ok(())
}
pub(crate) async fn reconcile_or_release_unacknowledged_start(
&self,
session: &ClientSession,
run_id: &str,
) -> Result<bool, String> {
let durability_lock = self.run_durability_lock(run_id).await;
let _durability_guard = durability_lock.lock().await;
let expected = {
let runs = self.runs.lock().await;
let meta = runs
.get(run_id)
.ok_or_else(|| format!("unknown reserved run_id `{run_id}`"))?;
if meta.client_id != session.client_id {
return Err(format!(
"{} run `{run_id}` belongs to client_id `{}`",
car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX,
meta.client_id
));
}
if meta.start_committed {
return Ok(false);
}
if meta.is_terminal() || meta.pending_terminal.is_some() {
return Err(format!(
"uncommitted run `{run_id}` unexpectedly has terminal state"
));
}
car_proto::RunStarted {
run_id: meta.run_id.clone(),
client_id: Some(meta.client_id.clone()),
agent_id: meta.agent_id.clone(),
intent: meta.intent.clone(),
outcome_description: meta.outcome_description.clone(),
started_at: meta.started_at,
}
};
let store = self.run_store.clone();
let durable_run_id = run_id.to_string();
let durable = tokio::task::spawn_blocking(move || store.run_started(&durable_run_id))
.await
.map_err(|error| format!("RunStarted recovery read task failed: {error}"))?
.map_err(|error| format!("RunStarted recovery read failed: {error}"))?;
match durable {
Some(durable) => {
if durable != expected {
return Err(format!(
"durable RunStarted identity for `{run_id}` does not match its live reservation"
));
}
let store = self.run_store.clone();
let durable_expected = expected.clone();
tokio::task::spawn_blocking(move || store.write_started(&durable_expected))
.await
.map_err(|error| {
format!("RunStarted recovery durability task failed: {error}")
})?
.map_err(|error| format!("RunStarted recovery durability failed: {error}"))?;
session.append_run_started_event(&expected).await?;
self.commit_run_start(run_id).await?;
Ok(false)
}
None => {
let store = self.run_store.clone();
let durable_expected = expected.clone();
tokio::task::spawn_blocking(move || {
store.rollback_empty_run_start(&durable_expected)
})
.await
.map_err(|error| format!("empty RunStarted rollback task failed: {error}"))?
.map_err(|error| format!("empty RunStarted rollback failed: {error}"))?;
{
let mut runs = self.runs.lock().await;
let meta = runs.get(run_id).ok_or_else(|| {
format!("reserved run `{run_id}` disappeared during rollback")
})?;
let still_expected = meta.client_id == session.client_id
&& !meta.start_committed
&& !meta.is_terminal()
&& meta.pending_terminal.is_none()
&& meta.agent_id == expected.agent_id
&& meta.intent == expected.intent
&& meta.outcome_description == expected.outcome_description
&& meta.started_at == expected.started_at;
if !still_expected {
return Err(format!(
"reserved run `{run_id}` changed during empty-start rollback"
));
}
runs.remove(run_id);
}
session.clear_run_journal_binding(run_id).await?;
let mut current = session.current_run_id.lock().await;
if current.as_deref() == Some(run_id) {
*current = None;
}
Ok(true)
}
}
}
pub async fn start_run(&self, meta: RunMeta) -> Result<(), String> {
let run_id = meta.run_id.clone();
self.reserve_run(meta).await?;
self.persist_run_start(&run_id).await?;
self.commit_run_start(&run_id).await
}
pub async fn prepare_run_completion(
&self,
run_id: &str,
termination: car_proto::RunTermination,
) -> Result<car_proto::RunEnded, String> {
self.prepare_run_completion_for_active_owner(run_id, termination, None)
.await
}
pub(crate) async fn prepare_run_completion_for_active_owner(
&self,
run_id: &str,
termination: car_proto::RunTermination,
expected_active_client_id: Option<&str>,
) -> Result<car_proto::RunEnded, String> {
let completion_digest = run_completion_digest(&termination)?;
if expected_active_client_id.is_some() {
if let Some(gate) = &self.run_completion_fence_gate {
gate.wait_if_armed().await;
}
}
let durability_lock = self.run_durability_lock(run_id).await;
let _durability_guard = durability_lock.lock().await;
let ended = {
let mut runs = self.runs.lock().await;
let meta = runs
.get_mut(run_id)
.ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
if let Some(error) = &meta.trace_corruption {
return Err(error.clone());
}
if !meta.start_committed {
return Err(format!("run `{run_id}` start is not durably committed"));
}
if expected_active_client_id.is_some_and(|expected| meta.active_client_id != expected) {
return Err(format!(
"run `{run_id}` active owner changed before terminalization"
));
}
if let Some(requested) = &meta.cancellation_pending {
let matches_cancel = matches!(
&termination,
car_proto::RunTermination::Cancelled { cancellation }
if cancellation.run_id == requested.run_id
&& cancellation.idempotency_key == requested.idempotency_key
&& cancellation.reason_digest == requested.reason_digest
&& cancellation.principal == requested.principal
&& cancellation.action_id == requested.action_id
&& cancellation.request_id == requested.request_id
);
if !matches_cancel {
return Err(format!(
"run `{run_id}` has a durable cancellation request and is quarantined"
));
}
}
if let Some(existing) = &meta.termination {
if !same_run_termination(existing, &termination) {
return Err(format!("run `{run_id}` already has a different terminal"));
}
meta.resume_lease = None;
car_proto::RunEnded {
run_id: run_id.to_string(),
client_id: Some(meta.client_id.clone()),
agent_id: meta.agent_id.clone(),
termination,
completion_digest: Some(completion_digest),
ended_at: meta
.ended_at
.ok_or_else(|| "terminal run is missing ended_at".to_string())?,
}
} else if let Some(pending) = &meta.pending_terminal {
if !same_run_termination(&pending.termination, &termination)
|| pending.completion_digest.as_deref() != Some(completion_digest.as_str())
{
return Err(format!(
"run `{run_id}` has a different pending terminal transaction"
));
}
meta.resume_lease = None;
pending.clone()
} else {
let ended = car_proto::RunEnded {
run_id: run_id.to_string(),
client_id: Some(meta.client_id.clone()),
agent_id: meta.agent_id.clone(),
termination,
completion_digest: Some(completion_digest),
ended_at: chrono::Utc::now(),
};
meta.pending_terminal = Some(ended.clone());
meta.resume_lease = None;
meta.durability_generation = meta.durability_generation.wrapping_add(1);
ended
}
};
let store = self.run_store.clone();
let durable_ended = ended.clone();
let persisted = tokio::task::spawn_blocking(move || store.write_ended(&durable_ended))
.await
.map_err(|error| format!("RunEnded durability task failed: {error}"))?;
if let Err(error) = persisted {
if crate::run_store::is_trace_corruption_error(&error) {
let mut runs = self.runs.lock().await;
let message = self
.quarantine_run_trace_locked(&mut runs, run_id, error.to_string())
.await;
return Err(message);
}
return Err(format!("RunEnded durability failed: {error}"));
}
Ok(ended)
}
pub async fn commit_run_completion(&self, ended: &car_proto::RunEnded) -> Result<(), String> {
{
let mut runs = self.runs.lock().await;
let meta = runs
.get_mut(&ended.run_id)
.ok_or_else(|| format!("unknown run_id `{}`", ended.run_id))?;
if let Some(existing) = &meta.termination {
if same_run_termination(existing, &ended.termination)
&& meta.ended_at == Some(ended.ended_at)
{
return Ok(());
}
return Err(format!(
"run `{}` already has a different terminal",
ended.run_id
));
}
if !meta
.pending_terminal
.as_ref()
.is_some_and(|pending| same_run_ended(pending, ended))
{
return Err(format!(
"run `{}` terminal commit does not match its prepared transaction",
ended.run_id
));
}
meta.termination = Some(ended.termination.clone());
meta.ended_at = Some(ended.ended_at);
meta.pending_terminal = None;
meta.resume_lease = None;
meta.cancellation_pending = None;
meta.cancellation_receipt = None;
meta.durability_generation = meta.durability_generation.wrapping_add(1);
let cursor = meta.turn_cursor();
let status = meta.live_status();
let agent_id = meta.agent_id.clone();
let subs = self.run_subscribers.lock().await;
Self::fanout_locked(
&subs,
&ended.run_id,
&agent_id,
car_proto::RunRecord::Ended(ended.clone()),
cursor,
status,
);
}
self.clear_terminal_run_turns(&ended.run_id).await;
Ok(())
}
pub async fn complete_run(
&self,
run_id: &str,
termination: car_proto::RunTermination,
) -> Result<car_proto::RunEnded, String> {
let ended = self.prepare_run_completion(run_id, termination).await?;
self.commit_run_completion(&ended).await?;
Ok(ended)
}
pub async fn persist_run_cancellation_requested(
&self,
owner_session: &ClientSession,
requested: &car_proto::RunCancellationRequested,
) -> Result<(), String> {
let run_id = requested.run_id.as_str();
let durability_lock = self.run_durability_lock(run_id).await;
let _durability_guard = durability_lock.lock().await;
let agent_id = {
let runs = self.runs.lock().await;
let meta = runs
.get(run_id)
.ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
if meta.is_terminal() || meta.pending_terminal.is_some() {
return Err(format!("run `{run_id}` is already terminal"));
}
if meta.client_id != owner_session.client_id || !meta.start_committed {
return Err("run cancellation client/run binding mismatch".into());
}
if let Some(existing) = &meta.cancellation_pending {
if existing != requested {
return Err(format!(
"run `{run_id}` already has a different cancellation request"
));
}
return Ok(());
}
meta.agent_id.clone()
};
let store = self.run_store.clone();
let durable_agent = agent_id.clone();
let durable_requested = requested.clone();
tokio::task::spawn_blocking(move || {
store.write_cancellation_requested(&durable_agent, &durable_requested)
})
.await
.map_err(|error| format!("cancellation request durability task failed: {error}"))?
.map_err(|error| format!("cancellation request durability failed: {error}"))?;
owner_session
.append_run_cancellation_requested_event(requested)
.await?;
let mut runs = self.runs.lock().await;
let meta = runs
.get_mut(run_id)
.ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
meta.cancellation_pending = Some(requested.clone());
meta.durability_generation = meta.durability_generation.wrapping_add(1);
let cursor = meta.turn_cursor();
let agent_id = meta.agent_id.clone();
let subs = self.run_subscribers.lock().await;
Self::fanout_locked(
&subs,
run_id,
&agent_id,
car_proto::RunRecord::CancellationRequested(requested.clone()),
cursor,
car_proto::RunLiveStatus::CancellationPending,
);
Ok(())
}
pub async fn persist_run_cancellation_result(
&self,
owner_session: &ClientSession,
result: &car_proto::RunCancelResponse,
) -> Result<(), String> {
let run_id = result.run_id.as_str();
let durability_lock = self.run_durability_lock(run_id).await;
let _durability_guard = durability_lock.lock().await;
let agent_id = {
let runs = self.runs.lock().await;
let meta = runs
.get(run_id)
.ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
if meta.is_terminal() {
return Err(format!("run `{run_id}` is already terminal"));
}
let requested = meta
.cancellation_pending
.as_ref()
.ok_or_else(|| format!("run `{run_id}` has no cancellation request"))?;
if requested.idempotency_key != result.idempotency_key
|| requested.reason_digest != result.reason_digest
|| requested.principal != result.principal
{
return Err(format!("run `{run_id}` cancellation receipt mismatch"));
}
if let Some(existing) = &meta.cancellation_receipt {
if existing != result {
return Err(format!(
"run `{run_id}` has a different cancellation receipt"
));
}
return Ok(());
}
meta.agent_id.clone()
};
let store = self.run_store.clone();
let durable_agent = agent_id.clone();
let durable_result = result.clone();
tokio::task::spawn_blocking(move || {
store.write_cancellation_result(&durable_agent, &durable_result)
})
.await
.map_err(|error| format!("cancellation result durability task failed: {error}"))?
.map_err(|error| format!("cancellation result durability failed: {error}"))?;
owner_session
.append_run_cancellation_result_event(result)
.await?;
let mut runs = self.runs.lock().await;
let meta = runs
.get_mut(run_id)
.ok_or_else(|| format!("unknown run_id `{run_id}`"))?;
meta.cancellation_receipt = Some(result.clone());
meta.durability_generation = meta.durability_generation.wrapping_add(1);
let cursor = meta.turn_cursor();
let agent_id = meta.agent_id.clone();
let subs = self.run_subscribers.lock().await;
Self::fanout_locked(
&subs,
run_id,
&agent_id,
car_proto::RunRecord::CancellationResult(result.clone()),
cursor,
car_proto::RunLiveStatus::CancellationPending,
);
Ok(())
}
pub async fn persist_recovered_run_cancellation_result(
&self,
agent_id: &str,
result: &car_proto::RunCancelResponse,
) -> Result<(), String> {
if result.status != car_proto::RunCancellationStatus::TerminationUnconfirmed
|| result.terminal_digest.is_some()
{
return Err("recovered cancellation receipt must be termination_unconfirmed".into());
}
let run_id = result.run_id.as_str();
let durability_lock = self.run_durability_lock(run_id).await;
let _durability_guard = durability_lock.lock().await;
let store = self.run_store.clone();
let durable_agent = agent_id.to_string();
let durable_run = run_id.to_string();
let records = tokio::task::spawn_blocking(move || {
store.get_run_trace_for_checked(&durable_agent, &durable_run)
})
.await
.map_err(|error| format!("recovered cancellation trace task failed: {error}"))?
.map_err(|error| format!("recovered cancellation trace read failed: {error}"))?
.ok_or_else(|| "recovered cancellation durable owner is unavailable".to_string())?;
let started = records
.iter()
.find_map(|record| match record {
car_proto::RunRecord::Started(started) => Some(started),
_ => None,
})
.ok_or_else(|| "recovered cancellation is missing durable RunStarted".to_string())?;
if started.agent_id != agent_id {
return Err("recovered cancellation durable agent mismatch".into());
}
let client_id = started
.client_id
.as_deref()
.filter(|client_id| {
!client_id.is_empty()
&& !client_id.contains('/')
&& !client_id.contains('\\')
&& *client_id != "."
&& *client_id != ".."
})
.ok_or_else(|| {
"recovered cancellation has no authenticated durable journal owner".to_string()
})?
.to_string();
if records
.iter()
.any(|record| matches!(record, car_proto::RunRecord::Ended(_)))
{
return Err(format!("run `{run_id}` is already terminal"));
}
let requested = records
.iter()
.find_map(|record| match record {
car_proto::RunRecord::CancellationRequested(requested) => Some(requested),
_ => None,
})
.ok_or_else(|| format!("run `{run_id}` has no durable cancellation request"))?;
if requested.receipt_version != result.receipt_version
|| requested.run_id != result.run_id
|| requested.idempotency_key != result.idempotency_key
|| requested.reason_digest != result.reason_digest
|| requested.principal != result.principal
|| requested.action_id != result.action_id
|| requested.request_id != result.request_id
{
return Err(format!(
"run `{run_id}` recovered cancellation receipt mismatch"
));
}
if let Some(existing) = records.iter().find_map(|record| match record {
car_proto::RunRecord::CancellationResult(existing) => Some(existing),
_ => None,
}) {
if existing != result {
return Err(format!(
"run `{run_id}` has a different durable cancellation receipt"
));
}
}
{
let runs = self.runs.lock().await;
if let Some(meta) = runs.get(run_id) {
if meta.agent_id != agent_id
|| meta.client_id != client_id
|| !meta.start_committed
|| meta.is_terminal()
|| meta.cancellation_pending.as_ref() != Some(requested)
{
return Err(format!(
"run `{run_id}` live state does not authenticate recovered cancellation"
));
}
if meta
.cancellation_receipt
.as_ref()
.is_some_and(|existing| existing != result)
{
return Err(format!(
"run `{run_id}` has a different live cancellation receipt"
));
}
}
}
let store = self.run_store.clone();
let durable_agent = agent_id.to_string();
let durable_result = result.clone();
tokio::task::spawn_blocking(move || {
store.write_cancellation_result(&durable_agent, &durable_result)
})
.await
.map_err(|error| format!("recovered cancellation durability task failed: {error}"))?
.map_err(|error| format!("recovered cancellation durability failed: {error}"))?;
let journal_dir = self.journal_dir.clone();
let journal_failures = self.journal_failures.clone();
let journal_client = client_id.clone();
let journal_result = result.clone();
let journal_was_new = tokio::task::spawn_blocking(move || {
append_recovered_cancellation_result_journal(
journal_dir,
journal_failures,
journal_client,
journal_result,
)
})
.await
.map_err(|error| format!("recovered cancellation journal task failed: {error}"))??;
let mut runs = self.runs.lock().await;
let (cursor, fanout) = match runs.get_mut(run_id) {
Some(meta) => {
let fanout = meta.cancellation_receipt.is_none();
meta.cancellation_receipt = Some(result.clone());
if fanout {
meta.durability_generation = meta.durability_generation.wrapping_add(1);
}
(meta.turn_cursor(), fanout)
}
None => (
records
.iter()
.filter(|record| matches!(record, car_proto::RunRecord::Turn(_)))
.count(),
journal_was_new,
),
};
if fanout {
let subscribers = self.run_subscribers.lock().await;
Self::fanout_locked(
&subscribers,
run_id,
agent_id,
car_proto::RunRecord::CancellationResult(result.clone()),
cursor,
car_proto::RunLiveStatus::CancellationPending,
);
}
Ok(())
}
pub async fn prepare_run_incomplete(&self, run_id: &str) -> Option<car_proto::RunEnded> {
self.prepare_run_incomplete_for_active_owner(run_id, None)
.await
}
async fn prepare_run_incomplete_for_active_owner(
&self,
run_id: &str,
expected_active_client_id: Option<&str>,
) -> Option<car_proto::RunEnded> {
let termination = self
.runs
.lock()
.await
.get(run_id)
.and_then(|meta| meta.pending_terminal.as_ref())
.map(|ended| ended.termination.clone())
.unwrap_or(car_proto::RunTermination::Incomplete);
match self
.prepare_run_completion_for_active_owner(run_id, termination, expected_active_client_id)
.await
{
Ok(ended) => Some(ended),
Err(error) => {
tracing::error!(run_id, %error, "failed to prepare durable incomplete terminal");
None
}
}
}
pub async fn mark_run_incomplete(&self, run_id: &str) -> Option<car_proto::RunEnded> {
let ended = self.prepare_run_incomplete(run_id).await?;
if let Err(error) = self.commit_run_completion(&ended).await {
tracing::error!(run_id, %error, "failed to commit incomplete terminal");
return None;
}
Some(ended)
}
async fn clear_terminal_run_turns(&self, run_id: &str) {
let mut runs = self.runs.lock().await;
if let Some(meta) = runs.get_mut(run_id) {
if meta.is_terminal() {
meta.turns = Vec::new();
}
}
}
pub async fn run_meta(&self, run_id: &str) -> Option<RunMeta> {
self.runs.lock().await.get(run_id).cloned()
}
pub async fn run_header(&self, run_id: &str) -> Option<(String, bool, usize, Option<String>)> {
self.runs.lock().await.get(run_id).map(|m| {
(
m.agent_id.clone(),
m.is_terminal(),
m.turns.len(),
m.trace_corruption.clone(),
)
})
}
pub async fn run_lifecycle_binding(&self, run_id: &str) -> Option<(String, bool)> {
self.runs.lock().await.get(run_id).map(|meta| {
(
meta.active_client_id.clone(),
meta.is_terminal() || meta.trace_corruption.is_some(),
)
})
}
pub async fn run_lifecycle_state(&self, run_id: &str) -> Option<(String, bool, bool, bool)> {
self.run_lifecycle_state_with_corruption(run_id).await.map(
|(client_id, terminal, committed, pending, _)| {
(client_id, terminal, committed, pending)
},
)
}
pub async fn run_lifecycle_state_with_corruption(
&self,
run_id: &str,
) -> Option<(String, bool, bool, bool, Option<String>)> {
self.runs.lock().await.get(run_id).map(|meta| {
(
meta.active_client_id.clone(),
meta.is_terminal(),
meta.start_committed,
meta.pending_terminal.is_some() || meta.cancellation_pending.is_some(),
meta.trace_corruption.clone(),
)
})
}
pub(crate) async fn resume_run(
&self,
run_id: &str,
agent_id: &str,
replacement_client_id: &str,
) -> Result<RunResumeBinding, String> {
let durability_lock = self.run_durability_lock(run_id).await;
let _durability_guard = durability_lock.lock().await;
let _liveness_guard = self.run_resume_liveness.lock().await;
let live_clients = self
.sessions
.lock()
.await
.keys()
.cloned()
.collect::<std::collections::HashSet<_>>();
if !live_clients.contains(replacement_client_id) {
return Err("runs.resume replacement socket is no longer active".into());
}
let mut runs = self.runs.lock().await;
let meta = runs
.get_mut(run_id)
.ok_or_else(|| "run not found or not authorized".to_string())?;
if meta.agent_id != agent_id {
return Err("run not found or not authorized".into());
}
if meta.is_terminal() {
return Err(format!(
"{} run `{run_id}` is already terminal",
car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
));
}
if !meta.start_committed {
return Err(format!(
"{} run `{run_id}` start is not durably committed",
car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
));
}
if meta.pending_terminal.is_some()
|| meta.cancellation_pending.is_some()
|| meta.trace_corruption.is_some()
{
return Err(format!(
"{} run `{run_id}` is quarantined and cannot be resumed",
car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
));
}
if meta.active_client_id == replacement_client_id {
let resumed_from_client_id =
meta.resume_predecessor_client_id.clone().ok_or_else(|| {
format!(
"{} run `{run_id}` is already active on this socket",
car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
)
})?;
return Ok(RunResumeBinding {
run_id: meta.run_id.clone(),
agent_id: meta.agent_id.clone(),
active_client_id: meta.active_client_id.clone(),
resumed_from_client_id,
});
}
if live_clients.contains(&meta.active_client_id) {
return Err(format!(
"{} run `{run_id}` still has a live owner",
car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
));
}
let lease_is_current = meta.resume_lease.as_ref().is_some_and(|lease| {
lease.disconnected_client_id == meta.active_client_id
&& tokio::time::Instant::now() < lease.expires_at
});
if !lease_is_current {
return Err(format!(
"{} run `{run_id}` resume lease is unavailable or expired",
car_proto::RUN_OWNERSHIP_CONFLICT_MESSAGE_PREFIX
));
}
let resumed_from_client_id = std::mem::replace(
&mut meta.active_client_id,
replacement_client_id.to_string(),
);
meta.resume_predecessor_client_id = Some(resumed_from_client_id.clone());
meta.resume_lease = None;
meta.durability_generation = meta.durability_generation.wrapping_add(1);
Ok(RunResumeBinding {
run_id: meta.run_id.clone(),
agent_id: meta.agent_id.clone(),
active_client_id: meta.active_client_id.clone(),
resumed_from_client_id,
})
}
pub(crate) async fn run_owner_binding(&self, run_id: &str) -> Option<(String, String)> {
self.runs
.lock()
.await
.get(run_id)
.map(|meta| (meta.client_id.clone(), meta.active_client_id.clone()))
}
pub async fn record_run_turns(
self: &Arc<Self>,
run_id: &str,
records: Vec<car_proto::RunRecord>,
) -> RecordRunTurnsOutcome {
self.record_run_turns_for_owner(run_id, None, records).await
}
#[doc(hidden)]
pub async fn record_run_turns_for_active_owner(
self: &Arc<Self>,
run_id: &str,
active_client_id: &str,
records: Vec<car_proto::RunRecord>,
) -> RecordRunTurnsOutcome {
self.record_run_turns_for_owner(run_id, Some(active_client_id.to_string()), records)
.await
}
async fn record_run_turns_for_owner(
self: &Arc<Self>,
run_id: &str,
expected_active_client_id: Option<String>,
records: Vec<car_proto::RunRecord>,
) -> RecordRunTurnsOutcome {
let state = Arc::clone(self);
let run_id = run_id.to_string();
match tokio::spawn(async move {
state
.record_run_turns_owned(&run_id, expected_active_client_id.as_deref(), records)
.await
})
.await
{
Ok(outcome) => outcome,
Err(error) => RecordRunTurnsOutcome::PersistenceFailed(format!(
"durable turn append task failed: {error}"
)),
}
}
pub async fn ensure_proposal_run_turns(
self: &Arc<Self>,
pending: &crate::run_store::PendingProposalFinalization,
) -> Result<(), String> {
let state = Arc::clone(self);
let pending = pending.clone();
tokio::spawn(async move { state.ensure_proposal_run_turns_owned(&pending).await })
.await
.map_err(|error| format!("proposal trace durability task failed: {error}"))?
}
async fn ensure_proposal_run_turns_owned(
&self,
pending: &crate::run_store::PendingProposalFinalization,
) -> Result<(), String> {
let durability_lock = self.run_durability_lock(&pending.run_id).await;
let _durability_guard = durability_lock.lock().await;
let (agent_id, status, generation) = {
let runs = self.runs.lock().await;
let meta = runs
.get(&pending.run_id)
.ok_or_else(|| format!("active run `{}` is absent", pending.run_id))?;
if let Some(error) = &meta.trace_corruption {
return Err(error.clone());
}
if !meta.accepts_proposals() {
return Err(format!("active run `{}` is not writable", pending.run_id));
}
(
meta.agent_id.clone(),
meta.live_status(),
meta.durability_generation,
)
};
let store = self.run_store.clone();
let durable = pending.clone();
let durable_result =
tokio::task::spawn_blocking(move || store.ensure_proposal_turns(&durable))
.await
.map_err(|error| format!("proposal trace durability task failed: {error}"))?;
let mut runs = self.runs.lock().await;
let ensured = match durable_result {
Ok(ensured) => ensured,
Err(error) if crate::run_store::is_trace_corruption_error(&error) => {
let message = self
.quarantine_run_trace_locked(&mut runs, &pending.run_id, error.to_string())
.await;
return Err(message);
}
Err(error) => return Err(format!("proposal trace durability failed: {error}")),
};
let meta = runs.get(&pending.run_id).ok_or_else(|| {
format!(
"active run `{}` disappeared after durability",
pending.run_id
)
})?;
if let Some(error) = &meta.trace_corruption {
return Err(error.clone());
}
if meta.durability_generation != generation || !meta.accepts_proposals() {
return Err(format!(
"active run `{}` changed during proposal trace durability",
pending.run_id
));
}
let already_live = !ensured.appended
&& meta.turns.iter().any(|record| match record {
car_proto::RunRecord::Turn(turn) => {
turn.proposal_id.as_deref() == Some(pending.final_proposal_id.as_str())
}
_ => false,
});
if ensured.appended || !already_live {
let meta = runs
.get_mut(&pending.run_id)
.expect("run registry remains locked during proposal trace commit");
let base = ensured
.records
.first()
.and_then(|record| match record {
car_proto::RunRecord::Turn(turn) => Some(turn.index),
_ => None,
})
.unwrap_or(meta.turns.len());
meta.turns.extend(ensured.records.iter().cloned());
meta.durability_generation = meta.durability_generation.wrapping_add(1);
let subs = self.run_subscribers.lock().await;
for (offset, record) in ensured.records.into_iter().enumerate() {
Self::fanout_locked(
&subs,
&pending.run_id,
&agent_id,
record,
base + offset + 1,
status,
);
}
}
Ok(())
}
async fn record_run_turns_owned(
&self,
run_id: &str,
expected_active_client_id: Option<&str>,
mut records: Vec<car_proto::RunRecord>,
) -> RecordRunTurnsOutcome {
let durability_lock = self.run_durability_lock(run_id).await;
let _durability_guard = durability_lock.lock().await;
let (agent_id, base, new_total, status, generation) = {
let mut runs = self.runs.lock().await;
match runs.get_mut(run_id) {
Some(meta) => {
if let Some(error) = &meta.trace_corruption {
return RecordRunTurnsOutcome::PersistenceFailed(error.clone());
}
if expected_active_client_id
.is_some_and(|expected| meta.active_client_id != expected)
{
return RecordRunTurnsOutcome::UnknownOrTerminal;
}
if !meta.accepts_proposals() {
return RecordRunTurnsOutcome::UnknownOrTerminal;
}
let incoming_turns = records
.iter()
.filter(|r| matches!(r, car_proto::RunRecord::Turn(_)))
.count();
if meta.turns.len() + incoming_turns > RECORD_TURNS_RUN_CEILING {
return RecordRunTurnsOutcome::RefusedCeiling;
} else {
let base = meta.turns.len();
for (offset, record) in records.iter_mut().enumerate() {
if let car_proto::RunRecord::Turn(turn) = record {
turn.index = base + offset;
}
}
let agent_id = meta.agent_id.clone();
let status = meta.live_status();
(
agent_id,
base,
base + records.len(),
status,
meta.durability_generation,
)
}
}
None => return RecordRunTurnsOutcome::UnknownOrTerminal,
}
};
let store = self.run_store.clone();
let durable_agent = agent_id.clone();
let durable_run = run_id.to_string();
let durable_records = records.clone();
let append = tokio::task::spawn_blocking(move || {
store.append_turns(&durable_agent, &durable_run, &durable_records)
})
.await;
let mut runs = self.runs.lock().await;
match append {
Ok(Ok(())) => {}
Ok(Err(error)) => {
tracing::warn!(run_id, %error, "run-store: failed to persist turns");
if crate::run_store::is_trace_corruption_error(&error) {
let message = self
.quarantine_run_trace_locked(&mut runs, run_id, error.to_string())
.await;
return RecordRunTurnsOutcome::PersistenceFailed(message);
}
return RecordRunTurnsOutcome::PersistenceFailed(error.to_string());
}
Err(error) => {
tracing::warn!(run_id, %error, "run-store: durable turn task failed");
return RecordRunTurnsOutcome::PersistenceFailed(error.to_string());
}
}
let meta = runs
.get_mut(run_id)
.expect("run registry entry survives a committed turn append");
if let Some(error) = &meta.trace_corruption {
return RecordRunTurnsOutcome::PersistenceFailed(error.clone());
}
if meta.durability_generation != generation || !meta.accepts_proposals() {
return RecordRunTurnsOutcome::PersistenceFailed(format!(
"run `{run_id}` changed during durable turn append"
));
}
meta.turns.extend(records.iter().cloned());
meta.durability_generation = meta.durability_generation.wrapping_add(1);
let subs = self.run_subscribers.lock().await;
for (offset, record) in records.into_iter().enumerate() {
Self::fanout_locked(&subs, run_id, &agent_id, record, base + offset + 1, status);
}
RecordRunTurnsOutcome::Appended { new_total }
}
pub async fn run_turn_count(&self, run_id: &str) -> usize {
self.runs
.lock()
.await
.get(run_id)
.map(|m| m.turns.len())
.unwrap_or(0)
}
pub async fn run_turns(&self, run_id: &str) -> Vec<car_proto::RunRecord> {
self.runs
.lock()
.await
.get(run_id)
.map(|m| m.turns.clone())
.unwrap_or_default()
}
pub(crate) async fn subscribe_run_page(
&self,
run_id: &str,
host_client_id: &str,
channel: Arc<WsChannel>,
cursor: usize,
limit: usize,
) -> Result<Option<RunSubscribePageResult>, String> {
let subscriber =
crate::host::RunTraceSubscriber::spawn(host_client_id.to_string(), channel);
let (agent_id, mut expected_state) = {
let runs = self.runs.lock().await;
let Some(meta) = runs.get(run_id) else {
return Ok(None);
};
if let Some(error) = &meta.trace_corruption {
return Err(error.clone());
}
if meta.is_terminal() {
return Ok(Some(RunSubscribePageResult::Durable {
agent_id: meta.agent_id.clone(),
status: meta.live_status(),
}));
}
(
meta.agent_id.clone(),
(meta.turns.len(), meta.pending_terminal.is_some()),
)
};
for attempt in 0..=RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT {
let store = self.run_store.clone();
let durable_agent = agent_id.clone();
let durable_run = run_id.to_string();
let durable_corruption = tokio::task::spawn_blocking(move || {
store.run_trace_corruption_for(&durable_agent, &durable_run)
})
.await
.map_err(|error| format!("run trace corruption check task failed: {error}"))?
.map_err(|error| format!("run trace corruption check failed: {error}"))?;
let mut runs = self.runs.lock().await;
let Some(meta) = runs.get(run_id) else {
return Ok(None);
};
if let Some(error) = &meta.trace_corruption {
return Err(error.clone());
}
if meta.agent_id != agent_id {
return Err(format!("run `{run_id}` changed ownership during subscribe"));
}
if let Some(corruption) = durable_corruption {
let message = self
.quarantine_run_trace_locked(
&mut runs,
run_id,
format!("malformed run trace record at line {}", corruption.line),
)
.await;
return Err(message);
}
let status = meta.live_status();
if meta.is_terminal() {
return Ok(Some(RunSubscribePageResult::Durable { agent_id, status }));
}
let current_state = (meta.turns.len(), meta.pending_terminal.is_some());
if current_state != expected_state {
if attempt == RUN_SUBSCRIBE_SUMMARY_STATE_RETRY_LIMIT {
return Err(format!(
"run `{run_id}` changed repeatedly during durable subscribe validation; retry"
));
}
expected_state = current_state;
drop(runs);
continue;
}
let live_cursor = meta.turns.len();
if cursor > live_cursor {
return Err(format!(
"runs.subscribe cursor {cursor} exceeds live_cursor {live_cursor}"
));
}
let end = cursor.saturating_add(limit).min(live_cursor);
let turns = meta.turns[cursor..end].to_vec();
let next_cursor = (end < live_cursor).then_some(end);
let subscribed = next_cursor.is_none();
if subscribed {
let mut subs = self.run_subscribers.lock().await;
subs.insert((run_id.to_string(), host_client_id.to_string()), subscriber);
}
drop(runs);
return Ok(Some(RunSubscribePageResult::Ready(
car_proto::RunSubscribeResponse {
run_id: run_id.to_string(),
agent_id,
turns,
cursor,
limit,
next_cursor,
live_cursor,
subscribed,
status,
},
)));
}
unreachable!("bounded subscribe validation loop always returns")
}
pub async fn unsubscribe_run(&self, run_id: &str, host_client_id: &str) -> bool {
self.run_subscribers
.lock()
.await
.remove(&(run_id.to_string(), host_client_id.to_string()))
.is_some()
}
pub async fn drop_run_subscribers_for_client(&self, host_client_id: &str) {
self.run_subscribers
.lock()
.await
.retain(|(_run, client), _| client != host_client_id);
}
async fn sweep_runs_for_disconnect(&self, session: &ClientSession) {
let client_id = session.client_id.as_str();
let pending: Vec<(String, Option<tokio::time::Instant>)> = {
let runs = self.runs.lock().await;
runs.values()
.filter(|m| m.active_client_id == client_id && !m.is_terminal())
.map(|m| {
let resume_deadline = m.resume_lease.as_ref().and_then(|lease| {
(lease.disconnected_client_id == client_id).then_some(lease.expires_at)
});
(m.run_id.clone(), resume_deadline)
})
.collect()
};
if pending.is_empty() {
return;
}
for (run_id, resume_deadline) in pending {
match resume_deadline {
Some(deadline) => tokio::time::sleep_until(deadline).await,
None => tokio::time::sleep(RUN_COMPLETE_GRACE).await,
}
match self.run_lifecycle_state(&run_id).await {
None | Some((_, true, _, _)) => continue,
Some((owner, false, _, _)) if owner != client_id => {
continue;
}
Some((_, false, true, _)) => {}
Some((_, false, false, _)) => {
match self
.reconcile_or_release_unacknowledged_start(session, &run_id)
.await
{
Ok(true) => continue,
Ok(false) => {}
Err(error) => {
tracing::error!(run_id, client_id, %error, "failed to reconcile or release an unacknowledged RunStarted transaction");
continue;
}
}
}
}
match self.run_store.pending_proposal(&run_id) {
Ok(Some(_)) => {
tracing::warn!(
run_id,
client_id,
"leaving run open because proposal finalization is pending durable replay"
);
continue;
}
Err(error) => {
tracing::error!(run_id, client_id, %error, "leaving run open because proposal finalization state is unreadable and outcome is unknown");
continue;
}
Ok(None) => {}
}
match self.run_store.execution_marker(&run_id) {
Ok(Some(marker)) => {
tracing::warn!(
run_id,
client_id,
original_proposal_id = marker.original_proposal_id,
"leaving run open because proposal execution outcome is unknown"
);
continue;
}
Err(error) => {
tracing::error!(run_id, client_id, %error, "leaving run open because proposal execution marker is unreadable and outcome is unknown");
continue;
}
Ok(None) => {}
}
if let Some(ended) = self
.prepare_run_incomplete_for_active_owner(&run_id, Some(client_id))
.await
{
let durable_client_id = ended.client_id.as_deref().unwrap_or(client_id);
let append_result = match session
.append_run_terminal_event_once(&ended, durable_client_id)
.await
{
Err(error) if error.is_retry_safe() => {
tracing::warn!(
run_id,
client_id,
%error,
"disconnect terminal durability is unknown; retrying the exact journal row once"
);
session
.append_run_terminal_event_once(&ended, durable_client_id)
.await
}
result => result,
};
if let Err(error) = append_result {
tracing::error!(run_id, client_id, %error, "failed to bind disconnect terminal journal event");
continue;
}
if let Err(error) = self.commit_run_completion(&ended).await {
tracing::error!(run_id, client_id, %error, "failed to commit disconnect terminal");
continue;
}
if let Err(error) = session.clear_run_journal_binding(&run_id).await {
tracing::error!(run_id, client_id, %error, "failed to clear durable disconnect journal binding");
continue;
}
let mut current = session.current_run_id.lock().await;
if current.as_deref() == Some(run_id.as_str()) {
*current = None;
}
}
}
}
pub async fn create_session(
&self,
client_id: &str,
channel: Arc<WsChannel>,
) -> Result<Arc<ClientSession>, String> {
let journal_path = self.journal_dir.join(format!("{}.jsonl", client_id));
let event_log = match self.journal_failures.clone() {
Some(failures) => EventLog::with_journal_failure_injector(journal_path, failures),
None => EventLog::with_journal(journal_path),
};
let negotiated_capabilities =
Arc::new(std::sync::RwLock::new(std::collections::BTreeSet::new()));
let ws_executor = Arc::new(WsToolExecutor::new(
channel.clone(),
negotiated_capabilities.clone(),
));
let executor: Arc<dyn ToolExecutor> =
Arc::new(self.mcp_executor.share_with_fallback(ws_executor));
let outbound = Arc::new(car_messaging::outbound::OutboundRegistry::new());
outbound.register(Arc::new(
car_messaging::outbound::ImessageOutboundAdapter::new(
Arc::new(crate::messaging_orchestrator::RealMessageSender),
crate::messaging_config::MessagingConfigStore::from_home(),
),
));
outbound.set_fallback(Arc::new(crate::host_channel::HostChannelAdapter::new(
executor.clone(),
)));
let runtime = Runtime::new()
.with_event_log(event_log)
.with_executor(executor)
.with_trajectory_store(self.trajectory_store.clone())
.with_message_sink(outbound);
if let Some(car_dir) = car_home_dir() {
apply_project_policies(&runtime, &car_dir).await?;
}
let permission_gate = Arc::new(tokio::sync::RwLock::new(car_policy::PermissionGate::new(
car_policy::PermissionTier::SandboxEdit,
)));
let authenticated = Arc::new(std::sync::atomic::AtomicBool::new(false));
let agent_id = Arc::new(tokio::sync::Mutex::new(None));
let callback_tool_schema_digests = Arc::new(tokio::sync::RwLock::new(HashMap::new()));
let memgine = match &self.shared_memgine {
Some(eng) => eng.clone(),
None => Arc::new(Mutex::new(car_memgine::MemgineEngine::new(None))),
};
runtime
.register_admission_gate(Arc::new(car_engine::StaticVerificationGate::new(
runtime.tools.clone(),
)))
.await;
runtime
.register_admission_gate(Arc::new(crate::supervision::SupervisionGate::new(
self.supervision.clone(),
)))
.await;
runtime
.register_admission_gate(Arc::new(
crate::permission_gate::PermissionAdmissionGate::new(
permission_gate.clone(),
self.approval_ledger.clone(),
)
.with_authenticated_agent(authenticated.clone(), agent_id.clone())
.with_callback_tools(
callback_tool_schema_digests.clone(),
runtime.registry.clone(),
)
.with_skill_memgine(memgine.clone())
.with_event_log(runtime.log.clone()),
))
.await;
let session = Arc::new(ClientSession {
client_id: client_id.to_string(),
runtime: Arc::new(runtime),
channel,
host: self.host.clone(),
memgine,
browser: car_ffi_common::browser::BrowserSessionSlot::new(),
authenticated,
negotiated_protocol_version: std::sync::atomic::AtomicU32::new(0),
negotiated_capabilities,
inference_control: Arc::new(crate::inference_control::InferenceRegistry::default()),
is_host: std::sync::atomic::AtomicBool::new(false),
agent_id,
callback_tool_schema_digests,
memory_namespace: tokio::sync::Mutex::new(None),
bound_memgine: tokio::sync::Mutex::new(None),
current_run_id: tokio::sync::Mutex::new(None),
run_lifecycle_guard: Arc::new(tokio::sync::Mutex::new(())),
permission_gate,
evolution_guard: crate::evolution::CycleGuard::default(),
last_chat_turn: tokio::sync::Mutex::new(None),
chat_inflight: std::sync::atomic::AtomicUsize::new(0),
tenant: tokio::sync::Mutex::new(None),
tool_stream_subscribed: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
});
if self.connectors.get().is_some() {
for entry in self.connectors().enabled_tool_entries().await {
session.runtime.register_tool_entry(entry).await;
}
}
{
let _liveness_guard = self.run_resume_liveness.lock().await;
self.sessions
.lock()
.await
.insert(client_id.to_string(), session.clone());
}
Ok(session)
}
pub async fn bind_substrate_to_connector(
&self,
session: &Arc<ClientSession>,
slug: &str,
) -> Result<String, String> {
self.ensure_connectors_loaded().await;
let mcp_session = self
.mcp_executor
.session(slug)
.await
.ok_or_else(|| format!("connector '{slug}' is not connected"))?;
let substrate: Arc<dyn car_engine::Substrate> =
Arc::new(car_engine::McpSubstrate::new(mcp_session, slug.to_string()));
let name = substrate.name().to_string();
session.runtime.set_substrate(substrate).await;
session.runtime.register_agent_basics().await;
let ws_executor = Arc::new(WsToolExecutor::new(
session.channel.clone(),
session.negotiated_capabilities.clone(),
));
let composed: Arc<dyn ToolExecutor> =
Arc::new(self.mcp_executor.share_with_fallback(ws_executor));
let shadowed: Arc<dyn ToolExecutor> = Arc::new(SubstrateShadowExecutor::new(composed));
session.runtime.set_executor(shadowed).await;
Ok(name)
}
pub async fn remove_session(&self, client_id: &str) -> Option<Arc<ClientSession>> {
if let Some(session) = self.sessions.lock().await.get(client_id).cloned() {
let n = session.runtime.tool_handles.cancel_all().await;
if n > 0 {
tracing::info!(
client_id,
cancelled = n,
"cancelled detached tools at session teardown"
);
}
}
let removed = {
let _liveness_guard = self.run_resume_liveness.lock().await;
let removed = self.sessions.lock().await.remove(client_id);
let resume_negotiated = removed.as_ref().is_some_and(|session| {
session
.negotiated_capabilities
.read()
.map(|caps| caps.contains(car_proto::RUNS_RESUME_CAPABILITY))
.unwrap_or(false)
});
if resume_negotiated {
let expires_at = tokio::time::Instant::now() + self.run_resume_lease;
let mut runs = self.runs.lock().await;
for meta in runs
.values_mut()
.filter(|meta| meta.active_client_id == client_id && !meta.is_terminal())
{
meta.resume_lease = Some(RunResumeLease {
disconnected_client_id: client_id.to_string(),
expires_at,
});
}
}
removed
};
if let Some(session) = &removed {
let bound = session.agent_id.lock().await.clone();
if let Some(id) = bound {
let mut attached = self.attached_agents.lock().await;
if attached.get(&id).map(String::as_str) == Some(client_id) {
attached.remove(&id);
}
}
let bound_for_guards = session.agent_id.lock().await.clone();
if let Some(id) = bound_for_guards {
self.peer_guards.lock().await.remove(&id);
}
let bound_agent = session.agent_id.lock().await.clone();
let mut chats = self.chat_sessions.lock().await;
chats.retain(|_, s| {
if s.host_client_id == client_id {
return false;
}
if let Some(agent_id) = &bound_agent {
if &s.agent_id == agent_id {
return false;
}
}
true
});
drop(chats);
self.chat_collectors
.lock()
.await
.retain(|_, c| c.host_client_id != client_id);
self.drop_run_subscribers_for_client(client_id).await;
self.browser_views
.drop_subscriptions_for_client(client_id)
.await;
if session.is_host.load(std::sync::atomic::Ordering::Acquire) {
let host_connected = self.any_host_connected().await;
self.browser_views
.broadcast_host_connected(host_connected)
.await;
}
self.browser_views
.note_producer_disconnected(client_id)
.await;
crate::coder::rpc::drop_subscriptions_for_client(self, client_id).await;
crate::coder::discuss::drop_subscriptions_for_client(self, client_id).await;
let _run_guard = session.run_lifecycle_guard.lock().await;
self.sweep_runs_for_disconnect(session).await;
}
removed
}
}
#[cfg(test)]
mod durable_task_tests {
use super::*;
#[tokio::test]
async fn dropping_response_waiter_does_not_cancel_operation_or_release_its_guard() {
let temp = tempfile::tempdir().unwrap();
let state = Arc::new(ServerState::with_config(ServerStateConfig::new(
temp.path().to_path_buf(),
)));
let overlap_guard = Arc::new(tokio::sync::Mutex::new(()));
let operation_guard = overlap_guard.clone();
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
let response = state
.spawn_durable_operation("auth.test", async move {
let _held = operation_guard.lock().await;
let _ = started_tx.send(());
let _ = release_rx.await;
Ok::<_, String>(serde_json::json!({ "ok": true }))
})
.await;
started_rx.await.unwrap();
drop(response);
assert!(
overlap_guard.try_lock().is_err(),
"dropping only the connection waiter must not release the operation guard"
);
release_tx.send(()).unwrap();
let joined = tokio::time::timeout(
std::time::Duration::from_secs(1),
state.durable_tasks.lock().await.join_next(),
)
.await
.expect("daemon-owned operation should finish")
.expect("task should exist")
.expect("task should join");
assert_eq!(joined, "auth.test");
assert!(
overlap_guard.try_lock().is_ok(),
"the guard releases only after the daemon-owned operation finishes"
);
}
}
#[cfg(test)]
mod tool_timeout_tests {
use super::*;
#[test]
fn honors_action_timeout_over_default() {
assert_eq!(
tool_callback_timeout(Some(180_000)),
std::time::Duration::from_millis(180_000 + TOOL_TIMEOUT_GRACE_MS)
);
assert!(tool_callback_timeout(Some(600_000)) > std::time::Duration::from_secs(600));
assert!(tool_callback_timeout(Some(180_000)) >= std::time::Duration::from_millis(180_000));
}
#[test]
fn default_is_not_the_old_60s() {
if std::env::var_os("CAR_TOOL_TIMEOUT").is_none() {
assert_eq!(
tool_callback_timeout(None),
std::time::Duration::from_millis(DEFAULT_TOOL_TIMEOUT_MS)
);
assert!(
DEFAULT_TOOL_TIMEOUT_MS > 60_000,
"default must exceed the old 60s"
);
}
}
}
#[cfg(test)]
mod observer_mode_tests {
use super::*;
fn journal_dir() -> PathBuf {
let target = std::env::var_os("CARGO_TARGET_DIR")
.map(std::path::PathBuf::from)
.unwrap_or_else(|| {
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("..")
.join("target")
});
std::fs::create_dir_all(&target).ok();
let target = std::fs::canonicalize(&target).unwrap_or(target);
let tmp = tempfile::TempDir::new_in(&target).unwrap();
let p = tmp.path().to_path_buf();
std::mem::forget(tmp); p
}
#[test]
fn supervisor_returns_observer_error_when_marker_set() {
let state = ServerState::standalone(journal_dir());
let fake_manifest = PathBuf::from("/tmp/fake-manifest-for-test.json");
state
.install_observer_manifest(fake_manifest.clone())
.expect("install_observer_manifest succeeds on fresh state");
assert_eq!(state.observer_manifest_path(), Some(&fake_manifest));
let err = state.supervisor().map(|_| ()).unwrap_err();
assert!(
err.contains("observe-only"),
"error must mention observe-only mode: {err}"
);
assert!(
err.contains("fake-manifest-for-test.json"),
"error must surface the manifest path so operators know which daemon owns it: {err}"
);
}
#[test]
fn install_observer_manifest_is_idempotent_per_path_collision() {
let state = ServerState::standalone(journal_dir());
let p = PathBuf::from("/tmp/manifest-a.json");
let q = PathBuf::from("/tmp/manifest-b.json");
state.install_observer_manifest(p.clone()).unwrap();
let err = state.install_observer_manifest(q.clone()).unwrap_err();
assert_eq!(err, q);
assert_eq!(state.observer_manifest_path(), Some(&p));
}
#[test]
fn supervisor_if_installed_does_not_lazy_init() {
let state = ServerState::standalone(journal_dir());
assert!(state.supervisor_if_installed().is_none());
assert!(state.observer_manifest_path().is_none());
}
}
#[cfg(test)]
mod substrate_binding_tests {
use super::*;
use car_engine::{McpSession, McpToolInfo};
use car_ir::ActionProposal;
use serde_json::json;
use std::sync::Mutex as StdMutex;
fn journal_dir() -> PathBuf {
let target = std::env::var_os("CARGO_TARGET_DIR")
.map(std::path::PathBuf::from)
.unwrap_or_else(|| {
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("..")
.join("target")
});
std::fs::create_dir_all(&target).ok();
let target = std::fs::canonicalize(&target).unwrap_or(target);
let tmp = tempfile::TempDir::new_in(&target).unwrap();
let p = tmp.path().to_path_buf();
std::mem::forget(tmp);
p
}
struct FakeVmSession {
name: String,
files: Arc<StdMutex<std::collections::HashMap<String, String>>>,
}
#[async_trait::async_trait]
impl McpSession for FakeVmSession {
async fn list_tools(&mut self) -> Result<Vec<McpToolInfo>, String> {
Ok(vec![])
}
async fn call_tool(
&mut self,
name: &str,
args: serde_json::Value,
) -> Result<serde_json::Value, String> {
match name {
"write_text" => {
let p = args["path"].as_str().unwrap().to_string();
let c = args["content"].as_str().unwrap().to_string();
self.files.lock().unwrap().insert(p, c);
Ok(json!("ok"))
}
"read_text" => {
let p = args["path"].as_str().unwrap();
let c = self
.files
.lock()
.unwrap()
.get(p)
.cloned()
.ok_or_else(|| "not found".to_string())?;
Ok(json!(c))
}
"run_command" => {
let command = args["command"].as_str().unwrap_or_default();
if let Some(path) = command
.split_once("[ -e '")
.map(|(_, path)| path)
.and_then(|path| path.split_once("' ]").map(|(path, _)| path))
{
let exists = self.files.lock().unwrap().contains_key(path);
return Ok(json!({
"stdout": "",
"stderr": "",
"exit_code": if exists { 0 } else { 1 }
}));
}
Ok(json!({
"stdout": "from-vm",
"stderr": "",
"exit_code": 0
}))
}
other => Err(format!("unknown tool {other}")),
}
}
fn name(&self) -> &str {
&self.name
}
}
fn read_file_proposal(path: &str) -> ActionProposal {
serde_json::from_value(json!({
"source": "test",
"actions": [{
"id": "r0",
"type": "tool_call",
"tool": "read_file",
"parameters": { "path": path },
"dependencies": [],
}],
}))
.expect("proposal deserializes")
}
#[tokio::test]
async fn session_runtime_has_the_messaging_send_tool() {
let state = Arc::new(ServerState::standalone(journal_dir()));
let session = state
.create_session("c-messaging", Arc::new(WsChannel::test_stub()))
.await
.unwrap();
assert!(
session
.runtime
.tools
.read()
.await
.contains_key("messaging.send"),
"every session must be able to execute messaging.send"
);
}
#[tokio::test]
async fn default_session_substrate_is_local() {
let state = Arc::new(ServerState::standalone(journal_dir()));
let channel = Arc::new(WsChannel::test_stub());
let session = state.create_session("c-default", channel).await.unwrap();
let sub = session.runtime.substrate().await;
assert_eq!(
sub.name(),
"local",
"an un-bound session must keep the default LocalSubstrate"
);
assert!(sub.is_local(), "default substrate must be the host");
}
#[tokio::test]
async fn bind_unknown_connector_errors_and_keeps_local() {
let state = Arc::new(ServerState::standalone(journal_dir()));
let channel = Arc::new(WsChannel::test_stub());
let session = state.create_session("c-missing", channel).await.unwrap();
let err = state
.bind_substrate_to_connector(&session, "nope")
.await
.unwrap_err();
assert!(err.contains("not connected"), "got: {err}");
assert_eq!(session.runtime.substrate().await.name(), "local");
}
#[tokio::test]
async fn bound_session_routes_builtins_to_substrate() {
let state = Arc::new(ServerState::standalone(journal_dir()));
let channel = Arc::new(WsChannel::test_stub());
let session = state.create_session("c-vm", channel).await.unwrap();
let files = Arc::new(StdMutex::new(std::collections::HashMap::new()));
let fake: Arc<Mutex<dyn McpSession>> = Arc::new(Mutex::new(FakeVmSession {
name: "vm".into(),
files: files.clone(),
}));
state.mcp_executor.add_session("vm", fake).await;
let bound = state
.bind_substrate_to_connector(&session, "vm")
.await
.expect("bind succeeds for a connected connector");
assert_eq!(bound, "vm");
assert_eq!(session.runtime.substrate().await.name(), "vm");
let write = serde_json::from_value::<ActionProposal>(json!({
"source": "test",
"actions": [{
"id": "w0",
"type": "tool_call",
"tool": "write_file",
"parameters": { "path": "/vm/a.txt", "content": "vm-bytes" },
"dependencies": [],
}],
}))
.unwrap();
let wres = session.runtime.execute(&write).await;
assert!(
wres.results[0].error.is_none(),
"write_file via substrate failed: {:?}",
wres.results[0].error
);
assert_eq!(
files.lock().unwrap().get("/vm/a.txt").map(String::as_str),
Some("vm-bytes"),
"write_file must land on the bound VM substrate"
);
let rres = session
.runtime
.execute(&read_file_proposal("/vm/a.txt"))
.await;
assert!(
rres.results[0].error.is_none(),
"read_file via substrate failed: {:?}",
rres.results[0].error
);
let out = rres.results[0].output.clone().unwrap_or(Value::Null);
let content = out.get("content").and_then(|v| v.as_str()).unwrap_or("");
assert_eq!(
content, " 1\tvm-bytes",
"read_file must come from the VM substrate"
);
}
#[tokio::test]
async fn shadow_executor_only_shadows_builtin_names() {
struct Inner;
#[async_trait::async_trait]
impl ToolExecutor for Inner {
async fn execute(&self, tool: &str, _p: &Value) -> Result<Value, String> {
Ok(json!({ "delegated": tool }))
}
}
let shadow = SubstrateShadowExecutor::new(Arc::new(Inner));
for name in SUBSTRATE_OWNED_TOOLS {
let err = shadow.execute(name, &json!({})).await.unwrap_err();
assert!(
err.starts_with("unknown tool"),
"{name} must be shadowed so the engine falls through: {err}"
);
}
for passthrough in ["mcp_vm_run_command", "calculate", "browser.run"] {
let out = shadow.execute(passthrough, &json!({})).await.unwrap();
assert_eq!(out["delegated"], json!(passthrough));
}
}
}
#[cfg(test)]
mod project_policy_loading_tests {
use super::*;
fn runtime() -> Runtime {
Runtime::new()
}
#[tokio::test]
async fn a_missing_policies_directory_is_silent() {
let dir = tempfile::tempdir().unwrap();
apply_project_policies(&runtime(), &dir.path().join(".car"))
.await
.expect("a project with no rules must start normally");
}
#[tokio::test]
async fn an_empty_policies_directory_is_silent() {
let dir = tempfile::tempdir().unwrap();
let car = dir.path().join(".car");
std::fs::create_dir_all(car.join("policies")).unwrap();
apply_project_policies(&runtime(), &car).await.unwrap();
}
#[tokio::test]
async fn a_well_formed_rule_file_loads_and_takes_effect() {
let dir = tempfile::tempdir().unwrap();
let car = dir.path().join(".car");
std::fs::create_dir_all(car.join("policies")).unwrap();
std::fs::write(
car.join("policies").join("messaging.toml"),
"deny_tool = [\"messaging.send\"]\n\n\
[[deny_tool_param]]\n\
tool = \"messaging.send\"\n\
param = \"channel\"\n\
equals = \"slack\"\n",
)
.unwrap();
let rt = runtime();
apply_project_policies(&rt, &car).await.unwrap();
let names: Vec<String> = rt
.list_policies(None)
.await
.unwrap()
.into_iter()
.map(|(name, _)| name)
.collect();
assert!(
names.len() >= 2,
"both declarative rules must reach the policy engine, got {names:?}"
);
}
#[tokio::test]
async fn a_malformed_rule_file_fails_loudly_and_names_the_file() {
let dir = tempfile::tempdir().unwrap();
let car = dir.path().join(".car");
std::fs::create_dir_all(car.join("policies")).unwrap();
std::fs::write(
car.join("policies").join("broken.toml"),
"[[deny_tool]\ntool = \"shell\"",
)
.unwrap();
let err = apply_project_policies(&runtime(), &car)
.await
.expect_err("a malformed policy file must NOT be swallowed");
assert!(
err.contains("broken.toml"),
"the operator has to know which file to fix: {err}"
);
assert!(err.contains("refusing to start"), "{err}");
}
}
#[cfg(test)]
mod org_scope_wiring_tests {
use super::*;
fn granter_hex_for(secret: &[u8], user: &str) -> String {
let master = car_sync::StretchedMaster::from_issued_high_entropy(secret, user);
let vk = car_sync::ed25519_verifying(&car_sync::derive_ed25519_identity(&master, user));
vk.to_bytes().iter().map(|b| format!("{b:02x}")).collect()
}
fn granter_hex() -> String {
granter_hex_for(b"granter-login", "acc_granter")
}
#[test]
fn unset_env_is_the_personal_only_path() {
assert!(ServerState::parse_org_scope_config(None).unwrap().is_none());
assert!(ServerState::parse_org_scope_config(Some(String::new()))
.unwrap()
.is_none());
}
#[test]
fn wellformed_env_parses_org_and_granter() {
let hex = granter_hex();
let (org, granters) = ServerState::parse_org_scope_config(Some(format!("orgtest:{hex}")))
.unwrap()
.expect("well-formed opt-in parses");
assert_eq!(org, "orgtest");
assert_eq!(granters.len(), 1);
let expected_master =
car_sync::StretchedMaster::from_issued_high_entropy(b"granter-login", "acc_granter");
let expected = car_sync::ed25519_verifying(&car_sync::derive_ed25519_identity(
&expected_master,
"acc_granter",
));
assert_eq!(granters[0].to_bytes(), expected.to_bytes());
}
#[test]
fn multiple_granters_parse_in_order() {
let g1 = granter_hex_for(b"g1", "acc_g1");
let g2 = granter_hex_for(b"g2", "acc_g2");
let (org, granters) =
ServerState::parse_org_scope_config(Some(format!("orgtest:{g1},{g2}")))
.unwrap()
.expect("well-formed multi-granter opt-in parses");
assert_eq!(org, "orgtest");
assert_eq!(granters.len(), 2);
assert_ne!(granters[0].to_bytes(), granters[1].to_bytes());
let spaced = ServerState::parse_org_scope_config(Some(format!("orgtest:{g1} , {g2}")))
.unwrap()
.unwrap();
assert_eq!(spaced.1.len(), 2);
}
#[test]
fn one_bad_granter_in_the_list_fails_the_whole_value() {
let good = granter_hex();
assert!(
ServerState::parse_org_scope_config(Some(format!("orgtest:{good},not-hex"))).is_err(),
"a malformed granter among valid ones must be a hard error"
);
assert!(ServerState::parse_org_scope_config(Some("orgtest:".into())).is_err());
assert!(ServerState::parse_org_scope_config(Some("orgtest: , ".into())).is_err());
}
#[test]
fn malformed_env_is_a_hard_error_never_silent() {
let hex = granter_hex();
let bads = [
"no-colon-here".to_string(), ":deadbeef".to_string(), "orgtest:not-hex".to_string(), "orgtest:aabb".to_string(), format!("bad org:{hex}"), format!("org/evil:{hex}"), ];
for bad in &bads {
assert!(
ServerState::parse_org_scope_config(Some(bad.clone())).is_err(),
"malformed value {bad:?} must be a hard error"
);
}
}
#[test]
fn scope_routing_sends_only_matching_org_to_the_org_subsystem() {
use car_sync::Scope;
let acme = Scope::Shared { org: "acme".into() };
let globex = Scope::Shared {
org: "globex".into(),
};
assert_eq!(route_for_scope(&Scope::Personal, None), SyncRoute::User);
assert_eq!(route_for_scope(&acme, None), SyncRoute::User);
assert_eq!(route_for_scope(&acme, Some("acme")), SyncRoute::Org);
assert_eq!(
route_for_scope(&Scope::Personal, Some("acme")),
SyncRoute::User
);
assert_eq!(
route_for_scope(&globex, Some("acme")),
SyncRoute::User,
"a different org must NOT go to acme's delivery subsystem"
);
}
}