use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::Arc;
use std::time::Duration;
use std::{io::Read, io::Write};
use futures::{SinkExt, StreamExt};
use incurs_codemode::{
ExecutionState as CodeExecutionState, ExecutionStatus as CodeExecutionStatus, RuntimeStore,
Snippet,
};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sha2::{Digest, Sha256};
use tokio::net::{UnixListener, UnixStream};
use tokio::sync::RwLock;
use tokio_util::codec::{Framed, LengthDelimitedCodec};
use tokio_util::sync::CancellationToken;
use crate::code::{CodePool, capability_descriptors, execution_summary, external_connector};
use crate::execution::ExecutionManager;
use crate::model::{
AgentSession, CausalEvent, ConnectorConfig, ExecutionSpec, ExecutionStatus, Lease,
LeaseOwnerKind, ProcessController, Recipe, Reservation, ReservationKind, now_ms,
valid_provider_name,
};
use crate::process::{controller, port_owners};
use crate::run::RunManager;
use crate::store::{Store, runtime_dir, state_dir};
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct DaemonStatus {
pub running: bool,
#[serde(default)]
pub version: Option<String>,
#[serde(default)]
pub execution_schema_version: Option<u32>,
pub pid: u32,
pub started_at: u64,
pub socket: String,
pub database: String,
pub active_executions: usize,
pub code_workers: usize,
}
#[derive(Debug, Serialize, Deserialize)]
struct RpcRequest {
id: String,
method: String,
params: Value,
#[serde(default = "default_controller")]
controller: ProcessController,
}
fn default_controller() -> ProcessController {
ProcessController::capture(false, None)
}
#[derive(Debug, Serialize, Deserialize)]
struct RpcResponse {
id: String,
result: Option<Value>,
error: Option<RpcError>,
}
#[derive(Debug, Serialize, Deserialize)]
struct RpcError {
code: String,
message: String,
}
#[derive(Debug)]
enum DaemonCallError {
Unavailable(String),
RequestFailed(String),
}
#[derive(Deserialize)]
struct StopParams {
id: String,
timeout_ms: u64,
}
#[derive(Deserialize)]
struct IdParams {
id: String,
}
#[derive(Deserialize)]
struct SignalParams {
id: String,
signal: String,
}
#[derive(Deserialize)]
struct InputParams {
id: String,
data: String,
}
#[derive(Deserialize)]
struct LogsParams {
id: String,
tail_bytes: usize,
}
#[derive(Deserialize)]
struct WaitParams {
id: String,
timeout_ms: Option<u64>,
}
#[derive(Deserialize)]
struct ExecutionStartParams {
spec: ExecutionSpec,
parent_id: Option<String>,
session_id: Option<String>,
}
#[derive(Deserialize)]
struct ExecutionListParams {
limit: usize,
cursor: Option<String>,
status: Option<ExecutionStatus>,
}
#[derive(Deserialize)]
struct SessionOpenParams {
actor: Option<String>,
labels: BTreeMap<String, String>,
ttl_ms: u64,
}
#[derive(Deserialize)]
struct RenewParams {
id: String,
ttl_ms: u64,
}
#[derive(Deserialize)]
struct ActiveParams {
active: bool,
}
#[derive(Deserialize)]
struct ReservationAcquireParams {
kind: String,
key: String,
lease_id: String,
ttl_ms: Option<u64>,
}
#[derive(Deserialize)]
struct EventQueryParams {
after: u64,
limit: usize,
}
#[derive(Deserialize)]
struct EventWatchParams {
after: u64,
limit: usize,
timeout_ms: u64,
}
#[derive(Deserialize)]
struct PinParams {
id: String,
pinned: bool,
}
#[derive(Deserialize)]
struct PruneParams {
max_age_ms: u64,
}
#[derive(Deserialize)]
struct ReceiptListParams {
limit: usize,
}
#[derive(Deserialize)]
struct CodeRunParams {
code: String,
timeout_ms: u64,
}
#[derive(Deserialize)]
struct CodeParams {
code: String,
}
#[derive(Deserialize)]
struct SearchParams {
query: String,
}
#[derive(Deserialize)]
struct ArtifactParams {
execution: String,
artifact: String,
}
#[derive(Deserialize)]
struct RecipePutParams {
name: String,
description: String,
code: String,
input_schema: Option<Value>,
}
#[derive(Deserialize)]
struct RecipeRunParams {
name: String,
timeout_ms: u64,
input: Value,
}
pub struct Daemon {
store: Arc<Store>,
runs: RunManager,
executions: ExecutionManager,
code: RwLock<CodePool>,
code_workers: usize,
shutdown: CancellationToken,
started_at: u64,
socket: PathBuf,
}
impl Daemon {
pub fn open_default() -> Result<Self, String> {
let store = Arc::new(Store::open_default()?);
let runs = RunManager::new(store.clone())?;
let code = CodePool::new(store.clone())?;
let code_workers = code.workers();
Ok(Self {
executions: ExecutionManager::new(store.clone(), runs.clone()),
runs,
code: RwLock::new(code),
code_workers,
store,
shutdown: CancellationToken::new(),
started_at: now_ms(),
socket: socket_path(),
})
}
pub async fn serve(self) -> Result<(), String> {
let runtime = runtime_dir();
std::fs::create_dir_all(&runtime).map_err(|error| error.to_string())?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&runtime, std::fs::Permissions::from_mode(0o700))
.map_err(|error| error.to_string())?;
}
let _lock = acquire_daemon_lock(&runtime)?;
self.runs.recover().await?;
self.executions.recover()?;
if self.socket.exists() {
match UnixStream::connect(&self.socket).await {
Ok(_) => {
return Err(format!(
"An apoc daemon already owns {}",
self.socket.display()
));
}
Err(_) => {
std::fs::remove_file(&self.socket).map_err(|error| error.to_string())?;
}
}
}
let listener = UnixListener::bind(&self.socket).map_err(|error| error.to_string())?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&self.socket, std::fs::Permissions::from_mode(0o600))
.map_err(|error| error.to_string())?;
}
let daemon = Arc::new(self);
let mut cleanup = tokio::time::interval(Duration::from_secs(1));
cleanup.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tokio::select! {
_ = daemon.shutdown.cancelled() => break,
_ = cleanup.tick() => {
let controller = default_controller();
let _ = daemon
.cleanup_expired_leases(Duration::from_secs(5), &controller)
.await;
let _ = daemon.store.prune_causal_payloads(
now_ms().saturating_sub(30 * 24 * 60 * 60 * 1_000),
);
let _ = daemon
.store
.prune_causal_payloads_to_size(10 * 1024 * 1024 * 1024);
}
accepted = listener.accept() => {
let (stream, _) = accepted.map_err(|error| error.to_string())?;
let daemon = Arc::clone(&daemon);
tokio::spawn(async move {
let _ = daemon.connection(stream).await;
});
}
}
}
let _ = std::fs::remove_file(&daemon.socket);
Ok(())
}
async fn connection(&self, stream: UnixStream) -> Result<(), String> {
let mut framed = Framed::new(stream, codec());
while let Some(frame) = framed.next().await {
let frame = frame.map_err(|error| error.to_string())?;
let request =
serde_json::from_slice::<RpcRequest>(&frame).map_err(|error| error.to_string())?;
let response = match self
.dispatch(&request.method, request.params, &request.controller)
.await
{
Ok(result) => RpcResponse {
id: request.id,
result: Some(result),
error: None,
},
Err(message) => RpcResponse {
id: request.id,
result: None,
error: Some(RpcError {
code: "DAEMON_REQUEST_FAILED".to_string(),
message,
}),
},
};
framed
.send(
serde_json::to_vec(&response)
.map_err(|error| error.to_string())?
.into(),
)
.await
.map_err(|error| error.to_string())?;
}
Ok(())
}
async fn dispatch(
&self,
method: &str,
params: Value,
controller: &ProcessController,
) -> Result<Value, String> {
match method {
"daemon.status" | "daemon.ping" => json(self.status()?),
"daemon.shutdown" => {
self.shutdown.cancel();
Ok(serde_json::json!({ "stopping": true }))
}
"execution.start" => {
let mut params = parse::<ExecutionStartParams>(params)?;
normalize_execution_workspace(&mut params.spec, controller)?;
params.parent_id = params.parent_id.or_else(|| controller.execution_id.clone());
if params.spec.owner_kind == crate::model::LeaseOwnerKind::Execution {
params.spec.owner_id = params.parent_id.clone();
}
if params.spec.owner_kind == crate::model::LeaseOwnerKind::Session {
params.spec.owner_id = params.session_id.clone();
}
let (execution, replayed) = self
.executions
.start_with_replay(
params.spec,
controller.clone(),
params.parent_id,
params.session_id,
)
.await?;
json(serde_json::json!({
"execution": execution,
"replayed": replayed,
"logs": null
}))
}
"execution.command" => {
let mut params = parse::<ExecutionStartParams>(params)?;
normalize_execution_workspace(&mut params.spec, controller)?;
params.parent_id = params.parent_id.or_else(|| controller.execution_id.clone());
if params.spec.owner_kind == crate::model::LeaseOwnerKind::Execution {
params.spec.owner_id = params.parent_id.clone();
}
if params.spec.owner_kind == crate::model::LeaseOwnerKind::Session {
params.spec.owner_id = params.session_id.clone();
}
let (execution, replayed) = self
.executions
.start_with_replay(
params.spec,
controller.clone(),
params.parent_id,
params.session_id,
)
.await?;
let execution = self.executions.wait(&execution.id, None).await?;
let logs = self.executions.logs(&execution.id, 32 * 1024)?;
json(serde_json::json!({
"execution": execution,
"replayed": replayed,
"logs": logs
}))
}
"execution.list" => {
let params = parse::<ExecutionListParams>(params)?;
json(self.executions.list_page(
params.limit,
params.cursor.as_deref(),
params.status,
)?)
}
"execution.get" => {
let params = parse::<IdParams>(params)?;
json(self.executions.get(¶ms.id)?)
}
"execution.wait" => {
let params = parse::<WaitParams>(params)?;
if self.store.get_execution_record(¶ms.id)?.is_some() {
json(
self.executions
.wait(¶ms.id, params.timeout_ms.map(Duration::from_millis))
.await?,
)
} else {
let state = self
.code
.read()
.await
.wait(¶ms.id, params.timeout_ms.map(Duration::from_millis))
.await?;
json(self.record_code_execution(&state, controller)?)
}
}
"execution.cancel" => {
let params = parse::<StopParams>(params)?;
if self.store.get_execution_record(¶ms.id)?.is_some() {
json(
self.executions
.cancel(
¶ms.id,
Duration::from_millis(params.timeout_ms),
controller,
)
.await?,
)
} else {
let state = self.code.read().await.cancel(params.id).await?;
json(self.record_code_execution(&state, controller)?)
}
}
"execution.signal" => {
let params = parse::<SignalParams>(params)?;
json(
self.executions
.signal(¶ms.id, ¶ms.signal, controller)
.await?,
)
}
"execution.input" => {
let params = parse::<InputParams>(params)?;
json(
self.executions
.input(¶ms.id, params.data.as_bytes(), controller)
.await?,
)
}
"execution.logs" => {
let params = parse::<LogsParams>(params)?;
json(self.executions.logs(¶ms.id, params.tail_bytes)?)
}
"execution.attach" => {
let params = parse::<LogsParams>(params)?;
Ok(serde_json::json!({
"execution": self.executions.get(¶ms.id)?,
"logs": self.executions.logs(¶ms.id, params.tail_bytes)?,
"live_io": self.runs.is_active(¶ms.id).await
}))
}
"execution.remove" => {
let params = parse::<IdParams>(params)?;
let id = params.id;
Ok(serde_json::json!({
"id": id,
"removed": self.executions.remove(&id).await?
}))
}
"session.open" => {
let params = parse::<SessionOpenParams>(params)?;
if params.ttl_ms == 0 {
return Err("Session ttl_ms must be greater than zero".to_string());
}
let now = now_ms();
let session = AgentSession {
id: uuid::Uuid::now_v7().to_string(),
actor: params.actor.unwrap_or_else(|| controller.actor.clone()),
labels: params.labels,
created_at: now,
heartbeat_at: now,
expires_at: now.saturating_add(params.ttl_ms),
closed: false,
};
self.store.put_session(&session)?;
self.store.put_lease(&Lease {
id: session_lease_id(&session.id),
owner_kind: LeaseOwnerKind::Session,
owner_id: session.id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: Some(session.expires_at),
released: false,
})?;
self.lifecycle_event(
"session.opened",
&format!("session/{}", session.id),
controller,
None,
Some(session.id.clone()),
serde_json::json!({ "expires_at": session.expires_at }),
)?;
json(session)
}
"session.heartbeat" => {
let params = parse::<RenewParams>(params)?;
if params.ttl_ms == 0 {
return Err("Session ttl_ms must be greater than zero".to_string());
}
let mut session = self
.store
.get_session(¶ms.id)?
.ok_or_else(|| format!("Session \"{}\" not found", params.id))?;
if session.closed || session.expires_at <= now_ms() {
return Err(format!("Session \"{}\" is closed or expired", params.id));
}
session.heartbeat_at = now_ms();
session.expires_at = session.heartbeat_at.saturating_add(params.ttl_ms);
self.store.put_session(&session)?;
let mut lease = self
.store
.get_lease(&session_lease_id(&session.id))?
.ok_or_else(|| "Session root lease is missing".to_string())?;
lease.renewed_at = session.heartbeat_at;
lease.expires_at = Some(session.expires_at);
self.store.put_lease(&lease)?;
json(session)
}
"session.get" => {
let params = parse::<IdParams>(params)?;
json(
self.store
.get_session(¶ms.id)?
.ok_or_else(|| format!("Session \"{}\" not found", params.id))?,
)
}
"session.list" => {
let params = parse::<ActiveParams>(params)?;
json(self.store.list_sessions(params.active)?)
}
"session.close" => {
let params = parse::<StopParams>(params)?;
json(
self.close_session(
¶ms.id,
Duration::from_millis(params.timeout_ms),
controller,
)
.await?,
)
}
"lease.get" => {
let params = parse::<IdParams>(params)?;
json(
self.store
.get_lease(¶ms.id)?
.ok_or_else(|| format!("Lease \"{}\" not found", params.id))?,
)
}
"lease.list" => {
let params = parse::<ActiveParams>(params)?;
json(self.store.list_leases(params.active)?)
}
"lease.renew" => {
let params = parse::<RenewParams>(params)?;
if params.ttl_ms == 0 {
return Err("Lease ttl_ms must be greater than zero".to_string());
}
let mut lease = self
.store
.get_lease(¶ms.id)?
.ok_or_else(|| format!("Lease \"{}\" not found", params.id))?;
if lease.released {
return Err(format!("Lease \"{}\" is released", params.id));
}
lease.renewed_at = now_ms();
lease.expires_at = Some(lease.renewed_at.saturating_add(params.ttl_ms));
self.store.put_lease(&lease)?;
json(lease)
}
"lease.release" => {
let params = parse::<StopParams>(params)?;
let released = self
.release_lease(
¶ms.id,
Duration::from_millis(params.timeout_ms),
controller,
)
.await?;
Ok(serde_json::json!({ "id": params.id, "released": released }))
}
"lease.cleanup" => {
let params = parse::<StopParams>(params)?;
json(
self.cleanup_expired_leases(
Duration::from_millis(params.timeout_ms),
controller,
)
.await?,
)
}
"reservation.acquire" | "reservation.wait" => {
let params = parse::<ReservationAcquireParams>(params)?;
let wait = method == "reservation.wait";
json(self.acquire_reservation(params, wait).await?)
}
"reservation.get" => {
let params = parse::<IdParams>(params)?;
json(
self.store
.get_reservation(¶ms.id)?
.ok_or_else(|| format!("Reservation \"{}\" not found", params.id))?,
)
}
"reservation.list" => {
let params = parse::<ActiveParams>(params)?;
json(self.store.list_reservations(params.active)?)
}
"reservation.release" => {
let params = parse::<IdParams>(params)?;
let released = self.store.release_reservation(¶ms.id)?.is_some();
Ok(serde_json::json!({ "id": params.id, "released": released }))
}
"event.query" => {
let params = parse::<EventQueryParams>(params)?;
json(self.store.causal_events(params.after, params.limit)?)
}
"event.watch" => {
let params = parse::<EventWatchParams>(params)?;
let deadline =
tokio::time::Instant::now() + Duration::from_millis(params.timeout_ms);
loop {
let events = self.store.causal_events(params.after, params.limit)?;
if !events.is_empty() || tokio::time::Instant::now() >= deadline {
break json(events);
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
}
"event.payload" => {
let params = parse::<IdParams>(params)?;
let value = self
.store
.causal_payload(¶ms.id)?
.ok_or_else(|| format!("Payload \"{}\" is unavailable or pruned", params.id))?;
Ok(serde_json::json!({ "id": params.id, "value": value }))
}
"event.pin" => {
let params = parse::<PinParams>(params)?;
let updated = self.store.pin_causal_payload(¶ms.id, params.pinned)?;
Ok(serde_json::json!({
"id": params.id,
"pinned": params.pinned,
"updated": updated
}))
}
"event.prune-payloads" => {
let params = parse::<PruneParams>(params)?;
let before = now_ms().saturating_sub(params.max_age_ms);
let pruned = self.store.prune_causal_payloads(before)?;
Ok(serde_json::json!({ "pruned": pruned, "before": before }))
}
"receipt.get" => {
let params = parse::<IdParams>(params)?;
json(
self.store
.operation_receipt(¶ms.id)?
.ok_or_else(|| format!("Receipt \"{}\" not found", params.id))?,
)
}
"receipt.list" => {
let params = parse::<ReceiptListParams>(params)?;
json(self.store.operation_receipts(params.limit)?)
}
"code.run" => {
let params = parse::<CodeRunParams>(params)?;
let state = self
.code
.read()
.await
.run(
params.code,
controller.cwd.clone(),
Some(Duration::from_millis(params.timeout_ms)),
)
.await?;
self.record_code_execution(&state, controller)?;
Ok(execution_summary(&state))
}
"code.start" => {
let params = parse::<CodeParams>(params)?;
let state = self
.code
.read()
.await
.start(params.code, controller.cwd.clone())
.await?;
self.record_code_execution(&state, controller)?;
Ok(execution_summary(&state))
}
"code.get" => {
let params = parse::<IdParams>(params)?;
let state = self.code.read().await.get(params.id).await?;
self.record_code_execution(&state, controller)?;
json(state)
}
"code.cancel" => {
let params = parse::<IdParams>(params)?;
let state = self.code.read().await.cancel(params.id).await?;
self.record_code_execution(&state, controller)?;
json(state)
}
"code.search" => {
let params = parse::<SearchParams>(params)?;
json(self.code.read().await.search(params.query).await?)
}
"code.artifact" => {
let params = parse::<ArtifactParams>(params)?;
self.code
.read()
.await
.artifact(params.execution, params.artifact)
.await
}
"recipe.put" => {
let params = parse::<RecipePutParams>(params)?;
let now = now_ms();
let created_at = self
.store
.get_recipe(¶ms.name)?
.and_then(|value| serde_json::from_value::<Recipe>(value).ok())
.map(|recipe| recipe.created_at)
.unwrap_or(now);
let recipe = Recipe {
name: params.name,
description: params.description,
code: params.code,
input_schema: params.input_schema,
created_at,
updated_at: now,
};
self.store.put_recipe(
&recipe.name,
&serde_json::to_value(&recipe).map_err(|error| error.to_string())?,
)?;
self.store
.put_snippet(&Snippet {
name: recipe.name.clone(),
description: recipe.description.clone(),
code: recipe.code.clone(),
saved_at: now,
input_schema: recipe.input_schema.clone(),
connectors: Vec::new(),
})
.await?;
json(recipe)
}
"recipe.list" => json(
self.store
.list_recipes()?
.into_iter()
.map(serde_json::from_value::<Recipe>)
.collect::<Result<Vec<_>, _>>()
.map_err(|error| error.to_string())?,
),
"recipe.get" => {
let params = parse::<IdParams>(params)?;
self.store
.get_recipe(¶ms.id)?
.ok_or_else(|| format!("Recipe \"{}\" not found", params.id))
}
"recipe.remove" => {
let params = parse::<IdParams>(params)?;
self.store.delete_snippet(¶ms.id).await?;
Ok(serde_json::json!({
"id": params.id,
"removed": self.store.delete_recipe(¶ms.id)?
}))
}
"recipe.run" => {
let params = parse::<RecipeRunParams>(params)?;
let recipe = self
.store
.get_recipe(¶ms.name)?
.ok_or_else(|| format!("Recipe \"{}\" not found", params.name))
.and_then(|value| {
serde_json::from_value::<Recipe>(value).map_err(|error| error.to_string())
})?;
let code = format!(
"const input = {};\n{}",
serde_json::to_string(¶ms.input).map_err(|error| error.to_string())?,
recipe.code
);
Ok(execution_summary(
&self
.code
.read()
.await
.run(
code,
controller.cwd.clone(),
Some(Duration::from_millis(params.timeout_ms)),
)
.await?,
))
}
"provider.put" | "connector.put" => {
self.ensure_code_idle().await?;
let config = serde_json::from_value::<ConnectorConfig>(params)
.map_err(|error| error.to_string())?;
if !valid_provider_name(&config.name) {
return Err(format!(
"Provider \"{}\" is not a valid JavaScript namespace",
config.name
));
}
self.store.put_provider(
&config.name,
&serde_json::to_value(&config).map_err(|error| error.to_string())?,
)?;
self.reload_code().await?;
json(config)
}
"provider.list" | "connector.list" => json(
self.store
.list_providers()?
.into_iter()
.map(serde_json::from_value::<ConnectorConfig>)
.collect::<Result<Vec<_>, _>>()
.map_err(|error| error.to_string())?,
),
"provider.get" | "connector.get" => {
let params = parse::<IdParams>(params)?;
self.store
.get_provider(¶ms.id)?
.ok_or_else(|| format!("Provider \"{}\" not found", params.id))
}
"provider.doctor" => {
let params = parse::<IdParams>(params)?;
let value = self
.store
.get_provider(¶ms.id)?
.ok_or_else(|| format!("Provider \"{}\" not found", params.id))?;
let config = serde_json::from_value::<ConnectorConfig>(value)
.map_err(|error| error.to_string())?;
let connector = external_connector(config.clone(), self.store.clone())?;
let description = connector.describe().await?;
let capabilities = capability_descriptors(&config, &description);
Ok(serde_json::json!({
"provider": config.name,
"generation": config.generation,
"instructions": description.instructions,
"capabilities": capabilities
}))
}
"provider.refresh" => {
self.ensure_code_idle().await?;
let params = parse::<IdParams>(params)?;
let value = self
.store
.get_provider(¶ms.id)?
.ok_or_else(|| format!("Provider \"{}\" not found", params.id))?;
let mut config = serde_json::from_value::<ConnectorConfig>(value)
.map_err(|error| error.to_string())?;
let connector = external_connector(config.clone(), self.store.clone())?;
let description = connector.describe().await?;
let snapshot =
serde_json::to_vec(&description).map_err(|error| error.to_string())?;
config.generation = format!("sha256:{:x}", Sha256::digest(snapshot));
config.updated_at = now_ms();
let capabilities = capability_descriptors(&config, &description);
self.store.put_provider(
&config.name,
&serde_json::to_value(&config).map_err(|error| error.to_string())?,
)?;
self.reload_code().await?;
Ok(serde_json::json!({
"provider": config.name,
"generation": config.generation,
"capabilities": capabilities
}))
}
"provider.remove" | "connector.remove" => {
self.ensure_code_idle().await?;
let params = parse::<IdParams>(params)?;
let removed = self.store.delete_provider(¶ms.id)?;
self.reload_code().await?;
Ok(serde_json::json!({ "id": params.id, "removed": removed }))
}
_ => Err(format!("Unknown daemon method \"{method}\"")),
}
}
fn status(&self) -> Result<DaemonStatus, String> {
let active_executions = self
.executions
.list()?
.into_iter()
.filter(|execution| {
!matches!(
execution.status,
crate::model::ExecutionStatus::Completed
| crate::model::ExecutionStatus::Failed
| crate::model::ExecutionStatus::Canceled
)
})
.count();
Ok(DaemonStatus {
running: true,
version: Some(env!("CARGO_PKG_VERSION").to_string()),
execution_schema_version: Some(crate::model::SCHEMA_VERSION),
pid: std::process::id(),
started_at: self.started_at,
socket: self.socket.to_string_lossy().into_owned(),
database: self.runs.store().path().to_string_lossy().into_owned(),
active_executions,
code_workers: self.code_workers,
})
}
async fn close_session(
&self,
id: &str,
timeout: Duration,
controller: &ProcessController,
) -> Result<AgentSession, String> {
let mut session = self
.store
.get_session(id)?
.ok_or_else(|| format!("Session \"{id}\" not found"))?;
let executions = self
.store
.list_execution_records()?
.into_iter()
.filter(|execution| execution.session_id.as_deref() == Some(id))
.map(|execution| execution.id)
.collect::<Vec<_>>();
for execution in executions {
if self.runs.is_active(&execution).await {
let _ = self
.executions
.cancel(&execution, timeout, controller)
.await;
}
}
self.store.release_lease(&session_lease_id(id))?;
session.closed = true;
session.heartbeat_at = now_ms();
session.expires_at = session.heartbeat_at;
self.store.put_session(&session)?;
self.lifecycle_event(
"session.closed",
&format!("session/{id}"),
controller,
None,
Some(id.to_string()),
serde_json::json!({}),
)?;
Ok(session)
}
async fn release_lease(
&self,
id: &str,
timeout: Duration,
controller: &ProcessController,
) -> Result<bool, String> {
let Some(lease) = self.store.get_lease(id)? else {
return Ok(false);
};
if lease.released {
return Ok(true);
}
match lease.owner_kind {
LeaseOwnerKind::Execution => {
if self.runs.is_active(&lease.owner_id).await {
let _ = self
.executions
.cancel(&lease.owner_id, timeout, controller)
.await;
}
}
LeaseOwnerKind::Session => {
if self.store.get_session(&lease.owner_id)?.is_some() {
let _ = self
.close_session(&lease.owner_id, timeout, controller)
.await?;
}
}
LeaseOwnerKind::Persistent => {}
}
self.store.release_lease(id)?;
self.lifecycle_event(
"lease.released",
&format!("lease/{id}"),
controller,
None,
None,
serde_json::json!({ "owner_id": lease.owner_id }),
)?;
Ok(true)
}
async fn cleanup_expired_leases(
&self,
timeout: Duration,
controller: &ProcessController,
) -> Result<Vec<Lease>, String> {
let now = now_ms();
let expired = self
.store
.list_leases(false)?
.into_iter()
.filter(|lease| {
!lease.released && lease.expires_at.is_some_and(|expires_at| expires_at <= now)
})
.collect::<Vec<_>>();
let mut released = Vec::with_capacity(expired.len());
for lease in expired {
self.release_lease(&lease.id, timeout, controller).await?;
if let Some(lease) = self.store.get_lease(&lease.id)? {
released.push(lease);
}
}
Ok(released)
}
async fn acquire_reservation(
&self,
params: ReservationAcquireParams,
wait: bool,
) -> Result<Reservation, String> {
let lease = self
.store
.get_lease(¶ms.lease_id)?
.ok_or_else(|| format!("Lease \"{}\" not found", params.lease_id))?;
if lease.released
|| lease
.expires_at
.is_some_and(|expires_at| expires_at <= now_ms())
{
return Err(format!(
"Lease \"{}\" is released or expired",
params.lease_id
));
}
let kind = reservation_kind(¶ms.kind)?;
let key = reservation_key(kind, ¶ms.key)?;
let external_conflict = if kind == ReservationKind::Port {
let (protocol, port) = parse_reservation_port(&key)?;
!port_owners(port, Some(protocol))?.is_empty()
} else {
false
};
let reservation = Reservation {
id: uuid::Uuid::now_v7().to_string(),
kind,
key,
lease_id: params.lease_id,
created_at: now_ms(),
expires_at: params.ttl_ms.map(|ttl| now_ms().saturating_add(ttl)),
external_conflict,
released: false,
};
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
loop {
match self.store.acquire_reservation(&reservation) {
Ok(reservation) => return Ok(reservation),
Err(error)
if wait
&& error.starts_with("RESERVATION_CONFLICT:")
&& tokio::time::Instant::now() < deadline =>
{
tokio::time::sleep(Duration::from_millis(25)).await;
}
Err(error) => return Err(error),
}
}
}
fn lifecycle_event(
&self,
kind: &str,
subject: &str,
controller: &ProcessController,
execution_id: Option<String>,
session_id: Option<String>,
details: Value,
) -> Result<CausalEvent, String> {
self.store.append_causal_event(CausalEvent {
sequence: 0,
id: uuid::Uuid::now_v7().to_string(),
at: now_ms(),
kind: kind.to_string(),
subject: subject.to_string(),
actor: controller.actor.clone(),
correlation_id: execution_id
.clone()
.or_else(|| session_id.clone())
.unwrap_or_else(|| controller.invocation_id.clone()),
causation_id: None,
execution_id,
session_id,
payload_id: None,
details,
})
}
fn record_code_execution(
&self,
state: &CodeExecutionState,
controller: &ProcessController,
) -> Result<crate::model::Execution, String> {
let previous = self.store.get_execution_record(&state.id)?;
let mut workspace = previous
.as_ref()
.map(|execution| execution.spec.workspace.clone())
.unwrap_or_else(|| crate::model::WorkspaceContext {
cwd: controller.cwd.clone(),
..crate::model::WorkspaceContext::default()
});
crate::execution::enrich_workspace(&mut workspace);
let status = code_status(state.status);
let pass = crate::model::ExecutionPass {
id: format!("{}/pass/1", state.id),
number: 1,
started_at: state.created_at,
completed_at: code_terminal(state.status).then_some(state.updated_at),
status,
error_code: state
.error
.as_ref()
.map(|_| "CODE_EXECUTION_FAILED".to_string()),
error: state.error.clone(),
};
let execution = crate::model::Execution {
id: state.id.clone(),
parent_id: previous
.as_ref()
.and_then(|execution| execution.parent_id.clone())
.or_else(|| controller.execution_id.clone()),
session_id: None,
spec: crate::model::ExecutionSpec {
schema_version: crate::model::SCHEMA_VERSION,
idempotency_key: None,
name: "code".to_string(),
purpose: Some("JavaScript Code Mode execution".to_string()),
labels: BTreeMap::new(),
program: crate::model::ExecutionProgram::Javascript {
source: state.code.clone(),
input: Value::Null,
},
workspace,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
io: crate::model::IoMode::Pipes,
owner_kind: LeaseOwnerKind::Persistent,
owner_id: None,
expires_at: None,
retry: crate::model::RetryPolicy::default(),
budget: crate::model::ExecutionBudget::default(),
},
status,
passes: vec![pass],
attempts: Vec::new(),
created_at: state.created_at,
updated_at: state.updated_at,
last_progress_at: state.updated_at,
result: state.result.clone().or_else(|| {
state.error.as_ref().map(|error| {
serde_json::json!({
"error_code": "CODE_EXECUTION_FAILED",
"error": error
})
})
}),
};
self.store.put_execution_record(&execution)?;
let lease_id = format!("execution/{}", state.id);
if self.store.get_lease(&lease_id)?.is_none() {
self.store.put_lease(&Lease {
id: lease_id.clone(),
owner_kind: LeaseOwnerKind::Execution,
owner_id: state.id.clone(),
resources: Vec::new(),
created_at: state.created_at,
renewed_at: state.updated_at,
expires_at: None,
released: false,
})?;
}
if code_terminal(state.status) {
self.store.release_lease(&lease_id)?;
}
if previous.as_ref().map(|value| value.status) != Some(status) {
self.lifecycle_event(
&format!("execution.{}", model_status_name(status)),
&format!("execution/{}", state.id),
controller,
Some(state.id.clone()),
None,
serde_json::json!({ "runtime": "javascript" }),
)?;
}
Ok(execution)
}
async fn reload_code(&self) -> Result<(), String> {
*self.code.write().await = CodePool::new(self.store.clone())?;
Ok(())
}
async fn ensure_code_idle(&self) -> Result<(), String> {
if self.store.list_executions().await?.iter().any(|execution| {
matches!(
execution.status,
CodeExecutionStatus::Running | CodeExecutionStatus::Paused
)
}) {
return Err(
"External connectors cannot change while Code Mode executions are active"
.to_string(),
);
}
Ok(())
}
}
fn normalize_execution_workspace(
spec: &mut ExecutionSpec,
controller: &ProcessController,
) -> Result<(), String> {
let base = controller
.cwd
.as_deref()
.ok_or_else(|| "INVALID_WORKSPACE: caller working directory is unavailable".to_string())?;
let cwd = resolve_workspace_path(base, spec.workspace.cwd.as_deref())?;
spec.workspace.cwd = Some(cwd.clone());
if let crate::model::ExecutionProgram::Commands { commands, .. } = &mut spec.program {
for command in commands {
if command.cwd.is_some() {
command.cwd = Some(resolve_workspace_path(&cwd, command.cwd.as_deref())?);
}
}
}
Ok(())
}
fn resolve_workspace_path(base: &str, value: Option<&str>) -> Result<String, String> {
let path = match value {
Some(value) if Path::new(value).is_absolute() => PathBuf::from(value),
Some(value) => Path::new(base).join(value),
None => PathBuf::from(base),
};
let path = std::fs::canonicalize(&path).map_err(|error| {
format!(
"INVALID_WORKSPACE: could not resolve working directory {}: {error}",
path.display()
)
})?;
if !path.is_dir() {
return Err(format!(
"INVALID_WORKSPACE: working directory {} is not a directory",
path.display()
));
}
Ok(path.to_string_lossy().into_owned())
}
fn acquire_daemon_lock(runtime: &Path) -> Result<std::fs::File, String> {
try_acquire_lock(&runtime.join("apoc.lock"))?
.ok_or_else(|| format!("An apoc daemon already owns {}", runtime.display()))
}
fn try_acquire_lock(path: &Path) -> Result<Option<std::fs::File>, String> {
use std::os::fd::AsRawFd;
let file = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.read(true)
.write(true)
.open(path)
.map_err(|error| error.to_string())?;
if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == -1 {
let error = std::io::Error::last_os_error();
if error.kind() == std::io::ErrorKind::WouldBlock {
return Ok(None);
}
return Err(format!("Failed to lock {}: {error}", path.display()));
}
Ok(Some(file))
}
#[derive(Debug, Clone, Default)]
pub struct DaemonClient;
impl DaemonClient {
pub async fn call<T: for<'de> Deserialize<'de>>(
&self,
method: &str,
params: impl Serialize,
) -> Result<T, String> {
let params = serde_json::to_value(params).map_err(|error| error.to_string())?;
let value = self.call_value(method, params, true).await?;
serde_json::from_value(value).map_err(|error| error.to_string())
}
pub async fn call_existing<T: for<'de> Deserialize<'de>>(
&self,
method: &str,
params: impl Serialize,
) -> Result<T, String> {
let params = serde_json::to_value(params).map_err(|error| error.to_string())?;
let value = self.call_value(method, params, false).await?;
serde_json::from_value(value).map_err(|error| error.to_string())
}
async fn call_value(
&self,
method: &str,
params: Value,
auto_start: bool,
) -> Result<Value, String> {
let method = method.to_string();
let controller = controller(false);
match tokio::task::spawn_blocking({
let method = method.clone();
let params = params.clone();
let controller = controller.clone();
move || call_sync(&method, params, controller)
})
.await
.map_err(|error| error.to_string())?
{
Ok(value) => Ok(value),
Err(DaemonCallError::Unavailable(error)) if auto_start => {
start_daemon().await?;
match tokio::task::spawn_blocking(move || call_sync(&method, params, controller))
.await
.map_err(|next| next.to_string())?
{
Ok(value) => Ok(value),
Err(DaemonCallError::Unavailable(next)) => {
Err(format!("Daemon did not start: {error}; {next}"))
}
Err(DaemonCallError::RequestFailed(next)) => Err(next),
}
}
Err(DaemonCallError::Unavailable(error)) => {
Err(format!("apoc daemon is not running: {error}"))
}
Err(DaemonCallError::RequestFailed(error)) => Err(error),
}
}
}
pub fn socket_path() -> PathBuf {
runtime_dir().join("apoc.sock")
}
async fn start_daemon() -> Result<(), String> {
let runtime = runtime_dir();
std::fs::create_dir_all(&runtime).map_err(|error| error.to_string())?;
let log_path = state_dir()?.join("daemon.log");
let startup = try_acquire_lock(&runtime.join("apoc.start.lock"))?;
if startup.is_some() {
if let Some(parent) = log_path.parent() {
std::fs::create_dir_all(parent).map_err(|error| error.to_string())?;
}
let stdout = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&log_path)
.map_err(|error| error.to_string())?;
let stderr = stdout.try_clone().map_err(|error| error.to_string())?;
let executable = std::env::current_exe().map_err(|error| error.to_string())?;
let descriptors = inherited_descriptors()?;
let mut command = tokio::process::Command::new(executable);
command
.args(["daemon", "serve"])
.stdin(Stdio::null())
.stdout(Stdio::from(stdout))
.stderr(Stdio::from(stderr))
.kill_on_drop(false);
#[cfg(unix)]
unsafe {
command.pre_exec(move || {
for descriptor in &descriptors {
let flags = libc::fcntl(*descriptor, libc::F_GETFD);
if flags != -1
&& libc::fcntl(*descriptor, libc::F_SETFD, flags | libc::FD_CLOEXEC) == -1
{
return Err(std::io::Error::last_os_error());
}
}
if libc::setsid() == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
command.spawn().map_err(|error| error.to_string())?;
}
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
loop {
if UnixStream::connect(socket_path()).await.is_ok() {
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!(
"Timed out starting apoc daemon; inspect {}",
log_path.display()
));
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
}
fn inherited_descriptors() -> Result<Vec<i32>, String> {
let directory = ["/proc/self/fd", "/dev/fd"]
.into_iter()
.find(|path| Path::new(path).exists())
.ok_or_else(|| "Cannot enumerate inherited file descriptors".to_string())?;
Ok(std::fs::read_dir(directory)
.map_err(|error| error.to_string())?
.filter_map(|entry| {
entry
.ok()?
.file_name()
.to_str()?
.parse::<i32>()
.ok()
.filter(|descriptor| *descriptor > 2)
})
.collect())
}
fn codec() -> LengthDelimitedCodec {
LengthDelimitedCodec::builder()
.max_frame_length(16 * 1024 * 1024)
.new_codec()
}
fn call_sync(
method: &str,
params: Value,
controller: ProcessController,
) -> Result<Value, DaemonCallError> {
call_sync_at(&socket_path(), method, params, controller)
}
fn call_sync_at(
socket: &Path,
method: &str,
params: Value,
controller: ProcessController,
) -> Result<Value, DaemonCallError> {
let mut stream = std::os::unix::net::UnixStream::connect(socket)
.map_err(|error| DaemonCallError::Unavailable(error.to_string()))?;
stream
.set_read_timeout(Some(Duration::from_secs(10 * 60)))
.map_err(|error| DaemonCallError::RequestFailed(error.to_string()))?;
stream
.set_write_timeout(Some(Duration::from_secs(30)))
.map_err(|error| DaemonCallError::RequestFailed(error.to_string()))?;
let id = uuid::Uuid::now_v7().to_string();
let payload = serde_json::to_vec(&RpcRequest {
id: id.clone(),
method: method.to_string(),
params,
controller,
})
.map_err(|error| DaemonCallError::RequestFailed(error.to_string()))?;
if payload.len() > 16 * 1024 * 1024 {
return Err(DaemonCallError::RequestFailed(
"Daemon request exceeds 16 MiB".to_string(),
));
}
stream
.write_all(&(payload.len() as u32).to_be_bytes())
.and_then(|_| stream.write_all(&payload))
.map_err(|error| DaemonCallError::RequestFailed(error.to_string()))?;
let mut length = [0_u8; 4];
stream
.read_exact(&mut length)
.map_err(|error| DaemonCallError::RequestFailed(error.to_string()))?;
let length = u32::from_be_bytes(length) as usize;
if length > 16 * 1024 * 1024 {
return Err(DaemonCallError::RequestFailed(
"Daemon response exceeds 16 MiB".to_string(),
));
}
let mut payload = vec![0_u8; length];
stream
.read_exact(&mut payload)
.map_err(|error| DaemonCallError::RequestFailed(error.to_string()))?;
let response = serde_json::from_slice::<RpcResponse>(&payload)
.map_err(|error| DaemonCallError::RequestFailed(error.to_string()))?;
if response.id != id {
return Err(DaemonCallError::RequestFailed(
"Daemon returned a mismatched RPC response".to_string(),
));
}
match (response.result, response.error) {
(Some(result), None) => Ok(result),
(_, Some(error)) => Err(DaemonCallError::RequestFailed(format!(
"{}: {}",
error.code, error.message
))),
_ => Err(DaemonCallError::RequestFailed(
"Daemon returned an invalid RPC response".to_string(),
)),
}
}
fn parse<T: for<'de> Deserialize<'de>>(value: Value) -> Result<T, String> {
serde_json::from_value(value).map_err(|error| error.to_string())
}
fn json(value: impl Serialize) -> Result<Value, String> {
serde_json::to_value(value).map_err(|error| error.to_string())
}
fn session_lease_id(session_id: &str) -> String {
format!("session/{session_id}")
}
fn reservation_kind(value: &str) -> Result<ReservationKind, String> {
match value {
"port" => Ok(ReservationKind::Port),
"path" => Ok(ReservationKind::Path),
"process" => Ok(ReservationKind::Process),
"device" => Ok(ReservationKind::Device),
"custom" => Ok(ReservationKind::Custom),
_ => Err(format!("Unknown reservation kind \"{value}\"")),
}
}
fn reservation_key(kind: ReservationKind, value: &str) -> Result<String, String> {
if value.is_empty() {
return Err("Reservation keys cannot be empty".to_string());
}
match kind {
ReservationKind::Port => {
let value = value.to_ascii_lowercase();
let (protocol, port) = if let Some((protocol, port)) = value.split_once('/') {
if matches!(protocol, "tcp" | "udp") {
(protocol, port)
} else {
(port, protocol)
}
} else {
("tcp", value.as_str())
};
if !matches!(protocol, "tcp" | "udp") {
return Err("Port reservation protocol must be tcp or udp".to_string());
}
let port = port
.parse::<u16>()
.map_err(|_| format!("Invalid port reservation \"{value}\""))?;
Ok(format!("{protocol}/{port}"))
}
ReservationKind::Path => {
let path = PathBuf::from(value);
let path = if path.is_absolute() {
path
} else {
std::env::current_dir()
.map_err(|error| error.to_string())?
.join(path)
};
Ok(path.to_string_lossy().into_owned())
}
ReservationKind::Process => {
if !value.contains('@') {
return Err("Process reservations require an exact stable process key".to_string());
}
Ok(value.to_string())
}
ReservationKind::Device | ReservationKind::Custom => Ok(value.to_string()),
}
}
fn parse_reservation_port(value: &str) -> Result<(&str, u16), String> {
let (protocol, port) = value
.split_once('/')
.ok_or_else(|| format!("Invalid canonical port reservation \"{value}\""))?;
let port = port
.parse::<u16>()
.map_err(|_| format!("Invalid canonical port reservation \"{value}\""))?;
Ok((protocol, port))
}
fn code_status(status: CodeExecutionStatus) -> crate::model::ExecutionStatus {
match status {
CodeExecutionStatus::Running => crate::model::ExecutionStatus::Running,
CodeExecutionStatus::Paused => crate::model::ExecutionStatus::Interrupted,
CodeExecutionStatus::Completed => crate::model::ExecutionStatus::Completed,
CodeExecutionStatus::Cancelled => crate::model::ExecutionStatus::Canceled,
CodeExecutionStatus::Error
| CodeExecutionStatus::Rejected
| CodeExecutionStatus::RolledBack => crate::model::ExecutionStatus::Failed,
}
}
fn code_terminal(status: CodeExecutionStatus) -> bool {
matches!(
status,
CodeExecutionStatus::Completed
| CodeExecutionStatus::Error
| CodeExecutionStatus::Rejected
| CodeExecutionStatus::RolledBack
| CodeExecutionStatus::Cancelled
)
}
fn model_status_name(status: crate::model::ExecutionStatus) -> &'static str {
match status {
crate::model::ExecutionStatus::Queued => "queued",
crate::model::ExecutionStatus::Running => "running",
crate::model::ExecutionStatus::Stalled => "stalled",
crate::model::ExecutionStatus::Interrupted => "interrupted",
crate::model::ExecutionStatus::Completed => "completed",
crate::model::ExecutionStatus::Failed => "failed",
crate::model::ExecutionStatus::Canceled => "canceled",
}
}
#[cfg(test)]
mod tests {
use std::os::unix::net::UnixListener as StdUnixListener;
use std::thread;
use tempfile::tempdir;
use super::*;
#[test]
fn classifies_connection_failure_as_daemon_unavailability() {
let directory = tempdir().unwrap();
let result = call_sync_at(
&directory.path().join("missing.sock"),
"daemon.test",
Value::Null,
ProcessController::capture(true, None),
);
assert!(matches!(result, Err(DaemonCallError::Unavailable(_))));
}
#[test]
fn classifies_rpc_errors_as_request_failures() {
let directory = tempdir().unwrap();
let socket = directory.path().join("daemon.sock");
let listener = StdUnixListener::bind(&socket).unwrap();
let server = thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let request = read_request(&mut stream);
write_response(
&mut stream,
RpcResponse {
id: request.id,
result: None,
error: Some(RpcError {
code: "TEST_FAILURE".to_string(),
message: "request was rejected".to_string(),
}),
},
);
});
let result = call_sync_at(
&socket,
"daemon.test",
Value::Null,
ProcessController::capture(true, None),
);
server.join().unwrap();
assert!(matches!(
result,
Err(DaemonCallError::RequestFailed(message))
if message == "TEST_FAILURE: request was rejected"
));
}
#[test]
fn classifies_post_send_disconnect_as_a_request_failure() {
let directory = tempdir().unwrap();
let socket = directory.path().join("daemon.sock");
let listener = StdUnixListener::bind(&socket).unwrap();
let server = thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let _ = read_request(&mut stream);
});
let result = call_sync_at(
&socket,
"daemon.test",
Value::Null,
ProcessController::capture(true, None),
);
server.join().unwrap();
assert!(matches!(result, Err(DaemonCallError::RequestFailed(_))));
}
fn read_request(stream: &mut std::os::unix::net::UnixStream) -> RpcRequest {
let mut length = [0_u8; 4];
stream.read_exact(&mut length).unwrap();
let mut payload = vec![0_u8; u32::from_be_bytes(length) as usize];
stream.read_exact(&mut payload).unwrap();
serde_json::from_slice(&payload).unwrap()
}
fn write_response(stream: &mut std::os::unix::net::UnixStream, response: RpcResponse) {
let payload = serde_json::to_vec(&response).unwrap();
stream
.write_all(&(payload.len() as u32).to_be_bytes())
.unwrap();
stream.write_all(&payload).unwrap();
}
}