use std::collections::HashMap;
use std::convert::Infallible;
use std::sync::Arc;
use std::time::Duration;
use axum::extract::{Path, Query, State as AxumState};
use axum::http::{HeaderMap, StatusCode};
use axum::response::sse::{Event, KeepAlive, Sse};
use axum::response::{IntoResponse, Response};
use axum::routing::{delete, get, post, put};
use axum::{middleware, Extension, Json, Router};
use chrono::{DateTime, Utc};
use futures::Stream;
use rusty_agent_runtime::agents::{
AgentId, CapabilityManifest, CoordinationContract, DelegateContract, FanOutContract,
QuorumContract, RaceContract, StateScope, COORDINATION_RESULT_KIND,
};
use rusty_agent_runtime::capsule::{derive_capsule_id, CapsuleManifest, CapsuleResolution};
#[cfg(feature = "capsules")]
use rusty_agent_runtime::capsule::{CapsuleDenial, CapsuleOverlay, ResourceBudget};
use rusty_agent_runtime::checkpoint::{
Checkpoint, Checkpointer, InMemoryCheckpointer, JsonFileCheckpointer,
};
use rusty_agent_runtime::durable::{
resolve_retry_parameters, resolve_timeout_bound_ms, retry_decision_event,
timeout_decision_event, ResolvedRetryParameters, RetryDecision,
};
use rusty_agent_runtime::effects::ApprovalToken;
use rusty_agent_runtime::journal::{Clock, EventDraft, Journal, JournalSnapshot, RngSource};
use rusty_agent_runtime::learn::{
admit_promotion, candidate_effect_key, detect_policy_drift, evaluation_effect_key,
promotion_effect_key, rollback_effect_key, surface_for_kind, Candidate, CandidateContent,
CandidateId, CandidateKind, CandidateOverlay, CandidateRecord, CandidateStatus, DriftBaseline,
DriftThresholds, EnvironmentTag, EvaluationRequest, LearnError, PromotionReceipt,
PromotionRefusal, RollbackReceipt, VersionPointer,
};
use rusty_agent_runtime::llm::Usage;
use rusty_agent_runtime::memory::{
assemble, detect_conflicts, memory_effect_key, memory_forget_effect_key, memory_read_request,
plan_forget, Candidacy, ContextBudget, Correction, CorrectionTarget, ForgetReason,
MemoryEvidence, MemoryForgetTombstone, MemoryKind, MemoryProvenance, MemoryQuery, MemoryRecord,
MemoryScope, MemoryStore, ProvenanceAuthor, ScopeAddress, ValidityWindow,
};
use rusty_agent_runtime::record::{
derive_policy_version, sha256_hex, CapsuleVersion, DecisionEvent, Effect, EffectReceipt,
ExecutorPolicy, JournalRef, PayloadRef, PolicyVersion, RunEventKind,
};
use rusty_agent_runtime::registry::{diff_candidates, ArtifactRecord, RegistryError};
use rusty_agent_runtime::replay::{BranchDiff, ExactReplay, ReplayFixture, ReplayParams};
use rusty_agent_runtime::state::State;
use rusty_agent_runtime::team_trace::TeamTrace;
use serde::Deserialize;
use serde_json::{json, Value};
use tokio::sync::Mutex;
use crate::agents::{
self, ActivationMutation, ActivationOutcome, AgentRecord, MailboxClaim, MailboxClaimScope,
};
use crate::assistants::{
valid_version_id, ActivateVersionOutcome, AssistantRecord, AssistantVersionRecord,
AssistantVersionView, AssistantView, CreateVersionOutcome, SetLifecycleOutcome,
ASSISTANT_LINEAGE_BYTES_LIMIT, ASSISTANT_VERSION_BYTES_LIMIT,
};
use crate::auth::TenantContext;
use crate::capsules::{CapsuleRecord, CapsuleWrite};
use crate::coordination;
use crate::crons::{self, CronRecord, OnRunCompleted};
use crate::error::ApiError;
use crate::learn::ServerMemoryStore;
use crate::policy::{self, PolicyActivation, PolicyRecord, PolicySource, PolicyWrite};
use crate::runs::{
self, MultitaskStrategy, RunConfigPayload, RunDeps, RunManager, RunPayload, RunStatus,
};
use crate::server_store::{ArtifactCommitOutcome, CandidateTransition, JsonFileStore, ServerStore};
use crate::sse;
use crate::supervision;
use crate::tasks::{self, CancelOutcome, MutationOutcome, TaskRecord, TaskStatus};
use crate::threads::ThreadRecord;
use crate::triggers;
use crate::{store, GraphRegistry, ServerConfig, RESERVED_NAMES};
pub(crate) struct AppState {
pub registry: GraphRegistry,
pub config: ServerConfig,
pub checkpointer: Arc<dyn Checkpointer>,
pub run_deps: RunDeps,
pub server_store: Arc<dyn ServerStore>,
pub state_locks: Mutex<HashMap<String, Arc<Mutex<()>>>>,
pub shutdown: tokio_util::sync::CancellationToken,
pub trigger_debounce: Mutex<HashMap<String, crate::triggers::DebounceBuffer>>,
#[cfg(feature = "capsules")]
pub capsule_plane: Arc<crate::capsule_policy::CapsulePolicyPlane>,
pub receipt_keyring: Arc<crate::receipts::SigningKeyring>,
pub broker: Arc<crate::broker::Broker>,
pub artifact_retention: Arc<crate::artifacts::ArtifactRetention>,
pub deployment: Arc<crate::deploy::DeploymentControl>,
pub mcp_bridge: crate::mcp_bridge::McpBridgeState,
pub a2a_streams: Mutex<HashMap<String, tokio::sync::broadcast::Sender<Value>>>,
pub journal_locks: Mutex<HashMap<String, Arc<Mutex<()>>>>,
}
fn build_backends(config: &ServerConfig) -> (Arc<dyn Checkpointer>, Arc<dyn ServerStore>) {
#[cfg(feature = "postgres")]
if let Some(url) = &config.database_url {
let server_store: Arc<dyn ServerStore> =
Arc::new(crate::server_store::PostgresStore::new(url.clone()));
let checkpointer = policy::PolicyBindingCheckpointer::new(
Arc::new(crate::server_store::LazyPostgresCheckpointer::new(
url.clone(),
)),
Arc::clone(&server_store),
);
return (Arc::new(checkpointer), server_store);
}
#[cfg(not(feature = "postgres"))]
assert!(
config.database_url.is_none(),
"`ServerConfig::database_url` requires the `postgres` feature \
(rebuild rusty-agent-server with `--features postgres`)"
);
let server_store: Arc<dyn ServerStore> = Arc::new(JsonFileStore::load(&config.store_path));
let checkpointer = policy::PolicyBindingCheckpointer::new(
Arc::new(JsonFileCheckpointer::new(config.store_path.clone())),
Arc::clone(&server_store),
);
(Arc::new(checkpointer), server_store)
}
pub(crate) fn build_router(
registry: GraphRegistry,
config: ServerConfig,
shutdown: tokio_util::sync::CancellationToken,
) -> (Router, Arc<crate::broker::Broker>) {
let (checkpointer, server_store) = build_backends(&config);
let run_deps = RunDeps {
registry: registry.clone(),
checkpointer: Arc::clone(&checkpointer),
manager: RunManager::new(),
server_store: Arc::clone(&server_store),
queue_cap: config.max_concurrent_runs_per_thread.max(1),
log_capacity: config.event_log_capacity.max(16),
shutdown: shutdown.clone(),
default_environment_tag: config.default_environment_tag.clone(),
};
let outbox_relay_interval = config.outbox_relay_interval;
#[cfg(feature = "capsules")]
let capsule_plane = Arc::new(crate::capsule_policy::CapsulePolicyPlane::new());
let receipt_keyring = Arc::new(crate::receipts::SigningKeyring::new(
Arc::clone(&server_store),
config.store_path.clone(),
));
let mut broker = crate::broker::Broker::new(
Arc::clone(&server_store),
config.store_path.clone(),
config.broker_handle_ttl,
);
if let Some(provider) = &config.oauth_provider {
broker = broker.with_oauth_provider(Arc::clone(provider), config.broker_refresh_window);
}
let broker = Arc::new(broker);
let broker_sweep_interval = config
.broker_sweep_interval
.filter(|_| config.oauth_provider.is_some());
let artifact_sweep_interval = config.artifact_sweep_interval;
let artifact_retention = Arc::new(crate::artifacts::ArtifactRetention::new(Arc::clone(
&server_store,
)));
let deployment = Arc::new(crate::deploy::DeploymentControl::new(Arc::clone(
&server_store,
)));
let state = Arc::new(AppState {
registry,
config,
checkpointer,
run_deps,
server_store,
state_locks: Mutex::new(HashMap::new()),
shutdown,
trigger_debounce: Mutex::new(HashMap::new()),
#[cfg(feature = "capsules")]
capsule_plane,
receipt_keyring,
broker: Arc::clone(&broker),
artifact_retention,
deployment,
mcp_bridge: crate::mcp_bridge::McpBridgeState::new(),
a2a_streams: Mutex::new(HashMap::new()),
journal_locks: Mutex::new(HashMap::new()),
});
crons::spawn_scheduler(Arc::clone(&state));
#[cfg(feature = "capsules")]
if let Ok(handle) = tokio::runtime::Handle::try_current() {
if !state.config.capsule_policy_files.is_empty() {
handle.spawn(crate::capsule_policy::load_config_policies(
Arc::clone(&state.server_store),
state.config.capsule_policy_files.clone(),
));
}
handle.spawn(crate::capsule_policy::preload_active_policies(
Arc::clone(&state.server_store),
Arc::clone(&state.capsule_plane),
));
}
crate::outbox::spawn_relay(
Arc::clone(&state.server_store),
outbox_relay_interval,
state.shutdown.clone(),
);
if let Some(interval) = broker_sweep_interval {
crate::broker::spawn_sweeper(Arc::clone(&state.broker), interval, state.shutdown.clone());
}
if let Some(interval) = artifact_sweep_interval {
crate::artifacts::spawn_sweeper(
Arc::clone(&state.artifact_retention),
interval,
state.shutdown.clone(),
);
}
let authed = Router::new()
.route("/ok", get(ok))
.route("/info", get(info))
.route("/threads", post(create_thread))
.route("/threads/{thread_id}/fork", post(fork_thread))
.route(
"/threads/{thread_id}/state",
get(get_state).post(update_state),
)
.route("/threads/{thread_id}/history", post(history))
.route("/threads/{thread_id}/runs", post(create_run))
.route("/threads/{thread_id}/runs/wait", post(create_run_wait))
.route("/threads/{thread_id}/runs/stream", post(create_run_stream))
.route(
"/threads/{thread_id}/runs/{run_id}",
delete(delete_run_checkpoints),
)
.route("/runs/{run_id}", get(get_run))
.route("/runs/{run_id}/cancel", post(cancel_run))
.route("/runs/{run_id}/stream", get(get_run_stream))
.route("/runs/{run_id}/events", get(get_run_events))
.route("/runs/{run_id}/fixture", get(get_run_fixture))
.route("/runs/replay", post(replay_run))
.route("/runs/diff", get(diff_runs))
.route("/runs/{run_id}/receipt", get(get_run_receipt))
.route("/receipts/verify", post(verify_receipt_route))
.route("/receipt_keys", get(list_receipt_keys_route))
.route("/receipt_keys/rotate", post(rotate_receipt_key))
.route("/receipt_keys/journal", get(get_receipt_keys_journal))
.route(
"/connections",
post(register_connection).get(list_connections),
)
.route("/connections/health", get(list_connections_health))
.route(
"/connections/{connection_id}",
get(get_connection).delete(delete_connection),
)
.route(
"/connections/{connection_id}/consent",
post(consent_connection),
)
.route(
"/connections/{connection_id}/revoke",
post(revoke_connection),
)
.route(
"/connections/{connection_id}/health",
get(get_connection_health),
)
.route("/broker/journal", get(get_broker_journal))
.route("/assistants", post(create_assistant).get(list_assistants))
.route("/assistants/{assistant_id}", get(get_assistant))
.route(
"/assistants/{assistant_id}/archive",
post(archive_assistant),
)
.route(
"/assistants/{assistant_id}/restore",
post(restore_assistant),
)
.route(
"/assistants/{assistant_id}/versions",
post(create_assistant_version).get(list_assistant_versions),
)
.route(
"/assistants/{assistant_id}/versions/{version_id}",
get(get_assistant_version),
)
.route(
"/assistants/{assistant_id}/versions/{version_id}/activate",
post(activate_assistant_version),
)
.route("/crons", post(create_cron).get(list_crons))
.route("/crons/{cron_id}", delete(delete_cron))
.route("/store/{namespace}", get(list_store_namespace))
.route(
"/store/{namespace}/{key}",
put(put_store_item)
.get(get_store_item)
.delete(delete_store_item),
)
.route("/tasks", post(enqueue_task).get(list_tasks))
.route("/tasks/outbox", post(enqueue_task_outbox))
.route("/tasks/claim", post(claim_task))
.route("/tasks/metrics", get(task_metrics))
.route("/tasks/{task_id}", get(get_task))
.route("/tasks/{task_id}/heartbeat", post(heartbeat_task))
.route("/tasks/{task_id}/complete", post(complete_task))
.route("/tasks/{task_id}/fail", post(fail_task))
.route("/tasks/{task_id}/cancel", post(cancel_task))
.route("/agents", post(create_agent).get(list_agents))
.route("/agents/{agent_id}", get(get_agent))
.route("/agents/{agent_id}/mailbox", post(send_agent_message))
.route("/agents/{agent_id}/mailbox/next", post(claim_agent_message))
.route("/agents/{agent_id}/status", get(get_agent_status))
.route("/agents/{agent_id}/cancel", post(cancel_agent))
.route("/agents/{agent_id}/restart", post(restart_agent))
.route("/agents/{agent_id}/supervision", get(get_agent_supervision))
.route("/teams/{team_id}/cancel", post(cancel_team))
.route("/memory", post(write_memory))
.route("/memory/query", post(query_memory))
.route("/memory/corrections", post(submit_correction))
.route("/memory/consolidate", post(enqueue_consolidation))
.route("/memory/conflicts", post(list_memory_conflicts))
.route("/memory/forget", post(forget_memory))
.route("/memory/forget_scope", post(forget_memory_scope))
.route("/memory/{memory_id}", get(get_memory))
.route(
"/learn/candidates",
post(create_candidate).get(list_candidates),
)
.route("/learn/candidates/{candidate_id}", get(get_candidate))
.route(
"/learn/candidates/{candidate_id}/evaluate",
post(evaluate_candidate),
)
.route(
"/learn/candidates/{candidate_id}/promote",
post(promote_candidate),
)
.route(
"/learn/candidates/{candidate_id}/rollback",
post(rollback_candidate),
)
.route("/learn/versions", get(list_version_pointers))
.route(
"/registry/artifacts",
post(declare_artifact).get(list_artifacts),
)
.route("/registry/artifacts/{family}/{name}", get(get_artifact))
.route(
"/registry/artifacts/{family}/{name}/commits",
post(commit_artifact).get(list_artifact_commits),
)
.route(
"/registry/artifacts/{family}/{name}/diff",
get(diff_artifact),
)
.route("/artifacts", get(crate::artifacts::list_run_artifacts))
.route(
"/artifacts/commits",
post(crate::artifacts::commit_run_artifact),
)
.route(
"/artifacts/spills",
post(crate::artifacts::commit_spilled_artifact),
)
.route(
"/artifacts/names/{name}",
get(crate::artifacts::get_run_artifact_named),
)
.route(
"/artifacts/names/{name}/versions",
get(crate::artifacts::list_run_artifact_versions),
)
.route(
"/artifacts/{artifact_id}",
get(crate::artifacts::get_run_artifact),
)
.route(
"/artifacts/{artifact_id}/bytes",
get(crate::artifacts::get_run_artifact_bytes),
)
.route(
"/artifacts/{artifact_id}/preview",
get(crate::artifacts::get_run_artifact_preview),
)
.route(
"/artifacts/{artifact_id}/release",
post(crate::artifacts::release_run_artifact),
)
.route("/artifacts/sweep", post(crate::artifacts::sweep_artifacts))
.route(
"/artifacts/journal",
get(crate::artifacts::get_artifacts_journal),
)
.route(
"/deployments/revisions",
post(crate::deploy::create_revision).get(crate::deploy::list_revisions),
)
.route(
"/deployments/revisions/{revision_id}",
get(crate::deploy::get_revision),
)
.route(
"/deployments/environments",
post(crate::deploy::declare_environment).get(crate::deploy::list_environments),
)
.route(
"/deployments/environments/{name}",
get(crate::deploy::get_environment),
)
.route(
"/deployments/environments/{name}/promote",
post(crate::deploy::promote_revision),
)
.route(
"/deployments/environments/{name}/rollback",
post(crate::deploy::rollback_revision),
)
.route(
"/deployments/environments/{name}/pointer",
get(crate::deploy::get_environment_pointer),
)
.route(
"/deployments/environments/{name}/canary",
put(crate::deploy::declare_canary).delete(crate::deploy::clear_canary),
)
.route("/deployments/shadows", post(crate::deploy::create_shadow))
.route(
"/deployments/health",
get(crate::deploy::get_deployment_health),
)
.route(
"/deployments/journal",
get(crate::deploy::get_deployment_journal),
)
.route(
"/deployments/secrets",
put(crate::deploy::set_env_secret).get(crate::deploy::list_env_secrets),
)
.route(
"/deployments/secrets/resolve",
post(crate::deploy::resolve_env_secret),
)
.route(
"/deployments/secrets/{environment}/{name}",
delete(crate::deploy::revoke_env_secret),
)
.route("/policy/versions", post(register_policy).get(list_policies))
.route("/policy/versions/{version}", get(get_policy))
.route("/policy/activations", post(activate_policy))
.route("/policy/active", get(get_active_policy))
.route("/policy/epochs", get(list_policy_epochs))
.route("/policy/drift", get(get_policy_drift))
.route("/capsules", post(register_capsule).get(list_capsules))
.route("/capsules/{id}", get(get_capsule))
.route("/capsules/resolve", post(resolve_capsules))
.route(
"/capsule_policies/versions",
post(register_capsule_policy).get(list_capsule_policies),
)
.route(
"/capsule_policies/versions/{version}",
get(get_capsule_policy),
)
.route(
"/capsule_policies/active",
get(get_active_capsule_policy).post(activate_capsule_policy),
)
.route(
"/capsules/overlays",
post(attach_capsule_overlay).get(list_capsule_overlays),
)
.route("/capsules/overlays/{name}", get(get_capsule_overlay))
.route("/mcp", post(crate::mcp_bridge::handle))
.route(
crate::a2a::AGENT_CARD_PATH,
get(crate::a2a::agent_card_route),
)
.route("/a2a", post(crate::a2a::handle))
.route("/capsules/{id}/blob", put(put_capsule_blob_route))
.route("/coordination/delegate", post(submit_delegate))
.route("/coordination/fan_out", post(submit_fan_out))
.route("/coordination/race", post(submit_race))
.route("/coordination/quorum", post(submit_quorum))
.route("/coordination/{coordination_id}", get(get_coordination))
.route(
"/coordination/{coordination_id}/trace",
get(get_coordination_trace),
)
.route("/agents/{agent_id}/activate", post(activate_agent))
.route(
"/agents/{agent_id}/activate/heartbeat",
post(heartbeat_activation),
)
.route(
"/agents/{agent_id}/activate/release",
post(release_activation),
)
.route(
"/triggers",
post(triggers::create_trigger).get(triggers::list_triggers),
)
.route(
"/triggers/{trigger_id}",
get(triggers::get_trigger)
.patch(triggers::update_trigger)
.delete(triggers::delete_trigger),
)
.route(
"/triggers/{trigger_id}/events",
get(triggers::list_trigger_events),
)
.route(
"/triggers/{trigger_id}/dead-letter",
get(triggers::list_dead_letter),
)
.route(
"/triggers/{trigger_id}/events/{event_id}/replay",
post(triggers::replay_event),
)
.layer(middleware::from_fn_with_state(
Arc::clone(&state),
crate::auth::require_api_key,
));
let app = Router::new()
.route("/triggers/{trigger_id}/webhook", post(triggers::webhook))
.merge(authed)
.layer(tower_http::cors::CorsLayer::permissive())
.with_state(state);
(app, broker)
}
pub(crate) fn router_with_shutdown(
registry: GraphRegistry,
config: ServerConfig,
shutdown: tokio_util::sync::CancellationToken,
) -> Router {
build_router(registry, config, shutdown).0
}
pub(crate) fn internal_err<E: std::fmt::Display>(e: E) -> ApiError {
ApiError::internal(e.to_string())
}
async fn require_thread(
state: &AppState,
tenant: &TenantContext,
thread_id: &str,
) -> Result<ThreadRecord, ApiError> {
state
.server_store
.get_thread(&tenant.scope(thread_id))
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("thread `{thread_id}` not found")))
}
pub(crate) fn validate_client_id(kind: &str, id: &str) -> Result<(), ApiError> {
let ok = !id.is_empty()
&& id.len() <= 256
&& !id.contains('/')
&& !id.contains('\\')
&& !id.chars().all(|c| c == '.')
&& !RESERVED_NAMES.contains(&id);
if ok {
Ok(())
} else {
Err(ApiError::bad_request(format!(
"invalid {kind} `{id}` (must be non-empty, <= 256 chars, no path separators, not a reserved name)"
)))
}
}
async fn state_lock(state: &AppState, internal_id: &str) -> Arc<Mutex<()>> {
state
.state_locks
.lock()
.await
.entry(internal_id.to_string())
.or_default()
.clone()
}
fn checkpoint_ref(cp: &Checkpoint, tenant: &TenantContext) -> Value {
json!({
"checkpoint_id": cp.id,
"thread_id": tenant.unscope(&cp.thread_id).unwrap_or(&cp.thread_id),
"step": cp.step,
"created_at": cp.created_at,
"policy_version": cp.header.policy_version,
})
}
async fn ok() -> Json<Value> {
Json(json!({ "ok": true }))
}
async fn info(AxumState(state): AxumState<Arc<AppState>>) -> Json<Value> {
let graphs: Vec<Value> = state
.registry
.names()
.into_iter()
.map(|name| {
json!({
"name": name,
"channels": state.registry.channel_names(&name),
})
})
.collect();
let persistence = if state.config.database_url.is_some() {
"postgres"
} else {
"json_file"
};
Json(json!({
"service": "rusty-server",
"version": env!("CARGO_PKG_VERSION"),
"checkpointer": persistence,
"server_store": persistence,
"store_path": state.config.store_path,
"graphs": graphs,
}))
}
#[derive(Debug, Deserialize)]
struct CreateThreadPayload {
graph: String,
#[serde(default)]
metadata: Option<Value>,
#[serde(default)]
thread_id: Option<String>,
}
async fn create_thread(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<CreateThreadPayload>,
) -> Result<(StatusCode, Json<ThreadRecord>), ApiError> {
if !state.registry.contains(&payload.graph) {
return Err(ApiError::bad_request(format!(
"unknown graph `{}` (see GET /info for registered graphs)",
payload.graph
)));
}
let thread_id = payload
.thread_id
.clone()
.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
validate_client_id("thread_id", &thread_id)?;
let internal_id = tenant.scope(&thread_id);
let record = ThreadRecord {
thread_id: thread_id.clone(),
tenant: tenant.tenant().to_string(),
graph: payload.graph,
metadata: payload.metadata.unwrap_or(Value::Null),
created_at: Utc::now(),
};
let created = state
.server_store
.create_thread(&internal_id, &record)
.await
.map_err(internal_err)?;
if !created {
return Err(ApiError::conflict(format!(
"thread `{thread_id}` already exists"
)));
}
Ok((StatusCode::CREATED, Json(record)))
}
#[derive(Debug, Deserialize)]
struct ForkThreadPayload {
#[serde(default)]
new_thread_id: Option<String>,
#[serde(default)]
checkpoint_id: Option<String>,
}
async fn fork_thread(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(thread_id): Path<String>,
Json(payload): Json<ForkThreadPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let record = require_thread(&state, &tenant, &thread_id).await?;
let new_thread_id = payload
.new_thread_id
.clone()
.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
validate_client_id("new_thread_id", &new_thread_id)?;
let new_internal_id = tenant.scope(&new_thread_id);
if state
.server_store
.get_thread(&new_internal_id)
.await
.map_err(internal_err)?
.is_some()
{
return Err(ApiError::conflict(format!(
"thread `{new_thread_id}` already exists"
)));
}
let copied = state
.checkpointer
.fork_thread(
&tenant.scope(&thread_id),
&new_internal_id,
payload.checkpoint_id.as_deref(),
)
.await
.map_err(|e| {
let message = e.to_string();
if message.contains("unknown checkpoint id") {
ApiError::not_found(message)
} else {
ApiError::bad_request(message)
}
})?;
let fork = ThreadRecord {
thread_id: new_thread_id.clone(),
tenant: tenant.tenant().to_string(),
graph: record.graph,
metadata: json!({
"forked_from": thread_id,
"fork_checkpoint_id": payload.checkpoint_id,
}),
created_at: Utc::now(),
};
let created = state
.server_store
.create_thread(&new_internal_id, &fork)
.await
.map_err(internal_err)?;
if !created {
return Err(ApiError::conflict(format!(
"thread `{new_thread_id}` already exists"
)));
}
Ok((
StatusCode::CREATED,
Json(json!({
"thread_id": new_thread_id,
"checkpoints_copied": copied,
})),
))
}
async fn get_state(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(thread_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
require_thread(&state, &tenant, &thread_id).await?;
let latest = state
.checkpointer
.get_latest(&tenant.scope(&thread_id))
.await
.map_err(internal_err)?;
Ok(Json(match latest {
None => json!({ "values": {}, "next": [], "checkpoint": null }),
Some(cp) => json!({
"values": cp.state.to_value(),
"next": cp.next_nodes,
"checkpoint": checkpoint_ref(&cp, &tenant),
}),
}))
}
#[derive(Debug, Deserialize)]
struct UpdateStatePayload {
values: Value,
#[serde(default)]
as_node: Option<String>,
#[serde(default)]
next_nodes: Option<Vec<String>>,
#[serde(default)]
enqueue: Option<Vec<EnqueueTaskPayload>>,
}
async fn update_state(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(thread_id): Path<String>,
Json(payload): Json<UpdateStatePayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
require_thread(&state, &tenant, &thread_id).await?;
let UpdateStatePayload {
values,
as_node,
next_nodes,
enqueue,
} = payload;
let _ = as_node;
let outbox_tasks = enqueue
.map(|payloads| {
payloads
.into_iter()
.map(|p| build_task_record(p, &tenant))
.collect::<Result<Vec<_>, _>>()
})
.transpose()?;
if let Some(tasks) = &outbox_tasks {
if !tasks.is_empty() {
enforce_task_quota(&state, &tenant, tasks.len()).await?;
}
}
let internal_id = tenant.scope(&thread_id);
let new_state = State::from_value(values)
.map_err(|e| ApiError::bad_request(format!("`values` must be a JSON object: {e}")))?;
let lock = state_lock(&state, &internal_id).await;
let _guard = lock.lock().await;
let latest = state
.checkpointer
.get_latest(&internal_id)
.await
.map_err(internal_err)?;
let (step, prev_next) = latest
.map(|cp| (cp.step + 1, cp.next_nodes))
.unwrap_or((0, Vec::new()));
let cp = Checkpoint::new(
&internal_id,
step,
new_state,
next_nodes.unwrap_or(prev_next),
);
match &outbox_tasks {
Some(tasks) => state
.server_store
.checkpoint_and_enqueue(&cp, tasks)
.await
.map_err(internal_err)?,
None => state
.checkpointer
.put(cp.clone())
.await
.map_err(internal_err)?,
}
Ok((
StatusCode::CREATED,
Json(json!({
"values": cp.state.to_value(),
"next": cp.next_nodes,
"checkpoint": checkpoint_ref(&cp, &tenant),
})),
))
}
#[derive(Debug, Default, Deserialize)]
struct HistoryPayload {
#[serde(default)]
limit: Option<usize>,
#[serde(default)]
before: Option<String>,
}
async fn history(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(thread_id): Path<String>,
Json(payload): Json<HistoryPayload>,
) -> Result<Json<Value>, ApiError> {
require_thread(&state, &tenant, &thread_id).await?;
let mut checkpoints = state
.checkpointer
.list(&tenant.scope(&thread_id))
.await
.map_err(internal_err)?;
checkpoints.reverse();
if let Some(before) = &payload.before {
match checkpoints.iter().position(|cp| &cp.id == before) {
Some(pos) => {
checkpoints.drain(..=pos);
}
None => {
return Err(ApiError::bad_request(format!(
"unknown `before` checkpoint `{before}`"
)));
}
}
}
if let Some(limit) = payload.limit {
checkpoints.truncate(limit);
}
let items: Vec<Value> = checkpoints
.iter()
.map(|cp| {
json!({
"values": cp.state.to_value(),
"next": cp.next_nodes,
"checkpoint": checkpoint_ref(cp, &tenant),
})
})
.collect();
Ok(Json(Value::Array(items)))
}
async fn schedule_for_thread(
state: &Arc<AppState>,
tenant: &TenantContext,
thread_id: &str,
mut payload: RunPayload,
) -> Result<runs::Scheduled, ApiError> {
let record = require_thread(state, tenant, thread_id).await?;
let internal_id = tenant.scope(thread_id);
if let Some(input) = &payload.input {
if !input.is_object() {
return Err(ApiError::bad_request(
"`input` must be a JSON object".to_string(),
));
}
}
if let Some(assistant_id) = &payload.assistant_id {
validate_client_id("assistant_id", assistant_id)?;
let assistant = state
.server_store
.get_assistant(&tenant.scope(assistant_id))
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("assistant `{assistant_id}` not found")))?;
if assistant.archived_at.is_some() {
return Err(ApiError::new(
StatusCode::CONFLICT,
"assistant_archived",
format!(
"assistant `{assistant_id}` is archived; restore it before starting new work"
),
));
}
if assistant.graph != record.graph {
return Err(ApiError::bad_request(format!(
"assistant `{assistant_id}` is bound to graph `{}` but thread `{thread_id}` uses `{}`",
assistant.graph, record.graph
)));
}
let payload_limit = payload.config.as_ref().and_then(|c| c.recursion_limit);
if payload_limit.is_none() {
if let Some(limit) = assistant
.config
.get("recursion_limit")
.and_then(Value::as_u64)
{
payload
.config
.get_or_insert_with(RunConfigPayload::default)
.recursion_limit = Some(limit as usize);
}
}
}
if let Some(checkpoint) = &payload.checkpoint {
let found = state
.checkpointer
.get_by_id(&internal_id, &checkpoint.checkpoint_id)
.await
.map_err(internal_err)?;
if found.is_none() {
return Err(ApiError::not_found(format!(
"thread `{thread_id}` has no checkpoint `{}`",
checkpoint.checkpoint_id
)));
}
}
let strategy = MultitaskStrategy::parse(payload.multitask_strategy.as_deref())
.map_err(ApiError::bad_request)?;
runs::schedule(
&state.run_deps,
&internal_id,
thread_id,
&record.graph,
payload,
strategy,
)
.await
}
async fn create_run(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(thread_id): Path<String>,
Json(payload): Json<RunPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let scheduled = schedule_for_thread(&state, &tenant, &thread_id, payload).await?;
Ok((
StatusCode::ACCEPTED,
Json(json!({
"run_id": scheduled.run_id,
"thread_id": thread_id,
"status": scheduled.status.as_str(),
})),
))
}
const MAX_RUN_WAIT: Duration = Duration::from_secs(3600);
async fn create_run_wait(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(thread_id): Path<String>,
Json(payload): Json<RunPayload>,
) -> Result<Json<Value>, ApiError> {
let scheduled = schedule_for_thread(&state, &tenant, &thread_id, payload).await?;
let mut terminal = scheduled.terminal;
let result = tokio::time::timeout(MAX_RUN_WAIT, terminal.wait_for(|v| v.is_some()))
.await
.map_err(|_| {
ApiError::new(
StatusCode::GATEWAY_TIMEOUT,
"timeout",
format!(
"run did not reach a terminal state within {}s",
MAX_RUN_WAIT.as_secs()
),
)
})?
.map_err(|_| ApiError::internal("run ended without a terminal result".to_string()))?;
let value = result.clone().expect("wait_for predicate guarantees Some");
Ok(Json(value))
}
fn sse_response(
replay: Vec<runs::SseFrame>,
broadcast: tokio::sync::broadcast::Receiver<runs::SseFrame>,
skip_through_seq: u64,
) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {
Sse::new(sse::frame_stream(replay, broadcast, skip_through_seq)).keep_alive(
KeepAlive::new()
.interval(Duration::from_secs(15))
.text("keep-alive"),
)
}
async fn create_run_stream(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(thread_id): Path<String>,
Json(payload): Json<RunPayload>,
) -> Result<Sse<impl Stream<Item = Result<Event, Infallible>>>, ApiError> {
let scheduled = schedule_for_thread(&state, &tenant, &thread_id, payload).await?;
Ok(sse_response(scheduled.replay, scheduled.broadcast, 0))
}
async fn get_run_stream(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(run_id): Path<String>,
headers: HeaderMap,
) -> Result<Sse<impl Stream<Item = Result<Event, Infallible>>>, ApiError> {
let (replay, broadcast, internal_thread_id) = state
.run_deps
.manager
.stream_parts(&run_id)
.await
.ok_or_else(|| ApiError::not_found(format!("run `{run_id}` not found")))?;
if !tenant.owns(&internal_thread_id) {
return Err(ApiError::not_found(format!("run `{run_id}` not found")));
}
let last_seen =
sse::parse_last_event_id(headers.get("last-event-id").and_then(|v| v.to_str().ok()));
Ok(sse_response(replay, broadcast, last_seen))
}
async fn delete_run_checkpoints(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path((thread_id, run_id)): Path<(String, String)>,
) -> Result<Json<Value>, ApiError> {
require_thread(&state, &tenant, &thread_id).await?;
let (wire_thread_id, checkpoint_ids) = match state.run_deps.manager.info(&run_id).await {
Some(info) => {
if !tenant.owns(&info.thread_id) {
return Err(ApiError::not_found(format!("run `{run_id}` not found")));
}
if matches!(info.status, RunStatus::Pending | RunStatus::Running) {
return Err(ApiError::conflict(
"run is still active; rollback applies to finished runs".to_string(),
));
}
(
info.wire_thread_id,
runs::lock_recover(&info.checkpoint_ids).clone(),
)
}
None => {
let evidence = run_evidence(&state, &tenant, &run_id).await?;
(evidence.wire_thread_id, evidence.checkpoint_ids)
}
};
if wire_thread_id != thread_id {
return Err(ApiError::bad_request(format!(
"run `{run_id}` does not belong to thread `{thread_id}`"
)));
}
if state.config.database_url.is_some() {
return Err(ApiError::conflict(
"rollback is not supported with the Postgres checkpointer".to_string(),
));
}
let internal_id = tenant.scope(&thread_id);
if state.run_deps.manager.thread_busy(&internal_id).await {
return Err(ApiError::conflict(
"thread has an active or queued run; rollback applies to idle threads".to_string(),
));
}
let ids = checkpoint_ids;
let history = state
.checkpointer
.list(&internal_id)
.await
.map_err(internal_err)?;
let is_suffix = history.len() >= ids.len()
&& history[history.len() - ids.len()..]
.iter()
.map(|cp| cp.id.as_str())
.eq(ids.iter().map(String::as_str));
if !is_suffix {
return Err(ApiError::conflict(
"the run's checkpoints are not the latest on this thread; \
rollback would punch holes mid-history"
.to_string(),
));
}
let dir = state.config.store_path.join(&internal_id);
let mut deleted = 0usize;
for id in &ids {
let path = dir.join(format!("{id}.json"));
match tokio::fs::remove_file(&path).await {
Ok(()) => deleted += 1,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => {
return Err(ApiError::internal(format!(
"failed to delete `{}`: {e}",
path.display()
)))
}
}
}
let remaining = state
.checkpointer
.list(&internal_id)
.await
.map_err(internal_err)?;
let latest_path = dir.join("latest");
match remaining.last() {
Some(cp) => atomic_write(&latest_path, cp.id.as_bytes())
.await
.map_err(internal_err)?,
None => match tokio::fs::remove_file(&latest_path).await {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => {
tracing::warn!(path = %latest_path.display(), %e, "failed to remove latest pointer")
}
},
}
Ok(Json(json!({
"run_id": run_id,
"thread_id": thread_id,
"deleted_checkpoints": deleted,
"remaining_checkpoints": remaining.len(),
})))
}
async fn atomic_write(path: &std::path::Path, bytes: &[u8]) -> std::io::Result<()> {
let tmp = path.with_extension("tmp");
tokio::fs::write(&tmp, bytes).await?;
tokio::fs::rename(&tmp, path).await
}
async fn get_run(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(run_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
let info = state
.run_deps
.manager
.info(&run_id)
.await
.ok_or_else(|| ApiError::not_found(format!("run `{run_id}` not found")))?;
if !tenant.owns(&info.thread_id) {
return Err(ApiError::not_found(format!("run `{run_id}` not found")));
}
let mut body = json!({
"run_id": run_id,
"thread_id": info.wire_thread_id,
"graph": info.graph,
"attempt": info.attempt,
"status": info.status.as_str(),
});
if let Some(terminal) = info.terminal {
if let (Some(body), Some(terminal)) = (body.as_object_mut(), terminal.as_object()) {
for (key, value) in terminal {
body.insert(key.clone(), value.clone());
}
}
}
Ok(Json(body))
}
async fn cancel_run(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(run_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
let info = state
.run_deps
.manager
.info(&run_id)
.await
.ok_or_else(|| ApiError::not_found(format!("run `{run_id}` not found")))?;
if !tenant.owns(&info.thread_id) {
return Err(ApiError::not_found(format!("run `{run_id}` not found")));
}
let outcome = state
.server_store
.cancel_run_tasks(tenant.tenant(), &run_id, Utc::now())
.await
.map_err(internal_err)?;
let ids =
|tasks: Vec<TaskRecord>| -> Vec<String> { tasks.into_iter().map(|t| t.task_id).collect() };
Ok(Json(json!({
"run_id": run_id,
"cancelled": ids(outcome.cancelled),
"signalled": ids(outcome.signalled),
})))
}
struct RunEvidence {
graph: String,
internal_thread_id: String,
wire_thread_id: String,
journal: Option<JournalSnapshot>,
checkpoint_ids: Vec<String>,
complete: bool,
}
fn reverify_journal(run_id: &str, snapshot: JournalSnapshot) -> Result<JournalSnapshot, ApiError> {
Journal::from_snapshot(snapshot.clone(), Clock::System).map_err(|e| {
ApiError::internal(format!(
"stored journal for run `{run_id}` failed its integrity check: {e}"
))
})?;
Ok(snapshot)
}
async fn run_evidence(
state: &AppState,
tenant: &TenantContext,
run_id: &str,
) -> Result<RunEvidence, ApiError> {
if let Some(info) = state.run_deps.manager.info(run_id).await {
if !tenant.owns(&info.thread_id) {
return Err(ApiError::not_found(format!("run `{run_id}` not found")));
}
let journal = state
.server_store
.get_journal(run_id)
.await
.map_err(internal_err)?
.map(|snapshot| reverify_journal(run_id, snapshot))
.transpose()?;
return Ok(RunEvidence {
graph: info.graph,
internal_thread_id: info.thread_id,
wire_thread_id: info.wire_thread_id,
journal,
checkpoint_ids: runs::lock_recover(&info.checkpoint_ids).clone(),
complete: info.status.is_terminal(),
});
}
let Some(snapshot) = state
.server_store
.get_journal(run_id)
.await
.map_err(internal_err)?
else {
return Err(ApiError::not_found(format!("run `{run_id}` not found")));
};
let internal_thread_id = tenant.scope(&snapshot.thread_id);
let thread = state
.server_store
.get_thread(&internal_thread_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("run `{run_id}` not found")))?;
let journal = reverify_journal(run_id, snapshot)?;
let checkpoint_ids = journal
.events
.iter()
.filter(|event| event.kind == RunEventKind::CheckpointWritten)
.filter_map(|event| crate::replay::resolve(&journal, event.output.as_ref()))
.filter_map(|output| output.get("checkpoint_id")?.as_str().map(str::to_owned))
.collect();
Ok(RunEvidence {
graph: thread.graph,
internal_thread_id,
wire_thread_id: journal.thread_id.clone(),
journal: Some(journal),
checkpoint_ids,
complete: true,
})
}
async fn get_run_events(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(run_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
let evidence = run_evidence(&state, &tenant, &run_id).await?;
let events = evidence
.journal
.map(|snapshot| snapshot.events)
.unwrap_or_default();
Ok(Json(json!({
"run_id": run_id,
"events": events,
"complete": evidence.complete,
})))
}
async fn get_run_fixture(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(run_id): Path<String>,
) -> Result<Json<ReplayFixture>, ApiError> {
let evidence = run_evidence(&state, &tenant, &run_id).await?;
let snapshot = evidence.journal.ok_or_else(|| {
ApiError::conflict(format!(
"run `{run_id}` has no persisted journal yet (queued or pre-checkpoint)"
))
})?;
let (graph, _spec) = state.registry.get(&evidence.graph).ok_or_else(|| {
ApiError::conflict(format!(
"graph `{}` is no longer registered; cannot capture a fixture for run `{run_id}`",
evidence.graph
))
})?;
let final_checkpoint = match evidence.checkpoint_ids.last() {
Some(id) => state
.checkpointer
.get_by_id(&evidence.internal_thread_id, id)
.await
.map_err(internal_err)?
.map(|mut cp| {
cp.thread_id = evidence.wire_thread_id.clone();
cp
}),
None => None,
};
let fixture = ReplayFixture::capture(
format!("{} run {run_id}", evidence.graph),
&graph,
"unversioned",
snapshot,
final_checkpoint,
None,
None,
);
Ok(Json(fixture))
}
fn carries_servable_effects(snapshot: &JournalSnapshot) -> bool {
snapshot.events.iter().any(|event| {
matches!(
event.kind,
RunEventKind::ModelCall
| RunEventKind::ToolCall
| RunEventKind::RemoteCall
| RunEventKind::WasmCall
)
})
}
async fn get_run_receipt(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(run_id): Path<String>,
) -> Result<Json<rusty_agent_runtime::receipt::RunReceipt>, ApiError> {
let evidence = run_evidence(&state, &tenant, &run_id).await?;
let snapshot = evidence.journal.ok_or_else(|| {
ApiError::conflict(format!(
"run `{run_id}` has no persisted journal yet (queued or pre-checkpoint)"
))
})?;
let head = JournalRef {
events: snapshot.events.len() as u64,
sha256: snapshot.head_hash.clone(),
};
if let Some(stored) = state
.server_store
.get_run_receipt(&run_id)
.await
.map_err(internal_err)?
{
if stored.run_id == run_id && stored.journal_head == head {
return Ok(Json(stored));
}
}
let (manifest, executor_policy) = match evidence.checkpoint_ids.last() {
Some(checkpoint_id) => state
.checkpointer
.get_by_id(&evidence.internal_thread_id, checkpoint_id)
.await
.map_err(internal_err)?
.map(|checkpoint| {
(
checkpoint.header.manifest,
Some(checkpoint.header.policy_version),
)
})
.unwrap_or((None, None)),
None => (None, None),
};
let signing_key = state
.receipt_keyring
.active_key()
.await
.map_err(ApiError::internal)?;
let receipt = rusty_agent_runtime::receipt::mint_receipt(
&snapshot,
manifest,
executor_policy,
&signing_key,
)
.map_err(internal_err)?;
state
.server_store
.put_run_receipt(&run_id, &receipt)
.await
.map_err(internal_err)?;
Ok(Json(receipt))
}
#[derive(Debug, Deserialize)]
struct VerifyReceiptPayload {
snapshot: JournalSnapshot,
receipt: rusty_agent_runtime::receipt::RunReceipt,
key_id: Option<String>,
}
async fn verify_receipt_route(
AxumState(state): AxumState<Arc<AppState>>,
Extension(_tenant): Extension<TenantContext>,
Json(payload): Json<VerifyReceiptPayload>,
) -> Result<Json<rusty_agent_runtime::receipt::VerifiedRun>, ApiError> {
let key_id = payload
.key_id
.unwrap_or_else(|| payload.receipt.signer.clone());
let record = state
.server_store
.get_receipt_key(&key_id)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!(
"signing key `{key_id}` is not in this deployment's key history"
))
})?;
let public_key = rusty_agent_runtime::receipt::PublicKey::from_hex(&record.public_key)
.map_err(internal_err)?;
match rusty_agent_runtime::receipt::verify_receipt(
&payload.snapshot,
&payload.receipt,
&public_key,
) {
Ok(verified) => Ok(Json(verified)),
Err(rejection) => Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"receipt_verification_failed",
format!("{}: {rejection}", rejection.component()),
)),
}
}
async fn list_receipt_keys_route(
AxumState(state): AxumState<Arc<AppState>>,
Extension(_tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let mut keys = state
.server_store
.list_receipt_keys()
.await
.map_err(internal_err)?;
keys.sort_by_key(|record| record.registered_at);
let active = state
.receipt_keyring
.active_key()
.await
.map_err(ApiError::internal)?
.key_id();
Ok(Json(json!({
"keys": keys,
"active": active,
})))
}
async fn rotate_receipt_key(
AxumState(state): AxumState<Arc<AppState>>,
Extension(_tenant): Extension<TenantContext>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let outcome = state
.receipt_keyring
.rotate()
.await
.map_err(ApiError::internal)?;
Ok((
StatusCode::CREATED,
Json(json!({
"previous_key_id": outcome.previous_key_id,
"key_id": outcome.new_key_id,
"public_key": outcome.public_key,
"event_id": outcome.event_id,
})),
))
}
async fn get_receipt_keys_journal(
AxumState(state): AxumState<Arc<AppState>>,
Extension(_tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let events = match state
.server_store
.get_journal(crate::receipts::RECEIPTS_JOURNAL_RUN_ID)
.await
.map_err(internal_err)?
{
Some(snapshot) => {
reverify_journal(crate::receipts::RECEIPTS_JOURNAL_RUN_ID, snapshot)?.events
}
None => Vec::new(),
};
Ok(Json(json!({
"run_id": crate::receipts::RECEIPTS_JOURNAL_RUN_ID,
"events": events,
"complete": false,
})))
}
#[derive(Debug, Deserialize)]
struct RegisterConnectionPayload {
provider: rusty_agent_runtime::broker::ConnectionProvider,
#[serde(default)]
subject: Option<String>,
#[serde(default)]
scopes: std::collections::BTreeSet<String>,
#[serde(default)]
token: Option<rusty_agent_runtime::broker::TokenMaterial>,
#[serde(default)]
authorization_code: Option<String>,
}
#[derive(Debug, Deserialize)]
struct ConsentConnectionPayload {
#[serde(default)]
scopes: Option<std::collections::BTreeSet<String>>,
#[serde(default)]
token: Option<rusty_agent_runtime::broker::TokenMaterial>,
#[serde(default)]
authorization_code: Option<String>,
}
#[derive(Debug, Deserialize)]
struct RevokeConnectionPayload {
#[serde(default)]
reason: Option<String>,
}
fn invalid_connection_input(e: impl std::fmt::Display) -> ApiError {
ApiError::unprocessable(e.to_string())
}
async fn exchange_authorization_code(
state: &AppState,
code: &str,
scopes: &std::collections::BTreeSet<String>,
) -> Result<rusty_agent_runtime::broker::TokenMaterial, ApiError> {
if code.is_empty() {
return Err(ApiError::unprocessable(
"authorization_code must not be empty".to_owned(),
));
}
let Some((provider, _)) = state.broker.oauth() else {
return Err(ApiError::conflict(
"no OAuth provider is configured on this server — an authorization-code exchange \
requires one (`ServerConfig::with_oauth_provider`)"
.to_owned(),
));
};
let grant = provider
.exchange_code(code, scopes)
.await
.map_err(|failure| {
if failure.permanent {
ApiError::unprocessable(format!("the authorization code was refused: {failure}"))
} else {
ApiError::internal(format!("the authorization code exchange failed: {failure}"))
}
})?;
Ok(rusty_agent_runtime::broker::TokenMaterial {
access_token: grant.access_token,
refresh_token: grant.refresh_token,
client_secret: None,
expires_at: grant.expires_at,
})
}
async fn register_connection(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<RegisterConnectionPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
if payload.token.is_some() == payload.authorization_code.is_some() {
return Err(ApiError::unprocessable(
"exactly one of `token` or `authorization_code` registers a connection — the \
material is sealed verbatim, or the code is exchanged through the deployment's \
OAuth provider; never both, never neither"
.to_owned(),
));
}
let token = match (&payload.token, &payload.authorization_code) {
(Some(token), None) => {
if token.access_token.is_empty() {
return Err(ApiError::unprocessable(
"token.access_token must not be empty".to_owned(),
));
}
token.clone()
}
(None, Some(code)) => exchange_authorization_code(&state, code, &payload.scopes).await?,
_ => unreachable!("the exclusivity gate passed"),
};
rusty_agent_runtime::broker::validate_connection_fields(
payload.subject.as_deref(),
&payload.scopes,
)
.map_err(invalid_connection_input)?;
let record = state
.broker
.register(
tenant.tenant(),
payload.provider,
payload.subject,
payload.scopes,
&token,
)
.await
.map_err(internal_err)?;
Ok((StatusCode::CREATED, Json(json!({ "connection": record }))))
}
async fn list_connections(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let mut records = state
.broker
.list(tenant.tenant())
.await
.map_err(internal_err)?;
records.sort_by(|a, b| a.connection_id.cmp(&b.connection_id));
Ok(Json(json!({ "connections": records })))
}
async fn get_connection(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(connection_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
let record = state
.broker
.get(tenant.tenant(), &connection_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("connection `{connection_id}` not found")))?;
Ok(Json(json!({ "connection": record })))
}
async fn consent_connection(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(connection_id): Path<String>,
Json(payload): Json<ConsentConnectionPayload>,
) -> Result<Json<Value>, ApiError> {
if payload.scopes.is_none() && payload.token.is_none() && payload.authorization_code.is_none() {
return Err(ApiError::unprocessable(
"a consent act needs `scopes`, `token`, and/or `authorization_code`".to_owned(),
));
}
if payload.token.is_some() && payload.authorization_code.is_some() {
return Err(ApiError::unprocessable(
"`token` and `authorization_code` are mutually exclusive — the material is sealed \
verbatim, or the code is exchanged through the deployment's OAuth provider; \
never both"
.to_owned(),
));
}
if let Some(scopes) = &payload.scopes {
rusty_agent_runtime::broker::validate_connection_fields(None, scopes)
.map_err(invalid_connection_input)?;
}
let token = match (&payload.token, &payload.authorization_code) {
(None, None) => None,
(Some(token), None) => {
if token.access_token.is_empty() {
return Err(ApiError::unprocessable(
"token.access_token must not be empty".to_owned(),
));
}
Some(token.clone())
}
(None, Some(code)) => {
let scopes = match &payload.scopes {
Some(scopes) => scopes.clone(),
None => {
state
.broker
.get(tenant.tenant(), &connection_id)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!("connection `{connection_id}` not found"))
})?
.scopes
}
};
Some(exchange_authorization_code(&state, code, &scopes).await?)
}
_ => unreachable!("the exclusivity gate passed"),
};
match state
.broker
.record_consent(
tenant.tenant(),
&connection_id,
payload.scopes,
token.as_ref(),
)
.await
.map_err(internal_err)?
{
crate::broker::ConsentOutcome::Applied { record, journaled } => Ok(Json(
json!({ "connection": record, "journaled": journaled }),
)),
crate::broker::ConsentOutcome::Converged(record) => Ok(Json(
json!({ "connection": record, "journaled": Value::Null }),
)),
crate::broker::ConsentOutcome::Unknown => Err(ApiError::not_found(format!(
"connection `{connection_id}` not found"
))),
crate::broker::ConsentOutcome::Conflict => Err(ApiError::conflict(format!(
"connection `{connection_id}` changed under the consent; retry"
))),
}
}
async fn revoke_connection(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(connection_id): Path<String>,
Json(payload): Json<RevokeConnectionPayload>,
) -> Result<Json<Value>, ApiError> {
match state
.broker
.revoke(tenant.tenant(), &connection_id, payload.reason)
.await
.map_err(internal_err)?
{
crate::broker::RevokeOutcome::Applied { record, event_id } => {
Ok(Json(json!({ "connection": record, "event_id": event_id })))
}
crate::broker::RevokeOutcome::Converged(record) => {
Ok(Json(json!({ "connection": record })))
}
crate::broker::RevokeOutcome::Unknown => Err(ApiError::not_found(format!(
"connection `{connection_id}` not found"
))),
crate::broker::RevokeOutcome::Conflict => Err(ApiError::conflict(format!(
"connection `{connection_id}` changed under the revocation; retry"
))),
}
}
async fn delete_connection(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(connection_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
let deleted = state
.broker
.delete(tenant.tenant(), &connection_id)
.await
.map_err(internal_err)?;
if !deleted {
return Err(ApiError::not_found(format!(
"connection `{connection_id}` not found"
)));
}
Ok(Json(json!({ "deleted": true })))
}
async fn get_connection_health(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(connection_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
let record = state
.broker
.get(tenant.tenant(), &connection_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("connection `{connection_id}` not found")))?;
Ok(Json(json!({
"connection_id": record.connection_id,
"status": record.status,
"health": record.health,
})))
}
async fn list_connections_health(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let mut records = state
.broker
.list(tenant.tenant())
.await
.map_err(internal_err)?;
records.sort_by(|a, b| a.connection_id.cmp(&b.connection_id));
let connections = records
.iter()
.map(|record| {
json!({
"connection_id": record.connection_id,
"provider": record.provider,
"status": record.status,
"health": record.health,
})
})
.collect::<Vec<_>>();
Ok(Json(json!({ "connections": connections })))
}
async fn get_broker_journal(
AxumState(state): AxumState<Arc<AppState>>,
Extension(_tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let events = match state
.server_store
.get_journal(crate::broker::BROKER_JOURNAL_RUN_ID)
.await
.map_err(internal_err)?
{
Some(snapshot) => reverify_journal(crate::broker::BROKER_JOURNAL_RUN_ID, snapshot)?.events,
None => Vec::new(),
};
Ok(Json(json!({
"run_id": crate::broker::BROKER_JOURNAL_RUN_ID,
"events": events,
"complete": false,
})))
}
#[derive(Debug, Deserialize)]
struct ReplayRunPayload {
run_id: String,
}
async fn replay_run(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<ReplayRunPayload>,
) -> Result<Json<Value>, ApiError> {
let run_id = payload.run_id;
let evidence = run_evidence(&state, &tenant, &run_id).await?;
let snapshot = evidence.journal.ok_or_else(|| {
ApiError::conflict(format!(
"run `{run_id}` has no persisted journal yet (queued or pre-checkpoint)"
))
})?;
if !evidence.complete {
return Err(ApiError::conflict(format!(
"run `{run_id}` is still executing; replay verifies a run's final journal"
)));
}
let (graph, spec) = state.registry.get(&evidence.graph).ok_or_else(|| {
ApiError::unprocessable(format!(
"graph `{}` is not registered in this server process; cannot replay run `{run_id}`",
evidence.graph
))
})?;
if carries_servable_effects(&snapshot) {
return Err(ApiError::unprocessable(format!(
"run `{run_id}` journaled model/tool/remote/WASM calls; server-side replay \
re-executes node code and cannot serve recorded effects — download the fixture \
(GET /runs/{run_id}/fixture) and replay it in CI with ReplayFixture"
)));
}
if snapshot
.events
.first()
.is_some_and(|event| event.kind == RunEventKind::Resume)
{
return Err(ApiError::unprocessable(format!(
"run `{run_id}` resumed from a checkpoint; its journal begins mid-run against \
state it does not carry — replay the original run's journal instead"
)));
}
let replay = ExactReplay::new(snapshot.clone()).map_err(|e| {
ApiError::internal(format!(
"stored journal for run `{run_id}` failed its integrity check: {e}"
))
})?;
let initial = crate::replay::initial_state_from(&snapshot);
let journal = replay.fresh_journal(Clock::System);
let params = ReplayParams::new(journal.clone(), RngSource::default())
.with_checkpointer(Arc::new(InMemoryCheckpointer::new()));
let _ = replay.run(&graph, &spec, initial, params).await;
let replayed = journal.snapshot();
let report = crate::replay::compare_journals(&snapshot, &replayed);
Ok(Json(json!({
"run_id": run_id,
"verified": report.verified,
"expected_events": snapshot.events.len(),
"actual_events": replayed.events.len(),
"first_divergence": report.first_divergence,
})))
}
#[derive(Debug, Deserialize)]
struct DiffQuery {
base: String,
branch: String,
}
async fn require_journal(
state: &AppState,
tenant: &TenantContext,
run_id: &str,
) -> Result<JournalSnapshot, ApiError> {
let evidence = run_evidence(state, tenant, run_id).await?;
evidence.journal.ok_or_else(|| {
ApiError::conflict(format!(
"run `{run_id}` has no persisted journal yet (queued or pre-checkpoint)"
))
})
}
async fn diff_runs(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Query(query): Query<DiffQuery>,
) -> Result<Json<BranchDiff>, ApiError> {
let base = require_journal(&state, &tenant, &query.base).await?;
let branch = require_journal(&state, &tenant, &query.branch).await?;
Ok(Json(BranchDiff::between(&base, &branch)))
}
#[derive(Debug, Deserialize)]
struct CreateAssistantPayload {
name: String,
graph: String,
#[serde(default)]
assistant_id: Option<String>,
#[serde(default)]
config: Option<Value>,
#[serde(default)]
metadata: Option<Value>,
}
async fn create_assistant(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<CreateAssistantPayload>,
) -> Result<(StatusCode, Json<AssistantView>), ApiError> {
if payload.name.trim().is_empty() {
return Err(ApiError::bad_request(
"`name` must not be empty".to_string(),
));
}
if !state.registry.contains(&payload.graph) {
return Err(ApiError::bad_request(format!(
"unknown graph `{}` (see GET /info for registered graphs)",
payload.graph
)));
}
let assistant_id = payload
.assistant_id
.clone()
.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
validate_client_id("assistant_id", &assistant_id)?;
let record = AssistantRecord::new(
tenant.scope(&assistant_id),
payload.name,
payload.graph,
payload.config.unwrap_or(Value::Null),
payload.metadata.unwrap_or(Value::Null),
Utc::now(),
);
if record.versions[0].storage_size() > ASSISTANT_VERSION_BYTES_LIMIT
|| record.lineage_storage_size() > ASSISTANT_LINEAGE_BYTES_LIMIT
{
return Err(ApiError::unprocessable(format!(
"assistant configuration exceeds the {} KiB version boundary",
ASSISTANT_VERSION_BYTES_LIMIT / 1024
)));
}
let created = state
.server_store
.create_assistant(&record)
.await
.map_err(internal_err)?;
if !created {
return Err(ApiError::conflict(format!(
"assistant `{assistant_id}` already exists"
)));
}
Ok((StatusCode::CREATED, Json(record.view(assistant_id))))
}
async fn list_assistants(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let records = state
.server_store
.list_assistants()
.await
.map_err(internal_err)?;
let mut records: Vec<AssistantView> = records
.into_iter()
.filter_map(|mut record| {
let external = tenant.unscope(&record.assistant_id)?.to_string();
record.assistant_id = external;
Some(record)
})
.collect();
records.sort_by(|a, b| {
a.created_at
.cmp(&b.created_at)
.then_with(|| a.assistant_id.cmp(&b.assistant_id))
});
Ok(Json(json!(records)))
}
async fn get_assistant(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(assistant_id): Path<String>,
) -> Result<Json<AssistantView>, ApiError> {
state
.server_store
.get_assistant(&tenant.scope(&assistant_id))
.await
.map_err(internal_err)?
.map(|record| Json(record.view(assistant_id.clone())))
.ok_or_else(|| ApiError::not_found(format!("assistant `{assistant_id}` not found")))
}
#[derive(Debug, Deserialize)]
struct AssistantLifecyclePayload {
expected_active_version_id: String,
}
async fn archive_assistant(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(assistant_id): Path<String>,
Json(payload): Json<AssistantLifecyclePayload>,
) -> Result<Json<Value>, ApiError> {
set_assistant_lifecycle(state, tenant, assistant_id, payload, true).await
}
async fn restore_assistant(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(assistant_id): Path<String>,
Json(payload): Json<AssistantLifecyclePayload>,
) -> Result<Json<Value>, ApiError> {
set_assistant_lifecycle(state, tenant, assistant_id, payload, false).await
}
async fn set_assistant_lifecycle(
state: Arc<AppState>,
tenant: TenantContext,
assistant_id: String,
payload: AssistantLifecyclePayload,
archived: bool,
) -> Result<Json<Value>, ApiError> {
validate_client_id("assistant_id", &assistant_id)?;
if !valid_version_id(&payload.expected_active_version_id) {
return Err(ApiError::bad_request(
"`expected_active_version_id` must be an exact assistant version id".to_string(),
));
}
let outcome = state
.server_store
.set_assistant_archived(
&tenant.scope(&assistant_id),
&payload.expected_active_version_id,
archived,
Utc::now(),
)
.await
.map_err(internal_err)?;
let (changed, record) = match outcome {
SetLifecycleOutcome::Changed { record } => (true, record),
SetLifecycleOutcome::Already { record } => (false, record),
SetLifecycleOutcome::AssistantNotFound => {
return Err(ApiError::not_found(format!(
"assistant `{assistant_id}` not found"
)))
}
SetLifecycleOutcome::Stale { active_version_id } => {
return Err(ApiError::conflict(format!(
"assistant `{assistant_id}` now serves version `{active_version_id}`; refresh before changing lifecycle"
)))
}
};
Ok(Json(json!({
"assistant": record.view(assistant_id),
"changed": changed,
"lifecycle": if archived { "archived" } else { "active" },
})))
}
#[derive(Debug, Deserialize)]
struct CreateAssistantVersionPayload {
base_version_id: String,
name: String,
graph: String,
#[serde(default)]
config: Option<Value>,
#[serde(default)]
metadata: Option<Value>,
}
async fn create_assistant_version(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(assistant_id): Path<String>,
Json(payload): Json<CreateAssistantVersionPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
validate_client_id("assistant_id", &assistant_id)?;
if !valid_version_id(&payload.base_version_id) {
return Err(ApiError::bad_request(
"`base_version_id` must be an exact assistant version id".to_string(),
));
}
if payload.name.trim().is_empty() {
return Err(ApiError::bad_request(
"`name` must not be empty".to_string(),
));
}
if !state.registry.contains(&payload.graph) {
return Err(ApiError::bad_request(format!(
"unknown graph `{}` (see GET /info for registered graphs)",
payload.graph
)));
}
let version = AssistantVersionRecord::new(
Some(payload.base_version_id.clone()),
payload.name,
payload.graph,
payload.config.unwrap_or(Value::Null),
payload.metadata.unwrap_or(Value::Null),
Utc::now(),
);
let scoped_id = tenant.scope(&assistant_id);
let outcome = state
.server_store
.create_assistant_version(&scoped_id, &payload.base_version_id, &version)
.await
.map_err(internal_err)?;
let (created, record, stored) = match outcome {
CreateVersionOutcome::Created { record, version } => (true, record, version),
CreateVersionOutcome::Existing { record, version } => (false, record, version),
CreateVersionOutcome::AssistantNotFound => {
return Err(ApiError::not_found(format!(
"assistant `{assistant_id}` not found"
)))
}
CreateVersionOutcome::Stale { active_version_id } => {
return Err(ApiError::conflict(format!(
"assistant `{assistant_id}` now serves version `{active_version_id}`; refresh before creating a version"
)))
}
CreateVersionOutcome::LimitReached => {
return Err(ApiError::unprocessable(
"assistant version history reached its 256-version limit".to_string(),
))
}
CreateVersionOutcome::SizeLimitReached => {
return Err(ApiError::unprocessable(format!(
"assistant version exceeds the {} KiB per-version or {} KiB lineage boundary",
ASSISTANT_VERSION_BYTES_LIMIT / 1024,
ASSISTANT_LINEAGE_BYTES_LIMIT / 1024
)))
}
};
let active_id = record.active_version_id();
let status = if created {
StatusCode::CREATED
} else {
StatusCode::OK
};
Ok((
status,
Json(json!({
"assistant_id": assistant_id,
"created": created,
"active_version_id": active_id,
"version": AssistantVersionView {
active: stored.version_id == active_id,
version: stored,
},
})),
))
}
async fn list_assistant_versions(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(assistant_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
validate_client_id("assistant_id", &assistant_id)?;
let record = state
.server_store
.get_assistant(&tenant.scope(&assistant_id))
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("assistant `{assistant_id}` not found")))?;
let active_id = record.active_version_id();
let assistant = record.view(assistant_id.clone());
if record.versions.len() > crate::assistants::ASSISTANT_VERSION_LIMIT {
return Err(ApiError::internal(
"assistant version history exceeds the supported limit".to_string(),
));
}
let views = record.version_summaries();
Ok(Json(json!({
"assistant_id": assistant_id,
"active_version_id": active_id,
"assistant": assistant,
"versions": views,
})))
}
async fn get_assistant_version(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path((assistant_id, version_id)): Path<(String, String)>,
) -> Result<Json<Value>, ApiError> {
validate_client_id("assistant_id", &assistant_id)?;
if !valid_version_id(&version_id) {
return Err(ApiError::bad_request(
"`version_id` must be an exact assistant version id".to_string(),
));
}
let record = state
.server_store
.get_assistant(&tenant.scope(&assistant_id))
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("assistant `{assistant_id}` not found")))?;
let active_id = record.active_version_id();
let version = record.version(&version_id).ok_or_else(|| {
ApiError::not_found(format!(
"assistant version `{version_id}` not found for `{assistant_id}`"
))
})?;
Ok(Json(json!({
"assistant_id": assistant_id,
"active_version_id": active_id,
"version": AssistantVersionView {
active: version.version_id == active_id,
version,
},
})))
}
#[derive(Debug, Deserialize)]
struct ActivateAssistantVersionPayload {
expected_active_version_id: String,
}
async fn activate_assistant_version(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path((assistant_id, version_id)): Path<(String, String)>,
Json(payload): Json<ActivateAssistantVersionPayload>,
) -> Result<Json<Value>, ApiError> {
validate_client_id("assistant_id", &assistant_id)?;
if !valid_version_id(&version_id) || !valid_version_id(&payload.expected_active_version_id) {
return Err(ApiError::bad_request(
"version ids must be exact assistant version ids".to_string(),
));
}
let scoped_id = tenant.scope(&assistant_id);
let target = state
.server_store
.get_assistant(&scoped_id)
.await
.map_err(internal_err)?
.and_then(|record| record.version(&version_id))
.ok_or_else(|| {
ApiError::not_found(format!(
"assistant version `{version_id}` not found for `{assistant_id}`"
))
})?;
if !state.registry.contains(&target.graph) {
return Err(ApiError::unprocessable(format!(
"assistant version `{version_id}` requires graph `{}`, which is not registered on this server",
target.graph
)));
}
let outcome = state
.server_store
.activate_assistant_version(&scoped_id, &version_id, &payload.expected_active_version_id)
.await
.map_err(internal_err)?;
let (activated, record) = match outcome {
ActivateVersionOutcome::Activated { record } => (true, record),
ActivateVersionOutcome::AlreadyActive { record } => (false, record),
ActivateVersionOutcome::AssistantNotFound => {
return Err(ApiError::not_found(format!(
"assistant `{assistant_id}` not found"
)))
}
ActivateVersionOutcome::VersionNotFound => {
return Err(ApiError::not_found(format!(
"assistant version `{version_id}` not found for `{assistant_id}`"
)))
}
ActivateVersionOutcome::Stale { active_version_id } => {
return Err(ApiError::conflict(format!(
"assistant `{assistant_id}` now serves version `{active_version_id}`; refresh before activation"
)))
}
};
Ok(Json(json!({
"assistant": record.view(assistant_id),
"activated": activated,
})))
}
#[derive(Debug, Deserialize)]
struct CreateCronPayload {
graph: String,
#[serde(default)]
interval_secs: Option<u64>,
#[serde(default)]
cron_expr: Option<String>,
#[serde(default)]
input: Option<Value>,
#[serde(default)]
cron_id: Option<String>,
#[serde(default)]
metadata: Option<Value>,
#[serde(default)]
on_run_completed: Option<String>,
}
async fn create_cron(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<CreateCronPayload>,
) -> Result<(StatusCode, Json<CronRecord>), ApiError> {
if !state.registry.contains(&payload.graph) {
return Err(ApiError::bad_request(format!(
"unknown graph `{}` (see GET /info for registered graphs)",
payload.graph
)));
}
crons::validate_schedule(payload.interval_secs, payload.cron_expr.as_deref())
.map_err(ApiError::bad_request)?;
if let Some(input) = &payload.input {
if !input.is_object() {
return Err(ApiError::bad_request(
"`input` must be a JSON object".to_string(),
));
}
}
let on_run_completed = OnRunCompleted::parse(payload.on_run_completed.as_deref())
.map_err(ApiError::bad_request)?;
let cron_id = payload
.cron_id
.clone()
.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
validate_client_id("cron_id", &cron_id)?;
let record = CronRecord {
cron_id: tenant.scope(&cron_id),
graph: payload.graph,
interval_secs: payload.interval_secs,
cron_expr: payload.cron_expr,
input: payload.input,
metadata: payload.metadata.unwrap_or(Value::Null),
on_run_completed,
created_at: Utc::now(),
last_run_at: None,
runs_fired: 0,
};
let created = state
.server_store
.create_cron(&record)
.await
.map_err(internal_err)?;
if !created {
return Err(ApiError::conflict(format!(
"cron `{cron_id}` already exists"
)));
}
let mut wire = record;
wire.cron_id = cron_id;
Ok((StatusCode::CREATED, Json(wire)))
}
async fn list_crons(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let records = state
.server_store
.list_crons()
.await
.map_err(internal_err)?;
let mut records: Vec<CronRecord> = records
.into_iter()
.filter_map(|mut record| {
let external = tenant.unscope(&record.cron_id)?.to_string();
record.cron_id = external;
Some(record)
})
.collect();
records.sort_by(|a, b| {
a.created_at
.cmp(&b.created_at)
.then_with(|| a.cron_id.cmp(&b.cron_id))
});
Ok(Json(json!(records)))
}
async fn delete_cron(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(cron_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
if state
.server_store
.delete_cron(&tenant.scope(&cron_id))
.await
.map_err(internal_err)?
{
Ok(Json(json!({ "cron_id": cron_id, "deleted": true })))
} else {
Err(ApiError::not_found(format!("cron `{cron_id}` not found")))
}
}
async fn put_store_item(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path((namespace, key)): Path<(String, String)>,
Json(value): Json<Value>,
) -> Result<(StatusCode, Json<store::StoreItem>), ApiError> {
store::validate_segment("namespace", &namespace)?;
store::validate_segment("key", &key)?;
let (mut item, created) = state
.server_store
.kv_put(&tenant.scope(&namespace), &key, value)
.await
.map_err(internal_err)?;
item.namespace = namespace;
let status = if created {
StatusCode::CREATED
} else {
StatusCode::OK
};
Ok((status, Json(item)))
}
async fn get_store_item(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path((namespace, key)): Path<(String, String)>,
) -> Result<Json<store::StoreItem>, ApiError> {
store::validate_segment("namespace", &namespace)?;
store::validate_segment("key", &key)?;
state
.server_store
.kv_get(&tenant.scope(&namespace), &key)
.await
.map_err(internal_err)?
.map(|mut item| {
item.namespace = namespace.clone();
Json(item)
})
.ok_or_else(|| ApiError::not_found(format!("no store item at `{namespace}/{key}`")))
}
async fn delete_store_item(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path((namespace, key)): Path<(String, String)>,
) -> Result<Json<Value>, ApiError> {
store::validate_segment("namespace", &namespace)?;
store::validate_segment("key", &key)?;
if state
.server_store
.kv_delete(&tenant.scope(&namespace), &key)
.await
.map_err(internal_err)?
{
Ok(Json(
json!({ "namespace": namespace, "key": key, "deleted": true }),
))
} else {
Err(ApiError::not_found(format!(
"no store item at `{namespace}/{key}`"
)))
}
}
async fn list_store_namespace(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(namespace): Path<String>,
) -> Result<Json<Value>, ApiError> {
store::validate_segment("namespace", &namespace)?;
let items = state
.server_store
.kv_list(&tenant.scope(&namespace))
.await
.map_err(internal_err)?;
let items: Vec<store::StoreItem> = items
.into_iter()
.map(|mut item| {
item.namespace = namespace.clone();
item
})
.collect();
Ok(Json(json!(items)))
}
#[derive(Debug, Deserialize)]
struct EnqueueTaskPayload {
kind: String,
payload: Value,
#[serde(default)]
pool: Option<String>,
#[serde(default)]
max_attempts: Option<u32>,
#[serde(default)]
idempotency_key: Option<String>,
#[serde(default)]
effect: Option<String>,
#[serde(default)]
run_id: Option<String>,
#[serde(default)]
thread_id: Option<String>,
#[serde(default)]
deadline: Option<String>,
#[serde(default)]
worker_version: Option<String>,
#[serde(default)]
recipient: Option<String>,
#[serde(default)]
parent: Option<String>,
}
fn build_task_record(
payload: EnqueueTaskPayload,
tenant: &TenantContext,
) -> Result<TaskRecord, ApiError> {
tasks::validate_label("kind", &payload.kind, 256).map_err(ApiError::bad_request)?;
let pool = payload
.pool
.unwrap_or_else(|| tasks::DEFAULT_POOL.to_string());
tasks::validate_pool(&pool).map_err(ApiError::bad_request)?;
let max_attempts = payload.max_attempts.unwrap_or(tasks::DEFAULT_MAX_ATTEMPTS);
if !(1..=tasks::MAX_ATTEMPTS_LIMIT).contains(&max_attempts) {
return Err(ApiError::bad_request(format!(
"`max_attempts` must be within 1..={}",
tasks::MAX_ATTEMPTS_LIMIT
)));
}
if let Some(key) = &payload.idempotency_key {
tasks::validate_label("idempotency_key", key, 256).map_err(ApiError::bad_request)?;
}
let effect = payload
.effect
.as_deref()
.map(tasks::parse_effect)
.transpose()
.map_err(ApiError::bad_request)?;
if let Some(run_id) = &payload.run_id {
tasks::validate_label("run_id", run_id, 256).map_err(ApiError::bad_request)?;
}
if let Some(thread_id) = &payload.thread_id {
tasks::validate_label("thread_id", thread_id, 256).map_err(ApiError::bad_request)?;
}
let deadline = payload
.deadline
.as_deref()
.map(|raw| {
chrono::DateTime::parse_from_rfc3339(raw)
.map(|dt| dt.with_timezone(&Utc))
.map_err(|_| {
ApiError::bad_request(format!(
"`deadline` must be an RFC 3339 timestamp (got `{raw}`)"
))
})
})
.transpose()?;
if let Some(version) = &payload.worker_version {
tasks::validate_label("worker_version", version, 256).map_err(ApiError::bad_request)?;
}
if let Some(recipient) = &payload.recipient {
agents::validate_recipient(recipient).map_err(ApiError::bad_request)?;
}
if let Some(parent) = &payload.parent {
tasks::validate_label("parent", parent, 512).map_err(ApiError::bad_request)?;
}
Ok(TaskRecord::new(
tasks::NewTask {
task_id: uuid::Uuid::new_v4().to_string(),
tenant: tenant.tenant().to_string(),
kind: payload.kind,
payload: payload.payload,
pool,
max_attempts,
idempotency_key: payload.idempotency_key,
effect,
run_id: payload.run_id,
thread_id: payload.thread_id,
deadline,
worker_version: payload.worker_version,
recipient: payload.recipient,
parent: payload.parent,
},
Utc::now(),
))
}
async fn enforce_task_quota(
state: &AppState,
tenant: &TenantContext,
additional: usize,
) -> Result<(), ApiError> {
let quota = state.config.quota_for(tenant.tenant());
if quota.is_unlimited() {
return Ok(());
}
let usage = state
.server_store
.task_usage(tenant.tenant())
.await
.map_err(internal_err)?;
if let Some(max) = quota.max_queued {
if usage.queued as usize + additional > max {
return Err(ApiError::too_many_requests(format!(
"tenant task quota exceeded: {} tasks queued (+{additional} submitted) would pass the limit of {max} — let workers drain the queue or raise the quota",
usage.queued
)));
}
}
if let Some(max) = quota.max_in_flight {
if usage.in_flight as usize >= max {
return Err(ApiError::too_many_requests(format!(
"tenant task quota exceeded: {} tasks in flight at the limit of {max} — wait for workers to settle or raise the quota",
usage.in_flight
)));
}
}
if let Some(max) = quota.max_dlq {
if usage.dlq as usize >= max {
return Err(ApiError::too_many_requests(format!(
"tenant task quota exceeded: DLQ depth {} at the limit of {max} — inspect and re-drive the dead-letter queue before submitting more work",
usage.dlq
)));
}
}
Ok(())
}
pub(crate) async fn a2a_enqueue(
state: &AppState,
tenant: &TenantContext,
record: &TaskRecord,
) -> Result<(TaskRecord, bool), ApiError> {
enforce_task_quota(state, tenant, 1).await?;
state
.server_store
.enqueue_task(record)
.await
.map_err(internal_err)
}
async fn enqueue_task(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<EnqueueTaskPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let record = build_task_record(payload, &tenant)?;
enforce_task_quota(&state, &tenant, 1).await?;
let (task, deduplicated) = state
.server_store
.enqueue_task(&record)
.await
.map_err(internal_err)?;
let status = if deduplicated {
StatusCode::OK
} else {
StatusCode::CREATED
};
Ok((
status,
Json(json!({
"task_id": task.task_id,
"deduplicated": deduplicated,
})),
))
}
async fn enqueue_task_outbox(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<EnqueueTaskPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let record = build_task_record(payload, &tenant)?;
enforce_task_quota(&state, &tenant, 1).await?;
let (task, deduplicated) = state
.server_store
.outbox_enqueue(&record)
.await
.map_err(internal_err)?;
Ok((
StatusCode::ACCEPTED,
Json(json!({
"task_id": task.task_id,
"deduplicated": deduplicated,
})),
))
}
#[derive(Debug, Deserialize)]
struct ClaimTaskPayload {
worker_id: String,
#[serde(default)]
pools: Option<Vec<String>>,
#[serde(default)]
worker_version: Option<String>,
lease_ms: u64,
}
async fn claim_task(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<ClaimTaskPayload>,
) -> Result<Response, ApiError> {
tasks::validate_label("worker_id", &payload.worker_id, 256).map_err(ApiError::bad_request)?;
tasks::validate_lease_ms(payload.lease_ms).map_err(ApiError::bad_request)?;
if let Some(version) = &payload.worker_version {
tasks::validate_label("worker_version", version, 256).map_err(ApiError::bad_request)?;
}
let pools = payload
.pools
.unwrap_or_else(|| vec![tasks::DEFAULT_POOL.to_string()]);
if pools.is_empty() {
return Err(ApiError::bad_request(
"`pools` must name at least one pool".to_string(),
));
}
for pool in &pools {
tasks::validate_pool(pool).map_err(ApiError::bad_request)?;
}
let timeout_record = policy::active_policy_record(&state.server_store, tenant.tenant())
.await
.unwrap_or_else(|error| {
tracing::warn!(%error, "active policy unreadable; claiming on the static floor");
policy::static_floor_record()
});
let now = Utc::now();
let claimed = state
.server_store
.claim_task(
tenant.tenant(),
&payload.worker_id,
&tasks::ClaimScope {
pools: &pools,
pool_limits: &state.config.task_pool_limits,
worker_version: payload.worker_version.as_deref(),
timeout_policy: &timeout_record.policy,
},
payload.lease_ms,
now,
)
.await
.map_err(internal_err)?;
Ok(match claimed {
Some(task) => {
journal_timeout_decision(&state, &tenant, &task, &timeout_record, now).await;
Json(json!({ "task": task.wire() })).into_response()
}
None => StatusCode::NO_CONTENT.into_response(),
})
}
#[derive(Debug, Deserialize)]
struct HeartbeatTaskPayload {
worker_id: String,
lease_ms: u64,
}
#[derive(Debug, Deserialize)]
struct CompleteTaskPayload {
worker_id: String,
result: Value,
#[serde(default)]
receipt: Option<EffectReceipt>,
#[serde(default)]
tokens: Option<Usage>,
#[serde(default)]
cost_usd: Option<f64>,
}
#[derive(Debug, Deserialize)]
struct FailTaskPayload {
worker_id: String,
error_class: String,
message: String,
retryable: bool,
#[serde(default)]
tokens: Option<Usage>,
#[serde(default)]
cost_usd: Option<f64>,
}
fn lease_outcome(
outcome: MutationOutcome,
task_id: &str,
worker_id: &str,
) -> Result<TaskRecord, ApiError> {
match outcome {
MutationOutcome::Applied(task) => Ok(*task),
MutationOutcome::LeaseLost => Err(ApiError::conflict(format!(
"task `{task_id}` is not leased to worker `{worker_id}` (lost, expired and reclaimed, or already settled)"
))),
MutationOutcome::Unknown => {
Err(ApiError::not_found(format!("task `{task_id}` not found")))
}
}
}
async fn heartbeat_task(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(task_id): Path<String>,
Json(payload): Json<HeartbeatTaskPayload>,
) -> Result<Json<Value>, ApiError> {
tasks::validate_label("worker_id", &payload.worker_id, 256).map_err(ApiError::bad_request)?;
tasks::validate_lease_ms(payload.lease_ms).map_err(ApiError::bad_request)?;
let outcome = state
.server_store
.heartbeat_task(
tenant.tenant(),
&task_id,
&payload.worker_id,
payload.lease_ms,
Utc::now(),
)
.await
.map_err(internal_err)?;
let task = lease_outcome(outcome, &task_id, &payload.worker_id)?;
let expires_at = task.lease.as_ref().map(|lease| lease.expires_at);
Ok(Json(json!({
"lease_expires_at": expires_at,
"cancel_requested": task.cancel_requested,
})))
}
async fn complete_task(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(task_id): Path<String>,
Json(payload): Json<CompleteTaskPayload>,
) -> Result<Json<Value>, ApiError> {
tasks::validate_label("worker_id", &payload.worker_id, 256).map_err(ApiError::bad_request)?;
if let Some(receipt) = &payload.receipt {
if let Some(task) = state
.server_store
.get_task(tenant.tenant(), &task_id)
.await
.map_err(internal_err)?
{
if task.idempotency_key.as_ref() != Some(&receipt.idempotency_key) {
return Err(ApiError::bad_request(format!(
"`receipt.idempotency_key` `{}` does not match the task's idempotency key `{:?}`",
receipt.idempotency_key, task.idempotency_key
)));
}
}
}
let outcome = state
.server_store
.complete_task(
tenant.tenant(),
&task_id,
&payload.worker_id,
tasks::CompletionReport {
result: payload.result,
receipt: payload.receipt,
cost: tasks::SettlementCost {
tokens: payload.tokens,
cost_usd: payload.cost_usd,
},
},
Utc::now(),
)
.await
.map_err(internal_err)?;
let task = lease_outcome(outcome, &task_id, &payload.worker_id)?;
journal_effect_receipt(&state, &tenant, &task).await;
coordination::on_task_settled(
&state.server_store,
state.config.quota_for(tenant.tenant()),
&tenant,
&task,
Utc::now(),
)
.await
.map_err(internal_err)?;
crate::a2a::publish_task_update(&state, &task).await;
Ok(Json(task.wire()))
}
async fn journal_effect_receipt(state: &AppState, tenant: &TenantContext, task: &TaskRecord) {
let (Some(receipt), Some(run_id)) = (&task.receipt, &task.run_id) else {
return;
};
if let Err(error) = try_journal_effect_receipt(state, tenant, receipt, run_id).await {
tracing::warn!(
task_id = %task.task_id,
%run_id,
%error,
"effect receipt stays on the task record; journaling skipped"
);
}
}
async fn try_journal_effect_receipt(
state: &AppState,
tenant: &TenantContext,
receipt: &EffectReceipt,
run_id: &str,
) -> Result<(), String> {
let Some(snapshot) = state
.server_store
.get_journal(run_id)
.await
.map_err(|e| format!("load journal: {e}"))?
else {
return Err("run has no persisted journal yet".to_string());
};
let internal_thread_id = tenant.scope(&snapshot.thread_id);
let owned = state
.server_store
.get_thread(&internal_thread_id)
.await
.map_err(|e| format!("resolve thread: {e}"))?
.is_some();
if !owned {
return Err("run does not resolve in this tenant".to_string());
}
let journal = Journal::from_snapshot(snapshot, Clock::System)
.map_err(|e| format!("journal failed its integrity check: {e}"))?;
let parent = journal.events().last().map(|event| event.id.clone());
journal.record_effect_receipt(receipt, parent);
state
.server_store
.put_journal(&journal.snapshot())
.await
.map_err(|e| format!("persist journal: {e}"))
}
#[derive(Debug, Deserialize)]
struct WriteMemoryPayload {
kind: MemoryKind,
scope: ScopeAddress,
content: Value,
author: ProvenanceAuthor,
#[serde(default)]
key: Option<String>,
#[serde(default)]
tags: Vec<String>,
#[serde(default)]
priority: i64,
#[serde(default)]
evidence: Option<MemoryEvidence>,
#[serde(default)]
confidence: Option<f64>,
#[serde(default)]
written_at: Option<DateTime<Utc>>,
#[serde(default)]
valid_from: Option<DateTime<Utc>>,
#[serde(default)]
valid_until: Option<DateTime<Utc>>,
#[serde(default)]
expires_at: Option<DateTime<Utc>>,
#[serde(default)]
supersedes: Option<String>,
#[serde(default)]
run_id: Option<String>,
#[serde(default)]
parent: Option<String>,
}
async fn check_memory_scope_gate(
state: &AppState,
tenant: &TenantContext,
scope: &ScopeAddress,
allow_run: bool,
) -> Result<(), ApiError> {
tasks::validate_label("scope.id", &scope.id, 256).map_err(ApiError::bad_request)?;
match scope.scope {
MemoryScope::Run if !allow_run => Err(ApiError::bad_request(
"`run`-scoped memory is runtime-only: the runtime writes it on a run's \
behalf — the API accepts `agent`, `team`, `user`, and `tenant` scopes"
.to_string(),
)),
MemoryScope::Run => Ok(()),
MemoryScope::Agent => {
let scoped_agent = tenant.scope(&scope.id);
let agent = state
.server_store
.get_agent(&scoped_agent)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("agent `{}` not found", scope.id)))?;
if !agent.manifest.scopes.contains(&StateScope::Private) {
return Err(ApiError::new(
StatusCode::FORBIDDEN,
"forbidden",
format!(
"agent `{}` does not declare the `private` state scope in its manifest \
— agent-scoped memory is the agent's own, and the manifest is what \
grants it",
scope.id
),
));
}
Ok(())
}
MemoryScope::Tenant => {
if scope.id != tenant.tenant() {
return Err(ApiError::new(
StatusCode::FORBIDDEN,
"forbidden",
format!(
"tenant-scoped memory id `{}` is not the caller's tenant `{}` — \
tenant isolation is not a scope a caller can cross",
scope.id,
tenant.tenant()
),
));
}
Ok(())
}
MemoryScope::Team | MemoryScope::User => Ok(()),
}
}
async fn write_memory(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<WriteMemoryPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
check_memory_scope_gate(&state, &tenant, &payload.scope, false).await?;
let confidence = match (payload.confidence, &payload.author) {
(Some(confidence), _) => confidence,
(None, ProvenanceAuthor::Human { .. }) => 1.0,
(None, _) => {
return Err(ApiError::bad_request(
"`confidence` is required for non-human authors — human-authored records \
default to 1.0 (the claim is the person's); every other author must declare \
its confidence explicitly"
.to_string(),
));
}
};
if let Some(key) = &payload.key {
tasks::validate_label("key", key, 256).map_err(ApiError::bad_request)?;
}
let now = Utc::now();
let written_at = payload.written_at.unwrap_or(now);
let provenance = MemoryProvenance {
author: payload.author,
evidence: payload.evidence.unwrap_or_default(),
written_at,
};
let validity = ValidityWindow {
valid_from: payload.valid_from.unwrap_or(now),
valid_until: payload.valid_until,
};
let mut record = MemoryRecord::new(
payload.kind,
payload.scope,
provenance,
confidence,
validity,
written_at,
payload.content.clone(),
)
.map_err(|e| ApiError::bad_request(e.to_string()))?;
if let Some(key) = payload.key {
record = record.with_key(key);
}
if !payload.tags.is_empty() {
record = record.with_tags(payload.tags);
}
if payload.priority != 0 {
record = record.with_priority(payload.priority);
}
if let Some(expires_at) = payload.expires_at {
record = record.with_expires_at(expires_at);
}
if let Some(supersedes) = payload.supersedes {
record = record.with_supersedes(supersedes);
}
let created = state
.server_store
.put_memory(tenant.tenant(), &record, &payload.content)
.await
.map_err(internal_err)?;
if let Some(run_id) = &payload.run_id {
journal_memory_write(&state, &tenant, run_id, &record, payload.parent).await;
}
let stored = state
.server_store
.get_memory(tenant.tenant(), &record.memory_id)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::internal("memory record missing immediately after write".to_string())
})?;
let status = if created {
StatusCode::CREATED
} else {
StatusCode::OK
};
Ok((
status,
Json(json!({
"memory_id": stored.memory_id,
"created": created,
"record": stored,
})),
))
}
async fn get_memory(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(memory_id): Path<String>,
) -> Result<Json<MemoryRecord>, ApiError> {
state
.server_store
.get_memory(tenant.tenant(), &memory_id)
.await
.map_err(internal_err)?
.map(Json)
.ok_or_else(|| ApiError::not_found(format!("memory `{memory_id}` not found")))
}
#[derive(Debug, Deserialize)]
struct QueryMemoryPayload {
#[serde(flatten)]
query: MemoryQuery,
#[serde(default)]
budget: Option<ContextBudget>,
#[serde(default)]
run_id: Option<String>,
#[serde(default)]
parent: Option<String>,
}
async fn memory_read_store(
state: &AppState,
tenant: &TenantContext,
query: &MemoryQuery,
) -> Result<Arc<dyn MemoryStore>, ApiError> {
let base: Arc<dyn MemoryStore> = Arc::new(ServerMemoryStore::new(
Arc::clone(&state.server_store),
tenant.tenant(),
));
let Some(scope) = &query.scope else {
return Ok(base);
};
let surface = format!("memory:{}", scope.as_address());
let pointer = state
.server_store
.get_version_pointer(tenant.tenant(), &surface)
.await
.map_err(internal_err)?;
let Some(active) = pointer.and_then(|pointer| pointer.active) else {
return Ok(base);
};
let record = state
.server_store
.get_candidate(tenant.tenant(), active.as_str())
.await
.map_err(internal_err)?;
let Some(record) = record else {
tracing::warn!(
candidate = %active,
%surface,
"version pointer names a candidate that is gone; reading the live namespace"
);
return Ok(base);
};
match CandidateOverlay::new(base.clone(), &record.candidate) {
Ok(overlay) => Ok(Arc::new(overlay)),
Err(error) => {
tracing::warn!(
candidate = %active,
%surface,
%error,
"candidate on a memory surface does not apply as an overlay; reading the \
live namespace"
);
Ok(base)
}
}
}
async fn query_memory(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<QueryMemoryPayload>,
) -> Result<Json<Value>, ApiError> {
let mut query = payload.query;
let as_of = query.as_of.unwrap_or_else(Utc::now);
query.as_of = Some(as_of);
if payload.run_id.is_some() && payload.budget.is_none() {
return Err(ApiError::bad_request(
"`budget` is required with `run_id`: a journaled memory read is a budgeted \
read — the journaled request is the resolved query plus the budget it was \
assembled under"
.to_string(),
));
}
let store = memory_read_store(&state, &tenant, &query).await?;
let records = store.query(&query, as_of).await.map_err(internal_err)?;
match payload.budget {
Some(budget) => {
let assembly =
assemble(records, &budget).map_err(|e| ApiError::unprocessable(e.to_string()))?;
if let Some(run_id) = &payload.run_id {
journal_memory_read(
&state,
&tenant,
run_id,
&query,
&budget,
&assembly,
payload.parent,
)
.await;
}
serde_json::to_value(&assembly)
.map(Json)
.map_err(internal_err)
}
None => {
let ranked = assemble(records, &ContextBudget::new(u32::MAX)).map_err(internal_err)?;
Ok(Json(json!({ "records": ranked.records })))
}
}
}
async fn journal_memory_write(
state: &AppState,
tenant: &TenantContext,
run_id: &str,
record: &MemoryRecord,
parent: Option<String>,
) {
let draft = EventDraft::new(RunEventKind::MemoryWrite, Effect::Idempotent).input(json!({
"effect_key": memory_effect_key(&record.scope, &record.memory_id),
"memory_id": record.memory_id,
}));
let draft = match serde_json::to_value(record) {
Ok(output) => draft.output(output),
Err(error) => {
tracing::warn!(%run_id, %error, "memory record failed to serialize; journaling skipped");
return;
}
};
if let Err(error) = try_journal_memory_event(state, tenant, run_id, parent, draft).await {
tracing::warn!(
%run_id,
memory_id = %record.memory_id,
%error,
"memory write is durable in the store; journaling skipped"
);
}
}
async fn journal_memory_read(
state: &AppState,
tenant: &TenantContext,
run_id: &str,
query: &MemoryQuery,
budget: &ContextBudget,
assembly: &rusty_agent_runtime::memory::MemoryAssembly,
parent: Option<String>,
) {
let draft = EventDraft::new(RunEventKind::MemoryRead, Effect::ReadOnly)
.input(memory_read_request(query, budget));
let draft = match serde_json::to_value(assembly) {
Ok(output) => draft.output(output),
Err(error) => {
tracing::warn!(%run_id, %error, "memory assembly failed to serialize; journaling skipped");
return;
}
};
if let Err(error) = try_journal_memory_event(state, tenant, run_id, parent, draft).await {
tracing::warn!(
%run_id,
%error,
"memory read answered from the store; journaling skipped"
);
}
}
async fn try_journal_memory_event(
state: &AppState,
tenant: &TenantContext,
run_id: &str,
parent: Option<String>,
draft: EventDraft,
) -> Result<(), String> {
let Some(snapshot) = state
.server_store
.get_journal(run_id)
.await
.map_err(|e| format!("load journal: {e}"))?
else {
return Err("run has no persisted journal yet".to_string());
};
let internal_thread_id = tenant.scope(&snapshot.thread_id);
let owned = state
.server_store
.get_thread(&internal_thread_id)
.await
.map_err(|e| format!("resolve thread: {e}"))?
.is_some();
if !owned {
return Err("run does not resolve in this tenant".to_string());
}
let journal = Journal::from_snapshot(snapshot, Clock::System)
.map_err(|e| format!("journal failed its integrity check: {e}"))?;
let parent = parent.or_else(|| journal.events().last().map(|event| event.id.clone()));
let draft = match parent {
Some(parent) => draft.parent(parent),
None => draft,
};
journal.record(draft);
state
.server_store
.put_journal(&journal.snapshot())
.await
.map_err(|e| format!("persist journal: {e}"))
}
async fn journal_policy_decision(
state: &AppState,
tenant: &TenantContext,
task: &TaskRecord,
error_class: rusty_agent_runtime::durable::ErrorClass,
retryable: bool,
max_attempts: u32,
decided_at: DateTime<Utc>,
) {
let Some(run_id) = task.run_id.clone() else {
return;
};
if let Err(error) = try_journal_policy_decision(
state,
tenant,
&run_id,
task,
error_class,
retryable,
max_attempts,
decided_at,
)
.await
{
tracing::warn!(
%run_id,
task_id = %task.task_id,
%error,
"retry decision is settled in the task record; journaling skipped"
);
}
}
#[allow(clippy::too_many_arguments)]
async fn try_journal_policy_decision(
state: &AppState,
tenant: &TenantContext,
run_id: &str,
task: &TaskRecord,
error_class: rusty_agent_runtime::durable::ErrorClass,
retryable: bool,
max_attempts: u32,
decided_at: DateTime<Utc>,
) -> Result<(), String> {
let decision = match (&task.status, task.next_attempt_at) {
(TaskStatus::Failed, Some(next)) => RetryDecision::Retry {
after_ms: (next - decided_at).num_milliseconds().max(0) as u64,
},
(TaskStatus::Dead, _) => RetryDecision::Dead,
_ => RetryDecision::Fail,
};
let effect = task.effect.unwrap_or(if retryable {
Effect::Idempotent
} else {
Effect::NonIdempotent
});
let Some(snapshot) = state
.server_store
.get_journal(run_id)
.await
.map_err(|e| format!("load journal: {e}"))?
else {
return Err("run has no persisted journal yet".to_string());
};
let thread_id = snapshot.thread_id.clone();
let internal_thread_id = tenant.scope(&thread_id);
let owned = state
.server_store
.get_thread(&internal_thread_id)
.await
.map_err(|e| format!("resolve thread: {e}"))?
.is_some();
if !owned {
return Err("run does not resolve in this tenant".to_string());
}
let journal = Journal::from_snapshot(snapshot, Clock::System)
.map_err(|e| format!("journal failed its integrity check: {e}"))?;
let policy_version = state
.checkpointer
.get_latest(&internal_thread_id)
.await
.map_err(|e| format!("load checkpoint: {e}"))?
.map(|checkpoint| checkpoint.header.policy_version)
.unwrap_or_default();
let seq = journal
.events()
.iter()
.filter(|event| event.kind == RunEventKind::PolicyDecision)
.count() as u64;
let event = retry_decision_event(
run_id,
thread_id,
seq,
effect,
error_class,
task.attempt,
max_attempts,
None,
&decision,
&policy_version,
decided_at,
);
let draft = EventDraft::new(RunEventKind::PolicyDecision, Effect::Pure).output(
serde_json::to_value(&event).map_err(|e| format!("serialize decision event: {e}"))?,
);
let parent = journal.events().last().map(|event| event.id.clone());
let draft = match parent {
Some(parent) => draft.parent(parent),
None => draft,
};
journal.record(draft);
state
.server_store
.put_journal(&journal.snapshot())
.await
.map_err(|e| format!("persist journal: {e}"))
}
async fn acting_executor_policy(
state: &AppState,
tenant: &TenantContext,
task: &TaskRecord,
) -> ExecutorPolicy {
if let Some(run_id) = &task.run_id {
let thread_id = match task.thread_id.clone() {
Some(thread_id) => Some(thread_id),
None => state
.server_store
.get_journal(run_id)
.await
.ok()
.flatten()
.map(|snapshot| snapshot.thread_id),
};
let version = match thread_id {
Some(thread_id) => state
.checkpointer
.get_latest(&tenant.scope(&thread_id))
.await
.ok()
.flatten()
.map(|checkpoint| checkpoint.header.policy_version)
.unwrap_or_default(),
None => return ExecutorPolicy::static_v0(),
};
if version.as_str() == PolicyVersion::STATIC_V0 {
return ExecutorPolicy::static_v0();
}
return match state
.server_store
.get_policy(tenant.tenant(), version.as_str())
.await
{
Ok(Some(record)) => record.policy,
Ok(None) => {
tracing::warn!(
version = %version.as_str(),
"acting policy version is not registered; acting on the static floor"
);
ExecutorPolicy::static_v0()
}
Err(error) => {
tracing::warn!(%error, "policy registry read failed; acting on the static floor");
ExecutorPolicy::static_v0()
}
};
}
match policy::active_policy_record(&state.server_store, tenant.tenant()).await {
Ok(record) => record.policy,
Err(error) => {
tracing::warn!(%error, "active policy unreadable; acting on the static floor");
ExecutorPolicy::static_v0()
}
}
}
async fn journal_timeout_decision(
state: &AppState,
tenant: &TenantContext,
task: &TaskRecord,
record: &PolicyRecord,
decided_at: DateTime<Utc>,
) {
let Some(run_id) = task.run_id.clone() else {
return;
};
let Some(bound) = resolve_timeout_bound_ms(&record.policy, Some(&task.kind)) else {
return;
};
if let Err(error) = try_journal_timeout_decision(
state,
tenant,
&run_id,
task,
bound,
&record.version,
decided_at,
)
.await
{
tracing::warn!(
%run_id,
task_id = %task.task_id,
%error,
"timeout decision is settled in the lease; journaling skipped"
);
}
}
async fn try_journal_timeout_decision(
state: &AppState,
tenant: &TenantContext,
run_id: &str,
task: &TaskRecord,
bound: u64,
policy_version: &PolicyVersion,
decided_at: DateTime<Utc>,
) -> Result<(), String> {
let Some(snapshot) = state
.server_store
.get_journal(run_id)
.await
.map_err(|e| format!("load journal: {e}"))?
else {
return Err("run has no persisted journal yet".to_string());
};
let thread_id = snapshot.thread_id.clone();
let internal_thread_id = tenant.scope(&thread_id);
let owned = state
.server_store
.get_thread(&internal_thread_id)
.await
.map_err(|e| format!("resolve thread: {e}"))?
.is_some();
if !owned {
return Err("run does not resolve in this tenant".to_string());
}
let journal = Journal::from_snapshot(snapshot, Clock::System)
.map_err(|e| format!("journal failed its integrity check: {e}"))?;
let seq = journal
.events()
.iter()
.filter(|event| event.kind == RunEventKind::PolicyDecision)
.count() as u64;
let event = timeout_decision_event(
run_id,
thread_id,
seq,
Some(&task.kind),
task.effect.unwrap_or(Effect::NonIdempotent),
task.attempt,
None,
Some(bound),
&rusty_agent_runtime::twin::DEFAULT_TIMEOUT_LADDER,
policy_version,
decided_at,
);
let draft = EventDraft::new(RunEventKind::PolicyDecision, Effect::Pure).output(
serde_json::to_value(&event).map_err(|e| format!("serialize decision event: {e}"))?,
);
let parent = journal.events().last().map(|event| event.id.clone());
let draft = match parent {
Some(parent) => draft.parent(parent),
None => draft,
};
journal.record(draft);
state
.server_store
.put_journal(&journal.snapshot())
.await
.map_err(|e| format!("persist journal: {e}"))
}
#[derive(Debug, Deserialize)]
struct CorrectionPayload {
#[serde(flatten)]
correction: Correction,
#[serde(default)]
run_id: Option<String>,
#[serde(default)]
parent: Option<String>,
}
fn correction_event_input(
event: &rusty_agent_runtime::record::RunEvent,
snapshot: &JournalSnapshot,
) -> Option<Value> {
match event.input.as_ref()? {
PayloadRef::Inline(value) => Some(value.clone()),
PayloadRef::Artifact(reference) => snapshot.artifacts.get(&reference.sha256).cloned(),
}
}
async fn submit_correction(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<CorrectionPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let correction = payload.correction;
tasks::validate_label("correction_id", &correction.correction_id, 256)
.map_err(ApiError::bad_request)?;
tasks::validate_label("author", &correction.author, 256).map_err(ApiError::bad_request)?;
check_memory_scope_gate(&state, &tenant, &correction.scope, true).await?;
let prior = state
.server_store
.query_memory(
tenant.tenant(),
&MemoryQuery {
scope: Some(correction.scope.clone()),
include_expired: true,
include_superseded: true,
..MemoryQuery::default()
},
Utc::now(),
)
.await
.map_err(internal_err)?;
let names_correction = |record: &MemoryRecord| {
record.provenance.evidence.correction_id.as_deref()
== Some(correction.correction_id.as_str())
};
if let Some(prior_memory) = prior
.iter()
.find(|record| record.kind == MemoryKind::Fact && names_correction(record))
{
let example_id = prior
.iter()
.find(|record| record.kind == MemoryKind::Example && names_correction(record))
.map(|record| record.memory_id.clone());
return Ok((
StatusCode::OK,
Json(json!({
"correction_id": correction.correction_id,
"attribution": correction.attribution(),
"candidate": prior_memory.candidacy.is_some(),
"memory_id": prior_memory.memory_id,
"created": false,
"record": prior_memory,
"superseded": prior_memory.supersedes,
"example_id": example_id,
})),
));
}
let now = Utc::now();
let mut key = None;
let mut example_input = None;
let mut journal_run_id = payload.run_id.clone();
match &correction.target {
CorrectionTarget::Memory { memory_id } => {
let target = state
.server_store
.get_memory(tenant.tenant(), memory_id)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!("correction target memory `{memory_id}` not found"))
})?;
key = target.key;
}
CorrectionTarget::RunEvent { run_id, event_id } => {
let snapshot = state
.server_store
.get_journal(run_id)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!(
"correction target run `{run_id}` has no persisted journal"
))
})?;
let internal_thread_id = tenant.scope(&snapshot.thread_id);
let owned = state
.server_store
.get_thread(&internal_thread_id)
.await
.map_err(internal_err)?
.is_some();
if !owned {
return Err(ApiError::not_found(format!("run `{run_id}` not found")));
}
let event = snapshot
.events
.iter()
.find(|event| &event.id == event_id)
.ok_or_else(|| {
ApiError::not_found(format!(
"run `{run_id}` has no journaled event `{event_id}`"
))
})?;
example_input = Some(correction_event_input(event, &snapshot).unwrap_or(Value::Null));
journal_run_id = journal_run_id.or(Some(run_id.clone()));
}
CorrectionTarget::Prompt { .. } => {}
}
let mut supersedes = None;
if let Some(key) = &key {
let live = state
.server_store
.query_memory(
tenant.tenant(),
&MemoryQuery {
scope: Some(correction.scope.clone()),
key: Some(key.clone()),
..MemoryQuery::default()
},
now,
)
.await
.map_err(internal_err)?;
supersedes = assemble(live, &ContextBudget::new(u32::MAX))
.map_err(internal_err)?
.records
.into_iter()
.next()
.map(|record| record.memory_id);
}
let candidacy = correction.is_candidate().then_some(Candidacy::Pending);
let provenance = MemoryProvenance {
author: correction.author_as_provenance(),
evidence: correction.evidence(),
written_at: now,
};
let build = |kind: MemoryKind, content: Value| -> Result<MemoryRecord, ApiError> {
let mut record = MemoryRecord::new(
kind,
correction.scope.clone(),
provenance.clone(),
1.0,
ValidityWindow::starting(now),
now,
content,
)
.map_err(|e| ApiError::bad_request(e.to_string()))?;
if let Some(key) = &key {
record = record.with_key(key.clone());
}
if let Some(supersedes) = &supersedes {
record = record.with_supersedes(supersedes.clone());
}
if let Some(candidacy) = candidacy {
record = record.with_candidacy(candidacy);
}
Ok(record)
};
let memory = build(MemoryKind::Fact, correction.corrected.clone())?;
let created = state
.server_store
.put_memory(tenant.tenant(), &memory, &correction.corrected)
.await
.map_err(internal_err)?;
if let Some(run_id) = &journal_run_id {
journal_memory_write(&state, &tenant, run_id, &memory, payload.parent.clone()).await;
}
let mut example_id = None;
if let Some(input) = example_input {
let content = json!({
"input": input,
"corrected": correction.corrected,
});
let example = build(MemoryKind::Example, content.clone())?;
example_id = Some(example.memory_id.clone());
state
.server_store
.put_memory(tenant.tenant(), &example, &content)
.await
.map_err(internal_err)?;
if let Some(run_id) = &journal_run_id {
journal_memory_write(&state, &tenant, run_id, &example, payload.parent.clone()).await;
}
}
let stored = state
.server_store
.get_memory(tenant.tenant(), &memory.memory_id)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::internal(
"correction-derived record missing immediately after write".to_string(),
)
})?;
let status = if created {
StatusCode::CREATED
} else {
StatusCode::OK
};
Ok((
status,
Json(json!({
"correction_id": correction.correction_id,
"attribution": correction.attribution(),
"candidate": candidacy.is_some(),
"memory_id": stored.memory_id,
"created": created,
"record": stored,
"superseded": memory.supersedes,
"example_id": example_id,
})),
))
}
#[derive(Debug, Deserialize)]
struct ConsolidatePayload {
scope: ScopeAddress,
memory_ids: Vec<String>,
distiller: String,
#[serde(default)]
key: Option<String>,
#[serde(default)]
tags: Vec<String>,
#[serde(default)]
priority: i64,
#[serde(default)]
pool: Option<String>,
#[serde(default)]
run_id: Option<String>,
#[serde(default)]
parent: Option<String>,
}
async fn enqueue_consolidation(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<ConsolidatePayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
if payload.memory_ids.is_empty() {
return Err(ApiError::bad_request(
"`memory_ids` must name at least one record — a summary that names no \
sources is not a consolidation"
.to_string(),
));
}
tasks::validate_label("distiller", &payload.distiller, 256).map_err(ApiError::bad_request)?;
if let Some(key) = &payload.key {
tasks::validate_label("key", key, 256).map_err(ApiError::bad_request)?;
}
check_memory_scope_gate(&state, &tenant, &payload.scope, false).await?;
let mut sorted_ids = payload.memory_ids.clone();
sorted_ids.sort();
sorted_ids.dedup();
for memory_id in &sorted_ids {
let record = state
.server_store
.get_memory(tenant.tenant(), memory_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("memory `{memory_id}` not found")))?;
if record.scope != payload.scope {
return Err(ApiError::bad_request(format!(
"memory `{memory_id}` lives at `{}`, not the declared scope `{}` — a \
consolidation spans scopes never",
record.scope.as_address(),
payload.scope.as_address()
)));
}
}
let idempotency_key = format!(
"memory_consolidation:{}:{}",
payload.scope.as_address(),
sha256_hex(sorted_ids.join(",").as_bytes())
);
let now = Utc::now();
let record = build_task_record(
EnqueueTaskPayload {
kind: tasks::MEMORY_CONSOLIDATION_KIND.to_string(),
payload: json!({
"scope": payload.scope,
"memory_ids": sorted_ids,
"distiller": payload.distiller,
"key": payload.key,
"tags": payload.tags,
"priority": payload.priority,
"written_at": now,
"run_id": payload.run_id,
"parent": payload.parent,
}),
pool: payload.pool,
max_attempts: None,
idempotency_key: Some(idempotency_key),
effect: None,
run_id: payload.run_id,
thread_id: None,
deadline: None,
worker_version: None,
recipient: None,
parent: payload.parent,
},
&tenant,
)?;
enforce_task_quota(&state, &tenant, 1).await?;
let (task, deduplicated) = state
.server_store
.enqueue_task(&record)
.await
.map_err(internal_err)?;
let status = if deduplicated {
StatusCode::OK
} else {
StatusCode::CREATED
};
Ok((
status,
Json(json!({
"task_id": task.task_id,
"deduplicated": deduplicated,
"kind": tasks::MEMORY_CONSOLIDATION_KIND,
})),
))
}
#[derive(Debug, Default, Deserialize)]
struct ConflictsPayload {
#[serde(default)]
scope: Option<ScopeAddress>,
}
async fn list_memory_conflicts(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<ConflictsPayload>,
) -> Result<Json<Value>, ApiError> {
let universe = memory_universe(&state, &tenant).await?;
let mut conflicts = detect_conflicts(&universe, Utc::now());
if let Some(scope) = &payload.scope {
conflicts.retain(|conflict| &conflict.scope == scope);
}
Ok(Json(json!({ "conflicts": conflicts })))
}
#[derive(Debug, Deserialize)]
struct ForgetPayload {
memory_id: String,
reason: ForgetReason,
#[serde(default)]
run_id: Option<String>,
#[serde(default)]
parent: Option<String>,
}
async fn memory_universe(
state: &AppState,
tenant: &TenantContext,
) -> Result<Vec<MemoryRecord>, ApiError> {
state
.server_store
.query_memory(
tenant.tenant(),
&MemoryQuery {
include_expired: true,
include_superseded: true,
..MemoryQuery::default()
},
Utc::now(),
)
.await
.map_err(internal_err)
}
async fn forget_memory(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<ForgetPayload>,
) -> Result<Json<Value>, ApiError> {
let record = state
.server_store
.get_memory(tenant.tenant(), &payload.memory_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("memory `{}` not found", payload.memory_id)))?;
let universe = memory_universe(&state, &tenant).await?;
let plan = plan_forget(&universe, std::slice::from_ref(&payload.memory_id));
for memory_id in plan.forgotten.iter().chain(plan.invalidated.iter()) {
state
.server_store
.delete_memory(tenant.tenant(), memory_id)
.await
.map_err(internal_err)?;
}
let tombstone = MemoryForgetTombstone {
memory_id: payload.memory_id.clone(),
scope: record.scope.clone(),
reason: payload.reason,
invalidated: plan.invalidated,
};
if let Some(run_id) = &payload.run_id {
journal_memory_forget(&state, &tenant, run_id, &tombstone, payload.parent).await;
}
Ok(Json(json!({
"forgotten": plan.forgotten,
"invalidated": tombstone.invalidated,
"tombstone": tombstone,
})))
}
#[derive(Debug, Deserialize)]
struct ForgetScopePayload {
scope: ScopeAddress,
reason: ForgetReason,
#[serde(default)]
run_id: Option<String>,
#[serde(default)]
parent: Option<String>,
}
async fn forget_memory_scope(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<ForgetScopePayload>,
) -> Result<Json<Value>, ApiError> {
tasks::validate_label("scope.id", &payload.scope.id, 256).map_err(ApiError::bad_request)?;
if payload.scope.scope == MemoryScope::Tenant && payload.scope.id != tenant.tenant() {
return Err(ApiError::new(
StatusCode::FORBIDDEN,
"forbidden",
format!(
"tenant-scoped erasure id `{}` is not the caller's tenant `{}` — \
tenant isolation is not a scope a caller can cross",
payload.scope.id,
tenant.tenant()
),
));
}
let universe = memory_universe(&state, &tenant).await?;
let targets: Vec<String> = universe
.iter()
.filter(|record| record.scope == payload.scope)
.map(|record| record.memory_id.clone())
.collect();
if targets.is_empty() {
return Ok(Json(json!({
"forgotten": [],
"invalidated": [],
"tombstones": 0,
})));
}
let scopes: HashMap<&str, ScopeAddress> = universe
.iter()
.map(|record| (record.memory_id.as_str(), record.scope.clone()))
.collect();
let plan = plan_forget(&universe, &targets);
for memory_id in plan.forgotten.iter().chain(plan.invalidated.iter()) {
state
.server_store
.delete_memory(tenant.tenant(), memory_id)
.await
.map_err(internal_err)?;
}
let mut tombstones = Vec::with_capacity(plan.forgotten.len());
for memory_id in &plan.forgotten {
let single = plan_forget(&universe, std::slice::from_ref(memory_id));
let scope = scopes
.get(memory_id.as_str())
.expect("forgotten ids come from the universe")
.clone();
tombstones.push(MemoryForgetTombstone {
memory_id: memory_id.clone(),
scope,
reason: payload.reason,
invalidated: single.invalidated,
});
}
if let Some(run_id) = &payload.run_id {
for tombstone in &tombstones {
journal_memory_forget(&state, &tenant, run_id, tombstone, payload.parent.clone()).await;
}
}
Ok(Json(json!({
"forgotten": plan.forgotten,
"invalidated": plan.invalidated,
"tombstones": tombstones.len(),
})))
}
async fn journal_memory_forget(
state: &AppState,
tenant: &TenantContext,
run_id: &str,
tombstone: &MemoryForgetTombstone,
parent: Option<String>,
) {
let draft = EventDraft::new(RunEventKind::MemoryForget, Effect::Idempotent).input(json!({
"effect_key": memory_forget_effect_key(&tombstone.scope, &tombstone.memory_id),
"memory_id": tombstone.memory_id,
}));
let draft = match serde_json::to_value(tombstone) {
Ok(output) => draft.output(output),
Err(error) => {
tracing::warn!(%run_id, %error, "memory tombstone failed to serialize; journaling skipped");
return;
}
};
if let Err(error) = try_journal_memory_event(state, tenant, run_id, parent, draft).await {
tracing::warn!(
%run_id,
memory_id = %tombstone.memory_id,
%error,
"forgetting is durable in the store; tombstone journaling skipped"
);
}
}
fn status_wire(status: CandidateStatus) -> String {
serde_json::to_value(status)
.ok()
.and_then(|value| value.as_str().map(str::to_owned))
.unwrap_or_else(|| format!("{status:?}"))
}
fn refusal_error(refusal: &PromotionRefusal) -> ApiError {
match refusal {
PromotionRefusal::RequiresApproval { .. } | PromotionRefusal::ApprovalMismatch { .. } => {
ApiError::new(StatusCode::FORBIDDEN, "forbidden", refusal.to_string())
}
_ => ApiError::unprocessable(refusal.to_string()),
}
}
fn learn_error(error: &LearnError) -> ApiError {
match error {
LearnError::InvalidTransition { .. } => ApiError::conflict(error.to_string()),
LearnError::Refused(refusal) => refusal_error(refusal),
_ => ApiError::unprocessable(error.to_string()),
}
}
fn journal_gate_error(error: String) -> ApiError {
if error.contains("no persisted journal") || error.contains("does not resolve") {
ApiError::not_found(error)
} else {
ApiError::unprocessable(error)
}
}
fn transition_outcome(outcome: CandidateTransition, candidate_id: &str) -> Result<(), ApiError> {
match outcome {
CandidateTransition::Applied => Ok(()),
CandidateTransition::Unknown => Err(ApiError::not_found(format!(
"candidate `{candidate_id}` not found"
))),
CandidateTransition::Conflict(live) => Err(ApiError::conflict(format!(
"candidate `{candidate_id}` is `{}` — a concurrent transition won the race; \
retry against the settled state",
status_wire(live)
))),
}
}
#[derive(Debug, Deserialize)]
struct CreateCandidatePayload {
candidate: Candidate,
run_id: String,
#[serde(default)]
parent: Option<String>,
}
async fn create_candidate(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<CreateCandidatePayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
payload
.candidate
.verify_address()
.map_err(|e| learn_error(&e))?;
let candidate_id = payload.candidate.candidate_id.to_string();
if let Some(existing) = state
.server_store
.get_candidate(tenant.tenant(), &candidate_id)
.await
.map_err(internal_err)?
{
return Ok((
StatusCode::OK,
Json(json!({
"candidate_id": candidate_id,
"created": false,
"record": existing,
})),
));
}
let draft = EventDraft::new(RunEventKind::CandidateCreated, Effect::Idempotent)
.input(json!({
"effect_key": candidate_effect_key(&payload.candidate.candidate_id),
"candidate_id": candidate_id,
}))
.output(
serde_json::to_value(&payload.candidate)
.map_err(|e| ApiError::internal(format!("serialize candidate: {e}")))?,
);
try_journal_memory_event(&state, &tenant, &payload.run_id, payload.parent, draft)
.await
.map_err(journal_gate_error)?;
let record = CandidateRecord::new(payload.candidate);
state
.server_store
.put_candidate(tenant.tenant(), &record)
.await
.map_err(internal_err)?;
Ok((
StatusCode::CREATED,
Json(json!({
"candidate_id": candidate_id,
"created": true,
"record": record,
})),
))
}
async fn list_candidates(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let mut records = state
.server_store
.list_candidates(tenant.tenant())
.await
.map_err(internal_err)?;
records.sort_by(|a, b| a.candidate.candidate_id.cmp(&b.candidate.candidate_id));
Ok(Json(json!({ "candidates": records })))
}
async fn get_candidate(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(candidate_id): Path<String>,
) -> Result<Json<CandidateRecord>, ApiError> {
state
.server_store
.get_candidate(tenant.tenant(), &candidate_id)
.await
.map_err(internal_err)?
.map(Json)
.ok_or_else(|| ApiError::not_found(format!("candidate `{candidate_id}` not found")))
}
#[derive(Debug, Deserialize)]
struct EvaluateCandidatePayload {
request: EvaluationRequest,
run_id: String,
#[serde(default)]
parent: Option<String>,
}
async fn evaluate_candidate(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(candidate_id): Path<String>,
Json(payload): Json<EvaluateCandidatePayload>,
) -> Result<Json<Value>, ApiError> {
let mut record = state
.server_store
.get_candidate(tenant.tenant(), &candidate_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("candidate `{candidate_id}` not found")))?;
let expect = record.status;
let Some(evaluator) = state.config.candidate_evaluator.clone() else {
return Err(ApiError::conflict(
"no candidate evaluator is configured on this server — promotion is gated on \
evidence, and evidence requires an evaluator \
(`ServerConfig::with_candidate_evaluator`)"
.to_string(),
));
};
let evaluation = evaluator
.evaluate(&record.candidate, &payload.request)
.await
.map_err(|e| ApiError::unprocessable(format!("candidate evaluation failed: {e}")))?;
if evaluation.candidate_id != record.candidate.candidate_id
|| evaluation.dataset_version != payload.request.dataset_version
{
return Err(ApiError::unprocessable(
"the evaluator returned an evaluation naming a different candidate or dataset \
version — the CandidateEvaluator contract requires both to match the request"
.to_string(),
));
}
record
.apply_evaluation(evaluation.clone())
.map_err(|e| learn_error(&e))?;
let draft = EventDraft::new(RunEventKind::CandidateEvaluated, Effect::Idempotent)
.input(json!({
"effect_key": evaluation_effect_key(
&record.candidate.candidate_id,
&evaluation.dataset_version,
),
"candidate_id": candidate_id,
}))
.output(
serde_json::to_value(&evaluation)
.map_err(|e| ApiError::internal(format!("serialize evaluation: {e}")))?,
);
try_journal_memory_event(&state, &tenant, &payload.run_id, payload.parent, draft)
.await
.map_err(journal_gate_error)?;
transition_outcome(
state
.server_store
.transition_candidate(tenant.tenant(), &candidate_id, expect, &record, None)
.await
.map_err(internal_err)?,
&candidate_id,
)?;
Ok(Json(json!({
"candidate_id": candidate_id,
"status": status_wire(record.status),
"evaluation": evaluation,
})))
}
#[derive(Debug, Deserialize)]
struct PromoteCandidatePayload {
run_id: String,
#[serde(default)]
approval: Option<ApprovalToken>,
#[serde(default)]
tag: Option<EnvironmentTag>,
#[serde(default)]
parent: Option<String>,
}
async fn promote_candidate(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(candidate_id): Path<String>,
Json(payload): Json<PromoteCandidatePayload>,
) -> Result<Json<Value>, ApiError> {
let mut record = state
.server_store
.get_candidate(tenant.tenant(), &candidate_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("candidate `{candidate_id}` not found")))?;
let decision = admit_promotion(
&state.config.promotion_envelope,
&record.candidate,
record.evaluation.as_ref(),
payload.approval.as_ref(),
)
.map_err(|e| learn_error(&e))?;
let surface = match &payload.tag {
Some(tag) => record.candidate.surface().tagged(tag),
None => record.candidate.surface(),
};
let pointer = state
.server_store
.get_version_pointer(tenant.tenant(), surface.as_str())
.await
.map_err(internal_err)?
.unwrap_or_else(|| VersionPointer::new(surface.clone()));
let receipt = PromotionReceipt {
candidate_id: record.candidate.candidate_id.clone(),
surface: surface.clone(),
previous: pointer.active.clone(),
decision,
promoted_at: Utc::now(),
};
record
.apply_promotion(receipt.clone())
.map_err(|e| learn_error(&e))?;
let moved = pointer.promoted(&receipt);
let draft = EventDraft::new(RunEventKind::CandidatePromoted, Effect::Idempotent)
.input(json!({
"effect_key": promotion_effect_key(&record.candidate.candidate_id),
"candidate_id": candidate_id,
}))
.output(
serde_json::to_value(&receipt)
.map_err(|e| ApiError::internal(format!("serialize promotion receipt: {e}")))?,
);
try_journal_memory_event(&state, &tenant, &payload.run_id, payload.parent, draft)
.await
.map_err(journal_gate_error)?;
transition_outcome(
state
.server_store
.transition_candidate(
tenant.tenant(),
&candidate_id,
CandidateStatus::Evaluated,
&record,
Some(&moved),
)
.await
.map_err(internal_err)?,
&candidate_id,
)?;
if payload.tag.is_none() && receipt.decision.canary.is_none() {
if let CandidateContent::Policy { family, parameters } = &record.candidate.content {
activate_policy_from_candidate(
&state,
&tenant,
&candidate_id,
*family,
parameters.clone(),
)
.await?;
}
}
Ok(Json(json!({
"candidate_id": candidate_id,
"status": status_wire(record.status),
"receipt": receipt,
"pointer": moved,
})))
}
#[derive(Debug, Deserialize)]
struct RollbackCandidatePayload {
run_id: String,
cause: String,
#[serde(default)]
tag: Option<EnvironmentTag>,
#[serde(default)]
parent: Option<String>,
}
async fn rollback_candidate(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(candidate_id): Path<String>,
Json(payload): Json<RollbackCandidatePayload>,
) -> Result<Json<Value>, ApiError> {
let mut record = state
.server_store
.get_candidate(tenant.tenant(), &candidate_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("candidate `{candidate_id}` not found")))?;
if record.status != CandidateStatus::Promoted {
return Err(ApiError::conflict(format!(
"candidate `{candidate_id}` is `{}` — only a promoted candidate can roll back",
status_wire(record.status)
)));
}
let surface = match &payload.tag {
Some(tag) => record.candidate.surface().tagged(tag),
None => record.candidate.surface(),
};
let pointer = state
.server_store
.get_version_pointer(tenant.tenant(), surface.as_str())
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::conflict(format!(
"surface `{surface}` has no version pointer — the candidate is not serving"
))
})?;
let serves_active = pointer.active.as_ref() == Some(record.candidate_id());
let serves_canary = pointer
.canary
.as_ref()
.is_some_and(|binding| &binding.candidate_id == record.candidate_id());
if !serves_active && !serves_canary {
return Err(ApiError::conflict(format!(
"candidate `{candidate_id}` is marked promoted but surface `{surface}` does not \
serve it — the pointer moved on; roll back what serves"
)));
}
let to = if serves_active {
record
.promotion
.as_ref()
.and_then(|receipt| receipt.previous.clone())
} else {
None
};
let receipt = RollbackReceipt {
surface: surface.clone(),
from: record.candidate.candidate_id.clone(),
to,
cause: payload.cause,
rolled_back_at: Utc::now(),
};
record
.apply_rollback(receipt.clone())
.map_err(|e| learn_error(&e))?;
let moved = pointer.rolled_back(&receipt);
let draft = EventDraft::new(RunEventKind::CandidateRolledBack, Effect::Idempotent)
.input(json!({
"effect_key": rollback_effect_key(&surface, &record.candidate.candidate_id),
"candidate_id": candidate_id,
}))
.output(
serde_json::to_value(&receipt)
.map_err(|e| ApiError::internal(format!("serialize rollback receipt: {e}")))?,
);
try_journal_memory_event(&state, &tenant, &payload.run_id, payload.parent, draft)
.await
.map_err(journal_gate_error)?;
transition_outcome(
state
.server_store
.transition_candidate(
tenant.tenant(),
&candidate_id,
CandidateStatus::Promoted,
&record,
Some(&moved),
)
.await
.map_err(internal_err)?,
&candidate_id,
)?;
if payload.tag.is_none() && serves_active {
if let CandidateContent::Policy { .. } = &record.candidate.content {
revert_policy_from_rollback(&state, &tenant, &receipt).await?;
}
}
Ok(Json(json!({
"candidate_id": candidate_id,
"status": status_wire(record.status),
"receipt": receipt,
"pointer": moved,
})))
}
async fn list_version_pointers(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let mut pointers = state
.server_store
.list_version_pointers(tenant.tenant())
.await
.map_err(internal_err)?;
pointers.sort_by(|a, b| a.surface.as_str().cmp(b.surface.as_str()));
Ok(Json(json!({ "versions": pointers })))
}
fn registry_error(error: &RegistryError) -> ApiError {
match error {
RegistryError::UndiffableContent(_) => ApiError::internal(error.to_string()),
_ => ApiError::unprocessable(error.to_string()),
}
}
fn parse_family(family: &str) -> Result<CandidateKind, ApiError> {
serde_json::from_value(Value::String(family.to_owned())).map_err(|_| {
ApiError::bad_request(format!(
"unknown registry family `{family}` — expected one of `prompt`, `policy`, \
`memory_set`, `tool_permission`, `tool_contract`, `model_settings`, \
`memory_configuration`, `middleware_composition`"
))
})
}
fn artifact_surface(family: &str, name: &str) -> Result<(CandidateKind, String), ApiError> {
let family = parse_family(family)?;
Ok((family, surface_for_kind(family, name).to_string()))
}
#[derive(Debug, Deserialize)]
struct DeclareArtifactPayload {
family: CandidateKind,
name: String,
owner: ProvenanceAuthor,
}
async fn declare_artifact(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<DeclareArtifactPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let record = ArtifactRecord::new(payload.family, payload.name, payload.owner, Utc::now())
.map_err(|e| registry_error(&e))?;
let surface = record.surface.to_string();
if state
.server_store
.put_artifact(tenant.tenant(), &record)
.await
.map_err(internal_err)?
{
return Ok((
StatusCode::CREATED,
Json(json!({
"surface": surface,
"created": true,
"artifact": record,
})),
));
}
let existing = state
.server_store
.get_artifact(tenant.tenant(), &surface)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::internal("artifact declared but not readable".to_string()))?;
if existing.family == record.family && existing.owner == record.owner {
return Ok((
StatusCode::OK,
Json(json!({
"surface": surface,
"created": false,
"artifact": existing,
})),
));
}
Err(ApiError::conflict(format!(
"surface `{surface}` is already declared as a `{}` artifact owned by `{}` — artifact \
identity is immutable; a different artifact needs a different name",
existing.family,
existing.owner.as_id_string()
)))
}
#[derive(Debug, Deserialize)]
struct ListArtifactsQuery {
family: Option<String>,
}
async fn list_artifacts(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Query(query): Query<ListArtifactsQuery>,
) -> Result<Json<Value>, ApiError> {
let mut records = state
.server_store
.list_artifacts(tenant.tenant())
.await
.map_err(internal_err)?;
if let Some(family) = &query.family {
let family = parse_family(family)?;
records.retain(|record| record.family == family);
}
records.sort_by(|a, b| a.surface.as_str().cmp(b.surface.as_str()));
Ok(Json(json!({ "artifacts": records })))
}
async fn get_artifact(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path((family, name)): Path<(String, String)>,
) -> Result<Json<ArtifactRecord>, ApiError> {
let (_, surface) = artifact_surface(&family, &name)?;
state
.server_store
.get_artifact(tenant.tenant(), &surface)
.await
.map_err(internal_err)?
.map(Json)
.ok_or_else(|| ApiError::not_found(format!("artifact `{surface}` not found")))
}
#[derive(Debug, Deserialize)]
struct CommitArtifactPayload {
candidate_id: String,
#[serde(default)]
committed_at: Option<DateTime<Utc>>,
}
async fn commit_artifact(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path((family, name)): Path<(String, String)>,
Json(payload): Json<CommitArtifactPayload>,
) -> Result<Json<Value>, ApiError> {
let (_, surface) = artifact_surface(&family, &name)?;
let artifact = state
.server_store
.get_artifact(tenant.tenant(), &surface)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("artifact `{surface}` not found")))?;
let candidate = state
.server_store
.get_candidate(tenant.tenant(), &payload.candidate_id)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!("candidate `{}` not found", payload.candidate_id))
})?;
if artifact
.find_commit(&candidate.candidate.candidate_id)
.is_some()
{
return Ok(Json(json!({
"surface": surface,
"committed": false,
"commits": artifact.commits.len(),
})));
}
let commit = artifact
.admit_commit(
&candidate.candidate,
payload.committed_at.unwrap_or_else(Utc::now),
)
.map_err(|e| registry_error(&e))?;
match state
.server_store
.commit_artifact(tenant.tenant(), &surface, artifact.commits.len(), &commit)
.await
.map_err(internal_err)?
{
ArtifactCommitOutcome::Applied => Ok(Json(json!({
"surface": surface,
"committed": true,
"commit": commit,
"commits": artifact.commits.len() + 1,
}))),
ArtifactCommitOutcome::Unknown => Err(ApiError::not_found(format!(
"artifact `{surface}` not found"
))),
ArtifactCommitOutcome::Conflict(live) => Err(ApiError::conflict(format!(
"artifact `{surface}` has {live} commits, not {} — a concurrent commit won the \
race; retry against the settled history",
artifact.commits.len()
))),
}
}
async fn list_artifact_commits(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path((family, name)): Path<(String, String)>,
) -> Result<Json<Value>, ApiError> {
let (_, surface) = artifact_surface(&family, &name)?;
let artifact = state
.server_store
.get_artifact(tenant.tenant(), &surface)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("artifact `{surface}` not found")))?;
let mut commits = Vec::with_capacity(artifact.commits.len());
for commit in &artifact.commits {
let candidate = state
.server_store
.get_candidate(tenant.tenant(), commit.candidate_id.as_str())
.await
.map_err(internal_err)?;
commits.push(json!({
"candidate_id": commit.candidate_id,
"committed_at": commit.committed_at,
"author": candidate.as_ref().map(|record| &record.candidate.distilled_by),
"status": candidate.as_ref().map(|record| status_wire(record.status)),
}));
}
Ok(Json(json!({
"surface": artifact.surface,
"family": artifact.family,
"owner": artifact.owner,
"commits": commits,
})))
}
#[derive(Debug, Deserialize)]
struct ArtifactDiffQuery {
from: String,
to: String,
}
async fn diff_artifact(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path((family, name)): Path<(String, String)>,
Query(query): Query<ArtifactDiffQuery>,
) -> Result<Json<Value>, ApiError> {
let (_, surface) = artifact_surface(&family, &name)?;
let artifact = state
.server_store
.get_artifact(tenant.tenant(), &surface)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("artifact `{surface}` not found")))?;
for id in [&query.from, &query.to] {
if artifact
.find_commit(&CandidateId::from(id.clone()))
.is_none()
{
return Err(registry_error(&RegistryError::NotCommitted {
candidate_id: CandidateId::from(id.clone()),
}));
}
}
let load = |id: &str| {
let state = state.clone();
let tenant = tenant.clone();
let id = id.to_owned();
async move { state.server_store.get_candidate(tenant.tenant(), &id).await }
};
let from = load(&query.from)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("candidate `{}` not found", query.from)))?;
let to = load(&query.to)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("candidate `{}` not found", query.to)))?;
let diff = diff_candidates(&from.candidate, &to.candidate).map_err(|e| registry_error(&e))?;
Ok(Json(json!({
"surface": surface,
"from": query.from,
"to": query.to,
"diff": diff,
})))
}
#[derive(Debug, Deserialize)]
struct RegisterPolicyPayload {
#[serde(default)]
version: Option<String>,
policy: ExecutorPolicy,
}
async fn register_policy(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<RegisterPolicyPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let version = match &payload.version {
Some(version) => {
policy::validate_policy_version(version).map_err(ApiError::bad_request)?;
PolicyVersion::new(version.clone())
}
None => derive_policy_version(&payload.policy).map_err(internal_err)?,
};
let record = PolicyRecord {
version,
policy: payload.policy,
source: PolicySource::Api,
registered_at: Utc::now(),
};
let write = state
.server_store
.put_policy(tenant.tenant(), &record)
.await
.map_err(internal_err)?;
match write {
PolicyWrite::Created => Ok((
StatusCode::CREATED,
Json(json!({ "version": record.version, "record": record })),
)),
PolicyWrite::Converged => Ok((
StatusCode::OK,
Json(json!({ "version": record.version, "record": record })),
)),
PolicyWrite::Conflict => Err(ApiError::conflict(format!(
"policy version `{}` already names a different body — versions are immutable; \
register the changed body under a new (or its derived) version",
record.version.as_str()
))),
}
}
async fn list_policies(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let mut policies = state
.server_store
.list_policies(tenant.tenant())
.await
.map_err(internal_err)?;
policies.sort_by(|a, b| a.version.as_str().cmp(b.version.as_str()));
Ok(Json(json!({ "policies": policies })))
}
async fn get_policy(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(version): Path<String>,
) -> Result<Json<Value>, ApiError> {
if version == PolicyVersion::STATIC_V0 {
return Ok(Json(
json!({ "version": version, "record": policy::static_floor_record() }),
));
}
let record = state
.server_store
.get_policy(tenant.tenant(), &version)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("policy version `{version}` not found")))?;
Ok(Json(json!({ "version": record.version, "record": record })))
}
#[derive(Debug, Deserialize)]
struct ActivatePolicyPayload {
version: String,
}
async fn activate_policy(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<ActivatePolicyPayload>,
) -> Result<Json<Value>, ApiError> {
let record = if payload.version == PolicyVersion::STATIC_V0 {
policy::static_floor_record()
} else {
state
.server_store
.get_policy(tenant.tenant(), &payload.version)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::unprocessable(format!(
"policy version `{}` is not registered — only registered bodies (and the \
static floor) can be activated",
payload.version
))
})?
};
let activation = PolicyActivation {
version: record.version.clone(),
activated_at: Utc::now(),
};
state
.server_store
.append_policy_activation(tenant.tenant(), &activation)
.await
.map_err(internal_err)?;
Ok(Json(
json!({ "version": activation.version, "active": record }),
))
}
async fn get_active_policy(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let record = policy::active_policy_record(&state.server_store, tenant.tenant())
.await
.map_err(internal_err)?;
Ok(Json(json!({ "version": record.version, "record": record })))
}
async fn list_policy_epochs(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let activations = state
.server_store
.list_policy_activations(tenant.tenant())
.await
.map_err(internal_err)?;
let bindings = state
.server_store
.list_policy_bindings(tenant.tenant())
.await
.map_err(internal_err)?;
let epochs = policy::derive_epochs(activations, bindings);
Ok(Json(json!({ "epochs": epochs })))
}
#[derive(Debug, Deserialize)]
struct PolicyDriftQuery {
version: Option<String>,
}
async fn get_policy_drift(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Query(query): Query<PolicyDriftQuery>,
) -> Result<Json<Value>, ApiError> {
let version = match query.version {
Some(version) => {
if version != PolicyVersion::STATIC_V0 {
policy::validate_policy_version(&version).map_err(ApiError::bad_request)?;
}
PolicyVersion::new(version)
}
None => state
.server_store
.list_policy_activations(tenant.tenant())
.await
.map_err(internal_err)?
.last()
.map(|activation| activation.version.clone())
.unwrap_or_default(),
};
if version.as_str() == PolicyVersion::STATIC_V0 {
return Err(ApiError::unprocessable(
"the static floor was never promoted; there is no promotion baseline to drift from"
.to_string(),
));
}
let record = state
.server_store
.get_policy(tenant.tenant(), version.as_str())
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!(
"policy version `{}` is not registered",
version.as_str()
))
})?;
let PolicySource::Candidate { candidate_id } = &record.source else {
return Err(ApiError::unprocessable(format!(
"policy version `{}` was registered through the API; only candidate-derived \
versions carry a promotion baseline",
version.as_str()
)));
};
let baseline = state
.server_store
.get_candidate(tenant.tenant(), candidate_id)
.await
.map_err(internal_err)?
.and_then(|record| record.evaluation)
.and_then(|evaluation| DriftBaseline::from_twin_report(&evaluation.baseline_report))
.ok_or_else(|| {
ApiError::unprocessable(format!(
"policy version `{}` carries no twin baseline — its candidate was never \
twin-evaluated",
version.as_str()
))
})?;
let tasks = state
.server_store
.list_tasks(tenant.tenant(), None)
.await
.map_err(internal_err)?;
let mut run_ids: Vec<String> = tasks
.iter()
.filter_map(|task| task.run_id.clone())
.collect();
run_ids.sort();
run_ids.dedup();
let mut decisions = Vec::new();
for run_id in run_ids {
let Some(snapshot) = state
.server_store
.get_journal(&run_id)
.await
.map_err(internal_err)?
else {
continue;
};
for event in &snapshot.events {
if event.kind != RunEventKind::PolicyDecision {
continue;
}
let Some(PayloadRef::Inline(value)) = &event.output else {
continue;
};
match serde_json::from_value::<DecisionEvent>(value.clone()) {
Ok(decision) => decisions.push(decision),
Err(error) => {
tracing::warn!(%run_id, %error, "skipping undecodable policy decision event")
}
}
}
}
let report = detect_policy_drift(&decisions, &version, &baseline, &DriftThresholds::default());
Ok(Json(json!({ "report": report })))
}
#[derive(Debug, Deserialize)]
struct RegisterCapsulePayload {
manifest: CapsuleManifest,
}
async fn register_capsule(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<RegisterCapsulePayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
payload
.manifest
.validate()
.map_err(|e| ApiError::unprocessable(format!("invalid capsule manifest: {e}")))?;
let capsule_id = derive_capsule_id(&payload.manifest).map_err(internal_err)?;
#[cfg(feature = "capsules")]
crate::capsule_policy::authorize_registration(
&state.server_store,
&state.capsule_plane,
tenant.tenant(),
&crate::capsule_policy::prospective_record(capsule_id.clone(), &payload.manifest),
)
.await
.map_err(admission_refusal_error)?;
let record = CapsuleRecord {
capsule_id,
manifest: payload.manifest,
registered_at: Utc::now(),
};
let write = state
.server_store
.put_capsule(tenant.tenant(), &record)
.await
.map_err(internal_err)?;
match write {
CapsuleWrite::Created => Ok((
StatusCode::CREATED,
Json(json!({ "capsule_id": record.capsule_id, "record": record })),
)),
CapsuleWrite::Converged => Ok((
StatusCode::OK,
Json(json!({ "capsule_id": record.capsule_id, "record": record })),
)),
CapsuleWrite::Conflict => Err(ApiError::conflict(format!(
"capsule `{}` version `{}` is already registered under a different content address \
(or the address collides with a different manifest) — versions are immutable; \
register the changed declaration under a new version",
record.manifest.identity.name, record.manifest.version
))),
}
}
async fn list_capsules(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let mut capsules = state
.server_store
.list_capsules(tenant.tenant())
.await
.map_err(internal_err)?;
capsules.sort_by(|a, b| a.capsule_id.as_str().cmp(b.capsule_id.as_str()));
Ok(Json(json!({ "capsules": capsules })))
}
async fn get_capsule(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(capsule_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
let record = state
.server_store
.get_capsule(tenant.tenant(), &capsule_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("capsule `{capsule_id}` not found")))?;
Ok(Json(
json!({ "capsule_id": record.capsule_id, "record": record }),
))
}
async fn put_capsule_blob_route(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(capsule_id): Path<String>,
body: axum::body::Bytes,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let record = state
.server_store
.get_capsule(tenant.tenant(), &capsule_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("capsule `{capsule_id}` not found")))?;
let digest = sha256_hex(&body);
if digest != record.manifest.build_digest {
return Err(ApiError::unprocessable(format!(
"blob digest `{digest}` does not match the manifest's build_digest `{}` — \
the registry stores only the bytes the manifest commits to",
record.manifest.build_digest
)));
}
let len = body.len();
state
.server_store
.put_capsule_blob(tenant.tenant(), &capsule_id, &body)
.await
.map_err(|e| {
if e.starts_with("capsule blob conflict") {
ApiError::conflict(e)
} else {
internal_err(e)
}
})?;
Ok((
StatusCode::CREATED,
Json(json!({
"capsule_id": capsule_id,
"sha256": digest,
"bytes": len,
})),
))
}
#[derive(Debug, Deserialize)]
struct ResolveCapsulesPayload {
pins: std::collections::BTreeMap<String, String>,
run_id: String,
#[serde(default)]
parent: Option<String>,
#[cfg(feature = "capsules")]
#[serde(default)]
budget: Option<ResourceBudget>,
}
async fn resolve_capsules(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<ResolveCapsulesPayload>,
) -> Result<Json<Value>, ApiError> {
#[cfg(feature = "capsules")]
let mut records = Vec::with_capacity(payload.pins.len());
let mut resolutions = Vec::with_capacity(payload.pins.len());
for (name, version) in &payload.pins {
let record = state
.server_store
.get_capsule_by_version(tenant.tenant(), name, version)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!(
"capsule pin `{name}` version `{version}` is not registered"
))
})?;
let recomputed = derive_capsule_id(&record.manifest).map_err(internal_err)?;
if recomputed != record.capsule_id {
return Err(ApiError::unprocessable(format!(
"capsule `{}` (pin `{name}` version `{version}`) fails its integrity check: \
the stored manifest re-derives to {recomputed} — the registry record is \
corrupt; re-register the manifest",
record.capsule_id
)));
}
resolutions.push(CapsuleResolution {
name: name.clone(),
version: CapsuleVersion::new(version.clone()),
capsule_id: record.capsule_id.clone(),
build_digest: record.manifest.build_digest.clone(),
policy_version: None,
overlays: None,
effective_grants: None,
clamped_budget: None,
});
#[cfg(feature = "capsules")]
records.push(record);
}
#[cfg(feature = "capsules")]
for (record, resolution) in records.iter().zip(resolutions.iter_mut()) {
match crate::capsule_policy::compose_admission(
&state.server_store,
&state.capsule_plane,
tenant.tenant(),
record,
payload.budget.as_ref(),
state.config.capsule_budget_ceiling.as_ref(),
)
.await
{
Ok(outcome) => {
resolution.policy_version = outcome.policy_version;
resolution.overlays = outcome.overlays;
resolution.effective_grants = outcome.effective_grants;
resolution.clamped_budget = outcome.clamped_budget;
}
Err(crate::capsule_policy::AdmissionRefusal::Policy {
version,
forbidden,
detail,
}) => {
for grant in &forbidden {
let denial = CapsuleDenial {
capsule_id: record.capsule_id.clone(),
capability: grant.capability_kind(),
absent_grant: grant.clone(),
detail: detail.clone(),
policy_version: Some(version.clone()),
};
let draft = EventDraft::new(RunEventKind::CapsuleDenied, Effect::Pure).output(
serde_json::to_value(&denial)
.map_err(|e| ApiError::internal(format!("serialize denial: {e}")))?,
);
try_journal_memory_event(
&state,
&tenant,
&payload.run_id,
payload.parent.clone(),
draft,
)
.await
.map_err(journal_gate_error)?;
}
return Err(ApiError::forbidden(detail));
}
Err(refusal) => return Err(admission_refusal_error(refusal)),
}
}
for resolution in &resolutions {
let draft = EventDraft::new(RunEventKind::CapsuleResolved, Effect::ReadOnly).output(
serde_json::to_value(resolution)
.map_err(|e| ApiError::internal(format!("serialize resolution: {e}")))?,
);
try_journal_memory_event(
&state,
&tenant,
&payload.run_id,
payload.parent.clone(),
draft,
)
.await
.map_err(journal_gate_error)?;
}
Ok(Json(json!({ "resolutions": resolutions })))
}
#[cfg(feature = "capsules")]
fn admission_refusal_error(refusal: crate::capsule_policy::AdmissionRefusal) -> ApiError {
match refusal {
crate::capsule_policy::AdmissionRefusal::Policy { detail, .. } => {
ApiError::forbidden(detail)
}
crate::capsule_policy::AdmissionRefusal::Budget {
field,
declared,
bound,
} => ApiError::unprocessable(format!(
"capsule budget field `{field}` declares {declared}, exceeding the tightest \
enclosing bound {bound} — token and cost bounds cannot be retrofitted mid-run; \
declare within the bound or raise it"
)),
crate::capsule_policy::AdmissionRefusal::Internal(detail) => ApiError::internal(detail),
}
}
#[cfg(feature = "capsules")]
#[derive(Debug, Deserialize)]
struct RegisterCapsulePolicyPayload {
policy_text: String,
#[serde(default)]
version: Option<String>,
}
#[cfg(feature = "capsules")]
async fn register_capsule_policy(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<RegisterCapsulePolicyPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
crate::capsule_policy::CedarEngine::parse(&payload.policy_text)
.map_err(|e| ApiError::unprocessable(format!("capsule policy does not parse: {e}")))?;
let version = match &payload.version {
Some(version) => {
crate::capsule_policy::validate_capsule_policy_version(version)
.map_err(|e| ApiError::unprocessable(format!("invalid policy version: {e}")))?;
version.clone()
}
None => crate::capsule_policy::derive_capsule_policy_version(&payload.policy_text),
};
let record = crate::capsule_policy::CapsulePolicyRecord {
version: version.clone(),
policy_text: payload.policy_text,
registered_at: Utc::now(),
};
match state
.server_store
.put_capsule_policy(tenant.tenant(), &record)
.await
.map_err(internal_err)?
{
crate::capsule_policy::CapsulePolicyWrite::Created => Ok((
StatusCode::CREATED,
Json(json!({ "version": version, "record": record })),
)),
crate::capsule_policy::CapsulePolicyWrite::Converged => Ok((
StatusCode::OK,
Json(json!({ "version": version, "record": record })),
)),
crate::capsule_policy::CapsulePolicyWrite::Conflict => Err(ApiError::conflict(format!(
"capsule policy version `{version}` is already registered under a different body — \
versions are immutable; register the changed policy under a new version"
))),
}
}
#[cfg(not(feature = "capsules"))]
async fn register_capsule_policy() -> Result<Json<Value>, ApiError> {
Err(crate::capsule_policy::plane_unavailable())
}
#[cfg(feature = "capsules")]
async fn list_capsule_policies(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let mut policies = state
.server_store
.list_capsule_policies(tenant.tenant())
.await
.map_err(internal_err)?;
policies.sort_by(|a, b| a.version.cmp(&b.version));
Ok(Json(json!({ "policies": policies })))
}
#[cfg(not(feature = "capsules"))]
async fn list_capsule_policies() -> Result<Json<Value>, ApiError> {
Err(crate::capsule_policy::plane_unavailable())
}
#[cfg(feature = "capsules")]
async fn get_capsule_policy(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(version): Path<String>,
) -> Result<Json<Value>, ApiError> {
let record = state
.server_store
.get_capsule_policy(tenant.tenant(), &version)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("capsule policy `{version}` not found")))?;
Ok(Json(json!({ "version": record.version, "record": record })))
}
#[cfg(not(feature = "capsules"))]
async fn get_capsule_policy(Path(_version): Path<String>) -> Result<Json<Value>, ApiError> {
Err(crate::capsule_policy::plane_unavailable())
}
#[cfg(feature = "capsules")]
async fn get_active_capsule_policy(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let record = state
.server_store
.active_capsule_policy(tenant.tenant())
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!(
"tenant `{}` has no active capsule policy — admission is unenforced until the \
first activation",
tenant.tenant()
))
})?;
Ok(Json(json!({ "version": record.version, "record": record })))
}
#[cfg(not(feature = "capsules"))]
async fn get_active_capsule_policy() -> Result<Json<Value>, ApiError> {
Err(crate::capsule_policy::plane_unavailable())
}
#[cfg(feature = "capsules")]
#[derive(Debug, Deserialize)]
struct ActivateCapsulePolicyPayload {
version: String,
}
#[cfg(feature = "capsules")]
async fn activate_capsule_policy(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<ActivateCapsulePolicyPayload>,
) -> Result<Json<Value>, ApiError> {
state
.server_store
.activate_capsule_policy(
tenant.tenant(),
&crate::capsule_policy::activation(&payload.version),
)
.await
.map_err(|e| {
if e.contains("not registered") {
ApiError::unprocessable(e)
} else {
internal_err(e)
}
})?;
state
.capsule_plane
.refresh(&state.server_store, tenant.tenant())
.await
.map_err(internal_err)?;
Ok(Json(json!({ "active": payload.version })))
}
#[cfg(not(feature = "capsules"))]
async fn activate_capsule_policy() -> Result<Json<Value>, ApiError> {
Err(crate::capsule_policy::plane_unavailable())
}
#[cfg(feature = "capsules")]
#[derive(Debug, Deserialize)]
struct AttachCapsuleOverlayPayload {
overlay: CapsuleOverlay,
}
#[cfg(feature = "capsules")]
async fn attach_capsule_overlay(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<AttachCapsuleOverlayPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
payload
.overlay
.validate()
.map_err(|e| ApiError::unprocessable(format!("invalid capsule overlay: {e}")))?;
crate::capsule_policy::authorize_overlay_attach(
&state.server_store,
tenant.tenant(),
&payload.overlay,
)
.await
.map_err(admission_refusal_error)?;
let record = crate::capsule_policy::CapsuleOverlayRecord {
overlay: payload.overlay,
author: tenant.tenant().to_string(),
attached_at: Utc::now(),
};
let created = state
.server_store
.put_capsule_overlay(tenant.tenant(), &record)
.await
.map_err(internal_err)?;
let status = if created {
StatusCode::CREATED
} else {
StatusCode::OK
};
Ok((status, Json(json!({ "overlay": record }))))
}
#[cfg(not(feature = "capsules"))]
async fn attach_capsule_overlay() -> Result<Json<Value>, ApiError> {
Err(crate::capsule_policy::plane_unavailable())
}
#[cfg(feature = "capsules")]
async fn list_capsule_overlays(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let mut overlays = state
.server_store
.list_capsule_overlays(tenant.tenant())
.await
.map_err(internal_err)?;
overlays.sort_by(|a, b| a.overlay.name.cmp(&b.overlay.name));
Ok(Json(json!({ "overlays": overlays })))
}
#[cfg(not(feature = "capsules"))]
async fn list_capsule_overlays() -> Result<Json<Value>, ApiError> {
Err(crate::capsule_policy::plane_unavailable())
}
#[cfg(feature = "capsules")]
async fn get_capsule_overlay(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(name): Path<String>,
) -> Result<Json<Value>, ApiError> {
let record = state
.server_store
.get_capsule_overlay(tenant.tenant(), &name)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("capsule overlay `{name}` not found")))?;
Ok(Json(json!({ "overlay": record })))
}
#[cfg(not(feature = "capsules"))]
async fn get_capsule_overlay(Path(_name): Path<String>) -> Result<Json<Value>, ApiError> {
Err(crate::capsule_policy::plane_unavailable())
}
async fn activate_policy_from_candidate(
state: &AppState,
tenant: &TenantContext,
candidate_id: &str,
family: rusty_agent_runtime::record::DecisionFamily,
parameters: Value,
) -> Result<(), ApiError> {
let base = policy::active_policy_record(&state.server_store, tenant.tenant())
.await
.map_err(internal_err)?;
let policy = base
.policy
.with_family_parameters(family, parameters)
.map_err(|e| {
ApiError::unprocessable(format!(
"policy candidate `{candidate_id}` does not apply to the active policy: {e}"
))
})?;
let version = derive_policy_version(&policy).map_err(internal_err)?;
let record = PolicyRecord {
version: version.clone(),
policy,
source: PolicySource::Candidate {
candidate_id: candidate_id.to_string(),
},
registered_at: Utc::now(),
};
match state
.server_store
.put_policy(tenant.tenant(), &record)
.await
.map_err(internal_err)?
{
PolicyWrite::Created | PolicyWrite::Converged => {}
PolicyWrite::Conflict => {
return Err(ApiError::conflict(format!(
"derived policy version `{}` already names a different body",
version.as_str()
)));
}
}
let activation = PolicyActivation {
version,
activated_at: Utc::now(),
};
state
.server_store
.append_policy_activation(tenant.tenant(), &activation)
.await
.map_err(internal_err)
}
async fn revert_policy_from_rollback(
state: &AppState,
tenant: &TenantContext,
receipt: &RollbackReceipt,
) -> Result<(), ApiError> {
let version = match &receipt.to {
Some(previous_id) => {
let policies = state
.server_store
.list_policies(tenant.tenant())
.await
.map_err(internal_err)?;
let found = policies
.into_iter()
.find_map(|record| match &record.source {
PolicySource::Candidate { candidate_id }
if candidate_id == previous_id.as_str() =>
{
Some(record.version)
}
_ => None,
});
match found {
Some(version) => version,
None => {
tracing::warn!(
candidate = %previous_id,
"policy rollback: the displaced candidate's policy record is gone; \
reverting to the static floor"
);
PolicyVersion::default()
}
}
}
None => PolicyVersion::default(),
};
let activation = PolicyActivation {
version,
activated_at: Utc::now(),
};
state
.server_store
.append_policy_activation(tenant.tenant(), &activation)
.await
.map_err(internal_err)
}
async fn fail_task(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(task_id): Path<String>,
Json(payload): Json<FailTaskPayload>,
) -> Result<Json<Value>, ApiError> {
tasks::validate_label("worker_id", &payload.worker_id, 256).map_err(ApiError::bad_request)?;
let error_class =
tasks::parse_error_class(&payload.error_class).map_err(ApiError::bad_request)?;
tasks::validate_label("message", &payload.message, 4096).map_err(ApiError::bad_request)?;
let now = Utc::now();
let resolved = match state.server_store.get_task(tenant.tenant(), &task_id).await {
Ok(Some(task)) => {
let policy = acting_executor_policy(&state, &tenant, &task).await;
resolve_retry_parameters(&policy, error_class, task.max_attempts)
}
_ => ResolvedRetryParameters::floor(0),
};
let outcome = state
.server_store
.fail_task(
tenant.tenant(),
&task_id,
&payload.worker_id,
tasks::FailureReport {
error_class,
message: payload.message,
retryable: payload.retryable,
cost: tasks::SettlementCost {
tokens: payload.tokens,
cost_usd: payload.cost_usd,
},
retry: resolved,
},
now,
)
.await
.map_err(internal_err)?;
let task = lease_outcome(outcome, &task_id, &payload.worker_id)?;
if error_class != rusty_agent_runtime::durable::ErrorClass::Cancelled {
journal_policy_decision(
&state,
&tenant,
&task,
error_class,
payload.retryable,
resolved.max_attempts,
now,
)
.await;
}
let mut escalation = None;
if error_class != rusty_agent_runtime::durable::ErrorClass::Cancelled {
if let Some(agent_external) = task
.recipient
.as_deref()
.and_then(rusty_agent_runtime::agents::agent_id_from_recipient)
{
if let Some(agent) = state
.server_store
.get_agent(&tenant.scope(agent_external))
.await
.map_err(internal_err)?
{
let outcome = supervision::supervise(
&state.server_store,
&tenant,
agent_external,
agent,
supervision::Trigger::TurnFailed {
error_class,
message: task
.last_error
.clone()
.unwrap_or_else(|| "turn failed".to_string()),
task_id: task.task_id.clone(),
},
Utc::now(),
)
.await
.map_err(internal_err)?;
escalation = outcome.delivery.map(|delivery| match delivery {
supervision::EscalationDelivery::Mailbox {
task_id,
deduplicated,
} => json!({
"kind": "mailbox",
"task_id": task_id,
"deduplicated": deduplicated,
}),
supervision::EscalationDelivery::DeadLetter { task_id } => json!({
"kind": "dead_letter",
"task_id": task_id,
}),
});
}
}
}
if task.is_terminal() {
coordination::on_task_settled(
&state.server_store,
state.config.quota_for(tenant.tenant()),
&tenant,
&task,
Utc::now(),
)
.await
.map_err(internal_err)?;
}
crate::a2a::publish_task_update(&state, &task).await;
Ok(Json(json!({
"requeued": task.status == TaskStatus::Failed && task.next_attempt_at.is_some(),
"next_attempt_at": task.next_attempt_at,
"dead": task.status == TaskStatus::Dead,
"escalation": escalation,
})))
}
async fn cancel_task(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(task_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
let outcome = state
.server_store
.cancel_task(tenant.tenant(), &task_id, Utc::now())
.await
.map_err(internal_err)?;
match outcome {
CancelOutcome::Applied(task) => {
if task.is_terminal() {
coordination::on_task_settled(
&state.server_store,
state.config.quota_for(tenant.tenant()),
&tenant,
&task,
Utc::now(),
)
.await
.map_err(internal_err)?;
}
crate::a2a::publish_task_update(&state, &task).await;
Ok(Json(task.wire()))
}
CancelOutcome::Terminal(status) => Err(ApiError::conflict(format!(
"task `{task_id}` is already terminal ({}) and cannot be cancelled",
status.as_str()
))),
CancelOutcome::Unknown => Err(ApiError::not_found(format!("task `{task_id}` not found"))),
}
}
async fn task_metrics(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let now = Utc::now();
let stats = state
.server_store
.task_pool_stats(tenant.tenant(), now)
.await
.map_err(internal_err)?;
let mut pools: Vec<Value> = stats
.iter()
.map(|stat| {
let limit = state.config.task_pool_limits.get(&stat.pool).copied();
pool_metrics_json(stat, limit, now)
})
.collect();
for (pool, &limit) in &state.config.task_pool_limits {
if !stats.iter().any(|s| &s.pool == pool) {
pools.push(pool_metrics_json(
&tasks::PoolStat {
pool: pool.clone(),
queue_depth: 0,
leased: 0,
oldest_visible_at: None,
},
Some(limit),
now,
));
}
}
pools.sort_by(|a, b| a["pool"].as_str().cmp(&b["pool"].as_str()));
Ok(Json(json!({ "pools": pools, "now": now })))
}
fn pool_metrics_json(
stat: &tasks::PoolStat,
limit: Option<usize>,
now: chrono::DateTime<Utc>,
) -> Value {
let oldest_age_ms = stat
.oldest_visible_at
.map(|at| (now - at).num_milliseconds().max(0));
json!({
"pool": stat.pool,
"queue_depth": stat.queue_depth,
"leased": stat.leased,
"concurrency_limit": limit,
"lease_saturation": limit.map(|max| stat.leased as f64 / max.max(1) as f64),
"oldest_visible_task_age_ms": oldest_age_ms,
})
}
async fn get_task(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(task_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
state
.server_store
.get_task(tenant.tenant(), &task_id)
.await
.map_err(internal_err)?
.map(|task| Json(task.wire()))
.ok_or_else(|| ApiError::not_found(format!("task `{task_id}` not found")))
}
#[derive(Debug, Deserialize)]
struct ListTasksQuery {
#[serde(default)]
status: Option<String>,
}
async fn list_tasks(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Query(query): Query<ListTasksQuery>,
) -> Result<Json<Value>, ApiError> {
let status = query
.status
.as_deref()
.map(|s| {
TaskStatus::parse(s).ok_or_else(|| {
ApiError::bad_request(format!(
"unknown task status `{s}` (expected queued|leased|failed|completed|dead|cancelled)"
))
})
})
.transpose()?;
let tasks = state
.server_store
.list_tasks(tenant.tenant(), status)
.await
.map_err(internal_err)?;
let wire: Vec<Value> = tasks.iter().map(TaskRecord::wire).collect();
Ok(Json(json!(wire)))
}
#[derive(Debug, Deserialize)]
struct CreateAgentPayload {
#[serde(default)]
agent_id: Option<String>,
manifest: Value,
#[serde(default)]
team_id: Option<String>,
#[serde(default)]
metadata: Option<Value>,
}
fn activation_wire(agent_id: &str, lease: &agents::ActivationLease) -> Value {
json!({
"agent_id": agent_id,
"owner": lease.owner,
"fencing": lease.fencing,
"lease_expires_at": lease.expires_at,
"acquired_at": lease.acquired_at,
})
}
async fn create_agent(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<CreateAgentPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let manifest: CapabilityManifest = serde_json::from_value(payload.manifest)
.map_err(|e| ApiError::bad_request(format!("invalid `manifest`: {e}")))?;
tasks::validate_label("agent_kind", &manifest.agent_kind, 256)
.map_err(ApiError::bad_request)?;
tasks::validate_label("manifest_version", &manifest.manifest_version, 256)
.map_err(ApiError::bad_request)?;
let agent_id = payload
.agent_id
.clone()
.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
validate_client_id("agent_id", &agent_id)?;
if let Some(team_id) = &payload.team_id {
tasks::validate_pool(team_id)
.map_err(|e| ApiError::bad_request(e.replace("`pool`", "`team_id`")))?;
}
let record = AgentRecord {
agent_id: tenant.scope(&agent_id),
manifest,
team_id: payload.team_id,
metadata: payload.metadata.unwrap_or(Value::Null),
created_at: Utc::now(),
supervision: agents::AgentSupervision::default(),
};
let created = state
.server_store
.create_agent(&record)
.await
.map_err(internal_err)?;
if !created {
return Err(ApiError::conflict(format!(
"agent `{agent_id}` already exists"
)));
}
let mut wire = record;
wire.agent_id = agent_id;
Ok((StatusCode::CREATED, Json(wire.wire())))
}
async fn list_agents(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let records = state
.server_store
.list_agents()
.await
.map_err(internal_err)?;
let mut records: Vec<AgentRecord> = records
.into_iter()
.filter_map(|mut record| {
let external = tenant.unscope(&record.agent_id)?.to_string();
record.agent_id = external;
Some(record)
})
.collect();
records.sort_by(|a, b| {
a.created_at
.cmp(&b.created_at)
.then_with(|| a.agent_id.cmp(&b.agent_id))
});
let wire: Vec<Value> = records.iter().map(AgentRecord::wire).collect();
Ok(Json(json!(wire)))
}
async fn get_agent(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(agent_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
state
.server_store
.get_agent(&tenant.scope(&agent_id))
.await
.map_err(internal_err)?
.map(|mut record| {
record.agent_id = agent_id.clone();
Json(record.wire())
})
.ok_or_else(|| ApiError::not_found(format!("agent `{agent_id}` not found")))
}
#[derive(Debug, Deserialize)]
struct SendAgentMessagePayload {
kind: String,
payload: Value,
#[serde(default)]
max_attempts: Option<u32>,
#[serde(default)]
idempotency_key: Option<String>,
#[serde(default)]
effect: Option<String>,
#[serde(default)]
deadline: Option<String>,
}
async fn send_agent_message(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(agent_id): Path<String>,
Json(payload): Json<SendAgentMessagePayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let agent = state
.server_store
.get_agent(&tenant.scope(&agent_id))
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("agent `{agent_id}` not found")))?;
tasks::validate_label("kind", &payload.kind, 256).map_err(ApiError::bad_request)?;
if agent.manifest.accepts_kind(&payload.kind).is_none() {
let declared: Vec<&str> = agent.manifest.accepts.keys().map(String::as_str).collect();
return Err(ApiError::bad_request(format!(
"agent `{agent_id}` does not accept kind `{}` (manifest declares: {})",
payload.kind,
declared.join(", ")
)));
}
let mut record = build_task_record(
EnqueueTaskPayload {
kind: payload.kind,
payload: payload.payload,
pool: None,
max_attempts: payload.max_attempts,
idempotency_key: payload.idempotency_key,
effect: payload.effect,
run_id: None,
thread_id: None,
deadline: payload.deadline,
worker_version: None,
recipient: Some(AgentId::new(agent_id.as_str()).mailbox_recipient()),
parent: None,
},
&tenant,
)?;
if let Some(agent_deadline) = agent.manifest.budget.as_ref().and_then(|b| b.deadline) {
record.deadline = Some(
record
.deadline
.map_or(agent_deadline, |d| d.min(agent_deadline)),
);
}
enforce_task_quota(&state, &tenant, 1).await?;
let (task, deduplicated) = state
.server_store
.enqueue_task(&record)
.await
.map_err(internal_err)?;
let status = if deduplicated {
StatusCode::OK
} else {
StatusCode::CREATED
};
Ok((
status,
Json(json!({
"task_id": task.task_id,
"deduplicated": deduplicated,
})),
))
}
async fn get_agent_status(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(agent_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
let scoped = tenant.scope(&agent_id);
state
.server_store
.get_agent(&scoped)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("agent `{agent_id}` not found")))?;
let lease = state
.server_store
.get_activation(&scoped)
.await
.map_err(internal_err)?;
let recipient = AgentId::new(agent_id.as_str()).mailbox_recipient();
let tasks = state
.server_store
.list_tasks(tenant.tenant(), None)
.await
.map_err(internal_err)?;
let now = Utc::now();
let (mut queued, mut in_flight, mut dead) = (0u64, 0u64, 0u64);
for task in tasks
.iter()
.filter(|t| t.recipient.as_deref() == Some(recipient.as_str()))
{
match task.status {
TaskStatus::Queued | TaskStatus::Failed => queued += 1,
TaskStatus::Leased => {
if task.lease.as_ref().is_some_and(|l| l.expires_at > now) {
in_flight += 1;
} else {
queued += 1;
}
}
TaskStatus::Dead => dead += 1,
_ => {}
}
}
Ok(Json(json!({
"agent_id": agent_id,
"activation": lease
.as_ref()
.map(|l| activation_wire(&agent_id, l))
.unwrap_or(Value::Null),
"mailbox": {
"queued": queued,
"in_flight": in_flight,
"dead": dead,
},
})))
}
#[derive(Debug, Deserialize)]
struct ActivateAgentPayload {
worker_id: String,
lease_ms: u64,
}
async fn activate_agent(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(agent_id): Path<String>,
Json(payload): Json<ActivateAgentPayload>,
) -> Result<Response, ApiError> {
tasks::validate_label("worker_id", &payload.worker_id, 256).map_err(ApiError::bad_request)?;
tasks::validate_lease_ms(payload.lease_ms).map_err(ApiError::bad_request)?;
let scoped = tenant.scope(&agent_id);
state
.server_store
.get_agent(&scoped)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("agent `{agent_id}` not found")))?;
let outcome = state
.server_store
.claim_activation(&scoped, &payload.worker_id, payload.lease_ms, Utc::now())
.await
.map_err(internal_err)?;
Ok(match outcome {
ActivationOutcome::Claimed(lease) => {
Json(activation_wire(&agent_id, &lease)).into_response()
}
ActivationOutcome::Held(lease) => ApiError::conflict(format!(
"agent `{agent_id}` activation is held by `{}` (fencing {}, lease expires {})",
lease.owner,
lease.fencing,
lease.expires_at.to_rfc3339()
))
.into_response(),
})
}
#[derive(Debug, Deserialize)]
struct ActivationHeartbeatPayload {
worker_id: String,
fencing: u64,
lease_ms: u64,
}
async fn heartbeat_activation(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(agent_id): Path<String>,
Json(payload): Json<ActivationHeartbeatPayload>,
) -> Result<Response, ApiError> {
tasks::validate_label("worker_id", &payload.worker_id, 256).map_err(ApiError::bad_request)?;
tasks::validate_lease_ms(payload.lease_ms).map_err(ApiError::bad_request)?;
let outcome = state
.server_store
.renew_activation(
&tenant.scope(&agent_id),
&payload.worker_id,
payload.fencing,
payload.lease_ms,
Utc::now(),
)
.await
.map_err(internal_err)?;
Ok(match outcome {
ActivationMutation::Applied(lease) => {
Json(activation_wire(&agent_id, &lease)).into_response()
}
ActivationMutation::FencingLost => ApiError::conflict(format!(
"activation for agent `{agent_id}` is no longer held by this owner + fencing pair"
))
.into_response(),
ActivationMutation::Unknown => {
ApiError::not_found(format!("no activation lease for agent `{agent_id}`"))
.into_response()
}
})
}
#[derive(Debug, Deserialize)]
struct ActivationReleasePayload {
worker_id: String,
fencing: u64,
}
async fn release_activation(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(agent_id): Path<String>,
Json(payload): Json<ActivationReleasePayload>,
) -> Result<Response, ApiError> {
tasks::validate_label("worker_id", &payload.worker_id, 256).map_err(ApiError::bad_request)?;
let outcome = state
.server_store
.release_activation(
&tenant.scope(&agent_id),
&payload.worker_id,
payload.fencing,
Utc::now(),
)
.await
.map_err(internal_err)?;
Ok(match outcome {
ActivationMutation::Applied(_) => {
Json(json!({ "released": true, "agent_id": agent_id })).into_response()
}
ActivationMutation::FencingLost => ApiError::conflict(format!(
"activation for agent `{agent_id}` is no longer held by this owner + fencing pair"
))
.into_response(),
ActivationMutation::Unknown => {
ApiError::not_found(format!("no activation lease for agent `{agent_id}`"))
.into_response()
}
})
}
#[derive(Debug, Deserialize)]
struct ClaimAgentMessagePayload {
worker_id: String,
fencing: u64,
lease_ms: u64,
}
async fn claim_agent_message(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(agent_id): Path<String>,
Json(payload): Json<ClaimAgentMessagePayload>,
) -> Result<Response, ApiError> {
tasks::validate_label("worker_id", &payload.worker_id, 256).map_err(ApiError::bad_request)?;
tasks::validate_lease_ms(payload.lease_ms).map_err(ApiError::bad_request)?;
let scoped = tenant.scope(&agent_id);
let agent = state
.server_store
.get_agent(&scoped)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("agent `{agent_id}` not found")))?;
let now = Utc::now();
if !agent.supervision.deadline_breached
&& agent
.manifest
.budget
.as_ref()
.and_then(|b| b.deadline)
.is_some_and(|deadline| deadline <= now)
{
supervision::on_deadline_breach(&state.server_store, &tenant, &agent_id, agent, now)
.await
.map_err(internal_err)?;
}
let recipient = AgentId::new(agent_id.as_str()).mailbox_recipient();
let claimed = state
.server_store
.claim_agent_task(
tenant.tenant(),
&MailboxClaimScope {
agent_id: &scoped,
recipient: &recipient,
owner: &payload.worker_id,
fencing: payload.fencing,
},
payload.lease_ms,
now,
)
.await
.map_err(internal_err)?;
Ok(match claimed {
MailboxClaim::Claimed(task) => Json(json!({ "task": task.wire() })).into_response(),
MailboxClaim::Empty => StatusCode::NO_CONTENT.into_response(),
MailboxClaim::ActivationLost => ApiError::conflict(format!(
"activation for agent `{agent_id}` is not held by this owner + fencing pair"
))
.into_response(),
})
}
async fn cancel_one_agent(
state: &AppState,
tenant: &TenantContext,
agent_external: &str,
) -> Result<Value, ApiError> {
let now = Utc::now();
let recipient = AgentId::new(agent_external).mailbox_recipient();
let outcome = state
.server_store
.cancel_agent_tasks(tenant.tenant(), &recipient, now)
.await
.map_err(internal_err)?;
let thread = tenant.scope(&AgentId::new(agent_external).thread_id());
let runs = state.run_deps.manager.cancel_thread_runs(&thread).await;
let ids =
|tasks: Vec<TaskRecord>| -> Vec<String> { tasks.into_iter().map(|t| t.task_id).collect() };
let cancelled = ids(outcome.cancelled);
let signalled = ids(outcome.signalled);
let mut exit_event = Value::Null;
if !cancelled.is_empty() || !signalled.is_empty() || !runs.is_empty() {
exit_event = json!(supervision::journal_agent_exit(
&state.server_store,
tenant,
agent_external,
"cancelled",
json!({
"cancelled_messages": cancelled,
"signalled_messages": signalled,
"signalled_runs": runs.signalled,
"cancelled_runs": runs.cancelled,
}),
)
.await
.map_err(internal_err)?);
}
Ok(json!({
"agent_id": agent_external,
"cancelled": cancelled,
"signalled": signalled,
"runs": {
"signalled": runs.signalled,
"cancelled": runs.cancelled,
},
"exit_event": exit_event,
}))
}
async fn cancel_agent(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(agent_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
state
.server_store
.get_agent(&tenant.scope(&agent_id))
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("agent `{agent_id}` not found")))?;
Ok(Json(cancel_one_agent(&state, &tenant, &agent_id).await?))
}
#[derive(Debug, Default, Deserialize)]
struct RestartAgentPayload {
#[serde(default)]
reason: Option<String>,
}
async fn restart_agent(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(agent_id): Path<String>,
body: Option<Json<RestartAgentPayload>>,
) -> Result<Json<Value>, ApiError> {
let agent = state
.server_store
.get_agent(&tenant.scope(&agent_id))
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("agent `{agent_id}` not found")))?;
let reason = body
.and_then(|Json(payload)| payload.reason)
.unwrap_or_else(|| "manual restart".to_string());
let outcome = supervision::supervise(
&state.server_store,
&tenant,
&agent_id,
agent,
supervision::Trigger::ManualRestart { reason },
Utc::now(),
)
.await
.map_err(internal_err)?;
let ordinal = match outcome.decision {
supervision::Decision::Restart { ordinal } => ordinal,
_ => unreachable!("a manual restart trigger always decides restart"),
};
Ok(Json(json!({
"agent_id": agent_id,
"restarted": true,
"restart_ordinal": ordinal,
"event": outcome.event_id,
})))
}
async fn get_agent_supervision(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(agent_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
let agent = state
.server_store
.get_agent(&tenant.scope(&agent_id))
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("agent `{agent_id}` not found")))?;
let events = supervision::supervision_events(&state.server_store, &tenant, &agent_id)
.await
.map_err(internal_err)?;
Ok(Json(json!({
"agent_id": agent_id,
"policy": agent.manifest.supervision,
"escalated": agent.supervision.escalated,
"deadline_breached": agent.supervision.deadline_breached,
"suppressed_failures": agent.supervision.suppressed_failures,
"attempts": agent.supervision.attempts,
"journal_run_id": supervision::supervision_journal_run_id(tenant.tenant(), &agent_id),
"events": events,
})))
}
async fn cancel_team(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(team_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
tasks::validate_pool(&team_id)
.map_err(|e| ApiError::bad_request(e.replace("`pool`", "`team_id`")))?;
let members: Vec<String> = state
.server_store
.list_agents()
.await
.map_err(internal_err)?
.into_iter()
.filter(|record| record.team_id.as_deref() == Some(team_id.as_str()))
.filter_map(|record| tenant.unscope(&record.agent_id).map(str::to_owned))
.collect();
if members.is_empty() {
return Err(ApiError::not_found(format!(
"no agents registered for team `{team_id}`"
)));
}
let mut cancelled = Vec::with_capacity(members.len());
for agent_external in members {
cancelled.push(cancel_one_agent(&state, &tenant, &agent_external).await?);
}
Ok(Json(json!({
"team_id": team_id,
"members": cancelled,
})))
}
const RESERVED_MEMBER_NAMES: &[&str] = &["outcome", "race-dlq"];
#[derive(Debug, Deserialize)]
struct SubmitDelegatePayload {
#[serde(default)]
coordination_id: Option<String>,
#[serde(default)]
delegator: Option<String>,
#[serde(default)]
parent: Option<String>,
delegate: DelegateContract,
}
#[derive(Debug, Deserialize)]
struct SubmitFanOutPayload {
#[serde(default)]
coordination_id: Option<String>,
#[serde(default)]
delegator: Option<String>,
#[serde(default)]
parent: Option<String>,
fan_out: FanOutContract,
}
#[derive(Debug, Deserialize)]
struct SubmitRacePayload {
#[serde(default)]
coordination_id: Option<String>,
#[serde(default)]
delegator: Option<String>,
#[serde(default)]
parent: Option<String>,
race: RaceContract,
}
#[derive(Debug, Deserialize)]
struct SubmitQuorumPayload {
#[serde(default)]
coordination_id: Option<String>,
#[serde(default)]
delegator: Option<String>,
#[serde(default)]
parent: Option<String>,
quorum: QuorumContract,
}
async fn submit_delegate(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<SubmitDelegatePayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
submit_coordination(
&state,
&tenant,
payload.coordination_id,
payload.delegator,
payload.parent,
CoordinationContract::Delegate(Box::new(payload.delegate)),
)
.await
}
async fn submit_fan_out(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<SubmitFanOutPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
submit_coordination(
&state,
&tenant,
payload.coordination_id,
payload.delegator,
payload.parent,
CoordinationContract::FanOut(payload.fan_out),
)
.await
}
async fn submit_race(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<SubmitRacePayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
submit_coordination(
&state,
&tenant,
payload.coordination_id,
payload.delegator,
payload.parent,
CoordinationContract::Race(payload.race),
)
.await
}
async fn submit_quorum(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<SubmitQuorumPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
submit_coordination(
&state,
&tenant,
payload.coordination_id,
payload.delegator,
payload.parent,
CoordinationContract::Quorum(payload.quorum),
)
.await
}
async fn submit_coordination(
state: &Arc<AppState>,
tenant: &TenantContext,
coordination_id: Option<String>,
delegator: Option<String>,
parent: Option<String>,
contract: CoordinationContract,
) -> Result<(StatusCode, Json<Value>), ApiError> {
contract
.validate()
.map_err(|violation| ApiError::bad_request(violation.to_string()))?;
let coordination_id = coordination_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
validate_client_id("coordination_id", &coordination_id)?;
if let Some(parent) = &parent {
tasks::validate_label("parent", parent, 512).map_err(ApiError::bad_request)?;
}
for delegation in contract.members() {
coordination::validate_member_label("member", &delegation.member)
.map_err(ApiError::bad_request)?;
if RESERVED_MEMBER_NAMES.contains(&delegation.member.as_str()) {
return Err(ApiError::bad_request(format!(
"member name `{}` is reserved (it would collide with the pattern's own derived tasks)",
delegation.member
)));
}
tasks::validate_label("agent_id", &delegation.agent_id, 256)
.map_err(ApiError::bad_request)?;
let agent = state
.server_store
.get_agent(&tenant.scope(&delegation.agent_id))
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::bad_request(format!(
"member `{}` target agent `{}` is not registered",
delegation.member, delegation.agent_id
))
})?;
if agent.manifest.manifest_version != delegation.manifest_version {
return Err(ApiError::bad_request(format!(
"member `{}` pins manifest version `{}` but agent `{}` is registered at `{}` — the pin must match exactly",
delegation.member,
delegation.manifest_version,
delegation.agent_id,
agent.manifest.manifest_version
)));
}
if agent.manifest.accepts_kind(&delegation.kind).is_none() {
let declared: Vec<&str> = agent.manifest.accepts.keys().map(String::as_str).collect();
return Err(ApiError::bad_request(format!(
"member `{}` kind `{}` is not accepted by agent `{}` (manifest declares: {})",
delegation.member,
delegation.kind,
delegation.agent_id,
declared.join(", ")
)));
}
if let CoordinationContract::Delegate(delegate_contract) = &contract {
if let Some(grant) = &delegate_contract.context {
if !grant.narrows(&agent.manifest.scopes) {
return Err(ApiError::bad_request(format!(
"context grant widens agent `{}`'s declared scopes ({:?}) — grants may only narrow",
delegation.agent_id, agent.manifest.scopes
)));
}
}
}
}
if let Some(delegator) = &delegator {
tasks::validate_label("delegator", delegator, 256).map_err(ApiError::bad_request)?;
let agent = state
.server_store
.get_agent(&tenant.scope(delegator))
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::bad_request(format!("delegator agent `{delegator}` is not registered"))
})?;
if agent
.manifest
.accepts_kind(COORDINATION_RESULT_KIND)
.is_none()
{
return Err(ApiError::bad_request(format!(
"delegator `{delegator}` does not accept `{COORDINATION_RESULT_KIND}` (its manifest must declare the reserved kind — a delegator that cannot receive the outcome would strand every pattern it starts)"
)));
}
}
let now = Utc::now();
let record = coordination::CoordinationRecord {
coordination_id: tenant.scope(&coordination_id),
delegator,
parent,
members: contract
.members()
.into_iter()
.map(|delegation| coordination::MemberRecord {
member: delegation.member.clone(),
agent_id: delegation.agent_id.clone(),
manifest_version: delegation.manifest_version.clone(),
task_id: coordination::member_task_id(
tenant.tenant(),
&coordination_id,
&delegation.member,
),
submitted: false,
})
.collect(),
contract,
settled: false,
outcome: None,
outcome_delivered: false,
dlq_written: false,
created_at: now,
updated_at: now,
};
let created = state
.server_store
.create_coordination(&record)
.await
.map_err(internal_err)?;
if !created {
return Ok((
StatusCode::OK,
Json(json!({
"coordination_id": coordination_id,
"deduplicated": true,
})),
));
}
let initial_window = match &record.contract {
CoordinationContract::FanOut(contract) => {
(contract.max_in_flight as usize).min(contract.members.len())
}
_ => record.members.len(),
};
enforce_task_quota(state, tenant, initial_window).await?;
let driven = coordination::drive(
&state.server_store,
state.config.quota_for(tenant.tenant()),
tenant,
record,
now,
)
.await
.map_err(internal_err)?;
let start_event = coordination::load_journal(&state.server_store, tenant, &coordination_id)
.await
.map_err(internal_err)?
.and_then(|journal| journal.events().first().map(|event| event.id.clone()));
let submitted: Vec<Value> = driven
.record
.members
.iter()
.filter(|member| member.submitted)
.map(|member| json!({"member": member.member, "task_id": member.task_id}))
.collect();
Ok((
StatusCode::CREATED,
Json(json!({
"coordination_id": coordination_id,
"start_event": start_event,
"submitted": submitted,
})),
))
}
async fn get_coordination(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(coordination_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
validate_client_id("coordination_id", &coordination_id)?;
let Some(record) = state
.server_store
.get_coordination(&tenant.scope(&coordination_id))
.await
.map_err(internal_err)?
else {
return Err(ApiError::not_found(format!(
"coordination `{coordination_id}` not found"
)));
};
let driven = coordination::drive(
&state.server_store,
state.config.quota_for(tenant.tenant()),
&tenant,
record,
Utc::now(),
)
.await
.map_err(internal_err)?;
let journal = coordination::load_journal(&state.server_store, &tenant, &coordination_id)
.await
.map_err(internal_err)?;
let members: Vec<Value> = driven
.record
.members
.iter()
.map(|member| {
let disposition = driven
.dispositions
.iter()
.find(|d| d.member == member.member);
json!({
"member": member.member,
"agent_id": member.agent_id,
"manifest_version": member.manifest_version,
"task_id": member.task_id,
"submitted": member.submitted,
"disposition": disposition,
})
})
.collect();
let journal_wire = journal.map(|journal| {
json!({
"run_id": coordination::coordination_journal_run_id(tenant.tenant(), &coordination_id),
"events": journal.events(),
})
});
Ok(Json(json!({
"coordination_id": coordination_id,
"delegator": driven.record.delegator,
"parent": driven.record.parent,
"contract": driven.record.contract,
"members": members,
"settled": driven.record.settled,
"outcome": driven.record.outcome,
"journal": journal_wire,
"created_at": driven.record.created_at,
"updated_at": driven.record.updated_at,
})))
}
async fn get_coordination_trace(
AxumState(state): AxumState<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Path(coordination_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
validate_client_id("coordination_id", &coordination_id)?;
let Some(record) = state
.server_store
.get_coordination(&tenant.scope(&coordination_id))
.await
.map_err(internal_err)?
else {
return Err(ApiError::not_found(format!(
"coordination `{coordination_id}` not found"
)));
};
let driven = coordination::drive(
&state.server_store,
state.config.quota_for(tenant.tenant()),
&tenant,
record,
Utc::now(),
)
.await
.map_err(internal_err)?;
let mut snapshots = Vec::new();
if let Some(journal) =
coordination::load_journal(&state.server_store, &tenant, &coordination_id)
.await
.map_err(internal_err)?
{
snapshots.push(journal.snapshot());
}
for member in &driven.record.members {
let Some(task) = state
.server_store
.get_task(tenant.tenant(), &member.task_id)
.await
.map_err(internal_err)?
else {
continue;
};
let Some(run_id) = &task.run_id else {
continue;
};
if let Some(snapshot) = state
.server_store
.get_journal(run_id)
.await
.map_err(internal_err)?
{
snapshots.push(snapshot);
}
}
let trace = TeamTrace::assemble(&snapshots);
Ok(Json(json!({
"coordination_id": coordination_id,
"connected": trace.is_connected(),
"trace": trace,
})))
}