use std::collections::HashMap;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use axum::extract::{Path as AxumPath, Query, State};
use axum::http::StatusCode;
use axum::{Extension, Json};
use chrono::Utc;
use rusty_agent_runtime::broker::{
hex_decode, hex_encode, SealedCredential, SEALED_FORMAT_VERSION,
};
use rusty_agent_runtime::deploy::{
deployment_admission, deployment_surface, pin_set_digest, revision_promotion_effect_id,
scoped_secret_name, validate_secret_name, CanaryClearance, CanaryDeclaration, CanaryDeployment,
DeployError, DeploymentPointer, DeploymentResolved, DeploymentRevision, EnvSecretAct,
EnvSecretDenial, EnvSecretRecord, EnvSecretRevocation, Environment, EnvironmentDeclaration,
GateDecisionRecord, GateDeclaration, RegistryPin, RevisionContent, RevisionId,
RevisionPromotion, RevisionRegistration, RevisionRollback, ShadowRunOutcome, ShadowRunStarted,
ShadowVerdict, StoredEnvSecret,
};
use rusty_agent_runtime::effects::{
ApprovalToken, EffectAdmissionContext, ShadowRefusal, ShadowRefusalSink,
};
use rusty_agent_runtime::executor::{Executor, RunConfig};
use rusty_agent_runtime::journal::{Clock, EventDraft, Journal, JournalSnapshot};
use rusty_agent_runtime::learn::{EnvironmentTag, SurfaceKey};
use rusty_agent_runtime::memory::ProvenanceAuthor;
use rusty_agent_runtime::record::{
sha256_hex, DecisionRole, Effect, EventStatus, PayloadRef, RunEvent, RunEventKind,
};
use rusty_agent_runtime::replay::JournalShadowSource;
use rusty_agent_runtime::state::State as GraphState;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use tokio::sync::Mutex;
use crate::auth::TenantContext;
use crate::error::ApiError;
use crate::routes::{internal_err, AppState};
use crate::server_store::{DeploymentTransition, ServerStore, StoreResult};
use chacha20poly1305::aead::{Aead, KeyInit, OsRng};
use chacha20poly1305::XChaCha20Poly1305;
pub(crate) fn revisions_dir(root: &Path) -> PathBuf {
root.join("deployments").join("revisions")
}
pub(crate) fn environments_dir(root: &Path) -> PathBuf {
root.join("deployments").join("environments")
}
pub(crate) fn pointers_dir(root: &Path) -> PathBuf {
root.join("deployments").join("pointers")
}
pub(crate) fn env_secrets_dir(root: &Path) -> PathBuf {
root.join("env-secrets")
}
fn collect_json_files(root: &Path, out: &mut Vec<PathBuf>) {
let Ok(entries) = std::fs::read_dir(root) else {
return;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
collect_json_files(&path, out);
} else if path.extension().and_then(|e| e.to_str()) == Some("json") {
out.push(path);
}
}
}
fn path_scoped_key(dir: &Path, path: &Path) -> Option<String> {
path.strip_prefix(dir)
.ok()
.map(|relative| relative.with_extension(""))
.map(|relative| {
relative
.components()
.map(|component| component.as_os_str().to_string_lossy().into_owned())
.collect::<Vec<_>>()
.join("/")
})
}
async fn persist_record(dir: &Path, scoped_key: &str, record: &impl Serialize) -> io::Result<()> {
tokio::fs::create_dir_all(dir).await?;
let bytes = serde_json::to_vec_pretty(record)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let path = dir.join(format!("{scoped_key}.json"));
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent).await?;
}
let tmp = dir.join(format!("{scoped_key}.tmp"));
tokio::fs::write(&tmp, bytes).await?;
tokio::fs::rename(&tmp, path).await
}
pub(crate) async fn persist_revision(
root: &Path,
scoped_id: &str,
record: &DeploymentRevision,
) -> io::Result<()> {
persist_record(&revisions_dir(root), scoped_id, record).await
}
pub(crate) fn load_revisions(root: &Path) -> HashMap<String, DeploymentRevision> {
let dir = revisions_dir(root);
let mut out = HashMap::new();
let mut files = Vec::new();
collect_json_files(&dir, &mut files);
for path in files {
let scoped_id = path_scoped_key(&dir, &path);
let parsed = std::fs::read_to_string(&path)
.ok()
.and_then(|raw| serde_json::from_str::<DeploymentRevision>(&raw).ok());
match (scoped_id, parsed) {
(Some(id), Some(record)) => {
out.insert(id, record);
}
_ => {
tracing::warn!(path = %path.display(), "skipping unreadable revision file")
}
}
}
out
}
pub(crate) async fn persist_environment(
root: &Path,
scoped_name: &str,
record: &Environment,
) -> io::Result<()> {
persist_record(&environments_dir(root), scoped_name, record).await
}
pub(crate) fn load_environments(root: &Path) -> HashMap<String, Environment> {
let dir = environments_dir(root);
let mut out = HashMap::new();
let mut files = Vec::new();
collect_json_files(&dir, &mut files);
for path in files {
let scoped_name = path_scoped_key(&dir, &path);
let parsed = std::fs::read_to_string(&path)
.ok()
.and_then(|raw| serde_json::from_str::<Environment>(&raw).ok());
match (scoped_name, parsed) {
(Some(name), Some(record)) => {
out.insert(name, record);
}
_ => {
tracing::warn!(path = %path.display(), "skipping unreadable environment file")
}
}
}
out
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct KeyedFile<T> {
key: String,
record: T,
}
fn keyed_file_name(scoped_key: &str) -> String {
sha256_hex(scoped_key.as_bytes())
}
async fn persist_keyed<T: Serialize>(dir: &Path, scoped_key: &str, record: &T) -> io::Result<()> {
tokio::fs::create_dir_all(dir).await?;
let file = KeyedFile {
key: scoped_key.to_string(),
record,
};
let bytes = serde_json::to_vec_pretty(&file)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let name = keyed_file_name(scoped_key);
let tmp = dir.join(format!("{name}.tmp"));
tokio::fs::write(&tmp, bytes).await?;
tokio::fs::rename(&tmp, dir.join(format!("{name}.json"))).await
}
fn load_keyed<T: for<'de> Deserialize<'de>>(dir: &Path) -> HashMap<String, T> {
let mut out = HashMap::new();
let Ok(entries) = std::fs::read_dir(dir) else {
return out;
};
for entry in entries.flatten() {
let path = entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
let parsed = std::fs::read_to_string(&path)
.ok()
.and_then(|raw| serde_json::from_str::<KeyedFile<T>>(&raw).ok());
let matches_name = parsed.as_ref().is_some_and(|file| {
path.file_stem().and_then(|s| s.to_str()) == Some(&*keyed_file_name(&file.key))
});
match (parsed, matches_name) {
(Some(file), true) => {
out.insert(file.key, file.record);
}
_ => {
tracing::warn!(path = %path.display(), "skipping unreadable keyed record file")
}
}
}
out
}
pub(crate) async fn persist_pointer(
root: &Path,
scoped_surface: &str,
pointer: &DeploymentPointer,
) -> io::Result<()> {
persist_keyed(&pointers_dir(root), scoped_surface, pointer).await
}
pub(crate) fn load_pointers(root: &Path) -> HashMap<String, DeploymentPointer> {
load_keyed(&pointers_dir(root))
}
pub(crate) async fn persist_env_secret(
root: &Path,
scoped_key: &str,
record: &StoredEnvSecret,
) -> io::Result<()> {
persist_keyed(&env_secrets_dir(root), scoped_key, record).await
}
pub(crate) async fn delete_env_secret_file(root: &Path, scoped_key: &str) -> io::Result<bool> {
let path = env_secrets_dir(root).join(format!("{}.json", keyed_file_name(scoped_key)));
match tokio::fs::remove_file(&path).await {
Ok(()) => Ok(true),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false),
Err(e) => Err(e),
}
}
pub(crate) fn load_env_secrets(root: &Path) -> HashMap<String, StoredEnvSecret> {
load_keyed(&env_secrets_dir(root))
}
const MASTER_KEY_ID_PREFIX: &str = "esk-";
fn master_secret_path(root: &Path, key_id: &str) -> PathBuf {
crate::receipts::keys_dir(root).join(format!("env-secret-master.{key_id}.secret"))
}
async fn write_master_secret(root: &Path, key_id: &str, key: &[u8; 32]) -> io::Result<()> {
let dir = crate::receipts::keys_dir(root);
tokio::fs::create_dir_all(&dir).await?;
let path = master_secret_path(root, key_id);
#[cfg(unix)]
{
let mut options = tokio::fs::OpenOptions::new();
options.write(true).create_new(true).mode(0o600);
use tokio::io::AsyncWriteExt as _;
let mut file = options.open(&path).await?;
file.write_all(hex_encode(key).as_bytes()).await?;
}
#[cfg(not(unix))]
{
tokio::fs::write(&path, hex_encode(key)).await?;
}
Ok(())
}
type MasterKeys = Vec<(String, [u8; 32])>;
fn load_master_secrets(root: &Path) -> MasterKeys {
let mut out = Vec::new();
let Ok(entries) = std::fs::read_dir(crate::receipts::keys_dir(root)) else {
return out;
};
for entry in entries.flatten() {
let name = entry.file_name().to_string_lossy().into_owned();
let Some(rest) = name.strip_prefix("env-secret-master.") else {
continue;
};
let Some(key_id) = rest.strip_suffix(".secret") else {
continue;
};
let parsed = std::fs::read_to_string(entry.path())
.ok()
.and_then(|hex| hex_decode(hex.trim()))
.and_then(|bytes| <[u8; 32]>::try_from(bytes).ok());
match parsed {
Some(key) => out.push((key_id.to_owned(), key)),
None => {
tracing::warn!(path = %entry.path().display(), "skipping unreadable env-secret master key file")
}
}
}
out
}
pub(crate) async fn master_secrets(root: &Path) -> Result<MasterKeys, String> {
let held = load_master_secrets(root);
if !held.is_empty() {
return Ok(held);
}
let key: [u8; 32] = draw_random();
let key_id = format!(
"{}{}",
MASTER_KEY_ID_PREFIX,
hex_encode(&draw_random::<8>())
);
write_master_secret(root, &key_id, &key)
.await
.map_err(|e| format!("mint env-secret master key: {e}"))?;
Ok(vec![(key_id, key)])
}
pub(crate) fn master_for<'k>(keys: &'k MasterKeys, key_id: &str) -> Option<&'k [u8; 32]> {
keys.iter().find(|(id, _)| id == key_id).map(|(_, key)| key)
}
fn draw_random<const N: usize>() -> [u8; N] {
let mut bytes = [0u8; N];
use chacha20poly1305::aead::rand_core::RngCore as _;
OsRng.fill_bytes(&mut bytes);
bytes
}
pub(crate) fn seal_env_secret(
master_id: &str,
master: &[u8; 32],
scoped_id: &str,
plaintext: &[u8],
) -> Result<SealedCredential, String> {
let data_key = draw_random::<32>();
let aad = scoped_id.as_bytes();
let wrap_nonce_bytes = draw_random::<24>();
let wrapped = XChaCha20Poly1305::new(master.into())
.encrypt(
chacha20poly1305::XNonce::from_slice(&wrap_nonce_bytes),
chacha20poly1305::aead::Payload {
msg: &data_key,
aad,
},
)
.map_err(|_| "wrap data key: AEAD failure".to_string())?;
let nonce_bytes = draw_random::<24>();
let ciphertext = XChaCha20Poly1305::new((&data_key).into())
.encrypt(
chacha20poly1305::XNonce::from_slice(&nonce_bytes),
chacha20poly1305::aead::Payload {
msg: plaintext,
aad,
},
)
.map_err(|_| "seal secret value: AEAD failure".to_string())?;
Ok(SealedCredential {
format_version: SEALED_FORMAT_VERSION,
key_id: master_id.to_owned(),
wrapped_data_key: hex_encode(&wrapped),
wrap_nonce: hex_encode(&wrap_nonce_bytes),
nonce: hex_encode(&nonce_bytes),
ciphertext: hex_encode(&ciphertext),
sealed_at: chrono::Utc::now(),
})
}
pub(crate) fn open_env_secret(
master: &[u8; 32],
scoped_id: &str,
envelope: &SealedCredential,
) -> Result<Vec<u8>, String> {
if envelope.format_version != SEALED_FORMAT_VERSION {
return Err(format!(
"envelope format version {} is not {SEALED_FORMAT_VERSION}",
envelope.format_version
));
}
let aad = scoped_id.as_bytes();
let wrapped = hex_decode(&envelope.wrapped_data_key)
.ok_or_else(|| "corrupt envelope: wrapped key is not hex".to_string())?;
let wrap_nonce = hex_decode(&envelope.wrap_nonce)
.ok_or_else(|| "corrupt envelope: wrap nonce is not hex".to_string())?;
if wrap_nonce.len() != 24 {
return Err("corrupt envelope: wrap nonce is not 24 bytes".to_string());
}
let data_key = XChaCha20Poly1305::new(master.into())
.decrypt(
chacha20poly1305::XNonce::from_slice(&wrap_nonce),
chacha20poly1305::aead::Payload { msg: &wrapped, aad },
)
.map_err(|_| "the wrapped data key failed its authentication tag".to_string())?;
let data_key: [u8; 32] = <[u8; 32]>::try_from(data_key)
.map_err(|_| "corrupt envelope: data key is not 32 bytes".to_string())?;
let ciphertext = hex_decode(&envelope.ciphertext)
.ok_or_else(|| "corrupt envelope: ciphertext is not hex".to_string())?;
let nonce = hex_decode(&envelope.nonce)
.ok_or_else(|| "corrupt envelope: nonce is not hex".to_string())?;
if nonce.len() != 24 {
return Err("corrupt envelope: nonce is not 24 bytes".to_string());
}
XChaCha20Poly1305::new((&data_key).into())
.decrypt(
chacha20poly1305::XNonce::from_slice(&nonce),
chacha20poly1305::aead::Payload {
msg: &ciphertext,
aad,
},
)
.map_err(|_| "the sealed secret failed its authentication tag".to_string())
}
pub(crate) const DEPLOYMENT_JOURNAL_RUN_ID: &str = "deployment-control";
pub(crate) struct DeploymentControl {
store: Arc<dyn ServerStore>,
chain_lock: Mutex<()>,
}
pub(crate) enum DeploymentAct {
Promotion(RevisionPromotion),
Rollback(RevisionRollback),
CanaryDeclare(CanaryDeclaration),
CanaryClear(CanaryClearance),
}
impl DeploymentAct {
fn kind(&self) -> RunEventKind {
match self {
DeploymentAct::Promotion(_) => RunEventKind::RevisionPromoted,
DeploymentAct::Rollback(_) => RunEventKind::RevisionRolledBack,
DeploymentAct::CanaryDeclare(_) => RunEventKind::CanaryDeclared,
DeploymentAct::CanaryClear(_) => RunEventKind::CanaryCleared,
}
}
fn output(&self) -> Result<Value, String> {
match self {
DeploymentAct::Promotion(act) => serde_json::to_value(act).map_err(|e| e.to_string()),
DeploymentAct::Rollback(act) => serde_json::to_value(act).map_err(|e| e.to_string()),
DeploymentAct::CanaryDeclare(act) => {
serde_json::to_value(act).map_err(|e| e.to_string())
}
DeploymentAct::CanaryClear(act) => serde_json::to_value(act).map_err(|e| e.to_string()),
}
}
fn scope(&self) -> (&str, &EnvironmentTag) {
match self {
DeploymentAct::Promotion(act) => (&act.tenant, &act.environment),
DeploymentAct::Rollback(act) => (&act.tenant, &act.environment),
DeploymentAct::CanaryDeclare(act) => (&act.tenant, &act.environment),
DeploymentAct::CanaryClear(act) => (&act.tenant, &act.environment),
}
}
fn in_family(&self, kind: RunEventKind) -> bool {
match self {
DeploymentAct::Promotion(_) | DeploymentAct::Rollback(_) => {
matches!(
kind,
RunEventKind::RevisionPromoted | RunEventKind::RevisionRolledBack
)
}
DeploymentAct::CanaryDeclare(_) | DeploymentAct::CanaryClear(_) => {
matches!(
kind,
RunEventKind::CanaryDeclared | RunEventKind::CanaryCleared
)
}
}
}
fn is_last_for_environment(&self, events: &[RunEvent]) -> bool {
let (tenant, environment) = self.scope();
for event in events.iter().rev() {
if !self.in_family(event.kind) {
continue;
}
let Some(PayloadRef::Inline(value)) = &event.output else {
continue;
};
let recorded: Option<(bool, bool)> = match event.kind {
RunEventKind::RevisionPromoted => {
serde_json::from_value::<RevisionPromotion>(value.clone())
.ok()
.map(|r| {
(
r.tenant == tenant && r.environment == *environment,
matches!(self, DeploymentAct::Promotion(act) if
r.revision_id == act.revision_id
&& r.previous == act.previous
&& r.author == act.author),
)
})
}
RunEventKind::RevisionRolledBack => {
serde_json::from_value::<RevisionRollback>(value.clone())
.ok()
.map(|r| {
(
r.tenant == tenant && r.environment == *environment,
matches!(self, DeploymentAct::Rollback(act) if
r.from == act.from
&& r.to == act.to
&& r.author == act.author),
)
})
}
RunEventKind::CanaryDeclared => {
serde_json::from_value::<CanaryDeclaration>(value.clone())
.ok()
.map(|r| {
(
r.tenant == tenant && r.environment == *environment,
matches!(self, DeploymentAct::CanaryDeclare(act) if
r.revision_id == act.revision_id
&& r.fraction == act.fraction
&& r.author == act.author),
)
})
}
RunEventKind::CanaryCleared => {
serde_json::from_value::<CanaryClearance>(value.clone())
.ok()
.map(|r| {
(
r.tenant == tenant && r.environment == *environment,
matches!(self, DeploymentAct::CanaryClear(act) if
r.cleared_revision_id == act.cleared_revision_id
&& r.author == act.author),
)
})
}
_ => None,
};
let Some((same_environment, same_act)) = recorded else {
continue;
};
if same_environment {
return same_act;
}
}
false
}
}
pub(crate) enum DeploymentMove {
Applied {
event_id: Option<String>,
journaled: bool,
},
Conflict(Option<RevisionId>),
}
impl DeploymentControl {
pub(crate) fn new(store: Arc<dyn ServerStore>) -> Self {
Self {
store,
chain_lock: Mutex::new(()),
}
}
async fn load_chain(&self) -> StoreResult<Option<Journal>> {
match self.store.get_journal(DEPLOYMENT_JOURNAL_RUN_ID).await? {
Some(snapshot) => Ok(Some(
Journal::from_snapshot(snapshot, Clock::System).map_err(|e| {
format!("the deployment evidence chain failed its integrity check: {e}")
})?,
)),
None => Ok(None),
}
}
pub(crate) async fn chain_events(&self) -> StoreResult<Vec<RunEvent>> {
Ok(match self.load_chain().await? {
Some(journal) => journal.events(),
None => Vec::new(),
})
}
pub(crate) async fn journal_act(
&self,
kind: RunEventKind,
output: Value,
) -> StoreResult<String> {
let _guard = self.chain_lock.lock().await;
let journal = match self.load_chain().await? {
Some(journal) => journal,
None => Journal::new(
DEPLOYMENT_JOURNAL_RUN_ID,
DEPLOYMENT_JOURNAL_RUN_ID,
Clock::System,
),
};
let event_id = journal.record(EventDraft::new(kind, Effect::Pure).output(output));
self.store.put_journal(&journal.snapshot()).await?;
Ok(event_id)
}
pub(crate) async fn journal_denial(&self, output: Value) {
let attempt = self
.journal_act(RunEventKind::EnvSecretDenied, output)
.await;
if let Err(e) = attempt {
tracing::warn!(%e, "env-secret denial could not be journaled");
}
}
pub(crate) async fn transition(
&self,
tenant: &str,
surface: &str,
expect: Option<RevisionId>,
next: &DeploymentPointer,
act: &DeploymentAct,
) -> StoreResult<DeploymentMove> {
let _guard = self.chain_lock.lock().await;
let journal = match self.load_chain().await? {
Some(journal) => journal,
None => Journal::new(
DEPLOYMENT_JOURNAL_RUN_ID,
DEPLOYMENT_JOURNAL_RUN_ID,
Clock::System,
),
};
let journaled = !act.is_last_for_environment(&journal.events());
let event_id = if journaled {
Some(journal.record(EventDraft::new(act.kind(), Effect::Pure).output(act.output()?)))
} else {
None
};
match self
.store
.transition_deployment(tenant, surface, expect, next, &journal.snapshot())
.await?
{
DeploymentTransition::Applied => Ok(DeploymentMove::Applied {
event_id,
journaled,
}),
DeploymentTransition::Conflict(live) => Ok(DeploymentMove::Conflict(live)),
}
}
}
fn rollback_target(
events: &[RunEvent],
tenant: &str,
environment: &EnvironmentTag,
from: &RevisionId,
) -> Option<Option<RevisionId>> {
let mut active: Option<RevisionId> = None;
let mut stack: Vec<Option<RevisionId>> = Vec::new();
let mut installed_stack: Option<Vec<Option<RevisionId>>> = None;
for event in events {
let Some(PayloadRef::Inline(value)) = &event.output else {
continue;
};
match event.kind {
RunEventKind::RevisionPromoted => {
let Ok(promotion) = serde_json::from_value::<RevisionPromotion>(value.clone())
else {
continue;
};
if promotion.tenant == tenant && &promotion.environment == environment {
stack.push(promotion.previous);
active = Some(promotion.revision_id.clone());
if &promotion.revision_id == from {
installed_stack = Some(stack.clone());
}
}
}
RunEventKind::RevisionRolledBack => {
let Ok(rollback) = serde_json::from_value::<RevisionRollback>(value.clone()) else {
continue;
};
if rollback.tenant == tenant && &rollback.environment == environment {
let _ = stack.pop();
active = rollback.to.clone();
if rollback.to.as_ref() == Some(from) {
installed_stack = Some(stack.clone());
}
}
}
_ => {}
}
}
if active.as_ref() == Some(from) {
return stack.last().cloned();
}
installed_stack.and_then(|stack| stack.last().cloned())
}
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct DeploymentRunBinding {
pub environment: EnvironmentTag,
}
#[derive(Debug, Clone)]
pub(crate) struct DeploymentAdmission {
pub resolution: DeploymentResolved,
}
pub(crate) async fn resolve_admission(
store: &Arc<dyn ServerStore>,
tenant: &str,
run_id: &str,
binding: &DeploymentRunBinding,
graph: &str,
graph_hash: &str,
) -> Result<DeploymentAdmission, ApiError> {
let internal = |e: String| ApiError::internal(format!("deployment admission read: {e}"));
let environment = &binding.environment;
state_must_declare(store, tenant, environment).await?;
let surface = deployment_surface(environment);
let pointer = store
.get_deployment_pointer(tenant, surface.as_str())
.await
.map_err(internal)?
.ok_or_else(|| {
ApiError::not_found(format!(
"environment `{environment}` serves nothing — nothing was ever promoted into \
it; an unpromoted environment binds no revision"
))
})?;
let (revision_id, slot) = deployment_admission(&pointer, run_id).ok_or_else(|| {
ApiError::not_found(format!(
"environment `{environment}` serves nothing — its pointer has no active revision \
and this run's draw did not admit a canary"
))
})?;
let revision = store
.get_revision(tenant, revision_id.as_str())
.await
.map_err(internal)?
.ok_or_else(|| {
ApiError::unprocessable(format!(
"the deployment pointer for `{environment}` names revision `{}`, which the \
store does not hold — corrupt control-plane state; refused, never served",
revision_id.as_str()
))
})?;
revision
.verify_address()
.map_err(|e| ApiError::unprocessable(e.to_string()))?;
if revision.content.graph != graph {
return Err(ApiError::unprocessable(format!(
"revision `{}` serves graph `{}`; this run targets `{graph}` — a revision binds \
one graph, and admission will not approximate",
revision_id.as_str(),
revision.content.graph
)));
}
if revision.content.graph_hash != graph_hash {
return Err(ApiError::unprocessable(format!(
"revision `{}` records topology hash `{}` for graph `{graph}`, but the registered \
graph hashes to `{graph_hash}` — the build drifted from the revision's record; \
register a fresh revision, never drift silently",
revision_id.as_str(),
revision.content.graph_hash
)));
}
Ok(DeploymentAdmission {
resolution: DeploymentResolved {
environment: environment.clone(),
revision_id,
pointer: slot,
pin_set_digest: pin_set_digest(&revision.content.pins),
},
})
}
async fn state_must_declare(
store: &Arc<dyn ServerStore>,
tenant: &str,
environment: &EnvironmentTag,
) -> Result<Environment, ApiError> {
store
.get_environment(tenant, environment.as_str())
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!(
"environment `{environment}` is not declared — declare it before deploying to \
it"
))
})
}
fn deploy_err(error: &DeployError) -> ApiError {
match error {
DeployError::UnaddressableContent(_) => ApiError::internal(error.to_string()),
_ => ApiError::unprocessable(error.to_string()),
}
}
#[derive(Deserialize)]
pub(crate) struct CreateRevisionPayload {
graph: String,
#[serde(default)]
assistant: Option<String>,
source_environment: EnvironmentTag,
#[serde(default)]
surfaces: Vec<String>,
author: ProvenanceAuthor,
}
pub(crate) async fn create_revision(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<CreateRevisionPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let tenant_id = tenant.tenant();
let (graph, _spec) = state.registry.get(&payload.graph).ok_or_else(|| {
ApiError::not_found(format!(
"graph `{}` is not registered — a revision exists to name what serves",
payload.graph
))
})?;
let graph_hash = graph.topology_hash();
if let Some(assistant) = &payload.assistant {
state
.server_store
.get_assistant(&tenant.scope(assistant))
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!(
"assistant `{assistant}` not found — a revision binds only what exists"
))
})?;
}
state_must_declare(&state.server_store, tenant_id, &payload.source_environment).await?;
let mut pins = Vec::with_capacity(payload.surfaces.len());
for surface in &payload.surfaces {
let surface = SurfaceKey::new(surface.clone());
let target = surface.tagged(&payload.source_environment);
let pointer = state
.server_store
.get_version_pointer(tenant_id, target.as_str())
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::unprocessable(format!(
"surface `{target}` has no version pointer — nothing was ever promoted \
for `{}`; an unresolvable pin is no pin",
payload.source_environment
))
})?;
let candidate_id = pointer.active.clone().ok_or_else(|| {
ApiError::unprocessable(format!(
"surface `{target}` serves nothing in `{}` — its pointer has no active \
version to freeze",
payload.source_environment
))
})?;
pins.push(RegistryPin {
surface,
candidate_id,
});
}
let revision = DeploymentRevision::new(
RevisionContent {
graph: payload.graph.clone(),
graph_hash,
assistant: payload.assistant.clone(),
source_environment: payload.source_environment.clone(),
pins,
},
payload.author.clone(),
Utc::now(),
)
.map_err(|e| deploy_err(&e))?;
if let Some(existing) = state
.server_store
.get_revision(tenant_id, revision.revision_id.as_str())
.await
.map_err(internal_err)?
{
return Ok((
StatusCode::OK,
Json(json!({ "created": false, "revision": existing })),
));
}
let registration = RevisionRegistration {
tenant: tenant_id.to_owned(),
revision: revision.clone(),
};
let output = serde_json::to_value(®istration).map_err(internal_err)?;
state
.deployment
.journal_act(RunEventKind::RevisionRegistered, output)
.await
.map_err(internal_err)?;
let created = state
.server_store
.put_revision(tenant_id, &revision)
.await
.map_err(internal_err)?;
Ok((
if created {
StatusCode::CREATED
} else {
StatusCode::OK
},
Json(json!({ "created": created, "revision": revision })),
))
}
pub(crate) async fn list_revisions(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let mut revisions = state
.server_store
.list_revisions(tenant.tenant())
.await
.map_err(internal_err)?;
revisions.sort_by(|a, b| a.revision_id.as_str().cmp(b.revision_id.as_str()));
Ok(Json(json!({ "revisions": revisions })))
}
pub(crate) async fn get_revision(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(revision_id): AxumPath<String>,
) -> Result<Json<Value>, ApiError> {
let revision = state
.server_store
.get_revision(tenant.tenant(), &revision_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("revision `{revision_id}` not found")))?;
Ok(Json(json!({ "revision": revision })))
}
#[derive(Deserialize)]
pub(crate) struct DeclareEnvironmentPayload {
name: String,
#[serde(default)]
gate: Option<GateDeclaration>,
#[serde(default)]
approval_required: bool,
author: ProvenanceAuthor,
}
pub(crate) async fn declare_environment(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<DeclareEnvironmentPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let tenant_id = tenant.tenant();
let tag = EnvironmentTag::new(payload.name.clone()).map_err(|e| {
ApiError::bad_request(format!("invalid environment name `{}`: {e}", payload.name))
})?;
if let Some(existing) = state
.server_store
.get_environment(tenant_id, tag.as_str())
.await
.map_err(internal_err)?
{
if existing.gate == payload.gate && existing.approval_required == payload.approval_required
{
return Ok((
StatusCode::OK,
Json(json!({ "created": false, "environment": existing })),
));
}
return Err(ApiError::conflict(format!(
"environment `{}` is already declared with a different rule — declarations are \
immutable; a changed declaration is a new environment, not an edit",
payload.name
)));
}
let environment = Environment {
name: tag,
gate: payload.gate,
approval_required: payload.approval_required,
created_by: payload.author,
created_at: Utc::now(),
};
let declaration = EnvironmentDeclaration {
tenant: tenant_id.to_owned(),
environment: environment.clone(),
};
let output = serde_json::to_value(&declaration).map_err(internal_err)?;
state
.deployment
.journal_act(RunEventKind::EnvironmentDeclared, output)
.await
.map_err(internal_err)?;
let created = state
.server_store
.put_environment(tenant_id, &environment)
.await
.map_err(internal_err)?;
Ok((
if created {
StatusCode::CREATED
} else {
StatusCode::OK
},
Json(json!({ "created": created, "environment": environment })),
))
}
pub(crate) async fn list_environments(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let mut environments = state
.server_store
.list_environments(tenant.tenant())
.await
.map_err(internal_err)?;
environments.sort_by(|a, b| a.name.as_str().cmp(b.name.as_str()));
Ok(Json(json!({ "environments": environments })))
}
pub(crate) async fn get_environment(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(name): AxumPath<String>,
) -> Result<Json<Value>, ApiError> {
let environment = state
.server_store
.get_environment(tenant.tenant(), &name)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("environment `{name}` is not declared")))?;
Ok(Json(json!({ "environment": environment })))
}
#[derive(Deserialize)]
pub(crate) struct PromotePayload {
revision_id: String,
author: ProvenanceAuthor,
#[serde(default)]
approval: Option<ApprovalToken>,
}
#[derive(Deserialize)]
pub(crate) struct RollbackPayload {
author: ProvenanceAuthor,
cause: String,
}
async fn run_gate(
state: &AppState,
tenant_id: &str,
tag: &EnvironmentTag,
revision: &DeploymentRevision,
gate: &GateDeclaration,
) -> Result<(), ApiError> {
let Some(evaluator) = state.config.revision_gate_evaluator.clone() else {
return Err(ApiError::conflict(format!(
"environment `{tag}` declares gate `{}` but no revision gate evaluator is \
configured — a gate that cannot run is a gate that did not pass",
gate.policy
)));
};
let surface = deployment_surface(tag);
let pointer = state
.server_store
.get_deployment_pointer(tenant_id, surface.as_str())
.await
.map_err(internal_err)?;
let baseline = match pointer.as_ref().and_then(|p| p.active.as_ref()) {
Some(active) => {
let baseline = state
.server_store
.get_revision(tenant_id, active.as_str())
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::unprocessable(format!(
"the deployment pointer for `{tag}` names revision `{}`, which the \
store does not hold — the gate has no trustworthy baseline; refused",
active.as_str()
))
})?;
baseline
.verify_address()
.map_err(|e| ApiError::unprocessable(e.to_string()))?;
Some(baseline)
}
None => None,
};
let evaluation = evaluator
.evaluate(revision, baseline.as_ref(), gate)
.await
.map_err(|e| {
ApiError::unprocessable(format!(
"gate `{}` could not evaluate revision `{}`: {e} — an unevaluated gate fails \
closed",
gate.policy,
revision.revision_id.as_str()
))
})?;
if evaluation.policy != gate.policy || evaluation.dataset_version != gate.dataset_version {
return Err(ApiError::unprocessable(format!(
"the gate evaluator answered for policy `{}` / dataset `{}`, not the declared \
`{}` / `{}` — an evaluation that cannot say what it evaluated is not evidence \
about this promotion",
evaluation.policy, evaluation.dataset_version, gate.policy, gate.dataset_version
)));
}
let record = GateDecisionRecord {
tenant: tenant_id.to_owned(),
environment: tag.clone(),
revision_id: revision.revision_id.clone(),
baseline_revision_id: baseline.as_ref().map(|b| b.revision_id.clone()),
policy: evaluation.policy.clone(),
dataset_version: evaluation.dataset_version.clone(),
outcome: evaluation.outcome,
checks: evaluation.checks.clone(),
decided_at: Utc::now(),
};
state
.deployment
.journal_act(
RunEventKind::GateDecisionRecorded,
serde_json::to_value(&record).map_err(|e| internal_err(e.to_string()))?,
)
.await
.map_err(internal_err)?;
if !evaluation.allowed() {
let failures: Vec<String> = evaluation
.checks
.iter()
.filter(|check| !check.passed)
.map(|check| check.detail.clone())
.collect();
return Err(ApiError::unprocessable(format!(
"gate `{}` refused revision `{}` for `{tag}` (verdict {:?}): {} — the decision \
is journaled on the deployment chain",
gate.policy,
revision.revision_id.as_str(),
record.outcome,
failures.join("; ")
)));
}
Ok(())
}
async fn move_pointer(
state: &AppState,
tenant_id: &str,
name: &str,
tag: &EnvironmentTag,
build: impl Fn(
Option<&DeploymentPointer>,
&[RunEvent],
) -> Result<(DeploymentAct, DeploymentPointer, bool), ApiError>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let surface = deployment_surface(tag);
for attempt in 0..2 {
let pointer = state
.server_store
.get_deployment_pointer(tenant_id, surface.as_str())
.await
.map_err(internal_err)?;
let current = pointer.as_ref().and_then(|p| p.active.clone());
let events = state
.deployment
.chain_events()
.await
.map_err(internal_err)?;
let (act, next, converged) = build(pointer.as_ref(), &events)?;
if converged {
let pointer = pointer.unwrap_or_else(|| DeploymentPointer::new(surface.clone()));
return Ok((
StatusCode::OK,
Json(json!({ "applied": false, "journaled": false, "pointer": pointer })),
));
}
let outcome = state
.deployment
.transition(tenant_id, surface.as_str(), current, &next, &act)
.await
.map_err(internal_err)?;
match outcome {
DeploymentMove::Applied {
event_id,
journaled,
} => {
return Ok((
if journaled {
StatusCode::CREATED
} else {
StatusCode::OK
},
Json(json!({
"applied": true,
"journaled": journaled,
"event_id": event_id,
"pointer": next,
})),
));
}
DeploymentMove::Conflict(live) => {
if attempt == 1 {
let serving = live
.as_ref()
.map(|id| id.as_str().to_owned())
.unwrap_or_else(|| "nothing".to_owned());
return Err(ApiError::conflict(format!(
"environment `{name}` moved under this act — the live pointer serves \
`{serving}`; re-read and retry (never a lost move)"
)));
}
}
}
}
unreachable!("the conflict arm returns on the second attempt")
}
pub(crate) async fn promote_revision(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(name): AxumPath<String>,
Json(payload): Json<PromotePayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let tenant_id = tenant.tenant();
let tag = EnvironmentTag::new(name.clone())
.map_err(|e| ApiError::bad_request(format!("invalid environment name `{name}`: {e}")))?;
let environment = state_must_declare(&state.server_store, tenant_id, &tag).await?;
let revision = state
.server_store
.get_revision(tenant_id, &payload.revision_id)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!("revision `{}` not found", payload.revision_id))
})?;
revision
.verify_address()
.map_err(|e| ApiError::unprocessable(e.to_string()))?;
if let Some(gate) = &environment.gate {
run_gate(&state, tenant_id, &tag, &revision, gate).await?;
}
if environment.approval_required {
let effect_id = revision_promotion_effect_id(&tag, &revision);
let admitted = payload
.approval
.as_ref()
.is_some_and(|token| token.admits(&effect_id));
if !admitted {
return Err(ApiError::forbidden(format!(
"environment `{name}` requires an approval token scoped to revision `{}`'s \
promotion effect id — a token minted for any other revision admits nothing \
here",
revision.revision_id.as_str()
)));
}
}
let surface = deployment_surface(&tag);
move_pointer(&state, tenant_id, &name, &tag, |pointer, _events| {
let base = pointer
.cloned()
.unwrap_or_else(|| DeploymentPointer::new(surface.clone()));
let promotion = RevisionPromotion {
tenant: tenant_id.to_owned(),
environment: tag.clone(),
revision_id: revision.revision_id.clone(),
previous: base.active.clone(),
author: payload.author.clone(),
promoted_at: Utc::now(),
};
let next = base.promoted(&promotion);
let converged = base.active.as_ref() == Some(&revision.revision_id);
Ok((DeploymentAct::Promotion(promotion), next, converged))
})
.await
}
pub(crate) async fn rollback_revision(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(name): AxumPath<String>,
Json(payload): Json<RollbackPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let tenant_id = tenant.tenant();
let tag = EnvironmentTag::new(name.clone())
.map_err(|e| ApiError::bad_request(format!("invalid environment name `{name}`: {e}")))?;
state_must_declare(&state.server_store, tenant_id, &tag).await?;
if payload.cause.trim().is_empty() {
return Err(ApiError::bad_request(
"a rollback names its cause — the operator's note, the incident id".to_owned(),
));
}
let surface = deployment_surface(&tag);
move_pointer(&state, tenant_id, &name, &tag, |pointer, events| {
let base = pointer
.cloned()
.unwrap_or_else(|| DeploymentPointer::new(surface.clone()));
let Some(from) = base.active.clone() else {
return Err(ApiError::conflict(format!(
"environment `{name}` serves nothing — there is no serving revision to roll \
back"
)));
};
let Some(to) = rollback_target(events, tenant_id, &tag, &from) else {
return Err(ApiError::conflict(format!(
"revision `{}` never served in `{name}` per the deployment chain — there is \
no journaled history to restore to",
from.as_str()
)));
};
let rollback = RevisionRollback {
tenant: tenant_id.to_owned(),
environment: tag.clone(),
from,
to: to.clone(),
cause: payload.cause.clone(),
author: payload.author.clone(),
rolled_back_at: Utc::now(),
};
let next = base.rolled_back(&rollback);
let converged = base.active == to;
Ok((DeploymentAct::Rollback(rollback), next, converged))
})
.await
}
#[derive(Deserialize)]
pub(crate) struct CanaryPayload {
revision_id: String,
fraction: f64,
author: ProvenanceAuthor,
}
#[derive(Deserialize)]
pub(crate) struct ClearCanaryPayload {
author: ProvenanceAuthor,
}
pub(crate) async fn declare_canary(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(name): AxumPath<String>,
Json(payload): Json<CanaryPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let tenant_id = tenant.tenant();
let tag = EnvironmentTag::new(name.clone())
.map_err(|e| ApiError::bad_request(format!("invalid environment name `{name}`: {e}")))?;
let environment = state_must_declare(&state.server_store, tenant_id, &tag).await?;
let revision = state
.server_store
.get_revision(tenant_id, &payload.revision_id)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!("revision `{}` not found", payload.revision_id))
})?;
revision
.verify_address()
.map_err(|e| ApiError::unprocessable(e.to_string()))?;
if let Some(gate) = &environment.gate {
run_gate(&state, tenant_id, &tag, &revision, gate).await?;
}
let canary = CanaryDeployment::new(revision.revision_id.clone(), payload.fraction)
.map_err(|e| deploy_err(&e))?;
let surface = deployment_surface(&tag);
move_pointer(&state, tenant_id, &name, &tag, |pointer, _events| {
let base = pointer
.cloned()
.unwrap_or_else(|| DeploymentPointer::new(surface.clone()));
let declaration = CanaryDeclaration {
tenant: tenant_id.to_owned(),
environment: tag.clone(),
revision_id: revision.revision_id.clone(),
fraction: canary.fraction,
author: payload.author.clone(),
declared_at: Utc::now(),
};
let next = base.canaried(&declaration);
let converged = base.canary.as_ref() == Some(&canary);
Ok((DeploymentAct::CanaryDeclare(declaration), next, converged))
})
.await
}
pub(crate) async fn clear_canary(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(name): AxumPath<String>,
Json(payload): Json<ClearCanaryPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let tenant_id = tenant.tenant();
let tag = EnvironmentTag::new(name.clone())
.map_err(|e| ApiError::bad_request(format!("invalid environment name `{name}`: {e}")))?;
state_must_declare(&state.server_store, tenant_id, &tag).await?;
let surface = deployment_surface(&tag);
let pointer = state
.server_store
.get_deployment_pointer(tenant_id, surface.as_str())
.await
.map_err(internal_err)?;
if pointer.as_ref().and_then(|p| p.canary.as_ref()).is_none() {
let pointer = pointer.unwrap_or_else(|| DeploymentPointer::new(surface.clone()));
return Ok((
StatusCode::OK,
Json(json!({ "applied": false, "journaled": false, "pointer": pointer })),
));
}
move_pointer(&state, tenant_id, &name, &tag, |pointer, _events| {
let base = pointer
.cloned()
.unwrap_or_else(|| DeploymentPointer::new(surface.clone()));
let Some(canary) = base.canary.clone() else {
return Err(ApiError::conflict(format!(
"environment `{name}` has no canary to clear — the slot moved under this \
act; re-read and retry"
)));
};
let clearance = CanaryClearance {
tenant: tenant_id.to_owned(),
environment: tag.clone(),
cleared_revision_id: canary.revision_id,
author: payload.author.clone(),
cleared_at: Utc::now(),
};
let next = base.cleared_canary();
Ok((DeploymentAct::CanaryClear(clearance), next, false))
})
.await
}
#[derive(Deserialize)]
pub(crate) struct ShadowPayload {
revision_id: String,
source: JournalSnapshot,
#[serde(default)]
input: Option<Value>,
author: ProvenanceAuthor,
}
fn shadow_id_part(raw: &str) -> String {
raw.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || c == '-' {
c
} else {
'-'
}
})
.collect()
}
pub(crate) async fn create_shadow(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<ShadowPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let tenant_id = tenant.tenant();
let revision = state
.server_store
.get_revision(tenant_id, &payload.revision_id)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!("revision `{}` not found", payload.revision_id))
})?;
revision
.verify_address()
.map_err(|e| ApiError::unprocessable(e.to_string()))?;
let Some((graph, spec)) = state.registry.get(&revision.content.graph) else {
return Err(ApiError::unprocessable(format!(
"revision `{}` serves graph `{}`, which is not registered — a shadow runs the \
same code admission would serve, and that graph is absent",
revision.revision_id.as_str(),
revision.content.graph
)));
};
let actual_hash = graph.topology_hash();
if actual_hash != revision.content.graph_hash {
return Err(ApiError::unprocessable(format!(
"revision `{}` records topology hash `{}` for graph `{}`, but the registered \
graph hashes to `{actual_hash}` — the build drifted from the revision's record; \
register a fresh revision, never drift silently",
revision.revision_id.as_str(),
revision.content.graph_hash,
revision.content.graph
)));
}
Journal::from_snapshot(payload.source.clone(), Clock::System).map_err(|e| {
ApiError::unprocessable(format!(
"the source journal failed its integrity check: {e} — refused, never replayed"
))
})?;
let shadow_run_id = format!(
"shadow-{}-{}-{}",
shadow_id_part(&payload.source.run_id),
&revision.revision_id.as_str()[..12],
&uuid::Uuid::new_v4().simple().to_string()[..8]
);
let journal = Journal::new(shadow_run_id.clone(), shadow_run_id.clone(), Clock::System);
let started = ShadowRunStarted {
tenant: tenant_id.to_owned(),
shadow_run_id: shadow_run_id.clone(),
source_run_id: payload.source.run_id.clone(),
revision_id: revision.revision_id.clone(),
role: DecisionRole::Shadow,
author: payload.author.clone(),
started_at: Utc::now(),
};
journal.record(
EventDraft::new(RunEventKind::ShadowRunStarted, Effect::Pure).output(json!(started)),
);
let source = Arc::new(JournalShadowSource::new(&payload.source));
let sink_journal = journal.clone();
let sink: ShadowRefusalSink = Arc::new(move |refusal: &ShadowRefusal| {
sink_journal.record(
EventDraft::new(RunEventKind::ShadowEffectRefused, Effect::Pure).output(json!(refusal)),
);
});
let context = EffectAdmissionContext::shadow(shadow_run_id.clone(), source.clone(), sink);
let initial = payload
.input
.clone()
.and_then(|v| GraphState::from_value(v).ok())
.unwrap_or_default();
let config = RunConfig::new(shadow_run_id.clone())
.with_journal(journal.clone())
.with_effect_admission_context(context);
let result = Executor::new().run(&graph, &spec, initial, config).await;
let refusals: Vec<ShadowRefusal> = journal
.events()
.iter()
.filter(|event| event.kind == RunEventKind::ShadowEffectRefused)
.filter_map(|event| match &event.output {
Some(PayloadRef::Inline(value)) => {
serde_json::from_value::<ShadowRefusal>(value.clone()).ok()
}
_ => None,
})
.collect();
let matched = refusals.iter().filter(|refusal| refusal.served).count();
let outcome = match &result {
Ok(_) => ShadowRunOutcome::Completed,
Err(e) => ShadowRunOutcome::Failed {
error: e.to_string(),
},
};
let verdict = ShadowVerdict {
tenant: tenant_id.to_owned(),
shadow_run_id: shadow_run_id.clone(),
source_run_id: payload.source.run_id.clone(),
revision_id: revision.revision_id.clone(),
refusals: refusals.clone(),
matched,
unserved: refusals.len() - matched,
unrequested: source.unrequested(),
outcome,
completed_at: Utc::now(),
};
journal
.record(EventDraft::new(RunEventKind::ShadowVerdict, Effect::Pure).output(json!(verdict)));
state
.server_store
.put_journal(&journal.snapshot())
.await
.map_err(internal_err)?;
match result {
Ok(_) => Ok((
StatusCode::CREATED,
Json(json!({
"shadow_run_id": shadow_run_id,
"verdict": verdict,
})),
)),
Err(e) => Err(ApiError::unprocessable(format!(
"shadow run `{shadow_run_id}` failed: {e} — the verdict is journaled under the \
shadow's id"
))),
}
}
pub(crate) async fn get_deployment_health(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let tenant_id = tenant.tenant();
let mut environments = state
.server_store
.list_environments(tenant_id)
.await
.map_err(internal_err)?;
environments.sort_by(|a, b| a.name.as_str().cmp(b.name.as_str()));
let chain_head = state
.server_store
.get_journal(DEPLOYMENT_JOURNAL_RUN_ID)
.await
.map_err(internal_err)?
.map(|snapshot| snapshot.head_hash);
let chain_events = state
.deployment
.chain_events()
.await
.map_err(internal_err)?;
let journals = state
.server_store
.list_journals()
.await
.map_err(internal_err)?;
let mut tallies: HashMap<(String, String), (usize, usize, usize)> = HashMap::new();
for snapshot in journals {
if snapshot.run_id == DEPLOYMENT_JOURNAL_RUN_ID {
continue;
}
let Ok(journal) = Journal::from_snapshot(snapshot, Clock::System) else {
continue;
};
let events = journal.events();
let Some(resolution) = events.iter().find_map(|event| {
if event.kind != RunEventKind::DeploymentResolved {
return None;
}
match &event.output {
Some(PayloadRef::Inline(value)) => {
serde_json::from_value::<DeploymentResolved>(value.clone()).ok()
}
_ => None,
}
}) else {
continue;
};
let slot = match resolution.pointer {
rusty_agent_runtime::registry::PointerBinding::Active => "active",
rusty_agent_runtime::registry::PointerBinding::Canary => "canary",
};
let entry = tallies
.entry((resolution.environment.to_string(), slot.to_owned()))
.or_insert((0, 0, 0));
entry.0 += 1;
if events.iter().any(|e| e.status == EventStatus::Error) {
entry.1 += 1;
}
if events.iter().any(|e| e.status == EventStatus::Interrupted) {
entry.2 += 1;
}
}
let tally_json = |environment: &str, slot: &str| {
let (runs, errored, interrupted) = tallies
.get(&(environment.to_owned(), slot.to_owned()))
.copied()
.unwrap_or((0, 0, 0));
json!({ "runs": runs, "errored": errored, "interrupted": interrupted })
};
let mut board = Vec::with_capacity(environments.len());
for environment in &environments {
let surface = deployment_surface(&environment.name);
let pointer = state
.server_store
.get_deployment_pointer(tenant_id, surface.as_str())
.await
.map_err(internal_err)?;
let last_gate_decision = chain_events.iter().rev().find_map(|event| {
if event.kind != RunEventKind::GateDecisionRecorded {
return None;
}
match &event.output {
Some(PayloadRef::Inline(value)) => {
match serde_json::from_value::<GateDecisionRecord>(value.clone()) {
Ok(record) if record.environment == environment.name => Some(value.clone()),
_ => None,
}
}
_ => None,
}
});
let name = environment.name.to_string();
board.push(json!({
"environment": environment.name,
"gate": environment.gate,
"approval_required": environment.approval_required,
"active_revision": pointer.as_ref().and_then(|p| p.active.clone()),
"canary": pointer.as_ref().and_then(|p| p.canary.clone()),
"last_gate_decision": last_gate_decision,
"recent_runs": {
"active": tally_json(&name, "active"),
"canary": tally_json(&name, "canary"),
},
}));
}
Ok(Json(json!({
"environments": board,
"deployment_chain_head": chain_head,
})))
}
pub(crate) async fn get_environment_pointer(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(name): AxumPath<String>,
) -> Result<Json<Value>, ApiError> {
let tenant_id = tenant.tenant();
let tag = EnvironmentTag::new(name.clone())
.map_err(|e| ApiError::bad_request(format!("invalid environment name `{name}`: {e}")))?;
state_must_declare(&state.server_store, tenant_id, &tag).await?;
let surface = deployment_surface(&tag);
let pointer = state
.server_store
.get_deployment_pointer(tenant_id, surface.as_str())
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!(
"environment `{name}` serves nothing — nothing was ever promoted into it"
))
})?;
Ok(Json(json!({ "pointer": pointer })))
}
pub(crate) async fn get_deployment_journal(
State(state): State<Arc<AppState>>,
Extension(_tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let Some(snapshot) = state
.server_store
.get_journal(DEPLOYMENT_JOURNAL_RUN_ID)
.await
.map_err(internal_err)?
else {
return Ok(Json(json!({
"run_id": DEPLOYMENT_JOURNAL_RUN_ID,
"events": [],
"complete": false,
})));
};
let journal = Journal::from_snapshot(snapshot, Clock::System).map_err(|e| {
ApiError::unprocessable(format!(
"the deployment evidence chain failed its integrity check: {e} — refusing to \
serve a chain that does not verify"
))
})?;
Ok(Json(json!({
"run_id": DEPLOYMENT_JOURNAL_RUN_ID,
"events": journal.events(),
"complete": false,
})))
}
#[derive(Deserialize)]
pub(crate) struct SetSecretPayload {
name: String,
environment: EnvironmentTag,
value: Value,
author: ProvenanceAuthor,
}
pub(crate) async fn set_env_secret(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<SetSecretPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let tenant_id = tenant.tenant();
validate_secret_name(&payload.name).map_err(|e| deploy_err(&e))?;
state_must_declare(&state.server_store, tenant_id, &payload.environment).await?;
let existing = state
.server_store
.get_env_secret(tenant_id, &payload.name, payload.environment.as_str())
.await
.map_err(internal_err)?;
let now = Utc::now();
let record = EnvSecretRecord {
name: payload.name.clone(),
environment: payload.environment.clone(),
set_by: payload.author.clone(),
created_at: existing
.as_ref()
.map(|s| s.record.created_at)
.unwrap_or(now),
rotated_at: existing.as_ref().map(|_| Some(now)).unwrap_or(None),
};
let plaintext = serde_json::to_vec(&payload.value).map_err(internal_err)?;
let scoped = scoped_secret_name(&payload.name, &payload.environment);
let keys = master_secrets(&state.config.store_path)
.await
.map_err(internal_err)?;
let (key_id, master) = keys
.last()
.ok_or_else(|| ApiError::internal("no env-secret master key after mint".to_owned()))?;
let envelope = seal_env_secret(key_id, master, &scoped, &plaintext).map_err(internal_err)?;
let stored = StoredEnvSecret {
record: record.clone(),
envelope,
};
let act = EnvSecretAct {
tenant: tenant_id.to_owned(),
record: record.clone(),
};
let output = serde_json::to_value(&act).map_err(internal_err)?;
state
.deployment
.journal_act(RunEventKind::EnvSecretSet, output)
.await
.map_err(internal_err)?;
let created = state
.server_store
.set_env_secret(tenant_id, &stored)
.await
.map_err(internal_err)?;
Ok((
if created {
StatusCode::CREATED
} else {
StatusCode::OK
},
Json(json!({ "created": created, "record": record })),
))
}
#[derive(Deserialize)]
pub(crate) struct ListSecretsQuery {
#[serde(default)]
environment: Option<EnvironmentTag>,
}
pub(crate) async fn list_env_secrets(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Query(query): Query<ListSecretsQuery>,
) -> Result<Json<Value>, ApiError> {
let mut records = state
.server_store
.list_env_secrets(tenant.tenant())
.await
.map_err(internal_err)?;
if let Some(environment) = &query.environment {
records.retain(|record| &record.environment == environment);
}
records.sort_by(|a, b| {
(a.environment.as_str(), a.name.as_str()).cmp(&(b.environment.as_str(), b.name.as_str()))
});
Ok(Json(json!({ "secrets": records })))
}
#[derive(Deserialize)]
pub(crate) struct ResolveSecretPayload {
name: String,
environment: EnvironmentTag,
holder: EnvironmentTag,
}
pub(crate) async fn resolve_env_secret(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<ResolveSecretPayload>,
) -> Result<Json<Value>, ApiError> {
let tenant_id = tenant.tenant();
if payload.holder != payload.environment {
let denial = EnvSecretDenial {
tenant: tenant_id.to_owned(),
name: payload.name.clone(),
requested_environment: payload.environment.clone(),
held_environment: payload.holder.clone(),
denied_at: Utc::now(),
};
if let Ok(output) = serde_json::to_value(&denial) {
state.deployment.journal_denial(output).await;
}
return Err(ApiError::new(
StatusCode::FORBIDDEN,
"environment_scope_denied",
format!(
"secret `{}` is scoped to `{}`; the holder admits `{}` — a value never \
crosses its environment's boundary",
payload.name, payload.environment, payload.holder,
),
));
}
let scoped = scoped_secret_name(&payload.name, &payload.environment);
let stored = state
.server_store
.get_env_secret(tenant_id, &payload.name, payload.environment.as_str())
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("secret `{scoped}` not found")))?;
let keys = master_secrets(&state.config.store_path)
.await
.map_err(internal_err)?;
let Some(master) = master_for(&keys, &stored.envelope.key_id) else {
return Err(ApiError::internal(format!(
"secret `{scoped}` was sealed under key `{}`, which this host does not hold — \
failing closed",
stored.envelope.key_id
)));
};
let plaintext = open_env_secret(master, &scoped, &stored.envelope)
.map_err(|e| ApiError::internal(format!("secret `{scoped}` failed to open: {e}")))?;
let value: Value = serde_json::from_slice(&plaintext).map_err(internal_err)?;
Ok(Json(json!({
"name": payload.name,
"environment": payload.environment,
"value": value,
})))
}
#[derive(Deserialize)]
pub(crate) struct RevokeSecretPayload {
author: ProvenanceAuthor,
}
pub(crate) async fn revoke_env_secret(
State(state): State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath((environment, name)): AxumPath<(String, String)>,
Json(payload): Json<RevokeSecretPayload>,
) -> Result<StatusCode, ApiError> {
let tenant_id = tenant.tenant();
let tag = EnvironmentTag::new(environment.clone()).map_err(|e| {
ApiError::bad_request(format!("invalid environment name `{environment}`: {e}"))
})?;
state
.server_store
.get_env_secret(tenant_id, &name, tag.as_str())
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!(
"secret `{}` not found",
scoped_secret_name(&name, &tag)
))
})?;
let revocation = EnvSecretRevocation {
tenant: tenant_id.to_owned(),
name: name.clone(),
environment: tag.clone(),
revoked_by: payload.author,
revoked_at: Utc::now(),
};
let output = serde_json::to_value(&revocation).map_err(internal_err)?;
state
.deployment
.journal_act(RunEventKind::EnvSecretRevoked, output)
.await
.map_err(internal_err)?;
state
.server_store
.delete_env_secret(tenant_id, &name, tag.as_str())
.await
.map_err(internal_err)?;
Ok(StatusCode::NO_CONTENT)
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::{DateTime, Utc};
use rusty_agent_runtime::deploy::{EnvSecretRecord, RevisionContent};
use rusty_agent_runtime::learn::EnvironmentTag;
use rusty_agent_runtime::memory::ProvenanceAuthor;
fn ts(millis: i64) -> DateTime<Utc> {
DateTime::<Utc>::from_timestamp_millis(millis).unwrap()
}
fn tag(name: &str) -> EnvironmentTag {
EnvironmentTag::new(name).unwrap()
}
fn temp_root() -> PathBuf {
std::env::temp_dir().join(format!("rusty-deploy-test-{}", uuid::Uuid::new_v4()))
}
fn revision() -> DeploymentRevision {
DeploymentRevision::new(
RevisionContent {
graph: "pipeline".into(),
graph_hash: "a".repeat(64),
assistant: None,
source_environment: tag("staging"),
pins: Vec::new(),
},
ProvenanceAuthor::Human {
human_id: "amjad".into(),
},
ts(1_760_000_000_000),
)
.unwrap()
}
fn environment() -> Environment {
Environment {
name: tag("staging"),
gate: None,
approval_required: false,
created_by: ProvenanceAuthor::Human {
human_id: "amjad".into(),
},
created_at: ts(1_760_000_000_000),
}
}
#[tokio::test]
async fn revisions_and_environments_round_trip_with_corrupt_tolerance() {
let root = temp_root();
let revision = revision();
let scoped = revision.revision_id.to_string();
persist_revision(&root, &scoped, &revision).await.unwrap();
let tenant_scoped = format!("acme/{scoped}");
persist_revision(&root, &tenant_scoped, &revision)
.await
.unwrap();
std::fs::write(revisions_dir(&root).join("broken.json"), b"{nope").unwrap();
let loaded = load_revisions(&root);
assert_eq!(loaded.len(), 2, "corrupt files are skipped, not fatal");
assert!(loaded.contains_key(&scoped), "default tenant: bare key");
assert_eq!(
loaded[&tenant_scoped].revision_id, revision.revision_id,
"named tenant: the key comes from the path"
);
let environment = environment();
let scoped_name = format!("acme/{}", environment.name.as_str());
persist_environment(&root, environment.name.as_str(), &environment)
.await
.unwrap();
persist_environment(&root, &scoped_name, &environment)
.await
.unwrap();
let loaded = load_environments(&root);
assert_eq!(loaded.len(), 2);
assert!(loaded.contains_key(environment.name.as_str()));
assert!(loaded.contains_key(&scoped_name));
let _ = std::fs::remove_dir_all(root);
}
#[tokio::test]
async fn pointers_and_secrets_round_trip_through_hashed_filenames() {
let root = temp_root();
let pointer = DeploymentPointer::new(rusty_agent_runtime::learn::SurfaceKey::new(
"deployment:staging",
));
let scoped_surface = "acme/deployment:staging";
persist_pointer(&root, scoped_surface, &pointer)
.await
.unwrap();
let listing: Vec<String> = std::fs::read_dir(pointers_dir(&root))
.unwrap()
.flatten()
.map(|entry| entry.file_name().to_string_lossy().into_owned())
.collect();
assert_eq!(
listing,
vec![format!("{}.json", sha256_hex(scoped_surface.as_bytes()))]
);
let loaded = load_pointers(&root);
assert_eq!(loaded.len(), 1);
assert_eq!(loaded[scoped_surface], pointer);
let forged = pointers_dir(&root).join(format!("{}.json", sha256_hex(b"forged")));
std::fs::write(
forged,
serde_json::to_vec_pretty(&KeyedFile {
key: "acme/deployment:prod".to_string(),
record: pointer.clone(),
})
.unwrap(),
)
.unwrap();
assert_eq!(
load_pointers(&root).len(),
1,
"the forged-name file is skipped"
);
let _ = std::fs::remove_dir_all(root);
}
#[tokio::test]
async fn secrets_seal_open_and_fail_closed_across_scopes() {
let root = temp_root();
let keys = master_secrets(&root).await.unwrap();
assert_eq!(keys.len(), 1, "first use mints one key");
let (key_id, master) = &keys[0];
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = std::fs::metadata(master_secret_path(&root, key_id))
.unwrap()
.permissions()
.mode();
assert_eq!(mode & 0o777, 0o600);
}
let scoped = "database-url@staging";
let value = b"postgres://staging-db:5432/app";
let envelope = seal_env_secret(key_id, master, scoped, value).unwrap();
assert_eq!(open_env_secret(master, scoped, &envelope).unwrap(), value);
assert!(open_env_secret(master, "database-url@prod", &envelope).is_err());
let wrong: [u8; 32] = draw_random();
assert!(open_env_secret(&wrong, scoped, &envelope).is_err());
let mut tampered = envelope.clone();
tampered.ciphertext = hex_encode(b"forged");
assert!(open_env_secret(master, scoped, &tampered).is_err());
assert!(master_for(&keys, "esk-doesnotexist").is_none());
let stored = StoredEnvSecret {
record: EnvSecretRecord {
name: "database-url".into(),
environment: tag("staging"),
set_by: ProvenanceAuthor::Human {
human_id: "amjad".into(),
},
created_at: ts(1_760_000_000_000),
rotated_at: None,
},
envelope,
};
persist_env_secret(&root, scoped, &stored).await.unwrap();
let raw =
std::fs::read(env_secrets_dir(&root).join(format!("{}.json", keyed_file_name(scoped))))
.unwrap();
let needle = b"staging-db";
assert!(
!raw.windows(needle.len()).any(|w| w == needle),
"the store holds ciphertext only — a raw read finds no plaintext"
);
let loaded = load_env_secrets(&root);
assert_eq!(loaded.len(), 1);
let reopened = open_env_secret(master, scoped, &loaded[scoped].envelope).unwrap();
assert_eq!(reopened, value, "a restart resolves byte-exact");
let _ = std::fs::remove_dir_all(root);
}
}