use std::collections::{HashMap, HashSet};
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use axum::extract::{Path as AxumPath, Query};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use axum::{Extension, Json};
use chrono::{DateTime, Utc};
use rusty_agent_runtime::artifact::{
append_artifact_version, commit_artifact, derive_preview, ArtifactCommitment, ArtifactError,
ArtifactLineage, ArtifactPrune, ArtifactRelease, ArtifactUnavailability, CommitDeclaration,
MediaKind, PruneCause, RetentionPolicy, RunArtifact, UnavailabilitySurface,
};
use rusty_agent_runtime::broker::hex_decode;
use rusty_agent_runtime::journal::{Clock, EventDraft, FileArtifactStore, Journal};
use rusty_agent_runtime::receipt::{verify_receipt_prefix, PublicKey};
use rusty_agent_runtime::record::{
sha256_hex, ArtifactRef, Effect, PayloadRef, RunEvent, RunEventKind,
};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use tokio::sync::Mutex;
use crate::auth::TenantContext;
use crate::error::ApiError;
use crate::routes::{internal_err, AppState};
use crate::server_store::{RunArtifactVersionWrite, RunArtifactWrite, ServerStore, StoreResult};
pub(crate) fn records_dir(root: &Path) -> PathBuf {
root.join("artifacts").join("records")
}
pub(crate) fn names_dir(root: &Path) -> PathBuf {
root.join("artifacts").join("names")
}
pub(crate) fn blob_store(root: &Path) -> FileArtifactStore {
FileArtifactStore::new(root.join("artifacts").join("blobs"))
}
pub(crate) async fn persist_record(
root: &Path,
scoped_id: &str,
record: &RunArtifact,
) -> io::Result<()> {
let dir = records_dir(root);
tokio::fs::create_dir_all(&dir).await?;
let bytes = serde_json::to_vec_pretty(record)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let path = dir.join(format!("{scoped_id}.json"));
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent).await?;
}
let tmp = dir.join(format!("{scoped_id}.tmp"));
tokio::fs::write(&tmp, bytes).await?;
tokio::fs::rename(&tmp, path).await
}
fn collect_json_files(root: &Path, out: &mut Vec<PathBuf>) {
let Ok(entries) = std::fs::read_dir(root) else {
return;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
collect_json_files(&path, out);
} else if path.extension().and_then(|e| e.to_str()) == Some("json") {
out.push(path);
}
}
}
fn path_scoped_id(dir: &Path, path: &Path) -> Option<String> {
path.strip_prefix(dir)
.ok()
.map(|relative| relative.with_extension(""))
.map(|relative| {
relative
.components()
.map(|component| component.as_os_str().to_string_lossy().into_owned())
.collect::<Vec<_>>()
.join("/")
})
}
pub(crate) fn load_records(root: &Path) -> HashMap<String, RunArtifact> {
let dir = records_dir(root);
let mut out = HashMap::new();
let mut files = Vec::new();
collect_json_files(&dir, &mut files);
for path in files {
let Some(scoped_id) = path_scoped_id(&dir, &path) else {
continue;
};
let parsed = std::fs::read_to_string(&path)
.ok()
.and_then(|raw| serde_json::from_str::<RunArtifact>(&raw).ok());
let bare = scoped_id.rsplit('/').next().unwrap_or(&scoped_id);
match parsed {
Some(record) if record.artifact_id == bare => {
out.insert(scoped_id, record);
}
_ => {
tracing::warn!(path = %path.display(), "skipping unreadable artifact record file")
}
}
}
out
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
struct NameFile {
key: String,
artifact_id: String,
}
fn name_file_name(scoped_name: &str) -> String {
sha256_hex(scoped_name.as_bytes())
}
pub(crate) async fn persist_name(
root: &Path,
scoped_name: &str,
artifact_id: &str,
) -> io::Result<()> {
let dir = names_dir(root);
tokio::fs::create_dir_all(&dir).await?;
let file = NameFile {
key: scoped_name.to_string(),
artifact_id: artifact_id.to_string(),
};
let bytes = serde_json::to_vec_pretty(&file)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let name = name_file_name(scoped_name);
let tmp = dir.join(format!("{name}.tmp"));
tokio::fs::write(&tmp, bytes).await?;
tokio::fs::rename(&tmp, dir.join(format!("{name}.json"))).await
}
pub(crate) fn load_names(root: &Path) -> HashMap<String, String> {
let dir = names_dir(root);
let mut out = HashMap::new();
let mut files = Vec::new();
collect_json_files(&dir, &mut files);
for path in files {
let parsed = std::fs::read_to_string(&path)
.ok()
.and_then(|raw| serde_json::from_str::<NameFile>(&raw).ok());
let matches_name = parsed.as_ref().is_some_and(|file| {
path.file_stem().and_then(|s| s.to_str()) == Some(&*name_file_name(&file.key))
});
match (parsed, matches_name) {
(Some(file), true) => {
out.insert(file.key, file.artifact_id);
}
_ => {
tracing::warn!(path = %path.display(), "skipping unreadable artifact name file")
}
}
}
out
}
#[derive(Debug, Deserialize)]
pub(crate) struct LineagePayload {
run_id: String,
effect_id: String,
event_id: String,
}
fn artifact_error(error: &ArtifactError) -> ApiError {
ApiError::unprocessable(error.to_string())
}
fn is_lower_hex_digest(value: &str) -> bool {
value.len() == 64
&& value
.chars()
.all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase())
}
fn parse_lineage(payload: LineagePayload) -> Result<ArtifactLineage, ApiError> {
if !is_lower_hex_digest(&payload.effect_id) {
return Err(ApiError::unprocessable(format!(
"effect id `{}` is not 64 lowercase hex characters — lineage anchors to a \
derived effect id; anything else cannot re-derive at audit",
payload.effect_id
)));
}
Ok(ArtifactLineage {
run_id: payload.run_id,
effect_id: serde_json::from_value(Value::String(payload.effect_id))
.map_err(|e| ApiError::unprocessable(format!("effect id does not parse: {e}")))?,
event_id: payload.event_id,
})
}
async fn journal_commitment(
state: &AppState,
tenant: &TenantContext,
run_id: &str,
commitment: &ArtifactCommitment,
) -> Result<String, ApiError> {
let snapshot = state
.server_store
.get_journal(run_id)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!(
"run `{run_id}` has no persisted journal — a commit journals into the \
producing run; an unknown run commits nothing"
))
})?;
let internal_thread_id = tenant.scope(&snapshot.thread_id);
let owned = state
.server_store
.get_thread(&internal_thread_id)
.await
.map_err(internal_err)?
.is_some();
if !owned {
return Err(ApiError::not_found(format!(
"run `{run_id}` does not resolve in this tenant"
)));
}
let journal = Journal::from_snapshot(snapshot, Clock::System).map_err(|e| {
ApiError::unprocessable(format!(
"run `{run_id}`'s journal failed its integrity check: {e} — the commitment \
cannot join a chain that does not verify"
))
})?;
let parent = journal.events().last().map(|event| event.id.clone());
let output = serde_json::to_value(commitment)
.map_err(|e| ApiError::internal(format!("serialize commitment: {e}")))?;
let mut draft = EventDraft::new(RunEventKind::ArtifactCommitted, Effect::Pure).output(output);
if let Some(parent) = parent {
draft = draft.parent(parent);
}
let event_id = journal.record(draft);
state
.server_store
.put_journal(&journal.snapshot())
.await
.map_err(internal_err)?;
Ok(event_id)
}
async fn commit_shared(
state: &AppState,
tenant: &TenantContext,
bytes: &[u8],
declaration: CommitDeclaration,
) -> Result<(StatusCode, Json<Value>), ApiError> {
if let Some(existing) = state
.server_store
.get_run_artifact(tenant.tenant(), &declaration.reference.sha256)
.await
.map_err(internal_err)?
{
if existing.name == declaration.name {
return Ok((
StatusCode::OK,
Json(json!({
"artifact_id": existing.artifact_id,
"created": false,
"artifact": existing,
})),
));
}
return Err(ApiError::conflict(format!(
"artifact `{}` is already committed under name `{:?}` — one object carries one \
logical name; the bytes are shared, the metadata is not",
existing.artifact_id, existing.name
)));
}
if let Some(name) = declaration.name.clone() {
if let Some(head) = state
.server_store
.get_run_artifact_by_name(tenant.tenant(), &name)
.await
.map_err(internal_err)?
{
let (record, commitment) =
append_artifact_version(&head, declaration).map_err(|e| artifact_error(&e))?;
let event_id =
journal_commitment(state, tenant, &record.lineage.run_id, &commitment).await?;
let stored = state
.server_store
.put_run_artifact_bytes(bytes)
.await
.map_err(internal_err)?;
if stored.sha256 != record.artifact_id {
return Err(ApiError::internal(format!(
"byte store minted address `{}` for bytes the commit declared as `{}` — \
the commit aborts; the orphaned bytes are unlisted and eventually swept",
stored.sha256, record.artifact_id
)));
}
return match state
.server_store
.put_run_artifact_version(tenant.tenant(), &head.artifact_id, &record)
.await
.map_err(internal_err)?
{
RunArtifactVersionWrite::Versioned => Ok((
StatusCode::CREATED,
Json(json!({
"artifact_id": record.artifact_id,
"created": true,
"artifact": record,
"commitment": commitment,
"journal_event_id": event_id,
})),
)),
RunArtifactVersionWrite::HeadMoved(live) => Err(ApiError::conflict(format!(
"name `{name}`'s head moved to artifact `{live}` while this version \
committed — a concurrent commit won; re-read the name and retry against \
the new head (nothing was written)"
))),
RunArtifactVersionWrite::NameUnknown => Err(ApiError::conflict(format!(
"name `{name}` stopped resolving while this version committed — re-read \
the name and retry (nothing was written)"
))),
};
}
}
let (record, commitment) = commit_artifact(declaration).map_err(|e| artifact_error(&e))?;
let event_id = journal_commitment(state, tenant, &record.lineage.run_id, &commitment).await?;
let stored = state
.server_store
.put_run_artifact_bytes(bytes)
.await
.map_err(internal_err)?;
if stored.sha256 != record.artifact_id {
return Err(ApiError::internal(format!(
"byte store minted address `{}` for bytes the commit declared as `{}` — the \
commit aborts; the orphaned bytes are unlisted and eventually swept",
stored.sha256, record.artifact_id
)));
}
match state
.server_store
.put_run_artifact(tenant.tenant(), &record)
.await
.map_err(internal_err)?
{
RunArtifactWrite::Created => Ok((
StatusCode::CREATED,
Json(json!({
"artifact_id": record.artifact_id,
"created": true,
"artifact": record,
"commitment": commitment,
"journal_event_id": event_id,
})),
)),
RunArtifactWrite::Converged(existing) => Ok((
StatusCode::OK,
Json(json!({
"artifact_id": existing.artifact_id,
"created": false,
"artifact": existing,
})),
)),
RunArtifactWrite::NameTaken(other) => Err(ApiError::conflict(format!(
"name `{:?}` already points at artifact `{other}`",
record.name
))),
RunArtifactWrite::Conflict(existing) => Err(ApiError::conflict(format!(
"artifact `{}` is already committed under name `{:?}`",
existing.artifact_id, existing.name
))),
}
}
#[derive(Debug, Deserialize)]
pub(crate) struct CommitArtifactPayload {
bytes_hex: String,
#[serde(default)]
name: Option<String>,
media_kind: MediaKind,
#[serde(default)]
media_type: Option<String>,
#[serde(default)]
retention: Option<RetentionPolicy>,
lineage: LineagePayload,
#[serde(default)]
committed_at: Option<DateTime<Utc>>,
}
pub(crate) async fn commit_run_artifact(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<CommitArtifactPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let bytes = hex_decode(&payload.bytes_hex).ok_or_else(|| {
ApiError::bad_request(
"`bytes_hex` is not valid hex — bytes ride the wire hex-encoded (the broker \
envelope's codec)"
.to_owned(),
)
})?;
let reference = ArtifactRef {
sha256: sha256_hex(&bytes),
bytes: bytes.len() as u64,
};
let declaration = CommitDeclaration {
reference,
name: payload.name,
media_kind: payload.media_kind,
media_type: payload.media_type,
lineage: parse_lineage(payload.lineage)?,
retention: payload.retention.unwrap_or_default(),
committed_at: payload.committed_at.unwrap_or_else(Utc::now),
};
commit_shared(&state, &tenant, &bytes, declaration).await
}
#[derive(Debug, Deserialize)]
pub(crate) struct CommitSpillPayload {
run_id: String,
event_id: String,
effect_id: String,
#[serde(default)]
name: Option<String>,
media_kind: MediaKind,
#[serde(default)]
media_type: Option<String>,
#[serde(default)]
retention: Option<RetentionPolicy>,
#[serde(default)]
committed_at: Option<DateTime<Utc>>,
}
pub(crate) async fn commit_spilled_artifact(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Json(payload): Json<CommitSpillPayload>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let snapshot = state
.server_store
.get_journal(&payload.run_id)
.await
.map_err(internal_err)?
.ok_or_else(|| {
ApiError::not_found(format!(
"run `{}` has no persisted journal — a spill commit reads the producing \
run's journal; an unknown run spills nothing",
payload.run_id
))
})?;
let internal_thread_id = tenant.scope(&snapshot.thread_id);
let owned = state
.server_store
.get_thread(&internal_thread_id)
.await
.map_err(internal_err)?
.is_some();
if !owned {
return Err(ApiError::not_found(format!(
"run `{}` does not resolve in this tenant",
payload.run_id
)));
}
let event = snapshot
.events
.iter()
.find(|event| event.id == payload.event_id)
.ok_or_else(|| {
ApiError::unprocessable(format!(
"run `{}` has no event `{}` — a spill commit names the event whose output \
carried the reference",
payload.run_id, payload.event_id
))
})?;
let reference = match event.output.as_ref() {
Some(PayloadRef::Artifact(reference)) => reference.clone(),
_ => {
return Err(ApiError::unprocessable(format!(
"event `{}`'s output is not an artifact reference — only a spilled \
(content-addressed) output commits through this path",
payload.event_id
)))
}
};
let value = snapshot.artifacts.get(&reference.sha256).ok_or_else(|| {
ApiError::unprocessable(format!(
"event `{}` references artifact `{}`, whose bytes are absent from the journal \
snapshot — a truncated snapshot cannot source a commit",
payload.event_id, reference.sha256
))
})?;
let bytes = serde_json::to_vec(value)
.map_err(|e| ApiError::internal(format!("serialize spilled payload: {e}")))?;
let declaration = CommitDeclaration {
reference,
name: payload.name,
media_kind: payload.media_kind,
media_type: payload.media_type,
lineage: parse_lineage(LineagePayload {
run_id: payload.run_id,
effect_id: payload.effect_id,
event_id: payload.event_id,
})?,
retention: payload.retention.unwrap_or_default(),
committed_at: payload.committed_at.unwrap_or_else(Utc::now),
};
commit_shared(&state, &tenant, &bytes, declaration).await
}
pub(crate) async fn get_run_artifact(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(artifact_id): AxumPath<String>,
) -> Result<Json<RunArtifact>, ApiError> {
state
.server_store
.get_run_artifact(tenant.tenant(), &artifact_id)
.await
.map_err(internal_err)?
.map(Json)
.ok_or_else(|| ApiError::not_found(format!("artifact `{artifact_id}` not found")))
}
async fn read_bytes_fail_closed(
state: &AppState,
tenant: &TenantContext,
record: &RunArtifact,
surface: UnavailabilitySurface,
) -> Result<Vec<u8>, ApiError> {
match state
.server_store
.get_run_artifact_bytes(&record.artifact_id)
.await
{
Ok(bytes) => Ok(bytes),
Err(e) => {
if let Some(detail) = e.strip_prefix("artifact corrupt:") {
Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"artifact_corrupt",
format!(
"artifact `{}` failed its integrity check:{detail} — the stored \
bytes are corrupt; refusing to serve",
record.artifact_id
),
))
} else if let Some(detail) = e.strip_prefix("artifact unavailable:") {
state
.artifact_retention
.journal_miss(tenant.tenant(), record, surface)
.await;
Err(ApiError::new(
StatusCode::GONE,
"artifact_unavailable",
format!(
"artifact `{}` is a live record whose bytes are not in the \
store:{detail}",
record.artifact_id
),
))
} else {
Err(internal_err(e))
}
}
}
}
pub(crate) async fn get_run_artifact_bytes(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(artifact_id): AxumPath<String>,
) -> Result<Response, ApiError> {
let record = state
.server_store
.get_run_artifact(tenant.tenant(), &artifact_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("artifact `{artifact_id}` not found")))?;
let bytes =
read_bytes_fail_closed(&state, &tenant, &record, UnavailabilitySurface::Bytes).await?;
let content_type = record
.media_type
.clone()
.unwrap_or_else(|| "application/octet-stream".to_owned());
Ok(([(axum::http::header::CONTENT_TYPE, content_type)], bytes).into_response())
}
#[derive(Debug, Deserialize)]
pub(crate) struct ListArtifactsQuery {
name: Option<String>,
media_kind: Option<String>,
run_id: Option<String>,
}
pub(crate) async fn list_run_artifacts(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
Query(query): Query<ListArtifactsQuery>,
) -> Result<Json<Value>, ApiError> {
let media_kind = query
.media_kind
.as_deref()
.map(|kind| {
serde_json::from_value::<MediaKind>(Value::String(kind.to_owned())).map_err(|_| {
ApiError::bad_request(format!(
"unknown media kind `{kind}` — expected one of `file`, `image`, \
`audio`, `data`"
))
})
})
.transpose()?;
let mut records = state
.server_store
.list_run_artifacts(tenant.tenant())
.await
.map_err(internal_err)?;
if let Some(name) = &query.name {
records.retain(|record| record.name.as_deref() == Some(name.as_str()));
}
if let Some(kind) = media_kind {
records.retain(|record| record.media_kind == kind);
}
if let Some(run_id) = &query.run_id {
records.retain(|record| record.lineage.run_id == *run_id);
}
records.sort_by(|a, b| a.artifact_id.cmp(&b.artifact_id));
Ok(Json(json!({ "artifacts": records })))
}
pub(crate) async fn get_run_artifact_named(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(name): AxumPath<String>,
) -> Result<Json<RunArtifact>, ApiError> {
state
.server_store
.get_run_artifact_by_name(tenant.tenant(), &name)
.await
.map_err(internal_err)?
.map(Json)
.ok_or_else(|| ApiError::not_found(format!("artifact name `{name}` not found")))
}
pub(crate) async fn list_run_artifact_versions(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(name): AxumPath<String>,
) -> Result<Json<Value>, ApiError> {
let record = state
.server_store
.get_run_artifact_by_name(tenant.tenant(), &name)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("artifact name `{name}` not found")))?;
Ok(Json(json!({
"name": name,
"current": record.artifact_id,
"versions": record.versions,
})))
}
pub(crate) const ARTIFACTS_JOURNAL_RUN_ID: &str = "run-artifacts";
#[derive(Debug, Default)]
struct ChainState {
released: HashMap<(String, String), String>,
prune_intents: HashSet<String>,
misses: HashSet<(String, String)>,
}
fn parse_chain(events: &[RunEvent]) -> ChainState {
let mut state = ChainState::default();
for event in events {
let value = match &event.output {
Some(PayloadRef::Inline(value)) => value.clone(),
_ => {
tracing::warn!(event_id = %event.id, "artifact chain event's payload is not inline; skipped");
continue;
}
};
match event.kind {
RunEventKind::ArtifactRetentionReleased => {
match serde_json::from_value::<ArtifactRelease>(value) {
Ok(release) => {
state
.released
.entry((release.tenant, release.artifact_id))
.or_insert_with(|| event.id.clone());
}
Err(e) => {
tracing::warn!(event_id = %event.id, %e, "artifact chain carries an unparseable release event")
}
}
}
RunEventKind::ArtifactPruned => match serde_json::from_value::<ArtifactPrune>(value) {
Ok(prune) => {
state.prune_intents.insert(prune.artifact_id);
}
Err(e) => {
tracing::warn!(event_id = %event.id, %e, "artifact chain carries an unparseable prune event")
}
},
RunEventKind::ArtifactUnavailable => {
match serde_json::from_value::<ArtifactUnavailability>(value) {
Ok(miss) => {
state.misses.insert((miss.tenant, miss.artifact_id));
}
Err(e) => {
tracing::warn!(event_id = %event.id, %e, "artifact chain carries an unparseable miss event")
}
}
}
_ => {}
}
}
state
}
#[derive(Debug, Default)]
struct Coverage {
covered: HashSet<String>,
unverifiable: Vec<String>,
}
enum Protection {
Protected,
Prunable(PruneCause),
}
fn evaluate_record(
tenant: &str,
record: &RunArtifact,
chain: &ChainState,
coverage: &Coverage,
now: DateTime<Utc>,
) -> Protection {
if chain
.released
.contains_key(&(tenant.to_owned(), record.artifact_id.clone()))
{
return Protection::Prunable(PruneCause::Released);
}
let pin_all = !coverage.unverifiable.is_empty();
match record.retention {
RetentionPolicy::Pinned => Protection::Protected,
RetentionPolicy::Days { days } => {
let deadline = record.created_at + chrono::Duration::days(i64::from(days));
if deadline > now || pin_all || coverage.covered.contains(&record.artifact_id) {
Protection::Protected
} else {
Protection::Prunable(PruneCause::Expired)
}
}
RetentionPolicy::ReceiptBound => {
if pin_all || coverage.covered.contains(&record.artifact_id) {
Protection::Protected
} else {
Protection::Prunable(PruneCause::Unbound)
}
}
}
}
fn evaluate_address(
records: &[(String, RunArtifact)],
chain: &ChainState,
coverage: &Coverage,
now: DateTime<Utc>,
) -> Option<PruneCause> {
let mut saw_released = false;
let mut saw_expired = false;
let mut saw_prunable = false;
for (tenant, record) in records {
match evaluate_record(tenant, record, chain, coverage, now) {
Protection::Protected => return None,
Protection::Prunable(cause) => {
saw_prunable = true;
saw_released |= cause == PruneCause::Released;
saw_expired |= cause == PruneCause::Expired;
}
}
}
if !saw_prunable {
return None;
}
Some(if saw_released {
PruneCause::Released
} else if saw_expired {
PruneCause::Expired
} else {
PruneCause::Unbound
})
}
#[derive(Debug, Default, Clone, Serialize)]
pub(crate) struct ArtifactSweepReport {
pub scanned: usize,
pub records: usize,
pub pruned: usize,
pub already_gone: usize,
pub protected: usize,
pub failed: usize,
pub unverifiable_receipts: Vec<String>,
}
#[derive(Debug, Clone)]
pub(crate) struct ReleaseOutcome {
pub artifact_id: String,
pub converged: bool,
pub pruned: bool,
pub journal_event_id: String,
}
pub(crate) struct ArtifactRetention {
store: Arc<dyn ServerStore>,
chain_lock: Mutex<()>,
}
impl ArtifactRetention {
pub(crate) fn new(store: Arc<dyn ServerStore>) -> Self {
Self {
store,
chain_lock: Mutex::new(()),
}
}
async fn load_chain(&self) -> StoreResult<Option<Journal>> {
match self.store.get_journal(ARTIFACTS_JOURNAL_RUN_ID).await? {
Some(snapshot) => Ok(Some(
Journal::from_snapshot(snapshot, Clock::System).map_err(|e| {
format!("the artifact evidence chain failed its integrity check: {e}")
})?,
)),
None => Ok(None),
}
}
async fn chain_state(&self) -> StoreResult<ChainState> {
let chain = self.load_chain().await?;
let events = chain
.as_ref()
.map(|journal| journal.events())
.unwrap_or_default();
Ok(parse_chain(&events))
}
async fn journal_act(&self, draft: EventDraft) -> StoreResult<String> {
let _guard = self.chain_lock.lock().await;
let journal = match self.load_chain().await? {
Some(journal) => journal,
None => Journal::new(
ARTIFACTS_JOURNAL_RUN_ID,
ARTIFACTS_JOURNAL_RUN_ID,
Clock::System,
),
};
let event_id = journal.record(draft);
self.store.put_journal(&journal.snapshot()).await?;
Ok(event_id)
}
pub(crate) async fn journal_miss(
&self,
tenant: &str,
record: &RunArtifact,
surface: UnavailabilitySurface,
) {
let attempt = async {
let _guard = self.chain_lock.lock().await;
let chain = self.load_chain().await?;
let events = chain
.as_ref()
.map(|journal| journal.events())
.unwrap_or_default();
let state = parse_chain(&events);
if state
.misses
.contains(&(tenant.to_owned(), record.artifact_id.clone()))
{
return Ok::<_, String>(());
}
let miss = ArtifactUnavailability {
artifact_id: record.artifact_id.clone(),
tenant: tenant.to_owned(),
name: record.name.clone(),
surface,
observed_at: Utc::now(),
};
let output = serde_json::to_value(&miss).map_err(|e| e.to_string())?;
let journal = chain.unwrap_or_else(|| {
Journal::new(
ARTIFACTS_JOURNAL_RUN_ID,
ARTIFACTS_JOURNAL_RUN_ID,
Clock::System,
)
});
journal.record(
EventDraft::new(RunEventKind::ArtifactUnavailable, Effect::Pure).output(output),
);
self.store.put_journal(&journal.snapshot()).await?;
Ok(())
}
.await;
if let Err(e) = attempt {
tracing::warn!(artifact_id = %record.artifact_id, %e, "artifact miss could not be journaled");
}
}
async fn receipt_coverage(&self) -> StoreResult<Coverage> {
let mut coverage = Coverage::default();
let receipts = self.store.list_all_run_receipts().await?;
if receipts.is_empty() {
return Ok(coverage);
}
let keys = self.store.list_receipt_keys().await?;
for (run_id, receipt) in receipts {
let verified = async {
let snapshot = self
.store
.get_journal(&run_id)
.await?
.ok_or_else(|| format!("run `{run_id}` has no persisted journal"))?;
let key = keys
.iter()
.find(|key| key.key_id == receipt.signer)
.ok_or_else(|| {
format!("signer key `{}` is not in the key history", receipt.signer)
})?;
let public = PublicKey::from_hex(&key.public_key).map_err(|e| e.to_string())?;
verify_receipt_prefix(&snapshot, &receipt, &public)
.map_err(|e| format!("prefix verification failed: {e}"))?;
Ok::<_, String>(snapshot)
}
.await;
let snapshot = match verified {
Ok(snapshot) => snapshot,
Err(e) => {
tracing::warn!(run_id = %run_id, %e, "receipt coverage unverifiable; pinning every address this pass");
coverage.unverifiable.push(run_id);
continue;
}
};
let covered_count = (receipt.journal_head.events as usize).min(snapshot.events.len());
for event in &snapshot.events[..covered_count] {
if !matches!(event.kind, RunEventKind::ArtifactCommitted) {
continue;
}
let value = match &event.output {
Some(PayloadRef::Inline(value)) => Some(value.clone()),
Some(PayloadRef::Artifact(reference)) => {
snapshot.artifacts.get(&reference.sha256).cloned()
}
None => None,
};
let parsed = value
.and_then(|value| serde_json::from_value::<ArtifactCommitment>(value).ok());
match parsed {
Some(commitment) => {
coverage.covered.insert(commitment.artifact_id);
}
None => {
tracing::warn!(run_id = %run_id, event_id = %event.id, "covered commitment will not parse; pinning every address this pass");
coverage.unverifiable.push(run_id.clone());
break;
}
}
}
}
Ok(coverage)
}
async fn journal_prune_intent(
&self,
artifact_id: &str,
name: Option<String>,
cause: PruneCause,
chain: &ChainState,
now: DateTime<Utc>,
) -> StoreResult<()> {
if chain.prune_intents.contains(artifact_id) {
return Ok(());
}
let prune = ArtifactPrune {
artifact_id: artifact_id.to_owned(),
name,
cause,
swept_at: now,
};
let output = serde_json::to_value(&prune).map_err(|e| e.to_string())?;
self.journal_act(
EventDraft::new(RunEventKind::ArtifactPruned, Effect::Pure).output(output),
)
.await?;
Ok(())
}
async fn prune_if_unprotected(&self, artifact_id: &str) -> StoreResult<bool> {
let chain = self.chain_state().await?;
let coverage = self.receipt_coverage().await?;
let records: Vec<(String, RunArtifact)> = self
.store
.list_all_run_artifacts()
.await?
.into_iter()
.filter(|(_, record)| record.artifact_id == artifact_id)
.collect();
if records.is_empty() {
return Ok(false);
}
let now = Utc::now();
let Some(cause) = evaluate_address(&records, &chain, &coverage, now) else {
return Ok(false);
};
let name = records.iter().find_map(|(_, record)| record.name.clone());
if let Err(e) = self
.journal_prune_intent(artifact_id, name, cause, &chain, now)
.await
{
tracing::warn!(artifact_id = %artifact_id, %e, "release-time prune could not journal its intention; the sweeper will converge it");
return Ok(false);
}
match self.store.delete_run_artifact_bytes(artifact_id).await {
Ok(deleted) => Ok(deleted),
Err(e) => {
tracing::warn!(artifact_id = %artifact_id, %e, "release-time prune could not delete the bytes; the sweeper will converge it");
Ok(false)
}
}
}
pub(crate) async fn release(
&self,
tenant: &str,
artifact_id: &str,
released_by: String,
reason: Option<String>,
) -> StoreResult<Option<ReleaseOutcome>> {
let Some(record) = self.store.get_run_artifact(tenant, artifact_id).await? else {
return Ok(None);
};
let (event_id, converged) = {
let _guard = self.chain_lock.lock().await;
let chain = self.load_chain().await?;
let events = chain
.as_ref()
.map(|journal| journal.events())
.unwrap_or_default();
let state = parse_chain(&events);
if let Some(existing) = state
.released
.get(&(tenant.to_owned(), artifact_id.to_owned()))
{
(existing.clone(), true)
} else {
let release = ArtifactRelease {
artifact_id: artifact_id.to_owned(),
tenant: tenant.to_owned(),
name: record.name.clone(),
released_by,
reason,
released_at: Utc::now(),
};
let output = serde_json::to_value(&release).map_err(|e| e.to_string())?;
let journal = chain.unwrap_or_else(|| {
Journal::new(
ARTIFACTS_JOURNAL_RUN_ID,
ARTIFACTS_JOURNAL_RUN_ID,
Clock::System,
)
});
let event_id = journal.record(
EventDraft::new(RunEventKind::ArtifactRetentionReleased, Effect::Pure)
.output(output),
);
self.store.put_journal(&journal.snapshot()).await?;
(event_id, false)
}
};
let pruned = self.prune_if_unprotected(artifact_id).await?;
Ok(Some(ReleaseOutcome {
artifact_id: artifact_id.to_owned(),
converged,
pruned,
journal_event_id: event_id,
}))
}
pub(crate) async fn sweep_once(&self, now: DateTime<Utc>) -> StoreResult<ArtifactSweepReport> {
let mut report = ArtifactSweepReport::default();
let chain = self.chain_state().await?;
let coverage = self.receipt_coverage().await?;
report.unverifiable_receipts = coverage.unverifiable.clone();
if !coverage.unverifiable.is_empty() {
tracing::warn!(runs = ?coverage.unverifiable, "artifact sweep: unverifiable receipt coverage pins every address this pass");
}
let mut by_address: HashMap<String, Vec<(String, RunArtifact)>> = HashMap::new();
for (tenant, record) in self.store.list_all_run_artifacts().await? {
report.records += 1;
by_address
.entry(record.artifact_id.clone())
.or_default()
.push((tenant, record));
}
let mut addresses: Vec<&String> = by_address.keys().collect();
addresses.sort();
for address in addresses {
let records = &by_address[address];
report.scanned += 1;
let Some(cause) = evaluate_address(records, &chain, &coverage, now) else {
report.protected += 1;
continue;
};
let name = records.iter().find_map(|(_, record)| record.name.clone());
self.journal_prune_intent(address, name, cause, &chain, now)
.await?;
match self.store.delete_run_artifact_bytes(address).await {
Ok(true) => report.pruned += 1,
Ok(false) => report.already_gone += 1,
Err(e) => {
tracing::warn!(artifact_id = %address, %e, "artifact sweep could not delete the bytes; will retry next pass");
report.failed += 1;
}
}
}
Ok(report)
}
}
pub(crate) fn spawn_sweeper(
retention: Arc<ArtifactRetention>,
interval: Duration,
shutdown: tokio_util::sync::CancellationToken,
) {
tokio::spawn(async move {
let mut ticker = tokio::time::interval(interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
_ = ticker.tick() => {}
_ = shutdown.cancelled() => {
tracing::info!("artifact retention sweeper shutting down; due prunes run on the next process or the operator-triggered pass");
break;
}
}
match retention.sweep_once(Utc::now()).await {
Ok(report) if report.pruned > 0 || report.failed > 0 || report.already_gone > 0 => {
tracing::info!(?report, "artifact retention sweep completed");
}
Ok(_) => {}
Err(error) => {
tracing::warn!(%error, "artifact retention sweep failed; will retry");
}
}
}
});
}
pub(crate) async fn get_run_artifact_preview(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(artifact_id): AxumPath<String>,
) -> Result<Json<Value>, ApiError> {
let record = state
.server_store
.get_run_artifact(tenant.tenant(), &artifact_id)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("artifact `{artifact_id}` not found")))?;
let bytes =
read_bytes_fail_closed(&state, &tenant, &record, UnavailabilitySurface::Preview).await?;
let preview = derive_preview(record.media_kind, &bytes);
Ok(Json(json!({
"artifact_id": record.artifact_id,
"preview": preview,
})))
}
#[derive(Debug, Deserialize)]
pub(crate) struct ReleaseArtifactPayload {
released_by: String,
#[serde(default)]
reason: Option<String>,
}
pub(crate) async fn release_run_artifact(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
Extension(tenant): Extension<TenantContext>,
AxumPath(artifact_id): AxumPath<String>,
Json(payload): Json<ReleaseArtifactPayload>,
) -> Result<Json<Value>, ApiError> {
if payload.released_by.trim().is_empty() {
return Err(ApiError::unprocessable(
"`released_by` is empty — a release is a governance act with a name on it".to_owned(),
));
}
let outcome = state
.artifact_retention
.release(
tenant.tenant(),
&artifact_id,
payload.released_by,
payload.reason,
)
.await
.map_err(internal_err)?
.ok_or_else(|| ApiError::not_found(format!("artifact `{artifact_id}` not found")))?;
Ok(Json(json!({
"artifact_id": outcome.artifact_id,
"released": true,
"converged": outcome.converged,
"pruned": outcome.pruned,
"journal_event_id": outcome.journal_event_id,
})))
}
pub(crate) async fn sweep_artifacts(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
Extension(_tenant): Extension<TenantContext>,
) -> Result<Json<ArtifactSweepReport>, ApiError> {
let report = state
.artifact_retention
.sweep_once(Utc::now())
.await
.map_err(internal_err)?;
Ok(Json(report))
}
pub(crate) async fn get_artifacts_journal(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
Extension(_tenant): Extension<TenantContext>,
) -> Result<Json<Value>, ApiError> {
let Some(snapshot) = state
.server_store
.get_journal(ARTIFACTS_JOURNAL_RUN_ID)
.await
.map_err(internal_err)?
else {
return Ok(Json(json!({
"run_id": ARTIFACTS_JOURNAL_RUN_ID,
"events": [],
"complete": false,
})));
};
let journal = Journal::from_snapshot(snapshot, Clock::System).map_err(|e| {
ApiError::unprocessable(format!(
"the artifact evidence chain failed its integrity check: {e} — refusing to serve \
a chain that does not verify"
))
})?;
Ok(Json(json!({
"run_id": ARTIFACTS_JOURNAL_RUN_ID,
"events": journal.events(),
"complete": false,
})))
}
#[cfg(test)]
mod tests {
use super::*;
use rusty_agent_runtime::artifact::ArtifactVersion;
fn ts(millis: i64) -> DateTime<Utc> {
DateTime::<Utc>::from_timestamp_millis(millis).unwrap()
}
fn record(address: &str, name: Option<&str>) -> RunArtifact {
RunArtifact {
artifact_id: address.to_owned(),
name: name.map(str::to_owned),
media_kind: MediaKind::File,
media_type: None,
lineage: ArtifactLineage {
run_id: "run-1".into(),
effect_id: serde_json::from_value(Value::String("e".repeat(64))).unwrap(),
event_id: "run-1:3".into(),
},
versions: name
.map(|_| ArtifactVersion {
sha256: address.to_owned(),
bytes: 12,
committed_at: ts(1_760_000_000_000),
})
.into_iter()
.collect(),
retention: RetentionPolicy::default(),
created_at: ts(1_760_000_000_000),
}
}
#[tokio::test]
async fn records_and_names_round_trip_through_the_layout() {
let root =
std::env::temp_dir().join(format!("rusty-artifacts-test-{}", uuid::Uuid::new_v4()));
let named = record(&"a".repeat(64), Some("weekly-report"));
let unnamed = record(&"b".repeat(64), None);
persist_record(&root, &format!("acme/{}", named.artifact_id), &named)
.await
.unwrap();
persist_record(&root, &unnamed.artifact_id, &unnamed)
.await
.unwrap();
persist_name(&root, "acme/weekly-report", &named.artifact_id)
.await
.unwrap();
let records = load_records(&root);
assert_eq!(records.len(), 2);
assert_eq!(records[&format!("acme/{}", named.artifact_id)], named);
assert_eq!(records[&unnamed.artifact_id], unnamed);
let names = load_names(&root);
assert_eq!(names["acme/weekly-report"], named.artifact_id);
persist_record(&root, &format!("acme/{}", "c".repeat(64)), &named)
.await
.unwrap();
assert_eq!(load_records(&root).len(), 2);
let _ = std::fs::remove_dir_all(root);
}
}