use std::path::{Path, PathBuf};
use miette::{Result, miette};
use serde::{Deserialize, Serialize};
use tokio::fs;
use uuid::Uuid;
use crate::reasoning::runtime_error::RuntimeErrorCase;
use crate::{
daat_locus_paths::daat_locus_paths,
persistence::{PersistenceFileMode, write_bytes_atomic},
};
const EVALUATIONS_DIR_NAME: &str = "evaluations";
const RUNTIME_ERROR_CASES_DIR: &str = "runtime_error_cases";
const PROMPT_REFLECTIONS_DIR: &str = "prompt_reflections";
const RUNTIME_PROMPT_CANDIDATES_DIR: &str = "runtime_prompt_candidates";
const RUNTIME_PROMPT_CANDIDATE_EVALUATIONS_DIR: &str = "runtime_prompt_candidate_evaluations";
const MAX_ARTIFACT_FILE_STEM_LEN: usize = 96;
const LEGACY_RUNTIME_ERROR_CORRECTION_DIRS: &[&str] = &[
"failure_patterns",
"bootstrap_demos",
"stress_cases",
"instruction_hypotheses",
"runtime_demos",
"turn_demos",
];
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct EvaluationArtifactRuntimePromptCandidate {
pub compile_key: String,
pub title: String,
pub rationale: String,
#[serde(default)]
pub prompt_patches: Vec<String>,
#[serde(default)]
pub source_demo_titles: Vec<String>,
#[serde(default)]
pub source_hypotheses: Vec<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct EvaluationArtifactPromptReflection {
pub compile_key: String,
pub title: String,
pub rationale: String,
#[serde(default)]
pub missing_instructions: Vec<String>,
#[serde(default)]
pub over_constraints: Vec<String>,
#[serde(default)]
pub source_trace_ids: Vec<String>,
pub confidence: f64,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct EvaluationArtifactRuntimePromptCandidateEvaluation {
pub compile_key: String,
pub candidate_title: String,
pub rationale: String,
pub score: f64,
pub accepted: bool,
pub selected: bool,
pub regressions_detected: usize,
#[serde(default)]
pub source_trace_ids: Vec<String>,
}
pub struct RuntimeErrorCorrectionArtifacts<'a> {
pub runtime_error_cases: &'a [RuntimeErrorCase],
pub prompt_reflections: &'a [EvaluationArtifactPromptReflection],
pub runtime_prompt_candidates: &'a [EvaluationArtifactRuntimePromptCandidate],
pub runtime_prompt_candidate_evaluations:
&'a [EvaluationArtifactRuntimePromptCandidateEvaluation],
}
pub struct EvaluationArtifactsStore {
root: PathBuf,
}
impl EvaluationArtifactsStore {
pub async fn open() -> Result<Self> {
Self::open_scoped(None).await
}
pub async fn open_scoped(scope: Option<&str>) -> Result<Self> {
let mut root = daat_locus_paths().await.artifact_dir(EVALUATIONS_DIR_NAME);
if let Some(scope) = scope {
root = root.join(artifact_file_stem(scope));
}
ensure_dir(&root).await?;
ensure_dir(&root.join(RUNTIME_ERROR_CASES_DIR)).await?;
ensure_dir(&root.join(PROMPT_REFLECTIONS_DIR)).await?;
ensure_dir(&root.join(RUNTIME_PROMPT_CANDIDATES_DIR)).await?;
ensure_dir(&root.join(RUNTIME_PROMPT_CANDIDATE_EVALUATIONS_DIR)).await?;
Ok(Self { root })
}
pub async fn replace_runtime_error_cases(
&self,
artifacts: &[RuntimeErrorCase],
) -> Result<Vec<PathBuf>> {
let artifacts = artifacts
.iter()
.cloned()
.map(|artifact| (artifact.case_id.clone(), artifact))
.collect::<Vec<_>>();
replace_artifacts(&self.root.join(RUNTIME_ERROR_CASES_DIR), artifacts).await
}
pub async fn replace_runtime_prompt_candidates(
&self,
artifacts: &[EvaluationArtifactRuntimePromptCandidate],
) -> Result<Vec<PathBuf>> {
let artifacts = artifacts
.iter()
.cloned()
.map(|artifact| {
let slug = slugify(&artifact.title);
(format!("{}-{}", artifact.compile_key, slug), artifact)
})
.collect::<Vec<_>>();
replace_artifacts(&self.root.join(RUNTIME_PROMPT_CANDIDATES_DIR), artifacts).await
}
pub async fn replace_prompt_reflections(
&self,
artifacts: &[EvaluationArtifactPromptReflection],
) -> Result<Vec<PathBuf>> {
let artifacts = artifacts
.iter()
.cloned()
.map(|artifact| {
let slug = slugify(&artifact.title);
(format!("{}-{}", artifact.compile_key, slug), artifact)
})
.collect::<Vec<_>>();
replace_artifacts(&self.root.join(PROMPT_REFLECTIONS_DIR), artifacts).await
}
pub async fn replace_runtime_prompt_candidate_evaluations(
&self,
artifacts: &[EvaluationArtifactRuntimePromptCandidateEvaluation],
) -> Result<Vec<PathBuf>> {
let artifacts = artifacts
.iter()
.cloned()
.map(|artifact| {
let slug = slugify(&artifact.candidate_title);
(format!("{}-{}", artifact.compile_key, slug), artifact)
})
.collect::<Vec<_>>();
replace_artifacts(
&self.root.join(RUNTIME_PROMPT_CANDIDATE_EVALUATIONS_DIR),
artifacts,
)
.await
}
pub async fn replace_runtime_error_correction_artifacts(
&self,
artifacts: RuntimeErrorCorrectionArtifacts<'_>,
) -> Result<()> {
self.replace_runtime_error_cases(artifacts.runtime_error_cases)
.await?;
for dir_name in LEGACY_RUNTIME_ERROR_CORRECTION_DIRS {
reset_artifact_dir(&self.root.join(dir_name)).await?;
}
self.replace_prompt_reflections(artifacts.prompt_reflections)
.await?;
self.replace_runtime_prompt_candidates(artifacts.runtime_prompt_candidates)
.await?;
self.replace_runtime_prompt_candidate_evaluations(
artifacts.runtime_prompt_candidate_evaluations,
)
.await?;
Ok(())
}
}
async fn ensure_dir(path: &Path) -> Result<()> {
if !path.exists() {
fs::create_dir_all(path).await.map_err(|err| {
miette!(
"failed to create evaluation artifacts dir {}: {err}",
path.display()
)
})?;
}
Ok(())
}
async fn reset_artifact_dir(path: &Path) -> Result<()> {
if path.exists() {
fs::remove_dir_all(path).await.map_err(|err| {
miette!(
"failed to reset evaluation artifacts dir {}: {err}",
path.display()
)
})?;
}
ensure_dir(path).await
}
async fn save_artifact<T>(dir: &Path, stem: &str, artifact: &T) -> Result<PathBuf>
where
T: Serialize + Sync,
{
let file_name = format!("{}-{}.json", artifact_file_stem(stem), Uuid::new_v4());
let path = dir.join(file_name);
let bytes = serde_json::to_vec_pretty(artifact)
.map_err(|err| miette!("failed to serialize evaluation artifact: {err}"))?;
write_bytes_atomic(path.clone(), bytes, PersistenceFileMode::Default)
.await
.map_err(|err| {
miette!(
"failed to write evaluation artifact {}: {err}",
path.display()
)
})?;
Ok(path)
}
async fn replace_artifacts<S, T, I>(dir: &Path, artifacts: I) -> Result<Vec<PathBuf>>
where
T: Serialize + Send + Sync,
S: AsRef<str> + Send,
I: IntoIterator<Item = (S, T)> + Send,
I::IntoIter: Send,
{
if dir.exists() {
fs::remove_dir_all(dir).await.map_err(|err| {
miette!(
"failed to reset evaluation artifacts dir {}: {err}",
dir.display()
)
})?;
}
ensure_dir(dir).await?;
let mut paths = Vec::new();
for (stem, artifact) in artifacts {
let path = save_artifact(dir, stem.as_ref(), &artifact).await?;
paths.push(path);
}
Ok(paths)
}
fn slugify(value: &str) -> String {
let mut slug = String::with_capacity(value.len());
for ch in value.chars() {
if ch.is_ascii_alphanumeric() {
slug.push(ch.to_ascii_lowercase());
} else if matches!(ch, ' ' | '-' | '_' | '.') && !slug.ends_with('-') {
slug.push('-');
}
}
slug.trim_matches('-').to_string()
}
fn artifact_file_stem(value: &str) -> String {
let slug = slugify(value);
let slug = if slug.is_empty() {
"artifact"
} else {
slug.as_str()
};
slug.chars().take(MAX_ARTIFACT_FILE_STEM_LEN).collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn artifact_file_stem_is_bounded_and_non_empty() {
let stem = artifact_file_stem(
"tool call uses an unknown app id parameter and should be reported clearly",
);
assert!(!stem.is_empty());
assert!(stem.len() <= MAX_ARTIFACT_FILE_STEM_LEN);
}
}