use crate::core::hex_util::hex_lower;
use agentcarousel_core::{CaseStatus, Role, Run};
use agentcarousel_reporters::{fetch_run, list_runs};
use chrono::Utc;
use clap::Parser;
use flate2::write::GzEncoder;
use flate2::Compression;
use serde_json::json;
use sha2::{Digest, Sha256};
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use tar::Builder;
const SKILL_DEFINITION_SCHEMA: &str = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/schemas/skill-definition.schema.json"
));
use super::exit_codes::ExitCode;
use super::output::{JsonError, JsonOutput};
use super::GlobalOptions;
#[derive(Debug, Parser)]
pub struct ExportArgs {
#[arg(value_name = "RUN_ID")]
run_id_positional: Option<String>,
#[arg(short = 'l', long, value_name = "N")]
last: Option<usize>,
#[arg(short = 'o', long)]
out: Option<PathBuf>,
#[arg(short = 'd', long, value_name = "DIR")]
out_dir: Option<PathBuf>,
}
const EXPORT_LAST_MAX: usize = 50;
pub fn run_export(args: ExportArgs, globals: &GlobalOptions) -> i32 {
let json = globals.json;
let run_id = args.run_id_positional.as_ref();
match (run_id, args.last) {
(Some(_), Some(_)) => {
if json {
JsonOutput::err(
"export",
JsonError::new(
"invalid_args",
"specify either RUN_ID or --last N, not both",
),
)
.print();
} else {
eprintln!("error: specify either RUN_ID or --last N, not both");
}
ExitCode::RuntimeError.as_i32()
}
(None, None) => {
if json {
JsonOutput::err(
"export",
JsonError::new("invalid_args", "specify RUN_ID or --last N").with_suggestions(
vec!["Example: agc export --last 5 --out-dir ./evidence".to_string()],
),
)
.print();
} else {
eprintln!(
"error: specify RUN_ID or --last N (e.g. export --last 5 --out-dir ./evidence)"
);
}
ExitCode::RuntimeError.as_i32()
}
(Some(run_id), None) => {
if args.out_dir.is_some() {
if json {
JsonOutput::err(
"export",
JsonError::new("invalid_args", "--out-dir is only valid with --last"),
)
.print();
} else {
eprintln!("error: --out-dir is only valid with --last");
}
return ExitCode::RuntimeError.as_i32();
}
match export_run_artifact(run_id, args.out.as_deref()) {
Ok(path) => {
if json {
JsonOutput::ok(
"export",
serde_json::json!({ "paths": [path.display().to_string()] }),
)
.print();
} else {
println!("created {}", path.display());
}
ExitCode::Ok.as_i32()
}
Err(err) => {
if json {
JsonOutput::err("export", JsonError::new("runtime_error", err)).print();
} else {
eprintln!("error: {err}");
}
ExitCode::RuntimeError.as_i32()
}
}
}
(None, Some(n)) => {
if args.out.is_some() {
if json {
JsonOutput::err(
"export",
JsonError::new(
"invalid_args",
"with --last, use --out-dir for the output directory (not --out)",
),
)
.print();
} else {
eprintln!(
"error: with --last, use --out-dir for the output directory (not --out)"
);
}
return ExitCode::RuntimeError.as_i32();
}
match export_last_n(n, args.out_dir.as_deref()) {
Ok(paths) => {
if json {
let strs: Vec<String> =
paths.iter().map(|p| p.display().to_string()).collect();
JsonOutput::ok("export", serde_json::json!({ "paths": strs })).print();
} else {
for path in paths {
println!("created {}", path.display());
}
}
ExitCode::Ok.as_i32()
}
Err(err) => {
if json {
JsonOutput::err("export", JsonError::new("runtime_error", err)).print();
} else {
eprintln!("error: {err}");
}
ExitCode::RuntimeError.as_i32()
}
}
}
}
}
fn export_last_n(n: usize, out_dir: Option<&Path>) -> Result<Vec<PathBuf>, String> {
if n == 0 || n > EXPORT_LAST_MAX {
return Err(format!(
"--last N must be between 1 and {EXPORT_LAST_MAX} (got {n})"
));
}
let listings = list_runs(n).map_err(|e| e.to_string())?;
if listings.is_empty() {
println!("no runs recorded");
return Ok(Vec::new());
}
if listings.len() < n {
eprintln!(
"note: only {} run(s) in history (requested {})",
listings.len(),
n
);
}
let base = out_dir.unwrap_or_else(|| Path::new("."));
fs::create_dir_all(base).map_err(|e| e.to_string())?;
let mut paths = Vec::new();
for listing in listings {
let path = base.join(format!("agentcarousel-evidence-{}.tar.gz", listing.id));
paths.push(export_run_artifact(&listing.id, Some(&path))?);
}
Ok(paths)
}
pub(crate) fn export_run_artifact(run_id: &str, out: Option<&Path>) -> Result<PathBuf, String> {
let run = fetch_run(run_id).map_err(|err| err.to_string())?;
let root = std::env::temp_dir().join(format!("agentcarousel-evidence-{run_id}"));
if root.exists() {
fs::remove_dir_all(&root).map_err(|err| err.to_string())?;
}
fs::create_dir_all(&root).map_err(|err| err.to_string())?;
let run_json_path = root.join("run.json");
write_json(&run_json_path, &run)?;
let bundle_lock_path = root.join("fixture_bundle.lock");
let schema_hash = fixture_schema_sha256();
let bundle_lock = json!({
"fixture_bundle_id": run.fixture_bundle_id,
"fixture_bundle_version": run.fixture_bundle_version,
"schema_hash": schema_hash
});
write_json(&bundle_lock_path, &bundle_lock)?;
let env_path = root.join("environment_fingerprint.json");
let mut env_payload = serde_json::Map::new();
env_payload.insert(
"agentcarousel_version".to_string(),
json!(run.agentcarousel_version),
);
env_payload.insert("os".to_string(), json!(std::env::consts::OS));
env_payload.insert("arch".to_string(), json!(std::env::consts::ARCH));
env_payload.insert("rust_version".to_string(), json!("unknown"));
env_payload.insert("timestamp_utc".to_string(), json!(Utc::now().to_rfc3339()));
env_payload.insert("git_sha".to_string(), json!(run.git_sha));
env_payload.insert(
"fixture_bundle_id".to_string(),
json!(run.fixture_bundle_id),
);
env_payload.insert(
"fixture_bundle_version".to_string(),
json!(run.fixture_bundle_version),
);
if let Ok(v) = std::env::var("GITHUB_REF") {
let t = v.trim();
if !t.is_empty() {
env_payload.insert("github_ref".to_string(), json!(t));
}
}
if let Ok(v) = std::env::var("GITHUB_RUN_ID") {
let t = v.trim();
if !t.is_empty() {
env_payload.insert("github_run_id".to_string(), json!(t));
}
}
write_json(&env_path, &serde_json::Value::Object(env_payload))?;
let redaction_path = root.join("REDACTION_POLICY.md");
let mut file = fs::File::create(&redaction_path).map_err(|err| err.to_string())?;
file.write_all(b"Redaction policy: trace outputs are scrubbed of common secrets and tokens.\n")
.map_err(|err| err.to_string())?;
let report_md_path = root.join("report.md");
let report_md = render_markdown_report(&run);
let mut file = fs::File::create(&report_md_path).map_err(|err| err.to_string())?;
file.write_all(report_md.as_bytes())
.map_err(|err| err.to_string())?;
let manifest_path = root.join("MANIFEST.json");
let manifest = build_manifest(&root)?;
write_json(&manifest_path, &manifest)?;
let out_path = out
.map(|path| path.to_path_buf())
.unwrap_or_else(|| PathBuf::from(format!("agentcarousel-evidence-{run_id}.tar.gz")));
let archive = fs::File::create(&out_path).map_err(|err| err.to_string())?;
let encoder = GzEncoder::new(archive, Compression::default());
let mut tar = Builder::new(encoder);
tar.append_dir_all(format!("agentcarousel-evidence-{run_id}"), &root)
.map_err(|err| err.to_string())?;
tar.finish().map_err(|err| err.to_string())?;
fs::remove_dir_all(&root).ok();
Ok(out_path)
}
fn render_markdown_report(run: &Run) -> String {
use std::fmt::Write as _;
let mut md = String::new();
let skill = run.skill_or_agent.as_deref().unwrap_or("unknown");
let _ = writeln!(md, "# agentcarousel Evidence Report — {skill}");
let _ = writeln!(md);
let _ = writeln!(md, "**Run ID:** {}", run.id.0);
if let Some(cmd_line) = run.summary.command_line.as_ref() {
let _ = writeln!(md, "**Command:** `{cmd_line}`");
} else {
let _ = writeln!(md, "**Command:** {}", run.command);
}
let _ = writeln!(
md,
"**Started:** {}",
run.started_at.format("%Y-%m-%d %H:%M:%S UTC")
);
if let Some(finished) = run.finished_at {
let _ = writeln!(
md,
"**Finished:** {}",
finished.format("%Y-%m-%d %H:%M:%S UTC")
);
}
let _ = writeln!(
md,
"**agentcarousel version:** {}",
run.agentcarousel_version
);
let _ = writeln!(md);
let s = &run.summary;
let _ = writeln!(md, "## Summary");
let _ = writeln!(md);
let result_icon = if s.failed == 0 && s.errored == 0 && s.timed_out == 0 {
"✅"
} else {
"❌"
};
let pass_rate_pct = s.pass_rate * 100.0;
let _ = writeln!(
md,
"{result_icon} **{}/{} passed** ({pass_rate_pct:.1}%)",
s.passed, s.total
);
let _ = writeln!(md);
let _ = writeln!(md, "| Metric | Value |");
let _ = writeln!(md, "|--------|-------|");
let _ = writeln!(md, "| Total | {} |", s.total);
let _ = writeln!(md, "| Passed | {} |", s.passed);
let _ = writeln!(md, "| Failed | {} |", s.failed);
if s.errored > 0 {
let _ = writeln!(md, "| Errored | {} |", s.errored);
}
if s.timed_out > 0 {
let _ = writeln!(md, "| Timed out | {} |", s.timed_out);
}
if let Some(eff) = s.mean_effectiveness_score {
let _ = writeln!(md, "| Effectiveness score | {eff:.2} / 1.00 |");
}
let _ = writeln!(md, "| Mean latency | {:.0}ms |", s.mean_latency_ms);
if let (Some(p50), Some(p95), Some(p99)) =
(s.latency_p50_ms, s.latency_p95_ms, s.latency_p99_ms)
{
let _ = writeln!(
md,
"| Latency p50 / p95 / p99 | {p50:.0}ms / {p95:.0}ms / {p99:.0}ms |"
);
}
let _ = writeln!(md);
if s.generator_model.is_some() || s.judge_model.is_some() {
let _ = writeln!(md, "### Models");
let _ = writeln!(md);
if let Some(gen_model) = s.generator_model.as_ref() {
let gen_tokens = match (s.tokens_in, s.tokens_out) {
(Some(i), Some(o)) => format!(" ({i} in / {o} out tokens)"),
_ => String::new(),
};
let gen_cost = s
.gen_cost_usd
.map(|c| format!(" · ${c:.4}"))
.unwrap_or_default();
let _ = writeln!(md, "- **Generator:** `{gen_model}`{gen_tokens}{gen_cost}");
}
if let Some(judge_model) = s.judge_model.as_ref() {
let judge_tokens = match (s.judge_tokens_in, s.judge_tokens_out) {
(Some(i), Some(o)) => format!(" ({i} in / {o} out tokens)"),
_ => String::new(),
};
let judge_cost = s
.judge_cost_usd
.map(|c| format!(" · ${c:.4}"))
.unwrap_or_default();
let _ = writeln!(md, "- **Judge:** `{judge_model}`{judge_tokens}{judge_cost}");
}
if let Some(total_cost) = s.total_cost_usd {
let _ = writeln!(md, "- **Total cost:** ${total_cost:.4}");
}
let _ = writeln!(md);
} else {
if let (Some(gin), Some(gout)) = (s.tokens_in, s.tokens_out) {
let _ = writeln!(md, "- Generator tokens: {gin} in / {gout} out");
}
if let (Some(jin), Some(jout)) = (s.judge_tokens_in, s.judge_tokens_out) {
let _ = writeln!(md, "- Judge tokens: {jin} in / {jout} out");
}
if let Some(cost) = s.gen_cost_usd {
let _ = writeln!(md, "- Generator cost: ${cost:.4}");
}
if let Some(cost) = s.judge_cost_usd {
let _ = writeln!(md, "- Judge cost: ${cost:.4}");
}
if let Some(cost) = s.total_cost_usd {
let _ = writeln!(md, "- Total cost: ${cost:.4}");
}
let _ = writeln!(md);
}
let _ = writeln!(md, "---");
let _ = writeln!(md);
let _ = writeln!(md, "## Cases");
let _ = writeln!(md);
for case in &run.cases {
let id = &case.case_id.0;
let (status_icon, status_label) = match case.status {
CaseStatus::Passed => ("✅", "Passed"),
CaseStatus::Failed => ("❌", "Failed"),
CaseStatus::Error => ("⚠️", "Error"),
CaseStatus::TimedOut => ("⏱️", "Timed Out"),
CaseStatus::Skipped => ("⏭️", "Skipped"),
CaseStatus::Flaky => ("⚠️", "Flaky"),
};
let latency_ms = case.metrics.total_latency_ms;
let eff_str = case
.eval_scores
.as_ref()
.map(|s| format!(" · effectiveness {:.2}", s.effectiveness_score))
.unwrap_or_default();
let _ = writeln!(
md,
"### {status_icon} {id} <sup>{status_label} · {latency_ms}ms{eff_str}</sup>"
);
let _ = writeln!(md);
if !case.input.is_empty() {
let _ = writeln!(md, "**Input:**");
let _ = writeln!(md);
for msg in &case.input {
let role = match msg.role {
Role::User => "user",
Role::Assistant => "assistant",
Role::System => "system",
Role::Tool => "tool",
};
let _ = writeln!(md, "**[{role}]**");
for line in msg.content.trim().lines() {
let _ = writeln!(md, "> {line}");
}
let _ = writeln!(md);
}
}
if let Some(reply) = case.trace.final_output.as_ref() {
let reply = reply.trim();
if !reply.is_empty() {
let _ = writeln!(md, "**Agent replied:**");
let _ = writeln!(md);
for line in reply.lines() {
let _ = writeln!(md, "> {line}");
}
let _ = writeln!(md);
}
}
if let Some(scores) = case.eval_scores.as_ref() {
if !scores.rubric_scores.is_empty() {
let _ = writeln!(md, "**Rubric:**");
let _ = writeln!(md);
let _ = writeln!(md, "| | Criterion | Score | Weight |");
let _ = writeln!(md, "|--|-----------|-------|--------|");
for rs in &scores.rubric_scores {
let icon = if rs.score >= 0.9 {
"✅"
} else if rs.score >= 0.5 {
"⚠️"
} else {
"❌"
};
let _ = writeln!(
md,
"| {icon} | {} | {:.2} | {:.2} |",
rs.rubric_id, rs.score, rs.weight
);
}
let _ = writeln!(md);
}
if let Some(rationale) = scores.judge_rationale.as_ref() {
let rationale = rationale.trim();
if !rationale.is_empty() {
let _ = writeln!(md, "**Judge:** {rationale}");
let _ = writeln!(md);
}
}
let case_result_icon = match case.status {
CaseStatus::Passed => "✅",
CaseStatus::Failed => "❌",
_ => "⚠️",
};
let _ = writeln!(
md,
"{case_result_icon} **Effectiveness: {:.2}**",
scores.effectiveness_score
);
let _ = writeln!(md);
}
if let Some(err) = case.error.as_ref() {
if !err.is_empty() {
let _ = writeln!(md, "**Error:** {err}");
let _ = writeln!(md);
}
}
let _ = writeln!(md, "---");
let _ = writeln!(md);
}
md
}
fn write_json<T: serde::Serialize>(path: &Path, value: &T) -> Result<(), String> {
let payload = serde_json::to_string_pretty(value).map_err(|err| err.to_string())?;
let mut file = fs::File::create(path).map_err(|err| err.to_string())?;
file.write_all(payload.as_bytes())
.map_err(|err| err.to_string())?;
Ok(())
}
fn fixture_schema_sha256() -> String {
let mut hasher = Sha256::new();
hasher.update(SKILL_DEFINITION_SCHEMA.as_bytes());
format!("sha256:{}", hex_lower(hasher.finalize().as_ref()))
}
fn sha256_file_hex(path: &Path) -> Result<String, String> {
let bytes = fs::read(path).map_err(|err| err.to_string())?;
let mut hasher = Sha256::new();
hasher.update(&bytes);
Ok(format!("sha256:{}", hex_lower(hasher.finalize().as_ref())))
}
fn build_manifest(root: &Path) -> Result<serde_json::Value, String> {
let tracked = [
"run.json",
"fixture_bundle.lock",
"environment_fingerprint.json",
"REDACTION_POLICY.md",
"report.md",
];
let mut files = Vec::new();
for name in tracked {
let path = root.join(name);
let digest = sha256_file_hex(&path)?;
files.push(json!({
"path": name,
"sha256": digest
}));
}
Ok(json!({
"manifest_version": 1,
"files": files
}))
}