1use std::collections::HashMap;
33use std::io;
34use std::path::{Path, PathBuf};
35
36use rusty_agent_runtime::learn::{CandidateRecord, VersionPointer};
37use rusty_agent_runtime::record::sha256_hex;
38use serde::{Deserialize, Serialize};
39
40pub(crate) fn candidates_dir(root: &Path) -> PathBuf {
43 root.join("learn").join("candidates")
44}
45
46pub(crate) fn versions_dir(root: &Path) -> PathBuf {
49 root.join("learn").join("versions")
50}
51
52pub(crate) async fn persist_candidate(
58 root: &Path,
59 scoped_id: &str,
60 record: &CandidateRecord,
61) -> io::Result<()> {
62 let dir = candidates_dir(root);
63 tokio::fs::create_dir_all(&dir).await?;
64 let bytes = serde_json::to_vec_pretty(record)
65 .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
66 let path = dir.join(format!("{scoped_id}.json"));
67 if let Some(parent) = path.parent() {
68 tokio::fs::create_dir_all(parent).await?;
69 }
70 let tmp = dir.join(format!("{scoped_id}.tmp"));
71 tokio::fs::write(&tmp, bytes).await?;
72 tokio::fs::rename(&tmp, path).await
73}
74
75fn collect_json_files(root: &Path, out: &mut Vec<PathBuf>) {
79 let Ok(entries) = std::fs::read_dir(root) else {
80 return;
81 };
82 for entry in entries.flatten() {
83 let path = entry.path();
84 if path.is_dir() {
85 collect_json_files(&path, out);
86 } else if path.extension().and_then(|e| e.to_str()) == Some("json") {
87 out.push(path);
88 }
89 }
90}
91
92fn path_scoped_id(dir: &Path, path: &Path) -> Option<String> {
97 path.strip_prefix(dir)
98 .ok()
99 .map(|relative| relative.with_extension(""))
100 .map(|relative| {
101 relative
102 .components()
103 .map(|component| component.as_os_str().to_string_lossy().into_owned())
104 .collect::<Vec<_>>()
105 .join("/")
106 })
107}
108
109pub(crate) fn load_candidates(root: &Path) -> HashMap<String, CandidateRecord> {
114 let dir = candidates_dir(root);
115 let mut out = HashMap::new();
116 let mut files = Vec::new();
117 collect_json_files(&dir, &mut files);
118 for path in files {
119 let scoped_id = path_scoped_id(&dir, &path);
120 let parsed = std::fs::read_to_string(&path)
121 .ok()
122 .and_then(|raw| serde_json::from_str::<CandidateRecord>(&raw).ok());
123 match (scoped_id, parsed) {
124 (Some(id), Some(record)) => {
125 out.insert(id, record);
126 }
127 _ => {
128 tracing::warn!(path = %path.display(), "skipping unreadable candidate file")
129 }
130 }
131 }
132 out
133}
134
135#[derive(Debug, Clone, Serialize, Deserialize)]
140struct VersionFile {
141 key: String,
144 pointer: VersionPointer,
146}
147
148fn version_file_name(scoped_surface: &str) -> String {
154 sha256_hex(scoped_surface.as_bytes())
155}
156
157pub(crate) async fn persist_version(
163 root: &Path,
164 scoped_surface: &str,
165 pointer: &VersionPointer,
166) -> io::Result<()> {
167 let dir = versions_dir(root);
168 tokio::fs::create_dir_all(&dir).await?;
169 let file = VersionFile {
170 key: scoped_surface.to_string(),
171 pointer: pointer.clone(),
172 };
173 let bytes = serde_json::to_vec_pretty(&file)
174 .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
175 let name = version_file_name(scoped_surface);
176 let tmp = dir.join(format!("{name}.tmp"));
177 tokio::fs::write(&tmp, bytes).await?;
178 tokio::fs::rename(&tmp, dir.join(format!("{name}.json"))).await
179}
180
181pub(crate) fn load_versions(root: &Path) -> HashMap<String, VersionPointer> {
188 let dir = versions_dir(root);
189 let mut out = HashMap::new();
190 let mut files = Vec::new();
191 collect_json_files(&dir, &mut files);
192 for path in files {
193 let parsed = std::fs::read_to_string(&path)
194 .ok()
195 .and_then(|raw| serde_json::from_str::<VersionFile>(&raw).ok());
196 let matches_name = parsed.as_ref().is_some_and(|file| {
197 path.file_stem().and_then(|s| s.to_str()) == Some(&*version_file_name(&file.key))
198 });
199 match (parsed, matches_name) {
200 (Some(file), true) => {
201 out.insert(file.key, file.pointer);
202 }
203 _ => {
204 tracing::warn!(path = %path.display(), "skipping unreadable version-pointer file")
205 }
206 }
207 }
208 out
209}
210
211use std::sync::Arc;
216
217use async_trait::async_trait;
218use chrono::{DateTime, Utc};
219use rusty_agent_runtime::error::{Result as RuntimeResult, RustyError};
220use rusty_agent_runtime::journal::{Journal, JournalSnapshot};
221use rusty_agent_runtime::learn::{
222 Candidate, CandidateContent, CandidateEvaluation, CandidateEvaluator, CandidateOverlay,
223 EvaluationRequest, EvaluationVerdict, ReplayDivergence, ReplaySummary,
224};
225use rusty_agent_runtime::memory::{MemoryQuery, MemoryRecord, MemoryStore, ProvenanceAuthor};
226use rusty_agent_runtime::record::PayloadRef;
227use rusty_agent_runtime::replay::ExactReplay;
228use rusty_eval::{
229 compare, CompareThresholds, Dataset, EvalCase, ExperimentConfig, ExperimentReport,
230 ExperimentRunner, PreparedRun,
231};
232
233use crate::server_store::ServerStore;
234
235fn invalid(message: impl Into<String>) -> RustyError {
239 RustyError::InvalidUpdate(message.into())
240}
241
242pub(crate) struct ServerMemoryStore {
248 store: Arc<dyn ServerStore>,
249 tenant: String,
250}
251
252impl std::fmt::Debug for ServerMemoryStore {
255 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
256 f.debug_struct("ServerMemoryStore")
257 .field("tenant", &self.tenant)
258 .finish_non_exhaustive()
259 }
260}
261
262impl ServerMemoryStore {
263 pub(crate) fn new(store: Arc<dyn ServerStore>, tenant: impl Into<String>) -> Self {
265 Self {
266 store,
267 tenant: tenant.into(),
268 }
269 }
270}
271
272#[async_trait]
273impl MemoryStore for ServerMemoryStore {
274 async fn put(&self, record: &MemoryRecord) -> RuntimeResult<bool> {
275 let content = match &record.content {
282 PayloadRef::Inline(value) => value.clone(),
283 PayloadRef::Artifact(_) => {
284 return Err(invalid(
285 "artifact-referenced memory content must be written through the governed \
286 write path",
287 ));
288 }
289 };
290 self.store
291 .put_memory(&self.tenant, record, &content)
292 .await
293 .map_err(|e| invalid(format!("memory put: {e}")))
294 }
295
296 async fn get(&self, memory_id: &str) -> RuntimeResult<Option<MemoryRecord>> {
297 self.store
298 .get_memory(&self.tenant, memory_id)
299 .await
300 .map_err(|e| invalid(format!("memory get: {e}")))
301 }
302
303 async fn all(&self) -> RuntimeResult<Vec<MemoryRecord>> {
304 let query = MemoryQuery {
308 include_superseded: true,
309 include_expired: true,
310 ..MemoryQuery::default()
311 };
312 self.store
313 .query_memory(&self.tenant, &query, Utc::now())
314 .await
315 .map_err(|e| invalid(format!("memory scan: {e}")))
316 }
317
318 async fn remove(&self, memory_id: &str) -> RuntimeResult<bool> {
319 self.store
320 .delete_memory(&self.tenant, memory_id)
321 .await
322 .map_err(|e| invalid(format!("memory delete: {e}")))
323 }
324
325 async fn query(
326 &self,
327 query: &MemoryQuery,
328 now: DateTime<Utc>,
329 ) -> RuntimeResult<Vec<MemoryRecord>> {
330 self.store
331 .query_memory(&self.tenant, query, now)
332 .await
333 .map_err(|e| invalid(format!("memory query: {e}")))
334 }
335}
336
337pub trait DatasetSource: Send + Sync + std::fmt::Debug {
342 fn load(&self, version: &str) -> Result<Dataset, String>;
344}
345
346#[derive(Debug)]
351pub struct DirectoryDatasetSource {
352 root: PathBuf,
353 dataset_name: String,
354}
355
356impl DirectoryDatasetSource {
357 pub fn new(root: impl Into<PathBuf>, dataset_name: impl Into<String>) -> Self {
359 Self {
360 root: root.into(),
361 dataset_name: dataset_name.into(),
362 }
363 }
364}
365
366impl DatasetSource for DirectoryDatasetSource {
367 fn load(&self, version: &str) -> Result<Dataset, String> {
368 let path = self
369 .root
370 .join(format!("{}@{version}.jsonl", self.dataset_name));
371 Dataset::load(&path).map_err(|e| e.to_string())
372 }
373}
374
375#[async_trait]
382pub trait EvaluationAgent: Send + Sync + std::fmt::Debug {
383 fn prepare(
389 &self,
390 case: &EvalCase,
391 journal: &Journal,
392 memory: Arc<dyn MemoryStore>,
393 ) -> RuntimeResult<PreparedRun>;
394
395 async fn redrive(
406 &self,
407 replay: &ExactReplay,
408 candidate: &Candidate,
409 memory: Arc<dyn MemoryStore>,
410 ) -> RuntimeResult<JournalSnapshot>;
411}
412
413fn metric_value(report: &ExperimentReport, metric: &str) -> Option<f64> {
418 if metric == "run_pass_rate" {
419 return Some(report.summary.run_pass_rate);
420 }
421 if metric == "case_pass_rate" {
422 return Some(report.summary.case_pass_rate);
423 }
424 if let Some(case_id) = metric.strip_prefix("case:") {
425 return report
426 .cases
427 .iter()
428 .find(|case| case.case_id == case_id)
429 .map(|case| case.pass_rate);
430 }
431 if let Some(key) = metric.strip_prefix("assertion:") {
432 return report
433 .summary
434 .assertions
435 .iter()
436 .find(|assertion| assertion.assertion == key)
437 .map(|assertion| assertion.rate);
438 }
439 None
440}
441
442#[derive(Debug)]
449pub struct EvalCandidateEvaluator {
450 baseline_memory: Arc<dyn MemoryStore>,
451 datasets: Arc<dyn DatasetSource>,
452 agent: Arc<dyn EvaluationAgent>,
453 runs_per_case: usize,
454 evaluated_by: ProvenanceAuthor,
455}
456
457impl EvalCandidateEvaluator {
458 pub fn new(
463 baseline_memory: Arc<dyn MemoryStore>,
464 datasets: Arc<dyn DatasetSource>,
465 agent: Arc<dyn EvaluationAgent>,
466 runs_per_case: usize,
467 evaluated_by: ProvenanceAuthor,
468 ) -> Self {
469 Self {
470 baseline_memory,
471 datasets,
472 agent,
473 runs_per_case: runs_per_case.max(1),
474 evaluated_by,
475 }
476 }
477}
478
479#[async_trait]
480impl CandidateEvaluator for EvalCandidateEvaluator {
481 async fn evaluate(
482 &self,
483 candidate: &Candidate,
484 request: &EvaluationRequest,
485 ) -> RuntimeResult<CandidateEvaluation> {
486 let dataset = self
487 .datasets
488 .load(&request.dataset_version)
489 .map_err(|e| invalid(format!("dataset `{}`: {e}", request.dataset_version)))?;
490
491 let candidate_memory: Arc<dyn MemoryStore> = match &candidate.content {
498 CandidateContent::MemorySet { .. } => Arc::new(CandidateOverlay::new(
499 self.baseline_memory.clone(),
500 candidate,
501 )?),
502 _ => self.baseline_memory.clone(),
503 };
504
505 let runner =
506 ExperimentRunner::new(ExperimentConfig::new().with_runs_per_case(self.runs_per_case));
507 let baseline_memory = self.baseline_memory.clone();
508 let baseline_agent = self.agent.clone();
509 let baseline = runner
510 .run(&dataset, |case, journal| {
511 baseline_agent.prepare(case, journal, baseline_memory.clone())
512 })
513 .await
514 .map_err(|e| invalid(format!("baseline experiment: {e}")))?;
515 let candidate_agent = self.agent.clone();
516 let overlay_memory = candidate_memory.clone();
517 let candidate_report = runner
518 .run(&dataset, move |case, journal| {
519 candidate_agent.prepare(case, journal, overlay_memory.clone())
520 })
521 .await
522 .map_err(|e| invalid(format!("candidate experiment: {e}")))?;
523
524 let comparison = compare(
525 &baseline,
526 &candidate_report,
527 &CompareThresholds {
528 max_pass_rate_drop: request.thresholds.max_pass_rate_drop,
529 max_latency_p95_ratio: request.thresholds.max_latency_p95_ratio,
530 },
531 );
532 let baseline_metric = metric_value(&baseline, &request.target_metric);
533 let candidate_metric = metric_value(&candidate_report, &request.target_metric);
534 let verdict = EvaluationVerdict {
535 regressed: comparison.regressed,
536 target_metric: request.target_metric.clone(),
537 baseline: baseline_metric,
538 candidate: candidate_metric,
539 delta: baseline_metric
540 .zip(candidate_metric)
541 .map(|(base, cand)| cand - base),
542 };
543
544 let mut fixture_ids = Vec::with_capacity(request.replay_evidence.len());
554 let mut matched = 0usize;
555 let mut divergences = Vec::new();
556 for fixture in &request.replay_evidence {
557 fixture_ids.push(fixture.run_id.clone());
558 let divergence = match ExactReplay::new(fixture.clone()) {
559 Ok(replay) => match self
560 .agent
561 .redrive(&replay, candidate, candidate_memory.clone())
562 .await
563 {
564 Ok(replayed) => match replay.verify(&replayed) {
565 Ok(()) => None,
566 Err(e) => Some(e.to_string()),
567 },
568 Err(e) => Some(e.to_string()),
569 },
570 Err(e) => Some(e.to_string()),
571 };
572 match divergence {
573 None => matched += 1,
574 Some(detail) => divergences.push(ReplayDivergence {
575 fixture_id: fixture.run_id.clone(),
576 detail,
577 }),
578 }
579 }
580 fixture_ids.sort();
581 divergences.sort_by(|a, b| a.fixture_id.cmp(&b.fixture_id));
582
583 Ok(CandidateEvaluation {
584 candidate_id: candidate.candidate_id.clone(),
585 dataset_version: dataset.version().to_owned(),
586 replay: ReplaySummary {
587 fixture_ids,
588 matched,
589 divergences,
590 },
591 baseline_report: serde_json::to_value(&baseline)?,
592 candidate_report: serde_json::to_value(&candidate_report)?,
593 verdict,
594 thresholds: request.thresholds,
595 evaluated_by: self.evaluated_by.clone(),
596 evaluated_at: Utc::now(),
597 })
598 }
599}
600
601#[cfg(test)]
602mod tests {
603 use super::*;
604 use chrono::{DateTime, Utc};
605 use rusty_agent_runtime::learn::{
606 Candidate, CandidateContent, EvidenceSpan, PromotionAuthority, PromotionDecision,
607 PromotionReceipt,
608 };
609 use rusty_agent_runtime::memory::ProvenanceAuthor;
610
611 fn ts(millis: i64) -> DateTime<Utc> {
612 DateTime::<Utc>::from_timestamp_millis(millis).unwrap()
613 }
614
615 fn candidate() -> Candidate {
616 Candidate::new(
617 CandidateContent::Prompt {
618 name: "system".into(),
619 prompt: "Answer tersely.".into(),
620 },
621 ProvenanceAuthor::Distiller {
622 name: "correction-loop".into(),
623 },
624 EvidenceSpan::default(),
625 ts(1_750_000_002_000),
626 )
627 .unwrap()
628 }
629
630 fn pointer(candidate: &Candidate) -> VersionPointer {
631 VersionPointer::new(candidate.surface()).promoted(&PromotionReceipt {
632 candidate_id: candidate.candidate_id.clone(),
633 surface: candidate.surface(),
634 previous: None,
635 decision: PromotionDecision {
636 authority: PromotionAuthority::Envelope {
637 envelope_version: "r0.8-default".into(),
638 },
639 canary: None,
640 },
641 promoted_at: ts(1_750_000_004_000),
642 })
643 }
644
645 #[tokio::test]
646 async fn candidates_round_trip_keyed_by_path_with_corrupt_tolerance() {
647 let root = std::env::temp_dir().join(format!("rusty-learn-test-{}", uuid::Uuid::new_v4()));
648 let record = CandidateRecord::new(candidate());
649 let scoped = record.candidate.candidate_id.to_string();
650 persist_candidate(&root, &scoped, &record).await.unwrap();
651 let tenant_scoped = format!("acme/{scoped}");
652 persist_candidate(&root, &tenant_scoped, &record)
653 .await
654 .unwrap();
655 std::fs::write(candidates_dir(&root).join("broken.json"), b"{nope").unwrap();
656
657 let loaded = load_candidates(&root);
658 assert_eq!(loaded.len(), 2, "corrupt files are skipped, not fatal");
659 assert!(loaded.contains_key(&scoped), "default tenant: bare key");
660 assert_eq!(
661 loaded[&tenant_scoped].candidate.candidate_id, record.candidate.candidate_id,
662 "named tenant: the key comes from the path, the record keeps the bare id"
663 );
664 let _ = std::fs::remove_dir_all(root);
665 }
666
667 #[tokio::test]
668 async fn versions_round_trip_through_hashed_filenames() {
669 let root = std::env::temp_dir().join(format!("rusty-learn-test-{}", uuid::Uuid::new_v4()));
670 let candidate = candidate();
671 let pointer = pointer(&candidate);
672 let scoped_surface = format!("acme/{}", candidate.surface().as_str());
673 persist_version(&root, &scoped_surface, &pointer)
674 .await
675 .unwrap();
676
677 let listing: Vec<String> = std::fs::read_dir(versions_dir(&root))
680 .unwrap()
681 .flatten()
682 .map(|entry| entry.file_name().to_string_lossy().into_owned())
683 .collect();
684 assert_eq!(
685 listing,
686 vec![format!("{}.json", sha256_hex(scoped_surface.as_bytes()))]
687 );
688
689 let loaded = load_versions(&root);
690 assert_eq!(loaded.len(), 1);
691 assert_eq!(loaded[&scoped_surface], pointer);
692
693 let stray = VersionFile {
697 key: "memory:user:someone-else".into(),
698 pointer: VersionPointer::new(rusty_agent_runtime::learn::SurfaceKey::new(
699 "memory:user:someone-else",
700 )),
701 };
702 let stray_bytes = serde_json::to_vec_pretty(&stray).unwrap();
703 std::fs::write(
704 versions_dir(&root).join(format!("{}.json", sha256_hex(b"memory:user:someone-else"))),
705 &stray_bytes,
706 )
707 .unwrap();
708 std::fs::write(
709 versions_dir(&root).join(format!("{}.json", sha256_hex(b"forged-name"))),
710 &stray_bytes,
711 )
712 .unwrap();
713 let loaded = load_versions(&root);
714 assert_eq!(loaded.len(), 2, "the forged-name file is skipped");
715 assert!(loaded.contains_key("memory:user:someone-else"));
716 let _ = std::fs::remove_dir_all(root);
717 }
718}