systemprompt_evaluation/lib.rs
1//! Evaluation domain crate for systemprompt.io.
2//!
3//! Closes the loop on the platform's AI request trace (`ai_requests` and its
4//! satellite tables, owned by `systemprompt-ai`): samples production traffic,
5//! scores it against rubrics with an LLM judge, and replays failures with a
6//! repair hint so repaired trajectories are scored and linked to the results
7//! they fix.
8//!
9//! Public surface is a typed [`EvaluationError`] boundary, repositories over
10//! the `eval_*` tables, and the services composing them:
11//! [`EvaluationService`], [`SamplerService`], [`JudgeService`],
12//! [`ReplayService`], and [`AutoImproveLoop`]. Judge and replay inference go
13//! through a [`DynAiProvider`](systemprompt_models::ai::DynAiProvider)
14//! supplied by the composition layer; it must be the platform's auditing
15//! implementation, which persists every request to `ai_requests`.
16//!
17//! Judge and replay requests are attributed with a job actor so sampling —
18//! which excludes `actor_kind = 'job'` — never grades the framework's own
19//! traffic.
20//!
21//! Copyright (c) systemprompt.io — Business Source License 1.1.
22//! See <https://systemprompt.io> for licensing details.
23
24pub mod error;
25pub mod extension;
26pub mod models;
27pub mod repository;
28pub mod services;
29
30pub use error::{EvaluationError, Result};
31pub use extension::EvaluationExtension;
32pub use models::{
33 CanonicalMessage, CanonicalPrompt, DimensionScore, EvalCase, EvalResult, EvalRun, EvalRunKind,
34 EvalRunStatus, JudgeVerdict, NewCaseParams, NewResultParams, NewRunParams, Rubric,
35 RubricDimension, SampleFilter, SampledRequest, TriggerSource, Verdict,
36};
37pub use repository::{
38 EvalCaseRepository, EvalJudgeCallRepository, EvalRepositories, EvalResultRepository,
39 EvalRubricRepository, EvalRunRepository, SamplingRepository,
40};
41pub use services::{
42 AutoImproveLoop, EvaluationService, JudgeService, LoopLimits, LoopReport, ReplayService,
43 RunRequest, SamplerService,
44};