rustev_surface/
eval.rs

1//! `eval` and `gate` (spec 006, 3.9), and the dataset and bundle handling
2//! `calibrate fit` shares, for every host.
3
4use std::collections::{BTreeMap, BTreeSet};
5
6use rustev_contract::Document;
7use rustev_contract::Identified;
8use rustev_contract::canonical::record_canonical_bytes;
9use rustev_contract::eval_report::{EvalReport, Split};
10use rustev_contract::limits::{DESCRIPTOR_V1, RECORD_V1, REPLAY_V1};
11use rustev_contract::replay::ReplayBundle;
12use rustev_eval::bundle::{BundleInput, BundleLoad, LoadFailure};
13use rustev_eval::config::EvaluatorConfig;
14use rustev_eval::dataset::{DatasetCase, DatasetManifest};
15use rustev_eval::metrics::TaskAdapter;
16use rustev_eval::report::{
17    Candidate, EvalDetail, Evaluated, GateResult, Inputs, evaluate, evaluate_gate,
18};
19
20use rustev_eval::replay::ReplayConfig;
21use rustev_eval::resolve::Resolver;
22
23use crate::adapter::TaskAdapterDoc;
24use crate::deps::{self, DepArgs, RulesHook};
25use crate::host::{Host, IoFail, ReadFail};
26use crate::out::{Done, Out, Res, code};
27use crate::plan::{dep_failed, load_plan, read_plan};
28
29/// A split by its name (spec 006, 3.9).
30pub fn split_named(name: &str) -> Option<Split> {
31    match name {
32        "training" => Some(Split::Training),
33        "model_selection" => Some(Split::ModelSelection),
34        "calibration" => Some(Split::Calibration),
35        "final_test" => Some(Split::FinalTest),
36        _ => None,
37    }
38}
39
40/// Read and parse one document of type `T` under `limit`, as
41/// `invalid_input` when it is refused.
42pub fn read_doc<T: Document>(
43    command: &str,
44    host: &dyn Host,
45    path: &str,
46    limit: usize,
47) -> Result<T, Done> {
48    let bytes = host.read(path, limit).map_err(|e| Done::io(command, e))?;
49    T::parse(&bytes).map_err(|e| Done::invalid(command, path, e.to_string()))
50}
51
52/// Case ids that map to bundle files (spec 006, 3.9.1): 1 to 128 bytes of
53/// `[A-Za-z0-9._-]` starting with an alphanumeric, distinct ignoring ASCII
54/// case.
55pub fn check_case_ids<'a>(
56    command: &str,
57    dataset: &str,
58    cases: impl Iterator<Item = &'a DatasetCase>,
59) -> Result<(), Done> {
60    let mut seen = BTreeSet::new();
61    for c in cases {
62        let id = c.id.as_str();
63        let ok = !id.is_empty()
64            && id.len() <= 128
65            && id.as_bytes()[0].is_ascii_alphanumeric()
66            && id
67                .bytes()
68                .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-'));
69        if !ok {
70            return Err(Done::invalid(
71                command,
72                dataset,
73                format!("case id {id:?} is not a bundle file name"),
74            ));
75        }
76        if !seen.insert(id.to_ascii_lowercase()) {
77            return Err(Done::invalid(
78                command,
79                dataset,
80                format!("case id {id:?} differs from another only by case"),
81            ));
82        }
83    }
84    Ok(())
85}
86
87/// The bundles of `cases` from `dir` (spec 015, 3.3): absent files are left
88/// out for the library to judge (`bundle-missing`); a file that exists but
89/// cannot be read, is past the document limit or does not parse is
90/// supplied as that case's load failure, and the command continues. Only
91/// an exhausted read budget stops it, because that depends on read order
92/// rather than on the bundle.
93pub fn read_bundles<'a>(
94    command: &str,
95    host: &dyn Host,
96    dir: &str,
97    cases: impl Iterator<Item = &'a DatasetCase>,
98) -> Result<BTreeMap<String, BundleInput>, Done> {
99    let mut bundles = BTreeMap::new();
100    for c in cases {
101        let path = host.join(dir, &format!("{}.json", c.id));
102        let input = match host.read_classified(&path, REPLAY_V1.max_bytes.saturating_add(1)) {
103            Ok(None) => continue,
104            Err(ReadFail::Budget(e)) => return Err(Done::io(command, e)),
105            Err(ReadFail::File(e)) => LoadFailure::new(BundleLoad::Inaccessible, &e.detail).into(),
106            Ok(Some(bytes)) if bytes.len() > REPLAY_V1.max_bytes => LoadFailure::new(
107                BundleLoad::Oversized,
108                &format!("more than {} bytes", REPLAY_V1.max_bytes),
109            )
110            .into(),
111            Ok(Some(bytes)) => match ReplayBundle::parse(&bytes) {
112                Ok(b) => b.into(),
113                Err(e) => LoadFailure::new(BundleLoad::Corrupt, &e.to_string()).into(),
114            },
115        };
116        bundles.insert(c.id.clone(), input);
117    }
118    Ok(bundles)
119}
120
121/// Write every `(name, bytes)` into `dir`, all or none. The bytes move
122/// into the host's list; nothing is copied.
123pub fn write_all(
124    command: &str,
125    host: &dyn Host,
126    dir: &str,
127    files: Vec<(&str, Vec<u8>)>,
128) -> Result<(), Done> {
129    let files: Vec<(String, Vec<u8>)> = files
130        .into_iter()
131        .map(|(n, b)| (host.join(dir, n), b))
132        .collect();
133    for (p, _) in &files {
134        host.ensure_absent(p).map_err(|e| Done::io(command, e))?;
135    }
136    host.write_all(&files).map_err(|e| Done::io(command, e))
137}
138
139pub const OUTPUTS: [&str; 4] = ["report.json", "detail.json", "config.json", "adapter.json"];
140
141/// What `eval` reads and writes.
142#[derive(Debug, Clone, Copy)]
143pub struct EvalArgs<'a> {
144    pub split: Split,
145    pub replay: &'a ReplayConfig,
146    pub out: &'a str,
147    pub dataset: &'a str,
148    pub config: &'a str,
149    pub adapter: &'a str,
150    pub bundles: &'a str,
151    pub candidate_plan: Option<&'a str>,
152    pub deps: DepArgs<'a>,
153}
154
155/// `eval`: a report, its detail, the configuration and the adapter,
156/// published in `out`.
157pub fn eval(
158    command: &str,
159    host: &dyn Host,
160    a: EvalArgs<'_>,
161    resolver: &dyn Resolver,
162    rules: &mut RulesHook<'_>,
163) -> Res {
164    let split = a.split;
165    let replay = a.replay;
166    (|| {
167        let out_dir = a.out;
168        for name in OUTPUTS {
169            host.ensure_absent(&host.join(out_dir, name))
170                .map_err(|e| Done::io(command, e))?;
171        }
172        let dataset_path = a.dataset;
173        let manifest: DatasetManifest = read_doc(command, host, dataset_path, REPLAY_V1.max_bytes)?;
174        let config_path = a.config;
175        let config: EvaluatorConfig = read_doc(command, host, config_path, REPLAY_V1.max_bytes)?;
176        config
177            .check()
178            .map_err(|e| Done::invalid(command, config_path, e.to_string()))?;
179        let adapter_path = a.adapter;
180        let adapter: TaskAdapterDoc =
181            read_doc(command, host, adapter_path, DESCRIPTOR_V1.max_bytes)?;
182        adapter
183            .check()
184            .map_err(|e| Done::invalid(command, adapter_path, e))?;
185        if config.adapter.reference() != adapter.adapter() {
186            return Err(Done::invalid(
187                command,
188                adapter_path,
189                "the configuration names another adapter",
190            ));
191        }
192        if config
193            .adapter
194            .rules
195            .as_ref()
196            .is_some_and(|r| *r != adapter.rules())
197        {
198            return Err(Done::invalid(
199                command,
200                adapter_path,
201                "the configuration binds other adapter rules",
202            ));
203        }
204        manifest
205            .check(&adapter.adapter(), |l| adapter.check_label(l))
206            .map_err(|e| Done::invalid(command, dataset_path, e.to_string()))?;
207        check_case_ids(command, dataset_path, manifest.cases.iter())?;
208        let bundles = read_bundles(command, host, a.bundles, manifest.split(split))?;
209        let candidate = match a.candidate_plan {
210            None => None,
211            Some(path) => {
212                let deps = deps::load(host, a.deps, rules).map_err(|e| dep_failed(command, e))?;
213                let plan = read_plan(command, host, path)?;
214                Some((load_plan(command, &plan, &deps)?, deps.calibrations))
215            }
216        };
217        let evaluated = evaluate(&Inputs {
218            manifest: &manifest,
219            split,
220            config: &config,
221            adapter: &adapter,
222            bundles: &bundles,
223            resolver,
224            replay,
225            candidate: candidate
226                .as_ref()
227                .map(|(plan, calibrations)| Candidate { plan, calibrations }),
228        })
229        .map_err(|e| Done::invalid(command, dataset_path, e.to_string()))?;
230        let bytes = |r: Result<Vec<u8>, rustev_contract::canonical::CanonicalError>| {
231            r.map_err(|e| Done::invalid(command, out_dir, e.to_string()))
232        };
233        let report_bytes = bytes(evaluated.report.record_canonical())?;
234        let detail_bytes = bytes(evaluated.detail.record_canonical())?;
235        let config_bytes = bytes(evaluated.config.canonical())?;
236        let adapter_bytes = bytes(adapter.canonical())?;
237        let metrics = bytes(record_canonical_bytes(&evaluated.report.metrics))?;
238        write_all(
239            command,
240            host,
241            out_dir,
242            vec![
243                (OUTPUTS[0], report_bytes),
244                (OUTPUTS[1], detail_bytes),
245                (OUTPUTS[2], config_bytes),
246                (OUTPUTS[3], adapter_bytes),
247            ],
248        )?;
249        let digest = |r: Result<rustev_contract::ids::ContentDigest, _>| {
250            r.map(|d| d.to_string()).unwrap_or_default()
251        };
252        Ok(Done::ok(
253            Out::new(command, "reported")
254                .set("report_digest", &digest(evaluated.report.record_digest()))
255                .set("dataset", &evaluated.report.dataset.id)
256                .set("split", rustev_eval::report::split_name(split))
257                .set("config", &evaluated.report.evaluator_config)
258                .set("adapter_digest", &digest(adapter.record_digest()))
259                .set("candidate", &candidate.is_some())
260                .set("out", out_dir)
261                .raw("metrics", metrics),
262        ))
263    })()
264}
265
266/// One side of a gate: the four documents `eval` wrote.
267fn read_side(command: &str, host: &dyn Host, dir: &str) -> Result<(Evaluated, Vec<u8>), Done> {
268    let report: EvalReport = read_doc(
269        command,
270        host,
271        &host.join(dir, OUTPUTS[0]),
272        RECORD_V1.max_bytes,
273    )?;
274    let detail: EvalDetail = read_doc(
275        command,
276        host,
277        &host.join(dir, OUTPUTS[1]),
278        REPLAY_V1.max_bytes,
279    )?;
280    let config: EvaluatorConfig = read_doc(
281        command,
282        host,
283        &host.join(dir, OUTPUTS[2]),
284        REPLAY_V1.max_bytes,
285    )?;
286    let adapter_path = host.join(dir, OUTPUTS[3]);
287    let adapter = host
288        .read(&adapter_path, DESCRIPTOR_V1.max_bytes)
289        .map_err(|e: IoFail| Done::io(command, e))?;
290    TaskAdapterDoc::parse(&adapter)
291        .map_err(|e| Done::invalid(command, &adapter_path, e.to_string()))?;
292    Ok((
293        Evaluated {
294            report,
295            detail,
296            config,
297        },
298        adapter,
299    ))
300}
301
302/// `gate`: one declared gate of the candidate's configuration over the two
303/// sides `eval` wrote.
304pub fn gate(command: &str, host: &dyn Host, baseline: &str, candidate: &str, name: &str) -> Res {
305    (|| {
306        let (baseline, badapter) = read_side(command, host, baseline)?;
307        let (candidate, cadapter) = read_side(command, host, candidate)?;
308        let result = if badapter != cadapter {
309            GateResult::Unknown {
310                reason: "the two sides were evaluated with different task adapters".into(),
311            }
312        } else {
313            match candidate.config.gates.iter().find(|g| g.name == name) {
314                None => GateResult::Unknown {
315                    reason: format!("the configuration declares no gate {name:?}"),
316                },
317                Some(g) => evaluate_gate(g, &baseline, &candidate),
318            }
319        };
320        let out = Out::new(command, "").set("gate", name);
321        Ok(match result {
322            GateResult::Pass => Done::ok(out.set("status", "pass")),
323            GateResult::Fail { reason } => {
324                Done::new(code::FAIL, out.set("status", "fail").set("reason", &reason))
325            }
326            GateResult::Unknown { reason } => Done::new(
327                code::UNKNOWN,
328                out.set("status", "unknown").set("reason", &reason),
329            ),
330        })
331    })()
332}
333
334/// Identity of a document, as text, for output.
335pub fn id_of<T: Identified>(doc: &T) -> String
336where
337    T::Id: std::fmt::Display,
338{
339    doc.id().map(|i| i.to_string()).unwrap_or_default()
340}
rustev_surface/eval.rs

rustev_surface/
eval.rs
