feagi_evolutionary/evaluation/
mod.rs

1// Copyright 2025 Neuraville Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4//! `GenomeEvaluation` v1 — one evaluated individual: a genome, its fitness, and its lineage.
5//!
6//! A completed Trainer protocol produces one record (ADR-016 in
7//! `docs/FEAGI_TRAINER_ADR_SET.md`). The record pins the content hash of the genome that was
8//! evaluated (the genotype), references the protocol's Trainer scorecards by id, carries the
9//! comparability key that decides which evaluations may be ranked against each other, the
10//! validation-split fitness, and the lineage used by evolutionary operators.
11//!
12//! Inheritance is genome-only: lineage links genome hashes, never trained connectomes. The
13//! record performs no I/O; hosts (Composer for feagi-desktop) store it.
14//!
15//! @cursor:ffi-safe — plain data, serde-only, no runtime reflection.
16
17use std::collections::BTreeSet;
18
19use feagi_dataset_contracts::{
20    BackendKind, ConnectomeHash, ContentHash, DatasetAssetId, EvaluationProtocolVersion, PluginRef,
21    ScorecardId, SplitId,
22};
23use serde::{Deserialize, Serialize};
24use thiserror::Error;
25
26#[cfg(test)]
27mod tests;
28
29/// Wire/format version of the `GenomeEvaluation` contract.
30pub const SCHEMA_VERSION: u32 = 1;
31
32/// Prefix of every content hash minted for genome snapshots and run configurations.
33const SHA256_PREFIX: &str = "sha256:";
34/// Hex digits in a SHA-256 digest.
35const SHA256_HEX_LEN: usize = 64;
36
37/// Identifies one `GenomeEvaluation` record.
38#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
39#[serde(transparent)]
40pub struct EvaluationId(pub String);
41
42/// Identifies the experiment (the environment genomes compete in).
43#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
44#[serde(transparent)]
45pub struct ExperimentId(pub String);
46
47/// Direction in which a fitness metric improves.
48#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
49#[serde(rename_all = "snake_case")]
50pub enum FitnessObjective {
51    /// Higher values are better (e.g. accuracy).
52    Maximize,
53    /// Lower values are better (e.g. error rate).
54    Minimize,
55}
56
57/// Which metric, on which split, is the fitness, and which way it improves.
58#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
59pub struct FitnessSpec {
60    /// Metric key as it appears in the scorecard `metrics` map.
61    pub metric: String,
62    /// Split the fitness is read from (validation; never the held-out test split).
63    pub split_id: SplitId,
64    /// Improvement direction.
65    pub objective: FitnessObjective,
66}
67
68/// Everything that must match for two evaluations to be ranked against each other.
69///
70/// Two evaluations are comparable exactly when their keys are equal (`==`).
71#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
72pub struct ComparabilityKey {
73    /// Experiment the genome was evaluated in.
74    pub experiment_id: ExperimentId,
75    /// Dataset asset scored against.
76    pub dataset_asset_id: DatasetAssetId,
77    /// Dataset version string.
78    pub dataset_version: String,
79    /// Content hash binding the evaluation to exact dataset bytes and labels.
80    pub dataset_content_hash: ContentHash,
81    /// Evaluation protocol semantics version.
82    pub evaluation_protocol_version: EvaluationProtocolVersion,
83    /// Metric pack that computed the metrics.
84    pub metric_pack: PluginRef,
85    /// Reward policy used during training.
86    pub reward_policy: PluginRef,
87    /// Fitness metric, split, and objective.
88    pub fitness: FitnessSpec,
89    /// Hash of the run settings (sample caps, class remap, bindings, burst frequency).
90    pub run_config_hash: ContentHash,
91    /// Genome schema version the genome was authored under.
92    pub genome_schema_version: u32,
93    /// feagi-core version of the runtime that ran the protocol.
94    pub feagi_core_version: String,
95    /// Execution backend.
96    pub backend: BackendKind,
97}
98
99/// Confidence interval of a repeated (N-seed) fitness estimate.
100#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
101pub struct ConfidenceInterval {
102    /// Lower bound.
103    pub low: f64,
104    /// Upper bound.
105    pub high: f64,
106    /// Confidence level in the open interval (0, 1), e.g. 0.95.
107    pub level: f64,
108}
109
110/// Fitness value with its repeat provenance.
111#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
112pub struct FitnessEstimate {
113    /// Point estimate (the mean when `n > 1`).
114    pub value: f64,
115    /// Number of fresh-development repeats the estimate is over.
116    pub n: u32,
117    /// Required when `n > 1`; absent when `n == 1`.
118    #[serde(default, skip_serializing_if = "Option::is_none")]
119    pub interval: Option<ConfidenceInterval>,
120}
121
122/// Whether the protocol produced a fitness value.
123#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
124#[serde(tag = "status", rename_all = "snake_case")]
125pub enum FitnessOutcome {
126    /// The fitness split ran to completion.
127    Scored(FitnessEstimate),
128    /// The protocol had no phase on the fitness split.
129    NoFitnessSplit,
130    /// The fitness split was skipped or ended early.
131    Incomplete,
132}
133
134/// How a genome came to exist.
135#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
136#[serde(rename_all = "snake_case")]
137pub enum GenomeOrigin {
138    /// Authored or edited by an operator.
139    Manual,
140    /// Brought in from outside the experiment (e.g. Brain Hub).
141    Imported,
142    /// Produced by mutating exactly one parent genome.
143    Mutation,
144    /// Produced by recombining two or more parent genomes.
145    Crossover,
146}
147
148/// Genome-only ancestry of the evaluated genome.
149#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
150pub struct Lineage {
151    /// How the genome was produced.
152    pub origin: GenomeOrigin,
153    /// Generation number; 0 for manual and imported genomes.
154    pub generation: u32,
155    /// Content hashes of parent genomes (never connectomes).
156    pub parents: Vec<ContentHash>,
157}
158
159/// One evaluated individual (ADR-016).
160#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
161pub struct GenomeEvaluation {
162    /// Wire/format version; must equal [`SCHEMA_VERSION`].
163    pub schema_version: u32,
164    /// Identity of this record.
165    pub evaluation_id: EvaluationId,
166    /// Content hash of the genome snapshot taken at protocol start.
167    pub genome_hash: ContentHash,
168    /// Comparability key.
169    pub key: ComparabilityKey,
170    /// Scorecards produced by the protocol's phases.
171    pub scorecard_ids: Vec<ScorecardId>,
172    /// Fitness outcome on `key.fitness.split_id`.
173    pub fitness: FitnessOutcome,
174    /// Genome-only ancestry.
175    pub lineage: Lineage,
176    /// Starting connectome, present only when the operator pinned this run.
177    #[serde(default, skip_serializing_if = "Option::is_none")]
178    pub pinned_connectome: Option<ConnectomeHash>,
179}
180
181/// Contract violation found by [`GenomeEvaluation::validate`].
182#[derive(Debug, Clone, PartialEq, Eq, Error)]
183pub enum EvaluationError {
184    /// `schema_version` is not [`SCHEMA_VERSION`].
185    #[error("unsupported schema_version {found}; expected {expected}")]
186    SchemaVersion {
187        /// Version on the record.
188        found: u32,
189        /// Version this crate reads.
190        expected: u32,
191    },
192    /// A required string field is empty or whitespace.
193    #[error("field `{0}` must not be empty")]
194    EmptyField(&'static str),
195    /// A content hash is not `sha256:` followed by 64 lowercase hex digits.
196    #[error("field `{field}` is not a sha256 content hash: {value}")]
197    InvalidHash {
198        /// Field holding the hash.
199        field: &'static str,
200        /// Offending value.
201        value: String,
202    },
203    /// No scorecard ids were recorded.
204    #[error("scorecard_ids must not be empty")]
205    NoScorecards,
206    /// A scorecard id appears more than once.
207    #[error("duplicate scorecard id {0}")]
208    DuplicateScorecard(String),
209    /// The fitness estimate breaks an invariant.
210    #[error("invalid fitness: {0}")]
211    InvalidFitness(String),
212    /// The lineage breaks an invariant.
213    #[error("invalid lineage: {0}")]
214    InvalidLineage(String),
215}
216
217impl GenomeEvaluation {
218    /// Returns the fitness estimate when this individual may take part in selection.
219    ///
220    /// Only fully scored evaluations are selectable; `NoFitnessSplit` and `Incomplete`
221    /// records stay in history but are not plotted as fitness points or used as parents.
222    pub fn selectable_fitness(&self) -> Option<&FitnessEstimate> {
223        match &self.fitness {
224            FitnessOutcome::Scored(estimate) => Some(estimate),
225            FitnessOutcome::NoFitnessSplit | FitnessOutcome::Incomplete => None,
226        }
227    }
228
229    /// Checks every contract invariant; hosts call this before storing a record.
230    ///
231    /// # Errors
232    /// Returns the first [`EvaluationError`] found.
233    pub fn validate(&self) -> Result<(), EvaluationError> {
234        if self.schema_version != SCHEMA_VERSION {
235            return Err(EvaluationError::SchemaVersion {
236                found: self.schema_version,
237                expected: SCHEMA_VERSION,
238            });
239        }
240        require_text("evaluation_id", &self.evaluation_id.0)?;
241        require_sha256("genome_hash", &self.genome_hash)?;
242        validate_key(&self.key)?;
243        validate_scorecards(&self.scorecard_ids)?;
244        if let FitnessOutcome::Scored(estimate) = &self.fitness {
245            validate_estimate(estimate)?;
246        }
247        validate_lineage(&self.lineage, &self.genome_hash)?;
248        if let Some(connectome) = &self.pinned_connectome {
249            require_text("pinned_connectome", &connectome.0)?;
250        }
251        Ok(())
252    }
253}
254
255fn require_text(field: &'static str, value: &str) -> Result<(), EvaluationError> {
256    if value.trim().is_empty() {
257        return Err(EvaluationError::EmptyField(field));
258    }
259    Ok(())
260}
261
262fn require_sha256(field: &'static str, hash: &ContentHash) -> Result<(), EvaluationError> {
263    let digest_ok = hash.0.strip_prefix(SHA256_PREFIX).is_some_and(|hex| {
264        hex.len() == SHA256_HEX_LEN
265            && hex
266                .bytes()
267                .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
268    });
269    if !digest_ok {
270        return Err(EvaluationError::InvalidHash {
271            field,
272            value: hash.0.clone(),
273        });
274    }
275    Ok(())
276}
277
278fn validate_key(key: &ComparabilityKey) -> Result<(), EvaluationError> {
279    require_text("key.experiment_id", &key.experiment_id.0)?;
280    require_text("key.dataset_asset_id", &key.dataset_asset_id.0)?;
281    require_text("key.dataset_version", &key.dataset_version)?;
282    require_text("key.dataset_content_hash", &key.dataset_content_hash.0)?;
283    require_text(
284        "key.evaluation_protocol_version",
285        &key.evaluation_protocol_version.0,
286    )?;
287    require_text("key.metric_pack.id", &key.metric_pack.id.0)?;
288    require_text("key.metric_pack.version", &key.metric_pack.version)?;
289    require_text("key.reward_policy.id", &key.reward_policy.id.0)?;
290    require_text("key.reward_policy.version", &key.reward_policy.version)?;
291    require_text("key.fitness.metric", &key.fitness.metric)?;
292    require_text("key.fitness.split_id", &key.fitness.split_id.0)?;
293    require_sha256("key.run_config_hash", &key.run_config_hash)?;
294    require_text("key.feagi_core_version", &key.feagi_core_version)?;
295    Ok(())
296}
297
298fn validate_scorecards(ids: &[ScorecardId]) -> Result<(), EvaluationError> {
299    if ids.is_empty() {
300        return Err(EvaluationError::NoScorecards);
301    }
302    let mut seen = BTreeSet::new();
303    for id in ids {
304        require_text("scorecard_ids[]", &id.0)?;
305        if !seen.insert(id.0.as_str()) {
306            return Err(EvaluationError::DuplicateScorecard(id.0.clone()));
307        }
308    }
309    Ok(())
310}
311
312fn validate_estimate(estimate: &FitnessEstimate) -> Result<(), EvaluationError> {
313    if !estimate.value.is_finite() {
314        return Err(EvaluationError::InvalidFitness(
315            "value must be finite".to_string(),
316        ));
317    }
318    match (estimate.n, &estimate.interval) {
319        (0, _) => Err(EvaluationError::InvalidFitness(
320            "n must be at least 1".to_string(),
321        )),
322        (1, None) => Ok(()),
323        (1, Some(_)) => Err(EvaluationError::InvalidFitness(
324            "a single run carries no interval".to_string(),
325        )),
326        (_, None) => Err(EvaluationError::InvalidFitness(
327            "a repeated estimate (n > 1) requires an interval".to_string(),
328        )),
329        (_, Some(interval)) => validate_interval(estimate.value, interval),
330    }
331}
332
333fn validate_interval(value: f64, interval: &ConfidenceInterval) -> Result<(), EvaluationError> {
334    let finite = interval.low.is_finite() && interval.high.is_finite();
335    if !finite || interval.low > value || value > interval.high {
336        return Err(EvaluationError::InvalidFitness(
337            "interval must be finite and contain the value".to_string(),
338        ));
339    }
340    if !(interval.level > 0.0 && interval.level < 1.0) {
341        return Err(EvaluationError::InvalidFitness(
342            "interval level must be in (0, 1)".to_string(),
343        ));
344    }
345    Ok(())
346}
347
348fn validate_lineage(lineage: &Lineage, genome_hash: &ContentHash) -> Result<(), EvaluationError> {
349    for parent in &lineage.parents {
350        require_sha256("lineage.parents[]", parent)?;
351    }
352    let unique: BTreeSet<&str> = lineage.parents.iter().map(|p| p.0.as_str()).collect();
353    if unique.len() != lineage.parents.len() {
354        return Err(EvaluationError::InvalidLineage(
355            "parents must be distinct".to_string(),
356        ));
357    }
358    if unique.contains(genome_hash.0.as_str()) {
359        return Err(EvaluationError::InvalidLineage(
360            "a genome cannot be its own parent".to_string(),
361        ));
362    }
363    let parent_count = lineage.parents.len();
364    match lineage.origin {
365        GenomeOrigin::Manual | GenomeOrigin::Imported => {
366            if parent_count != 0 || lineage.generation != 0 {
367                return Err(EvaluationError::InvalidLineage(
368                    "manual and imported genomes are generation 0 with no parents".to_string(),
369                ));
370            }
371        }
372        GenomeOrigin::Mutation => {
373            if parent_count != 1 || lineage.generation == 0 {
374                return Err(EvaluationError::InvalidLineage(
375                    "a mutation has exactly one parent and generation >= 1".to_string(),
376                ));
377            }
378        }
379        GenomeOrigin::Crossover => {
380            if parent_count < 2 || lineage.generation == 0 {
381                return Err(EvaluationError::InvalidLineage(
382                    "a crossover has at least two parents and generation >= 1".to_string(),
383                ));
384            }
385        }
386    }
387    Ok(())
388}
feagi_evolutionary/evaluation/mod.rs

feagi_evolutionary/evaluation/
mod.rs
