Skip to main content

glass_dev/
experiments.rs

1//! Isolated competing implementations ranked from observable evidence.
2
3use crate::DevelopmentWorkspace;
4use crate::agents::{AgentId, AgentRegistry, AgentSpec};
5use crate::git::{GitError, GitService};
6use crate::testing::TestRun;
7use glass_browser::development::{DevelopmentError, DevelopmentResult};
8use serde::{Deserialize, Serialize};
9use std::collections::{BTreeMap, BTreeSet};
10use std::path::{Path, PathBuf};
11use std::time::Duration;
12
13const MAX_EXPERIMENTS: usize = 8;
14const MAX_NOTES_BYTES: usize = 16 * 1024;
15
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
17#[serde(rename_all = "camelCase")]
18pub enum ExperimentState {
19    Ready,
20    Running,
21    Completed,
22    Failed,
23    Selected,
24}
25
26#[derive(Debug, Clone, Default, Serialize, Deserialize)]
27#[serde(rename_all = "camelCase")]
28pub struct ExperimentEvidence {
29    pub tests_passed: u64,
30    pub tests_failed: u64,
31    pub workflow_passed: Option<bool>,
32    pub semantic_regressions: u64,
33    pub visual_difference: Option<f64>,
34    pub lcp_ms: Option<f64>,
35    pub diagnostics: u64,
36    pub debugger_stops: u64,
37    pub changed_files: u64,
38    pub notes: String,
39}
40
41#[derive(Debug, Clone, Serialize, Deserialize)]
42#[serde(rename_all = "camelCase")]
43pub struct ExperimentSnapshot {
44    pub id: String,
45    pub branch: String,
46    pub worktree: PathBuf,
47    pub port: Option<u16>,
48    pub state: ExperimentState,
49    pub agent_id: Option<AgentId>,
50    pub evidence: ExperimentEvidence,
51}
52
53#[derive(Debug, Clone, Serialize, Deserialize)]
54#[serde(rename_all = "camelCase")]
55pub struct ExperimentRanking {
56    pub id: String,
57    pub score: i64,
58    pub reasons: Vec<String>,
59}
60
61#[derive(Debug, Clone, Serialize, Deserialize)]
62#[serde(rename_all = "camelCase")]
63pub struct ExperimentComparison {
64    pub rankings: Vec<ExperimentRanking>,
65    pub recommended: Option<String>,
66}
67
68struct Experiment {
69    snapshot: ExperimentSnapshot,
70    workspace: DevelopmentWorkspace,
71}
72
73/// Owns worktrees and resident runtime state for competing implementations.
74pub struct ExperimentManager {
75    repository: GitService,
76    worktree_root: PathBuf,
77    experiments: BTreeMap<String, Experiment>,
78    ports: BTreeSet<u16>,
79}
80
81impl ExperimentManager {
82    pub fn new(
83        repository_root: impl AsRef<Path>,
84        worktree_root: impl AsRef<Path>,
85    ) -> DevelopmentResult<Self> {
86        let repository = GitService::open(repository_root)
87            .map_err(|error| DevelopmentError::Process(error.to_string()))?;
88        let worktree_root = absolutize(worktree_root.as_ref())?;
89        if worktree_root == Path::new("/") || worktree_root == repository.root() {
90            return Err(DevelopmentError::InvalidInput(
91                "experiment worktree root must be an explicit directory outside the repository"
92                    .into(),
93            ));
94        }
95        std::fs::create_dir_all(&worktree_root)?;
96        Ok(Self {
97            repository,
98            worktree_root,
99            experiments: BTreeMap::new(),
100            ports: BTreeSet::new(),
101        })
102    }
103
104    pub fn create(
105        &mut self,
106        id: &str,
107        branch: &str,
108        port: Option<u16>,
109    ) -> DevelopmentResult<ExperimentSnapshot> {
110        validate_name(id, "experiment")?;
111        validate_name(branch, "experiment branch")?;
112        if self.experiments.contains_key(id) {
113            return Err(DevelopmentError::Conflict(format!(
114                "experiment {id} already exists"
115            )));
116        }
117        if self.experiments.len() >= MAX_EXPERIMENTS {
118            return Err(DevelopmentError::Conflict(format!(
119                "experiment limit is {MAX_EXPERIMENTS}"
120            )));
121        }
122        if let Some(port) = port
123            && (port == 0 || !self.ports.insert(port))
124        {
125            return Err(DevelopmentError::Conflict(
126                "experiment ports must be non-zero and unique".into(),
127            ));
128        }
129        let worktree = self.worktree_root.join(id);
130        if worktree.exists() {
131            if let Some(port) = port {
132                self.ports.remove(&port);
133            }
134            return Err(DevelopmentError::Conflict(format!(
135                "experiment path already exists: {}",
136                worktree.display()
137            )));
138        }
139        if let Err(error) = self.repository.create_worktree(&worktree, branch, true) {
140            if let Some(port) = port {
141                self.ports.remove(&port);
142            }
143            return Err(DevelopmentError::Process(error.to_string()));
144        }
145        let workspace = match DevelopmentWorkspace::open(&worktree) {
146            Ok(workspace) => workspace,
147            Err(error) => {
148                let _ = self.repository.remove_worktree(&worktree, true);
149                if let Some(port) = port {
150                    self.ports.remove(&port);
151                }
152                return Err(error);
153            }
154        };
155        let snapshot = ExperimentSnapshot {
156            id: id.into(),
157            branch: branch.into(),
158            worktree,
159            port,
160            state: ExperimentState::Ready,
161            agent_id: None,
162            evidence: ExperimentEvidence::default(),
163        };
164        self.experiments.insert(
165            id.into(),
166            Experiment {
167                snapshot: snapshot.clone(),
168                workspace,
169            },
170        );
171        Ok(snapshot)
172    }
173
174    pub fn assign_agent(
175        &mut self,
176        id: &str,
177        agents: &mut AgentRegistry,
178        mut spec: AgentSpec,
179    ) -> DevelopmentResult<AgentId> {
180        let experiment = self.experiment_mut(id)?;
181        if experiment.snapshot.agent_id.is_some() {
182            return Err(DevelopmentError::Conflict(format!(
183                "experiment {id} already has an agent"
184            )));
185        }
186        spec.worktree = Some(experiment.snapshot.worktree.clone());
187        let agent = agents.create(spec)?;
188        experiment.snapshot.agent_id = Some(agent.clone());
189        experiment.snapshot.state = ExperimentState::Running;
190        Ok(agent)
191    }
192
193    pub fn start_process(
194        &mut self,
195        id: &str,
196        name: &str,
197        command: &str,
198    ) -> DevelopmentResult<serde_json::Value> {
199        let experiment = self.experiment_mut(id)?;
200        experiment.snapshot.state = ExperimentState::Running;
201        Ok(serde_json::to_value(
202            experiment
203                .workspace
204                .project_mut()
205                .start_process(name, command)?,
206        )?)
207    }
208
209    pub fn run_test(
210        &mut self,
211        id: &str,
212        run_id: &str,
213        suite_id: &str,
214        actor_id: &str,
215        timeout: Option<Duration>,
216    ) -> DevelopmentResult<TestRun> {
217        let experiment = self.experiment_mut(id)?;
218        let revision = experiment.workspace.project().revision();
219        experiment.snapshot.state = ExperimentState::Running;
220        experiment
221            .workspace
222            .tests_mut()
223            .start(run_id, suite_id, actor_id, revision, timeout)
224            .map_err(|error| DevelopmentError::Process(error.to_string()))
225    }
226
227    pub fn poll_tests(&mut self, id: &str) -> DevelopmentResult<Vec<TestRun>> {
228        let experiment = self.experiment_mut(id)?;
229        let finished = experiment
230            .workspace
231            .tests_mut()
232            .poll()
233            .map_err(|error| DevelopmentError::Process(error.to_string()))?;
234        for run in &finished {
235            let failed = run.exit_code != Some(0);
236            let count = u64::try_from(run.cases.len().max(1)).unwrap_or(u64::MAX);
237            if failed {
238                experiment.snapshot.evidence.tests_failed = experiment
239                    .snapshot
240                    .evidence
241                    .tests_failed
242                    .saturating_add(count);
243            } else {
244                experiment.snapshot.evidence.tests_passed = experiment
245                    .snapshot
246                    .evidence
247                    .tests_passed
248                    .saturating_add(count);
249            }
250        }
251        Ok(finished)
252    }
253
254    pub fn record_evidence(
255        &mut self,
256        id: &str,
257        evidence: ExperimentEvidence,
258    ) -> DevelopmentResult<()> {
259        if evidence.notes.len() > MAX_NOTES_BYTES || evidence.notes.contains('\0') {
260            return Err(DevelopmentError::InvalidInput(
261                "experiment notes exceed their bound or contain NUL".into(),
262            ));
263        }
264        if evidence
265            .visual_difference
266            .is_some_and(|value| !value.is_finite() || value < 0.0)
267            || evidence
268                .lcp_ms
269                .is_some_and(|value| !value.is_finite() || value < 0.0)
270        {
271            return Err(DevelopmentError::InvalidInput(
272                "experiment metrics must be finite and non-negative".into(),
273            ));
274        }
275        let experiment = self.experiment_mut(id)?;
276        experiment.snapshot.evidence = evidence;
277        experiment.snapshot.state = ExperimentState::Completed;
278        Ok(())
279    }
280
281    pub fn refresh_changed_files(&mut self, id: &str) -> DevelopmentResult<u64> {
282        let experiment = self.experiment_mut(id)?;
283        let status = experiment
284            .workspace
285            .git()
286            .ok_or_else(|| DevelopmentError::NotFound("experiment Git repository".into()))?
287            .status()
288            .map_err(|error| DevelopmentError::Process(error.to_string()))?;
289        let changed = u64::try_from(status.entries.len()).unwrap_or(u64::MAX);
290        experiment.snapshot.evidence.changed_files = changed;
291        Ok(changed)
292    }
293
294    pub fn compare(&self) -> ExperimentComparison {
295        let mut rankings = self
296            .experiments
297            .values()
298            .map(|experiment| rank(&experiment.snapshot))
299            .collect::<Vec<_>>();
300        rankings.sort_by(|left, right| {
301            right
302                .score
303                .cmp(&left.score)
304                .then_with(|| left.id.cmp(&right.id))
305        });
306        let recommended = rankings
307            .first()
308            .filter(|ranking| {
309                self.experiments.get(&ranking.id).is_some_and(|experiment| {
310                    experiment.snapshot.state == ExperimentState::Completed
311                })
312            })
313            .map(|ranking| ranking.id.clone());
314        ExperimentComparison {
315            rankings,
316            recommended,
317        }
318    }
319
320    pub fn select(&mut self, id: &str) -> DevelopmentResult<ExperimentSnapshot> {
321        let comparison = self.compare();
322        if comparison.recommended.as_deref() != Some(id) {
323            return Err(DevelopmentError::Conflict(
324                "selected experiment is not the current evidence-derived recommendation".into(),
325            ));
326        }
327        for experiment in self.experiments.values_mut() {
328            if experiment.snapshot.state == ExperimentState::Selected {
329                experiment.snapshot.state = ExperimentState::Completed;
330            }
331        }
332        let experiment = self.experiment_mut(id)?;
333        experiment.snapshot.state = ExperimentState::Selected;
334        Ok(experiment.snapshot.clone())
335    }
336
337    pub fn snapshots(&self) -> Vec<ExperimentSnapshot> {
338        self.experiments
339            .values()
340            .map(|experiment| experiment.snapshot.clone())
341            .collect()
342    }
343
344    pub fn remove(&mut self, id: &str, force: bool) -> DevelopmentResult<()> {
345        let experiment = self
346            .experiments
347            .remove(id)
348            .ok_or_else(|| DevelopmentError::NotFound(format!("experiment {id}")))?;
349        if let Some(port) = experiment.snapshot.port {
350            self.ports.remove(&port);
351        }
352        drop(experiment.workspace);
353        self.repository
354            .remove_worktree(&experiment.snapshot.worktree, force)
355            .map_err(|error| DevelopmentError::Process(error.to_string()))
356    }
357
358    fn experiment_mut(&mut self, id: &str) -> DevelopmentResult<&mut Experiment> {
359        self.experiments
360            .get_mut(id)
361            .ok_or_else(|| DevelopmentError::NotFound(format!("experiment {id}")))
362    }
363}
364
365fn rank(snapshot: &ExperimentSnapshot) -> ExperimentRanking {
366    let evidence = &snapshot.evidence;
367    let mut score = 0_i64;
368    let mut reasons = Vec::new();
369    score = score.saturating_add((evidence.tests_passed.min(10_000) as i64) * 10);
370    score = score.saturating_sub((evidence.tests_failed.min(10_000) as i64) * 1_000);
371    match evidence.workflow_passed {
372        Some(true) => {
373            score = score.saturating_add(5_000);
374            reasons.push("workflow passed".into());
375        }
376        Some(false) => {
377            score = score.saturating_sub(20_000);
378            reasons.push("workflow failed".into());
379        }
380        None => reasons.push("workflow not measured".into()),
381    }
382    score = score.saturating_sub((evidence.semantic_regressions.min(10_000) as i64) * 2_000);
383    score = score.saturating_sub((evidence.diagnostics.min(10_000) as i64) * 200);
384    score = score.saturating_sub((evidence.changed_files.min(10_000) as i64) * 5);
385    if let Some(lcp) = evidence.lcp_ms {
386        score = score.saturating_sub(lcp.min(i64::MAX as f64) as i64);
387        reasons.push(format!("LCP {lcp:.2} ms"));
388    }
389    if evidence.tests_failed == 0 {
390        reasons.push(format!("{} tests passed", evidence.tests_passed));
391    } else {
392        reasons.push(format!("{} tests failed", evidence.tests_failed));
393    }
394    if evidence.semantic_regressions > 0 {
395        reasons.push(format!(
396            "{} semantic regressions",
397            evidence.semantic_regressions
398        ));
399    }
400    ExperimentRanking {
401        id: snapshot.id.clone(),
402        score,
403        reasons,
404    }
405}
406
407fn validate_name(value: &str, description: &str) -> DevelopmentResult<()> {
408    if value.is_empty()
409        || value.len() > 128
410        || !value
411            .chars()
412            .all(|character| character.is_ascii_alphanumeric() || "-_.".contains(character))
413    {
414        return Err(DevelopmentError::InvalidInput(format!(
415            "{description} must be 1..=128 ASCII letters, digits, '-', '_' or '.'"
416        )));
417    }
418    Ok(())
419}
420
421fn absolutize(path: &Path) -> DevelopmentResult<PathBuf> {
422    if path.is_absolute() {
423        Ok(path.to_path_buf())
424    } else {
425        Ok(std::env::current_dir()?.join(path))
426    }
427}
428
429impl From<GitError> for DevelopmentError {
430    fn from(error: GitError) -> Self {
431        Self::Process(error.to_string())
432    }
433}
434
435#[cfg(test)]
436mod tests {
437    use super::*;
438    use std::process::Command;
439    use std::sync::atomic::{AtomicU64, Ordering};
440
441    static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
442
443    fn repository() -> (PathBuf, PathBuf) {
444        let sequence = NEXT_ROOT.fetch_add(1, Ordering::Relaxed);
445        let base = std::env::temp_dir().join(format!(
446            "glass-experiments-{}-{sequence}",
447            std::process::id()
448        ));
449        let root = base.join("repository");
450        std::fs::create_dir_all(&root).unwrap();
451        std::fs::write(
452            root.join("Cargo.toml"),
453            "[package]\nname='fixture'\nversion='0.1.0'\n",
454        )
455        .unwrap();
456        for arguments in [
457            vec!["init"],
458            vec!["config", "user.name", "Glass Test"],
459            vec!["config", "user.email", "glass@example.invalid"],
460            vec!["add", "Cargo.toml"],
461            vec!["commit", "-m", "initial"],
462        ] {
463            assert!(
464                Command::new("git")
465                    .args(arguments)
466                    .current_dir(&root)
467                    .status()
468                    .unwrap()
469                    .success()
470            );
471        }
472        (base.clone(), root)
473    }
474
475    #[test]
476    fn isolated_worktrees_are_ranked_and_selected_from_evidence() {
477        let (base, root) = repository();
478        let worktrees = base.join("worktrees");
479        let mut manager = ExperimentManager::new(&root, &worktrees).unwrap();
480        let first = manager
481            .create("approach-a", "experiment-a", Some(3101))
482            .unwrap();
483        let second = manager
484            .create("approach-b", "experiment-b", Some(3102))
485            .unwrap();
486        assert!(first.worktree.exists());
487        assert!(second.worktree.exists());
488        std::fs::write(first.worktree.join("a.rs"), "fn a() {}\n").unwrap();
489        std::fs::write(second.worktree.join("b.rs"), "fn b() {}\n").unwrap();
490        assert_eq!(manager.refresh_changed_files("approach-a").unwrap(), 1);
491        assert_eq!(manager.refresh_changed_files("approach-b").unwrap(), 1);
492        manager
493            .record_evidence(
494                "approach-a",
495                ExperimentEvidence {
496                    tests_passed: 142,
497                    workflow_passed: Some(true),
498                    lcp_ms: Some(1_220.0),
499                    ..ExperimentEvidence::default()
500                },
501            )
502            .unwrap();
503        manager
504            .record_evidence(
505                "approach-b",
506                ExperimentEvidence {
507                    tests_passed: 141,
508                    tests_failed: 1,
509                    workflow_passed: Some(false),
510                    semantic_regressions: 1,
511                    lcp_ms: Some(1_080.0),
512                    ..ExperimentEvidence::default()
513                },
514            )
515            .unwrap();
516        let comparison = manager.compare();
517        assert_eq!(comparison.recommended.as_deref(), Some("approach-a"));
518        assert_eq!(
519            manager.select("approach-a").unwrap().state,
520            ExperimentState::Selected
521        );
522        assert!(manager.select("approach-b").is_err());
523        manager.remove("approach-a", true).unwrap();
524        manager.remove("approach-b", true).unwrap();
525        drop(manager);
526        std::fs::remove_dir_all(base).unwrap();
527    }
528}