Skip to main content

glass_dev/
experiments.rs

1//! Isolated competing implementations ranked from observable evidence.
2
3use crate::agents::{AgentId, AgentRegistry, AgentSpec};
4use crate::development::{DevelopmentError, DevelopmentResult, ProcessHealth};
5use crate::git::{GitError, GitService};
6use crate::testing::TestRun;
7use crate::{
8    DevelopmentWorkspace, LocalTrustDecision, WorkspaceIdentity, WorkspaceTrust,
9    WorkspaceTrustStore,
10};
11use serde::{Deserialize, Serialize};
12use serde_json::Value;
13use std::collections::{BTreeMap, BTreeSet};
14use std::path::{Path, PathBuf};
15use std::process::{Command, Stdio};
16use std::thread;
17use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
18
19const MAX_EXPERIMENTS: usize = 8;
20const MAX_NOTES_BYTES: usize = 16 * 1024;
21
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
23#[serde(rename_all = "camelCase")]
24pub enum ExperimentState {
25    Ready,
26    Running,
27    Completed,
28    Failed,
29    Selected,
30}
31
32impl ExperimentState {
33    /// Human-readable label for terminal and TUI surfaces.
34    pub fn label(&self) -> &'static str {
35        match self {
36            Self::Ready => "ready",
37            Self::Running => "running",
38            Self::Completed => "completed",
39            Self::Failed => "failed",
40            Self::Selected => "selected",
41        }
42    }
43}
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
46#[serde(rename_all = "camelCase")]
47pub enum ExperimentTrustPolicy {
48    /// Child worktrees start untrusted and cannot execute project code.
49    Reevaluate,
50    /// A trusted parent grants process-lifetime trust to child worktrees only.
51    InheritOnce,
52}
53
54#[derive(Debug, Clone, Default, Serialize, Deserialize)]
55#[serde(rename_all = "camelCase")]
56pub struct ExperimentEvidence {
57    pub tests_passed: u64,
58    pub tests_failed: u64,
59    pub workflow_passed: Option<bool>,
60    pub semantic_regressions: u64,
61    pub visual_difference: Option<f64>,
62    pub lcp_ms: Option<f64>,
63    pub diagnostics: u64,
64    pub debugger_stops: u64,
65    pub changed_files: u64,
66    pub build_passed: Option<bool>,
67    pub startup_healthy: Option<bool>,
68    pub process_crashes: u64,
69    pub notes: String,
70    #[serde(default)]
71    pub provenance: BTreeMap<String, EvidenceProvenance>,
72}
73
74#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
75#[serde(rename_all = "camelCase")]
76pub struct EvidenceProvenance {
77    pub producer: String,
78    pub timestamp_ms: u128,
79    pub workspace_revision: u64,
80    pub browser_revision: Option<u64>,
81    pub run_id: Option<String>,
82    pub measured: bool,
83    pub available: bool,
84    pub details: Value,
85}
86
87#[derive(Debug, Clone, Serialize, Deserialize)]
88#[serde(rename_all = "camelCase")]
89pub struct ExperimentSnapshot {
90    pub id: String,
91    pub branch: String,
92    pub worktree: PathBuf,
93    pub port: Option<u16>,
94    pub state: ExperimentState,
95    pub agent_id: Option<AgentId>,
96    pub evidence: ExperimentEvidence,
97}
98
99#[derive(Debug, Clone, Serialize, Deserialize)]
100#[serde(rename_all = "camelCase")]
101pub struct ExperimentRanking {
102    pub id: String,
103    pub score: i64,
104    pub reasons: Vec<String>,
105}
106
107#[derive(Debug, Clone, Serialize, Deserialize)]
108#[serde(rename_all = "camelCase")]
109pub struct ExperimentComparison {
110    pub rankings: Vec<ExperimentRanking>,
111    pub recommended: Option<String>,
112}
113
114#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
115#[serde(rename_all = "camelCase")]
116pub struct ExperimentWeights {
117    pub test_passed: i64,
118    pub test_failed: i64,
119    pub workflow_passed: i64,
120    pub workflow_failed: i64,
121    pub semantic_regression: i64,
122    pub diagnostic: i64,
123    pub changed_file: i64,
124    pub process_crash: i64,
125}
126
127impl Default for ExperimentWeights {
128    fn default() -> Self {
129        Self {
130            test_passed: 10,
131            test_failed: -1_000,
132            workflow_passed: 5_000,
133            workflow_failed: -20_000,
134            semantic_regression: -2_000,
135            diagnostic: -200,
136            changed_file: -5,
137            process_crash: -5_000,
138        }
139    }
140}
141
142struct Experiment {
143    snapshot: ExperimentSnapshot,
144    workspace: DevelopmentWorkspace,
145}
146
147/// Owns worktrees and resident runtime state for competing implementations.
148pub struct ExperimentManager {
149    repository: GitService,
150    worktree_root: PathBuf,
151    experiments: BTreeMap<String, Experiment>,
152    ports: BTreeSet<u16>,
153    trust_policy: ExperimentTrustPolicy,
154    weights: ExperimentWeights,
155}
156
157impl ExperimentManager {
158    pub fn new(
159        repository_root: impl AsRef<Path>,
160        worktree_root: impl AsRef<Path>,
161    ) -> DevelopmentResult<Self> {
162        let identity = WorkspaceIdentity::inspect(repository_root.as_ref())?;
163        let trust = WorkspaceTrustStore::platform_default()?.status(&identity)?;
164        Self::new_governed(repository_root, worktree_root, trust)
165    }
166
167    pub fn new_governed(
168        repository_root: impl AsRef<Path>,
169        worktree_root: impl AsRef<Path>,
170        parent_trust: WorkspaceTrust,
171    ) -> DevelopmentResult<Self> {
172        if !parent_trust.permits_project_execution() {
173            return Err(DevelopmentError::Conflict(
174                "experiments cannot create executable worktrees before workspace trust".into(),
175            ));
176        }
177        let repository = GitService::open(repository_root)
178            .map_err(|error| DevelopmentError::Process(error.to_string()))?;
179        let worktree_root = absolutize(worktree_root.as_ref())?;
180        if worktree_root == Path::new("/") || worktree_root == repository.root() {
181            return Err(DevelopmentError::InvalidInput(
182                "experiment worktree root must be an explicit directory outside the repository"
183                    .into(),
184            ));
185        }
186        std::fs::create_dir_all(&worktree_root)?;
187        Ok(Self {
188            repository,
189            worktree_root,
190            experiments: BTreeMap::new(),
191            ports: BTreeSet::new(),
192            trust_policy: ExperimentTrustPolicy::InheritOnce,
193            weights: ExperimentWeights::default(),
194        })
195    }
196
197    pub fn create(
198        &mut self,
199        id: &str,
200        branch: &str,
201        port: Option<u16>,
202    ) -> DevelopmentResult<ExperimentSnapshot> {
203        validate_name(id, "experiment")?;
204        validate_name(branch, "experiment branch")?;
205        if self.experiments.contains_key(id) {
206            return Err(DevelopmentError::Conflict(format!(
207                "experiment {id} already exists"
208            )));
209        }
210        if self.experiments.len() >= MAX_EXPERIMENTS {
211            return Err(DevelopmentError::Conflict(format!(
212                "experiment limit is {MAX_EXPERIMENTS}"
213            )));
214        }
215        if let Some(port) = port
216            && (port == 0 || !self.ports.insert(port))
217        {
218            return Err(DevelopmentError::Conflict(
219                "experiment ports must be non-zero and unique".into(),
220            ));
221        }
222        let worktree = self.worktree_root.join(id);
223        if worktree.exists() {
224            if let Some(port) = port {
225                self.ports.remove(&port);
226            }
227            return Err(DevelopmentError::Conflict(format!(
228                "experiment path already exists: {}",
229                worktree.display()
230            )));
231        }
232        if let Err(error) = self.repository.create_worktree(&worktree, branch, true) {
233            if let Some(port) = port {
234                self.ports.remove(&port);
235            }
236            return Err(DevelopmentError::Process(error.to_string()));
237        }
238        let workspace = match DevelopmentWorkspace::open(&worktree) {
239            Ok(mut workspace) => {
240                if self.trust_policy == ExperimentTrustPolicy::InheritOnce
241                    && let Err(error) =
242                        workspace.apply_local_trust_decision(LocalTrustDecision::TrustOnce)
243                {
244                    drop(workspace);
245                    let _ = self.repository.remove_worktree(&worktree, true);
246                    if let Some(port) = port {
247                        self.ports.remove(&port);
248                    }
249                    return Err(error);
250                }
251                workspace
252            }
253            Err(error) => {
254                let _ = self.repository.remove_worktree(&worktree, true);
255                if let Some(port) = port {
256                    self.ports.remove(&port);
257                }
258                return Err(error);
259            }
260        };
261        let snapshot = ExperimentSnapshot {
262            id: id.into(),
263            branch: branch.into(),
264            worktree,
265            port,
266            state: ExperimentState::Ready,
267            agent_id: None,
268            evidence: ExperimentEvidence::default(),
269        };
270        self.experiments.insert(
271            id.into(),
272            Experiment {
273                snapshot: snapshot.clone(),
274                workspace,
275            },
276        );
277        Ok(snapshot)
278    }
279
280    pub fn assign_agent(
281        &mut self,
282        id: &str,
283        agents: &mut AgentRegistry,
284        mut spec: AgentSpec,
285    ) -> DevelopmentResult<AgentId> {
286        let experiment = self.experiment_mut(id)?;
287        if experiment.snapshot.agent_id.is_some() {
288            return Err(DevelopmentError::Conflict(format!(
289                "experiment {id} already has an agent"
290            )));
291        }
292        spec.worktree = Some(experiment.snapshot.worktree.clone());
293        let agent = agents.create(spec)?;
294        experiment.snapshot.agent_id = Some(agent.clone());
295        experiment.snapshot.state = ExperimentState::Running;
296        Ok(agent)
297    }
298
299    pub fn start_process(
300        &mut self,
301        id: &str,
302        name: &str,
303        command: &str,
304    ) -> DevelopmentResult<serde_json::Value> {
305        let experiment = self.experiment_mut(id)?;
306        experiment.snapshot.state = ExperimentState::Running;
307        Ok(serde_json::to_value(
308            experiment
309                .workspace
310                .project_mut()
311                .start_process(name, command)?,
312        )?)
313    }
314
315    pub fn run_test(
316        &mut self,
317        id: &str,
318        run_id: &str,
319        suite_id: &str,
320        actor_id: &str,
321        timeout: Option<Duration>,
322    ) -> DevelopmentResult<TestRun> {
323        let experiment = self.experiment_mut(id)?;
324        let revision = experiment.workspace.project().revision();
325        experiment.snapshot.state = ExperimentState::Running;
326        experiment
327            .workspace
328            .tests_mut()
329            .start(run_id, suite_id, actor_id, revision, timeout)
330            .map_err(|error| DevelopmentError::Process(error.to_string()))
331    }
332
333    pub fn poll_tests(&mut self, id: &str) -> DevelopmentResult<Vec<TestRun>> {
334        let experiment = self.experiment_mut(id)?;
335        let finished = experiment
336            .workspace
337            .tests_mut()
338            .poll()
339            .map_err(|error| DevelopmentError::Process(error.to_string()))?;
340        for run in &finished {
341            let failed = run.exit_code != Some(0);
342            let count = u64::try_from(run.cases.len().max(1)).unwrap_or(u64::MAX);
343            if failed {
344                experiment.snapshot.evidence.tests_failed = experiment
345                    .snapshot
346                    .evidence
347                    .tests_failed
348                    .saturating_add(count);
349            } else {
350                experiment.snapshot.evidence.tests_passed = experiment
351                    .snapshot
352                    .evidence
353                    .tests_passed
354                    .saturating_add(count);
355            }
356        }
357        Ok(finished)
358    }
359
360    /// Collect every currently available evidence family from resident or
361    /// bounded project services. Missing providers are recorded as unavailable
362    /// rather than silently converted into favorable zeroes.
363    pub fn collect_automatic(&mut self, id: &str) -> DevelopmentResult<ExperimentEvidence> {
364        let experiment = self.experiment_mut(id)?;
365        experiment.snapshot.state = ExperimentState::Running;
366        let revision = experiment.workspace.project().revision();
367        let browser_state = experiment.workspace.browser().state().ok();
368        let browser_revision = browser_state
369            .as_ref()
370            .and_then(|state| state.get("browserRevision"))
371            .and_then(Value::as_u64);
372        let mut evidence = ExperimentEvidence::default();
373
374        let detection = experiment.workspace.project().detection().clone();
375        if let Some(command) = detection.build_command.as_deref() {
376            let measurement = run_measured_command(&experiment.snapshot.worktree, command, 600)?;
377            evidence.build_passed = Some(measurement.passed);
378            evidence.provenance.insert(
379                "buildPassed".into(),
380                measured_provenance(
381                    "project-build",
382                    revision,
383                    browser_revision,
384                    Some(measurement.run_id),
385                    serde_json::json!({
386                        "command":command,
387                        "exitCode":measurement.exit_code,
388                        "durationMs":measurement.duration_ms
389                    }),
390                ),
391            );
392        } else {
393            evidence.provenance.insert(
394                "buildPassed".into(),
395                unavailable_provenance("project-build", revision, browser_revision),
396            );
397        }
398
399        if let Some(command) = detection.test_command.as_deref() {
400            let measurement = run_measured_command(&experiment.snapshot.worktree, command, 600)?;
401            if measurement.passed {
402                evidence.tests_passed = 1;
403            } else {
404                evidence.tests_failed = 1;
405            }
406            evidence.provenance.insert(
407                "tests".into(),
408                measured_provenance(
409                    "project-test",
410                    revision,
411                    browser_revision,
412                    Some(measurement.run_id),
413                    serde_json::json!({
414                        "command":command,
415                        "exitCode":measurement.exit_code,
416                        "durationMs":measurement.duration_ms,
417                        "aggregateCommandResult":true
418                    }),
419                ),
420            );
421        } else {
422            evidence.provenance.insert(
423                "tests".into(),
424                unavailable_provenance("project-test", revision, browser_revision),
425            );
426        }
427
428        if let Some(git) = experiment.workspace.git() {
429            let status = git
430                .status()
431                .map_err(|error| DevelopmentError::Process(error.to_string()))?;
432            evidence.changed_files = u64::try_from(status.entries.len()).unwrap_or(u64::MAX);
433            evidence.provenance.insert(
434                "changedFiles".into(),
435                measured_provenance(
436                    "resident-git",
437                    revision,
438                    browser_revision,
439                    None,
440                    serde_json::json!({
441                        "entries":status.entries,
442                        "conflicts":status.conflicts
443                    }),
444                ),
445            );
446        }
447
448        let processes = experiment
449            .workspace
450            .project_mut()
451            .processes()
452            .list_checked()?;
453        evidence.process_crashes = processes
454            .iter()
455            .filter(|process| process.health == ProcessHealth::Failed)
456            .count() as u64;
457        evidence.startup_healthy = (!processes.is_empty()).then(|| {
458            evidence.process_crashes == 0
459                && processes.iter().all(|process| {
460                    matches!(
461                        process.health,
462                        ProcessHealth::Healthy | ProcessHealth::Starting
463                    )
464                })
465        });
466        evidence.provenance.insert(
467            "startupHealth".into(),
468            EvidenceProvenance {
469                producer: "resident-process-service".into(),
470                timestamp_ms: now_ms(),
471                workspace_revision: revision,
472                browser_revision,
473                run_id: None,
474                measured: true,
475                available: !processes.is_empty(),
476                details: serde_json::to_value(&processes)?,
477            },
478        );
479
480        let diagnostics = experiment
481            .workspace
482            .language()
483            .events(0)
484            .into_iter()
485            .filter(|event| event.operation == "diagnostics")
486            .filter_map(|event| event.result_count)
487            .sum::<usize>();
488        evidence.diagnostics = diagnostics as u64;
489        evidence.provenance.insert(
490            "diagnostics".into(),
491            EvidenceProvenance {
492                producer: "resident-lsp".into(),
493                timestamp_ms: now_ms(),
494                workspace_revision: revision,
495                browser_revision,
496                run_id: None,
497                measured: true,
498                available: experiment.workspace.language().names().next().is_some(),
499                details: serde_json::json!({"observedDiagnostics":diagnostics}),
500            },
501        );
502
503        let debugger_names = experiment
504            .workspace
505            .debugger_names()
506            .map(str::to_string)
507            .collect::<Vec<_>>();
508        let mut debugger_stops = 0_u64;
509        for debugger in &debugger_names {
510            debugger_stops = debugger_stops.saturating_add(
511                experiment
512                    .workspace
513                    .debugger_mut(debugger)
514                    .map_err(|error| DevelopmentError::Process(error.to_string()))?
515                    .poll_events()
516                    .map_err(|error| DevelopmentError::Process(error.to_string()))?
517                    .iter()
518                    .filter(|event| event.event == "stopped")
519                    .count() as u64,
520            );
521        }
522        evidence.debugger_stops = debugger_stops;
523        evidence.provenance.insert(
524            "debuggerStops".into(),
525            EvidenceProvenance {
526                producer: "resident-dap".into(),
527                timestamp_ms: now_ms(),
528                workspace_revision: revision,
529                browser_revision,
530                run_id: None,
531                measured: true,
532                available: !debugger_names.is_empty(),
533                details: serde_json::json!({"stops":debugger_stops,"sessions":debugger_names}),
534            },
535        );
536
537        if browser_state
538            .as_ref()
539            .and_then(|state| state.get("connected"))
540            .and_then(Value::as_bool)
541            == Some(true)
542        {
543            match experiment.workspace.browser().verify_workflow() {
544                Ok(result) => {
545                    evidence.workflow_passed = result.get("verified").and_then(Value::as_bool);
546                    evidence.provenance.insert(
547                        "workflowPassed".into(),
548                        measured_provenance(
549                            "resident-browser-workflow",
550                            revision,
551                            browser_revision,
552                            result
553                                .pointer("/result/runId")
554                                .and_then(Value::as_str)
555                                .map(str::to_string),
556                            result,
557                        ),
558                    );
559                }
560                Err(error) => {
561                    evidence.provenance.insert(
562                        "workflowPassed".into(),
563                        EvidenceProvenance {
564                            details: serde_json::json!({"error":error.to_string()}),
565                            ..unavailable_provenance(
566                                "resident-browser-workflow",
567                                revision,
568                                browser_revision,
569                            )
570                        },
571                    );
572                }
573            }
574            match experiment.workspace.browser().diff() {
575                Ok(result) => {
576                    evidence.semantic_regressions = result
577                        .get("changes")
578                        .and_then(Value::as_array)
579                        .map_or(0, |changes| changes.len() as u64);
580                    evidence.provenance.insert(
581                        "semanticRegressions".into(),
582                        measured_provenance(
583                            "resident-browser-semantic-diff",
584                            revision,
585                            browser_revision,
586                            None,
587                            result,
588                        ),
589                    );
590                }
591                Err(error) => {
592                    evidence.provenance.insert(
593                        "semanticRegressions".into(),
594                        EvidenceProvenance {
595                            details: serde_json::json!({"error":error.to_string()}),
596                            ..unavailable_provenance(
597                                "resident-browser-semantic-diff",
598                                revision,
599                                browser_revision,
600                            )
601                        },
602                    );
603                }
604            }
605        } else {
606            for (metric, producer) in [
607                ("workflowPassed", "resident-browser-workflow"),
608                ("semanticRegressions", "resident-browser-semantic-diff"),
609                ("visualDifference", "resident-browser-visual-compare"),
610                ("lcpMs", "resident-browser-performance"),
611            ] {
612                evidence.provenance.insert(
613                    metric.into(),
614                    unavailable_provenance(producer, revision, browser_revision),
615                );
616            }
617        }
618        experiment.snapshot.evidence = evidence.clone();
619        experiment.snapshot.state = ExperimentState::Completed;
620        Ok(evidence)
621    }
622
623    pub fn record_evidence(
624        &mut self,
625        id: &str,
626        mut evidence: ExperimentEvidence,
627    ) -> DevelopmentResult<()> {
628        if evidence.notes.len() > MAX_NOTES_BYTES || evidence.notes.contains('\0') {
629            return Err(DevelopmentError::InvalidInput(
630                "experiment notes exceed their bound or contain NUL".into(),
631            ));
632        }
633        if evidence
634            .visual_difference
635            .is_some_and(|value| !value.is_finite() || value < 0.0)
636            || evidence
637                .lcp_ms
638                .is_some_and(|value| !value.is_finite() || value < 0.0)
639        {
640            return Err(DevelopmentError::InvalidInput(
641                "experiment metrics must be finite and non-negative".into(),
642            ));
643        }
644        let experiment = self.experiment_mut(id)?;
645        let revision = experiment.workspace.project().revision();
646        let browser_revision = experiment
647            .workspace
648            .browser()
649            .state()
650            .ok()
651            .and_then(|state| state.get("browserRevision").and_then(Value::as_u64));
652        for metric in [
653            "tests",
654            "workflowPassed",
655            "semanticRegressions",
656            "visualDifference",
657            "lcpMs",
658            "diagnostics",
659            "debuggerStops",
660            "changedFiles",
661            "buildPassed",
662            "startupHealth",
663            "processCrashes",
664        ] {
665            evidence.provenance.insert(
666                metric.into(),
667                EvidenceProvenance {
668                    producer: "manual-external".into(),
669                    timestamp_ms: now_ms(),
670                    workspace_revision: revision,
671                    browser_revision,
672                    run_id: None,
673                    measured: false,
674                    available: true,
675                    details: Value::Null,
676                },
677            );
678        }
679        experiment.snapshot.evidence = evidence;
680        experiment.snapshot.state = ExperimentState::Completed;
681        Ok(())
682    }
683
684    pub fn refresh_changed_files(&mut self, id: &str) -> DevelopmentResult<u64> {
685        let experiment = self.experiment_mut(id)?;
686        let status = experiment
687            .workspace
688            .git()
689            .ok_or_else(|| DevelopmentError::NotFound("experiment Git repository".into()))?
690            .status()
691            .map_err(|error| DevelopmentError::Process(error.to_string()))?;
692        let changed = u64::try_from(status.entries.len()).unwrap_or(u64::MAX);
693        experiment.snapshot.evidence.changed_files = changed;
694        Ok(changed)
695    }
696
697    pub fn compare(&self) -> ExperimentComparison {
698        let mut rankings = self
699            .experiments
700            .values()
701            .map(|experiment| rank(&experiment.snapshot, &self.weights))
702            .collect::<Vec<_>>();
703        rankings.sort_by(|left, right| {
704            right
705                .score
706                .cmp(&left.score)
707                .then_with(|| left.id.cmp(&right.id))
708        });
709        let recommended = rankings
710            .first()
711            .filter(|ranking| {
712                self.experiments.get(&ranking.id).is_some_and(|experiment| {
713                    experiment.snapshot.state == ExperimentState::Completed
714                })
715            })
716            .map(|ranking| ranking.id.clone());
717        ExperimentComparison {
718            rankings,
719            recommended,
720        }
721    }
722
723    pub fn set_weights(
724        &mut self,
725        weights: ExperimentWeights,
726        trust: WorkspaceTrust,
727    ) -> DevelopmentResult<()> {
728        if !trust.permits_project_execution() {
729            return Err(DevelopmentError::Conflict(
730                "experiment ranking weights require a trusted workspace".into(),
731            ));
732        }
733        if [
734            weights.test_passed,
735            weights.test_failed,
736            weights.workflow_passed,
737            weights.workflow_failed,
738            weights.semantic_regression,
739            weights.diagnostic,
740            weights.changed_file,
741            weights.process_crash,
742        ]
743        .iter()
744        .any(|weight| weight.unsigned_abs() > 1_000_000)
745        {
746            return Err(DevelopmentError::InvalidInput(
747                "experiment weight magnitude exceeds 1000000".into(),
748            ));
749        }
750        self.weights = weights;
751        Ok(())
752    }
753
754    pub fn select(&mut self, id: &str) -> DevelopmentResult<ExperimentSnapshot> {
755        let comparison = self.compare();
756        if comparison.recommended.as_deref() != Some(id) {
757            return Err(DevelopmentError::Conflict(
758                "selected experiment is not the current evidence-derived recommendation".into(),
759            ));
760        }
761        for experiment in self.experiments.values_mut() {
762            if experiment.snapshot.state == ExperimentState::Selected {
763                experiment.snapshot.state = ExperimentState::Completed;
764            }
765        }
766        let experiment = self.experiment_mut(id)?;
767        experiment.snapshot.state = ExperimentState::Selected;
768        Ok(experiment.snapshot.clone())
769    }
770
771    pub fn snapshots(&self) -> Vec<ExperimentSnapshot> {
772        self.experiments
773            .values()
774            .map(|experiment| experiment.snapshot.clone())
775            .collect()
776    }
777
778    pub fn remove(&mut self, id: &str, force: bool) -> DevelopmentResult<()> {
779        let experiment = self
780            .experiments
781            .remove(id)
782            .ok_or_else(|| DevelopmentError::NotFound(format!("experiment {id}")))?;
783        if let Some(port) = experiment.snapshot.port {
784            self.ports.remove(&port);
785        }
786        drop(experiment.workspace);
787        self.repository
788            .remove_worktree(&experiment.snapshot.worktree, force)
789            .map_err(|error| DevelopmentError::Process(error.to_string()))
790    }
791
792    fn experiment_mut(&mut self, id: &str) -> DevelopmentResult<&mut Experiment> {
793        self.experiments
794            .get_mut(id)
795            .ok_or_else(|| DevelopmentError::NotFound(format!("experiment {id}")))
796    }
797}
798
799struct CommandMeasurement {
800    passed: bool,
801    exit_code: i32,
802    duration_ms: u128,
803    run_id: String,
804}
805
806fn run_measured_command(
807    worktree: &Path,
808    command: &str,
809    timeout_seconds: u64,
810) -> DevelopmentResult<CommandMeasurement> {
811    if command.is_empty() || command.len() > 16 * 1024 || command.contains('\0') {
812        return Err(DevelopmentError::InvalidInput(
813            "experiment command is empty, oversized, or contains NUL".into(),
814        ));
815    }
816    let started = Instant::now();
817    let mut process = if cfg!(windows) {
818        let mut process = Command::new("cmd.exe");
819        process.args(["/d", "/s", "/c", command]);
820        process
821    } else {
822        let mut process = Command::new("sh");
823        process.args(["-lc", command]);
824        process
825    };
826    let mut child = process
827        .current_dir(worktree)
828        .stdin(Stdio::null())
829        .stdout(Stdio::null())
830        .stderr(Stdio::null())
831        .spawn()?;
832    let deadline = Instant::now() + Duration::from_secs(timeout_seconds);
833    let exit_code = loop {
834        if let Some(status) = child.try_wait()? {
835            break status.code().unwrap_or(-1);
836        }
837        if Instant::now() >= deadline {
838            child.kill()?;
839            child.wait()?;
840            break -1;
841        }
842        thread::sleep(Duration::from_millis(10));
843    };
844    let timestamp = now_ms();
845    Ok(CommandMeasurement {
846        passed: exit_code == 0,
847        exit_code,
848        duration_ms: started.elapsed().as_millis(),
849        run_id: format!("experiment-command-{timestamp}"),
850    })
851}
852
853fn measured_provenance(
854    producer: &str,
855    workspace_revision: u64,
856    browser_revision: Option<u64>,
857    run_id: Option<String>,
858    details: Value,
859) -> EvidenceProvenance {
860    EvidenceProvenance {
861        producer: producer.into(),
862        timestamp_ms: now_ms(),
863        workspace_revision,
864        browser_revision,
865        run_id,
866        measured: true,
867        available: true,
868        details,
869    }
870}
871
872fn unavailable_provenance(
873    producer: &str,
874    workspace_revision: u64,
875    browser_revision: Option<u64>,
876) -> EvidenceProvenance {
877    EvidenceProvenance {
878        producer: producer.into(),
879        timestamp_ms: now_ms(),
880        workspace_revision,
881        browser_revision,
882        run_id: None,
883        measured: true,
884        available: false,
885        details: Value::Null,
886    }
887}
888
889fn now_ms() -> u128 {
890    SystemTime::now()
891        .duration_since(UNIX_EPOCH)
892        .unwrap_or_default()
893        .as_millis()
894}
895
896fn rank(snapshot: &ExperimentSnapshot, weights: &ExperimentWeights) -> ExperimentRanking {
897    let evidence = &snapshot.evidence;
898    let mut score = 0_i64;
899    let mut reasons = Vec::new();
900    score = score.saturating_add(
901        (evidence.tests_passed.min(10_000) as i64).saturating_mul(weights.test_passed),
902    );
903    score = score.saturating_add(
904        (evidence.tests_failed.min(10_000) as i64).saturating_mul(weights.test_failed),
905    );
906    match evidence.workflow_passed {
907        Some(true) => {
908            score = score.saturating_add(weights.workflow_passed);
909            reasons.push("workflow passed".into());
910        }
911        Some(false) => {
912            score = score.saturating_add(weights.workflow_failed);
913            reasons.push("workflow failed".into());
914        }
915        None => reasons.push("workflow not measured".into()),
916    }
917    score = score.saturating_add(
918        (evidence.semantic_regressions.min(10_000) as i64)
919            .saturating_mul(weights.semantic_regression),
920    );
921    score = score.saturating_add(
922        (evidence.diagnostics.min(10_000) as i64).saturating_mul(weights.diagnostic),
923    );
924    score = score.saturating_add(
925        (evidence.changed_files.min(10_000) as i64).saturating_mul(weights.changed_file),
926    );
927    score = score.saturating_add(
928        (evidence.process_crashes.min(10_000) as i64).saturating_mul(weights.process_crash),
929    );
930    match evidence.build_passed {
931        Some(true) => score = score.saturating_add(2_000),
932        Some(false) => score = score.saturating_sub(10_000),
933        None => {}
934    }
935    match evidence.startup_healthy {
936        Some(true) => score = score.saturating_add(1_000),
937        Some(false) => score = score.saturating_sub(5_000),
938        None => {}
939    }
940    if let Some(difference) = evidence.visual_difference {
941        score = score.saturating_sub((difference.clamp(0.0, 1.0) * 1_000.0) as i64);
942    }
943    if let Some(lcp) = evidence.lcp_ms {
944        score = score.saturating_sub(lcp.min(i64::MAX as f64) as i64);
945        reasons.push(format!("LCP {lcp:.2} ms"));
946    }
947    if evidence.tests_failed == 0 {
948        reasons.push(format!("{} tests passed", evidence.tests_passed));
949    } else {
950        reasons.push(format!("{} tests failed", evidence.tests_failed));
951    }
952    if evidence.semantic_regressions > 0 {
953        reasons.push(format!(
954            "{} semantic regressions",
955            evidence.semantic_regressions
956        ));
957    }
958    ExperimentRanking {
959        id: snapshot.id.clone(),
960        score,
961        reasons,
962    }
963}
964
965fn validate_name(value: &str, description: &str) -> DevelopmentResult<()> {
966    if value.is_empty()
967        || value.len() > 128
968        || !value
969            .chars()
970            .all(|character| character.is_ascii_alphanumeric() || "-_.".contains(character))
971    {
972        return Err(DevelopmentError::InvalidInput(format!(
973            "{description} must be 1..=128 ASCII letters, digits, '-', '_' or '.'"
974        )));
975    }
976    Ok(())
977}
978
979fn absolutize(path: &Path) -> DevelopmentResult<PathBuf> {
980    if path.is_absolute() {
981        Ok(path.to_path_buf())
982    } else {
983        Ok(std::env::current_dir()?.join(path))
984    }
985}
986
987impl From<GitError> for DevelopmentError {
988    fn from(error: GitError) -> Self {
989        Self::Process(error.to_string())
990    }
991}
992
993#[cfg(test)]
994mod tests {
995    use super::*;
996    use crate::AgentStatus;
997    use std::process::Command;
998    use std::sync::atomic::{AtomicU64, Ordering};
999
1000    static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
1001
1002    fn repository() -> (PathBuf, PathBuf) {
1003        let sequence = NEXT_ROOT.fetch_add(1, Ordering::Relaxed);
1004        let base = std::env::temp_dir().join(format!(
1005            "glass-experiments-{}-{sequence}",
1006            std::process::id()
1007        ));
1008        let root = base.join("repository");
1009        std::fs::create_dir_all(&root).unwrap();
1010        std::fs::write(
1011            root.join("Cargo.toml"),
1012            "[package]\nname='fixture'\nversion='0.1.0'\n",
1013        )
1014        .unwrap();
1015        std::fs::write(
1016            root.join("glass.toml"),
1017            "[commands]\nbuild='rustc --version'\ntest='rustc --version'\n",
1018        )
1019        .unwrap();
1020        for arguments in [
1021            vec!["init"],
1022            vec!["config", "user.name", "Glass Test"],
1023            vec!["config", "user.email", "glass@example.invalid"],
1024            vec!["add", "."],
1025            vec!["commit", "-m", "initial"],
1026        ] {
1027            assert!(
1028                Command::new("git")
1029                    .args(arguments)
1030                    .current_dir(&root)
1031                    .status()
1032                    .unwrap()
1033                    .success()
1034            );
1035        }
1036        (base.clone(), root)
1037    }
1038
1039    #[test]
1040    fn isolated_worktrees_are_ranked_and_selected_from_evidence() {
1041        let (base, root) = repository();
1042        let worktrees = base.join("worktrees");
1043        assert!(
1044            ExperimentManager::new_governed(&root, &worktrees, WorkspaceTrust::Untrusted).is_err()
1045        );
1046        let mut manager =
1047            ExperimentManager::new_governed(&root, &worktrees, WorkspaceTrust::TrustedOnce)
1048                .unwrap();
1049        let first = manager
1050            .create("approach-a", "experiment-a", Some(3101))
1051            .unwrap();
1052        let second = manager
1053            .create("approach-b", "experiment-b", Some(3102))
1054            .unwrap();
1055        let third = manager
1056            .create("approach-c", "experiment-c", Some(3103))
1057            .unwrap();
1058        assert!(first.worktree.exists());
1059        assert!(second.worktree.exists());
1060        assert!(third.worktree.exists());
1061        let pi_available = Command::new("pi")
1062            .arg("--version")
1063            .output()
1064            .is_ok_and(|output| output.status.success());
1065        let mut agents = AgentRegistry::new(&root).unwrap();
1066        let mut assigned = Vec::new();
1067        if pi_available {
1068            for id in ["approach-a", "approach-b", "approach-c"] {
1069                assigned.push(
1070                    manager
1071                        .assign_agent(
1072                            id,
1073                            &mut agents,
1074                            AgentSpec::new("experiment-implementer", format!("evaluate {id}")),
1075                        )
1076                        .unwrap(),
1077                );
1078            }
1079            for agent in &assigned {
1080                let deadline = Instant::now() + Duration::from_secs(20);
1081                loop {
1082                    let snapshot = agents.snapshot(agent).unwrap();
1083                    if snapshot.status == AgentStatus::Idle {
1084                        break;
1085                    }
1086                    assert_ne!(
1087                        snapshot.status,
1088                        AgentStatus::Failed,
1089                        "native Pi experiment worker failed: {:?}",
1090                        snapshot.last_error
1091                    );
1092                    if Instant::now() >= deadline {
1093                        panic!(
1094                            "native Pi experiment worker did not become idle: {:?}",
1095                            snapshot
1096                        );
1097                    }
1098                    thread::sleep(Duration::from_millis(25));
1099                }
1100            }
1101            assert!(manager.snapshots().iter().all(|snapshot| {
1102                snapshot
1103                    .agent_id
1104                    .as_ref()
1105                    .is_some_and(|agent| assigned.contains(agent))
1106            }));
1107        }
1108        std::fs::write(first.worktree.join("a.rs"), "fn a() {}\n").unwrap();
1109        std::fs::write(second.worktree.join("b.rs"), "fn b() {}\n").unwrap();
1110        std::fs::write(second.worktree.join("b-extra.rs"), "fn b_extra() {}\n").unwrap();
1111        std::fs::write(third.worktree.join("c.rs"), "fn c() {}\n").unwrap();
1112        std::fs::write(third.worktree.join("c-extra-1.rs"), "fn c1() {}\n").unwrap();
1113        std::fs::write(third.worktree.join("c-extra-2.rs"), "fn c2() {}\n").unwrap();
1114        assert!(
1115            manager
1116                .set_weights(ExperimentWeights::default(), WorkspaceTrust::Untrusted)
1117                .is_err()
1118        );
1119        for (id, changed_files) in [
1120            ("approach-a", 1_u64),
1121            ("approach-b", 2_u64),
1122            ("approach-c", 3_u64),
1123        ] {
1124            let automatic = manager.collect_automatic(id).unwrap();
1125            assert_eq!(automatic.build_passed, Some(true));
1126            assert_eq!(automatic.tests_passed, 1);
1127            assert_eq!(automatic.changed_files, changed_files);
1128            assert!(automatic.provenance["buildPassed"].measured);
1129            assert!(automatic.provenance["buildPassed"].available);
1130            assert!(automatic.provenance["tests"].measured);
1131            assert!(automatic.provenance["tests"].available);
1132            assert!(automatic.provenance["changedFiles"].measured);
1133            assert!(!automatic.provenance["workflowPassed"].available);
1134            assert!(!automatic.provenance["visualDifference"].available);
1135            assert!(!automatic.provenance["lcpMs"].available);
1136        }
1137        let comparison = manager.compare();
1138        assert_eq!(comparison.rankings.len(), 3);
1139        assert_eq!(comparison.recommended.as_deref(), Some("approach-a"));
1140        assert_eq!(
1141            manager.select("approach-a").unwrap().state,
1142            ExperimentState::Selected
1143        );
1144        assert!(manager.select("approach-b").is_err());
1145        manager
1146            .record_evidence("approach-b", ExperimentEvidence::default())
1147            .unwrap();
1148        let manual = manager
1149            .snapshots()
1150            .into_iter()
1151            .find(|snapshot| snapshot.id == "approach-b")
1152            .unwrap();
1153        assert_eq!(
1154            manual.evidence.provenance["tests"].producer,
1155            "manual-external"
1156        );
1157        assert!(!manual.evidence.provenance["tests"].measured);
1158        drop(agents);
1159        manager.remove("approach-a", true).unwrap();
1160        manager.remove("approach-b", true).unwrap();
1161        manager.remove("approach-c", true).unwrap();
1162        drop(manager);
1163        std::fs::remove_dir_all(base).unwrap();
1164    }
1165}