1use crate::DevelopmentWorkspace;
4use crate::agents::{AgentId, AgentRegistry, AgentSpec};
5use crate::git::{GitError, GitService};
6use crate::testing::TestRun;
7use glass_browser::development::{DevelopmentError, DevelopmentResult};
8use serde::{Deserialize, Serialize};
9use std::collections::{BTreeMap, BTreeSet};
10use std::path::{Path, PathBuf};
11use std::time::Duration;
12
13const MAX_EXPERIMENTS: usize = 8;
14const MAX_NOTES_BYTES: usize = 16 * 1024;
15
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
17#[serde(rename_all = "camelCase")]
18pub enum ExperimentState {
19 Ready,
20 Running,
21 Completed,
22 Failed,
23 Selected,
24}
25
26#[derive(Debug, Clone, Default, Serialize, Deserialize)]
27#[serde(rename_all = "camelCase")]
28pub struct ExperimentEvidence {
29 pub tests_passed: u64,
30 pub tests_failed: u64,
31 pub workflow_passed: Option<bool>,
32 pub semantic_regressions: u64,
33 pub visual_difference: Option<f64>,
34 pub lcp_ms: Option<f64>,
35 pub diagnostics: u64,
36 pub debugger_stops: u64,
37 pub changed_files: u64,
38 pub notes: String,
39}
40
41#[derive(Debug, Clone, Serialize, Deserialize)]
42#[serde(rename_all = "camelCase")]
43pub struct ExperimentSnapshot {
44 pub id: String,
45 pub branch: String,
46 pub worktree: PathBuf,
47 pub port: Option<u16>,
48 pub state: ExperimentState,
49 pub agent_id: Option<AgentId>,
50 pub evidence: ExperimentEvidence,
51}
52
53#[derive(Debug, Clone, Serialize, Deserialize)]
54#[serde(rename_all = "camelCase")]
55pub struct ExperimentRanking {
56 pub id: String,
57 pub score: i64,
58 pub reasons: Vec<String>,
59}
60
61#[derive(Debug, Clone, Serialize, Deserialize)]
62#[serde(rename_all = "camelCase")]
63pub struct ExperimentComparison {
64 pub rankings: Vec<ExperimentRanking>,
65 pub recommended: Option<String>,
66}
67
68struct Experiment {
69 snapshot: ExperimentSnapshot,
70 workspace: DevelopmentWorkspace,
71}
72
73pub struct ExperimentManager {
75 repository: GitService,
76 worktree_root: PathBuf,
77 experiments: BTreeMap<String, Experiment>,
78 ports: BTreeSet<u16>,
79}
80
81impl ExperimentManager {
82 pub fn new(
83 repository_root: impl AsRef<Path>,
84 worktree_root: impl AsRef<Path>,
85 ) -> DevelopmentResult<Self> {
86 let repository = GitService::open(repository_root)
87 .map_err(|error| DevelopmentError::Process(error.to_string()))?;
88 let worktree_root = absolutize(worktree_root.as_ref())?;
89 if worktree_root == Path::new("/") || worktree_root == repository.root() {
90 return Err(DevelopmentError::InvalidInput(
91 "experiment worktree root must be an explicit directory outside the repository"
92 .into(),
93 ));
94 }
95 std::fs::create_dir_all(&worktree_root)?;
96 Ok(Self {
97 repository,
98 worktree_root,
99 experiments: BTreeMap::new(),
100 ports: BTreeSet::new(),
101 })
102 }
103
104 pub fn create(
105 &mut self,
106 id: &str,
107 branch: &str,
108 port: Option<u16>,
109 ) -> DevelopmentResult<ExperimentSnapshot> {
110 validate_name(id, "experiment")?;
111 validate_name(branch, "experiment branch")?;
112 if self.experiments.contains_key(id) {
113 return Err(DevelopmentError::Conflict(format!(
114 "experiment {id} already exists"
115 )));
116 }
117 if self.experiments.len() >= MAX_EXPERIMENTS {
118 return Err(DevelopmentError::Conflict(format!(
119 "experiment limit is {MAX_EXPERIMENTS}"
120 )));
121 }
122 if let Some(port) = port
123 && (port == 0 || !self.ports.insert(port))
124 {
125 return Err(DevelopmentError::Conflict(
126 "experiment ports must be non-zero and unique".into(),
127 ));
128 }
129 let worktree = self.worktree_root.join(id);
130 if worktree.exists() {
131 if let Some(port) = port {
132 self.ports.remove(&port);
133 }
134 return Err(DevelopmentError::Conflict(format!(
135 "experiment path already exists: {}",
136 worktree.display()
137 )));
138 }
139 if let Err(error) = self.repository.create_worktree(&worktree, branch, true) {
140 if let Some(port) = port {
141 self.ports.remove(&port);
142 }
143 return Err(DevelopmentError::Process(error.to_string()));
144 }
145 let workspace = match DevelopmentWorkspace::open(&worktree) {
146 Ok(workspace) => workspace,
147 Err(error) => {
148 let _ = self.repository.remove_worktree(&worktree, true);
149 if let Some(port) = port {
150 self.ports.remove(&port);
151 }
152 return Err(error);
153 }
154 };
155 let snapshot = ExperimentSnapshot {
156 id: id.into(),
157 branch: branch.into(),
158 worktree,
159 port,
160 state: ExperimentState::Ready,
161 agent_id: None,
162 evidence: ExperimentEvidence::default(),
163 };
164 self.experiments.insert(
165 id.into(),
166 Experiment {
167 snapshot: snapshot.clone(),
168 workspace,
169 },
170 );
171 Ok(snapshot)
172 }
173
174 pub fn assign_agent(
175 &mut self,
176 id: &str,
177 agents: &mut AgentRegistry,
178 mut spec: AgentSpec,
179 ) -> DevelopmentResult<AgentId> {
180 let experiment = self.experiment_mut(id)?;
181 if experiment.snapshot.agent_id.is_some() {
182 return Err(DevelopmentError::Conflict(format!(
183 "experiment {id} already has an agent"
184 )));
185 }
186 spec.worktree = Some(experiment.snapshot.worktree.clone());
187 let agent = agents.create(spec)?;
188 experiment.snapshot.agent_id = Some(agent.clone());
189 experiment.snapshot.state = ExperimentState::Running;
190 Ok(agent)
191 }
192
193 pub fn start_process(
194 &mut self,
195 id: &str,
196 name: &str,
197 command: &str,
198 ) -> DevelopmentResult<serde_json::Value> {
199 let experiment = self.experiment_mut(id)?;
200 experiment.snapshot.state = ExperimentState::Running;
201 Ok(serde_json::to_value(
202 experiment
203 .workspace
204 .project_mut()
205 .start_process(name, command)?,
206 )?)
207 }
208
209 pub fn run_test(
210 &mut self,
211 id: &str,
212 run_id: &str,
213 suite_id: &str,
214 actor_id: &str,
215 timeout: Option<Duration>,
216 ) -> DevelopmentResult<TestRun> {
217 let experiment = self.experiment_mut(id)?;
218 let revision = experiment.workspace.project().revision();
219 experiment.snapshot.state = ExperimentState::Running;
220 experiment
221 .workspace
222 .tests_mut()
223 .start(run_id, suite_id, actor_id, revision, timeout)
224 .map_err(|error| DevelopmentError::Process(error.to_string()))
225 }
226
227 pub fn poll_tests(&mut self, id: &str) -> DevelopmentResult<Vec<TestRun>> {
228 let experiment = self.experiment_mut(id)?;
229 let finished = experiment
230 .workspace
231 .tests_mut()
232 .poll()
233 .map_err(|error| DevelopmentError::Process(error.to_string()))?;
234 for run in &finished {
235 let failed = run.exit_code != Some(0);
236 let count = u64::try_from(run.cases.len().max(1)).unwrap_or(u64::MAX);
237 if failed {
238 experiment.snapshot.evidence.tests_failed = experiment
239 .snapshot
240 .evidence
241 .tests_failed
242 .saturating_add(count);
243 } else {
244 experiment.snapshot.evidence.tests_passed = experiment
245 .snapshot
246 .evidence
247 .tests_passed
248 .saturating_add(count);
249 }
250 }
251 Ok(finished)
252 }
253
254 pub fn record_evidence(
255 &mut self,
256 id: &str,
257 evidence: ExperimentEvidence,
258 ) -> DevelopmentResult<()> {
259 if evidence.notes.len() > MAX_NOTES_BYTES || evidence.notes.contains('\0') {
260 return Err(DevelopmentError::InvalidInput(
261 "experiment notes exceed their bound or contain NUL".into(),
262 ));
263 }
264 if evidence
265 .visual_difference
266 .is_some_and(|value| !value.is_finite() || value < 0.0)
267 || evidence
268 .lcp_ms
269 .is_some_and(|value| !value.is_finite() || value < 0.0)
270 {
271 return Err(DevelopmentError::InvalidInput(
272 "experiment metrics must be finite and non-negative".into(),
273 ));
274 }
275 let experiment = self.experiment_mut(id)?;
276 experiment.snapshot.evidence = evidence;
277 experiment.snapshot.state = ExperimentState::Completed;
278 Ok(())
279 }
280
281 pub fn refresh_changed_files(&mut self, id: &str) -> DevelopmentResult<u64> {
282 let experiment = self.experiment_mut(id)?;
283 let status = experiment
284 .workspace
285 .git()
286 .ok_or_else(|| DevelopmentError::NotFound("experiment Git repository".into()))?
287 .status()
288 .map_err(|error| DevelopmentError::Process(error.to_string()))?;
289 let changed = u64::try_from(status.entries.len()).unwrap_or(u64::MAX);
290 experiment.snapshot.evidence.changed_files = changed;
291 Ok(changed)
292 }
293
294 pub fn compare(&self) -> ExperimentComparison {
295 let mut rankings = self
296 .experiments
297 .values()
298 .map(|experiment| rank(&experiment.snapshot))
299 .collect::<Vec<_>>();
300 rankings.sort_by(|left, right| {
301 right
302 .score
303 .cmp(&left.score)
304 .then_with(|| left.id.cmp(&right.id))
305 });
306 let recommended = rankings
307 .first()
308 .filter(|ranking| {
309 self.experiments.get(&ranking.id).is_some_and(|experiment| {
310 experiment.snapshot.state == ExperimentState::Completed
311 })
312 })
313 .map(|ranking| ranking.id.clone());
314 ExperimentComparison {
315 rankings,
316 recommended,
317 }
318 }
319
320 pub fn select(&mut self, id: &str) -> DevelopmentResult<ExperimentSnapshot> {
321 let comparison = self.compare();
322 if comparison.recommended.as_deref() != Some(id) {
323 return Err(DevelopmentError::Conflict(
324 "selected experiment is not the current evidence-derived recommendation".into(),
325 ));
326 }
327 for experiment in self.experiments.values_mut() {
328 if experiment.snapshot.state == ExperimentState::Selected {
329 experiment.snapshot.state = ExperimentState::Completed;
330 }
331 }
332 let experiment = self.experiment_mut(id)?;
333 experiment.snapshot.state = ExperimentState::Selected;
334 Ok(experiment.snapshot.clone())
335 }
336
337 pub fn snapshots(&self) -> Vec<ExperimentSnapshot> {
338 self.experiments
339 .values()
340 .map(|experiment| experiment.snapshot.clone())
341 .collect()
342 }
343
344 pub fn remove(&mut self, id: &str, force: bool) -> DevelopmentResult<()> {
345 let experiment = self
346 .experiments
347 .remove(id)
348 .ok_or_else(|| DevelopmentError::NotFound(format!("experiment {id}")))?;
349 if let Some(port) = experiment.snapshot.port {
350 self.ports.remove(&port);
351 }
352 drop(experiment.workspace);
353 self.repository
354 .remove_worktree(&experiment.snapshot.worktree, force)
355 .map_err(|error| DevelopmentError::Process(error.to_string()))
356 }
357
358 fn experiment_mut(&mut self, id: &str) -> DevelopmentResult<&mut Experiment> {
359 self.experiments
360 .get_mut(id)
361 .ok_or_else(|| DevelopmentError::NotFound(format!("experiment {id}")))
362 }
363}
364
365fn rank(snapshot: &ExperimentSnapshot) -> ExperimentRanking {
366 let evidence = &snapshot.evidence;
367 let mut score = 0_i64;
368 let mut reasons = Vec::new();
369 score = score.saturating_add((evidence.tests_passed.min(10_000) as i64) * 10);
370 score = score.saturating_sub((evidence.tests_failed.min(10_000) as i64) * 1_000);
371 match evidence.workflow_passed {
372 Some(true) => {
373 score = score.saturating_add(5_000);
374 reasons.push("workflow passed".into());
375 }
376 Some(false) => {
377 score = score.saturating_sub(20_000);
378 reasons.push("workflow failed".into());
379 }
380 None => reasons.push("workflow not measured".into()),
381 }
382 score = score.saturating_sub((evidence.semantic_regressions.min(10_000) as i64) * 2_000);
383 score = score.saturating_sub((evidence.diagnostics.min(10_000) as i64) * 200);
384 score = score.saturating_sub((evidence.changed_files.min(10_000) as i64) * 5);
385 if let Some(lcp) = evidence.lcp_ms {
386 score = score.saturating_sub(lcp.min(i64::MAX as f64) as i64);
387 reasons.push(format!("LCP {lcp:.2} ms"));
388 }
389 if evidence.tests_failed == 0 {
390 reasons.push(format!("{} tests passed", evidence.tests_passed));
391 } else {
392 reasons.push(format!("{} tests failed", evidence.tests_failed));
393 }
394 if evidence.semantic_regressions > 0 {
395 reasons.push(format!(
396 "{} semantic regressions",
397 evidence.semantic_regressions
398 ));
399 }
400 ExperimentRanking {
401 id: snapshot.id.clone(),
402 score,
403 reasons,
404 }
405}
406
407fn validate_name(value: &str, description: &str) -> DevelopmentResult<()> {
408 if value.is_empty()
409 || value.len() > 128
410 || !value
411 .chars()
412 .all(|character| character.is_ascii_alphanumeric() || "-_.".contains(character))
413 {
414 return Err(DevelopmentError::InvalidInput(format!(
415 "{description} must be 1..=128 ASCII letters, digits, '-', '_' or '.'"
416 )));
417 }
418 Ok(())
419}
420
421fn absolutize(path: &Path) -> DevelopmentResult<PathBuf> {
422 if path.is_absolute() {
423 Ok(path.to_path_buf())
424 } else {
425 Ok(std::env::current_dir()?.join(path))
426 }
427}
428
429impl From<GitError> for DevelopmentError {
430 fn from(error: GitError) -> Self {
431 Self::Process(error.to_string())
432 }
433}
434
435#[cfg(test)]
436mod tests {
437 use super::*;
438 use std::process::Command;
439 use std::sync::atomic::{AtomicU64, Ordering};
440
441 static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
442
443 fn repository() -> (PathBuf, PathBuf) {
444 let sequence = NEXT_ROOT.fetch_add(1, Ordering::Relaxed);
445 let base = std::env::temp_dir().join(format!(
446 "glass-experiments-{}-{sequence}",
447 std::process::id()
448 ));
449 let root = base.join("repository");
450 std::fs::create_dir_all(&root).unwrap();
451 std::fs::write(
452 root.join("Cargo.toml"),
453 "[package]\nname='fixture'\nversion='0.1.0'\n",
454 )
455 .unwrap();
456 for arguments in [
457 vec!["init"],
458 vec!["config", "user.name", "Glass Test"],
459 vec!["config", "user.email", "glass@example.invalid"],
460 vec!["add", "Cargo.toml"],
461 vec!["commit", "-m", "initial"],
462 ] {
463 assert!(
464 Command::new("git")
465 .args(arguments)
466 .current_dir(&root)
467 .status()
468 .unwrap()
469 .success()
470 );
471 }
472 (base.clone(), root)
473 }
474
475 #[test]
476 fn isolated_worktrees_are_ranked_and_selected_from_evidence() {
477 let (base, root) = repository();
478 let worktrees = base.join("worktrees");
479 let mut manager = ExperimentManager::new(&root, &worktrees).unwrap();
480 let first = manager
481 .create("approach-a", "experiment-a", Some(3101))
482 .unwrap();
483 let second = manager
484 .create("approach-b", "experiment-b", Some(3102))
485 .unwrap();
486 assert!(first.worktree.exists());
487 assert!(second.worktree.exists());
488 std::fs::write(first.worktree.join("a.rs"), "fn a() {}\n").unwrap();
489 std::fs::write(second.worktree.join("b.rs"), "fn b() {}\n").unwrap();
490 assert_eq!(manager.refresh_changed_files("approach-a").unwrap(), 1);
491 assert_eq!(manager.refresh_changed_files("approach-b").unwrap(), 1);
492 manager
493 .record_evidence(
494 "approach-a",
495 ExperimentEvidence {
496 tests_passed: 142,
497 workflow_passed: Some(true),
498 lcp_ms: Some(1_220.0),
499 ..ExperimentEvidence::default()
500 },
501 )
502 .unwrap();
503 manager
504 .record_evidence(
505 "approach-b",
506 ExperimentEvidence {
507 tests_passed: 141,
508 tests_failed: 1,
509 workflow_passed: Some(false),
510 semantic_regressions: 1,
511 lcp_ms: Some(1_080.0),
512 ..ExperimentEvidence::default()
513 },
514 )
515 .unwrap();
516 let comparison = manager.compare();
517 assert_eq!(comparison.recommended.as_deref(), Some("approach-a"));
518 assert_eq!(
519 manager.select("approach-a").unwrap().state,
520 ExperimentState::Selected
521 );
522 assert!(manager.select("approach-b").is_err());
523 manager.remove("approach-a", true).unwrap();
524 manager.remove("approach-b", true).unwrap();
525 drop(manager);
526 std::fs::remove_dir_all(base).unwrap();
527 }
528}