1use crate::agents::{AgentId, AgentRegistry, AgentSpec};
4use crate::development::{DevelopmentError, DevelopmentResult, ProcessHealth};
5use crate::git::{GitError, GitService};
6use crate::testing::TestRun;
7use crate::{
8 DevelopmentWorkspace, LocalTrustDecision, WorkspaceIdentity, WorkspaceTrust,
9 WorkspaceTrustStore,
10};
11use serde::{Deserialize, Serialize};
12use serde_json::Value;
13use std::collections::{BTreeMap, BTreeSet};
14use std::path::{Path, PathBuf};
15use std::process::{Command, Stdio};
16use std::thread;
17use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
18
19const MAX_EXPERIMENTS: usize = 8;
20const MAX_NOTES_BYTES: usize = 16 * 1024;
21
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
23#[serde(rename_all = "camelCase")]
24pub enum ExperimentState {
25 Ready,
26 Running,
27 Completed,
28 Failed,
29 Selected,
30}
31
32#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
33#[serde(rename_all = "camelCase")]
34pub enum ExperimentTrustPolicy {
35 Reevaluate,
37 InheritOnce,
39}
40
41#[derive(Debug, Clone, Default, Serialize, Deserialize)]
42#[serde(rename_all = "camelCase")]
43pub struct ExperimentEvidence {
44 pub tests_passed: u64,
45 pub tests_failed: u64,
46 pub workflow_passed: Option<bool>,
47 pub semantic_regressions: u64,
48 pub visual_difference: Option<f64>,
49 pub lcp_ms: Option<f64>,
50 pub diagnostics: u64,
51 pub debugger_stops: u64,
52 pub changed_files: u64,
53 pub build_passed: Option<bool>,
54 pub startup_healthy: Option<bool>,
55 pub process_crashes: u64,
56 pub notes: String,
57 #[serde(default)]
58 pub provenance: BTreeMap<String, EvidenceProvenance>,
59}
60
61#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
62#[serde(rename_all = "camelCase")]
63pub struct EvidenceProvenance {
64 pub producer: String,
65 pub timestamp_ms: u128,
66 pub workspace_revision: u64,
67 pub browser_revision: Option<u64>,
68 pub run_id: Option<String>,
69 pub measured: bool,
70 pub available: bool,
71 pub details: Value,
72}
73
74#[derive(Debug, Clone, Serialize, Deserialize)]
75#[serde(rename_all = "camelCase")]
76pub struct ExperimentSnapshot {
77 pub id: String,
78 pub branch: String,
79 pub worktree: PathBuf,
80 pub port: Option<u16>,
81 pub state: ExperimentState,
82 pub agent_id: Option<AgentId>,
83 pub evidence: ExperimentEvidence,
84}
85
86#[derive(Debug, Clone, Serialize, Deserialize)]
87#[serde(rename_all = "camelCase")]
88pub struct ExperimentRanking {
89 pub id: String,
90 pub score: i64,
91 pub reasons: Vec<String>,
92}
93
94#[derive(Debug, Clone, Serialize, Deserialize)]
95#[serde(rename_all = "camelCase")]
96pub struct ExperimentComparison {
97 pub rankings: Vec<ExperimentRanking>,
98 pub recommended: Option<String>,
99}
100
101#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
102#[serde(rename_all = "camelCase")]
103pub struct ExperimentWeights {
104 pub test_passed: i64,
105 pub test_failed: i64,
106 pub workflow_passed: i64,
107 pub workflow_failed: i64,
108 pub semantic_regression: i64,
109 pub diagnostic: i64,
110 pub changed_file: i64,
111 pub process_crash: i64,
112}
113
114impl Default for ExperimentWeights {
115 fn default() -> Self {
116 Self {
117 test_passed: 10,
118 test_failed: -1_000,
119 workflow_passed: 5_000,
120 workflow_failed: -20_000,
121 semantic_regression: -2_000,
122 diagnostic: -200,
123 changed_file: -5,
124 process_crash: -5_000,
125 }
126 }
127}
128
129struct Experiment {
130 snapshot: ExperimentSnapshot,
131 workspace: DevelopmentWorkspace,
132}
133
134pub struct ExperimentManager {
136 repository: GitService,
137 worktree_root: PathBuf,
138 experiments: BTreeMap<String, Experiment>,
139 ports: BTreeSet<u16>,
140 trust_policy: ExperimentTrustPolicy,
141 weights: ExperimentWeights,
142}
143
144impl ExperimentManager {
145 pub fn new(
146 repository_root: impl AsRef<Path>,
147 worktree_root: impl AsRef<Path>,
148 ) -> DevelopmentResult<Self> {
149 let identity = WorkspaceIdentity::inspect(repository_root.as_ref())?;
150 let trust = WorkspaceTrustStore::platform_default()?.status(&identity)?;
151 Self::new_governed(repository_root, worktree_root, trust)
152 }
153
154 pub fn new_governed(
155 repository_root: impl AsRef<Path>,
156 worktree_root: impl AsRef<Path>,
157 parent_trust: WorkspaceTrust,
158 ) -> DevelopmentResult<Self> {
159 if !parent_trust.permits_project_execution() {
160 return Err(DevelopmentError::Conflict(
161 "experiments cannot create executable worktrees before workspace trust".into(),
162 ));
163 }
164 let repository = GitService::open(repository_root)
165 .map_err(|error| DevelopmentError::Process(error.to_string()))?;
166 let worktree_root = absolutize(worktree_root.as_ref())?;
167 if worktree_root == Path::new("/") || worktree_root == repository.root() {
168 return Err(DevelopmentError::InvalidInput(
169 "experiment worktree root must be an explicit directory outside the repository"
170 .into(),
171 ));
172 }
173 std::fs::create_dir_all(&worktree_root)?;
174 Ok(Self {
175 repository,
176 worktree_root,
177 experiments: BTreeMap::new(),
178 ports: BTreeSet::new(),
179 trust_policy: ExperimentTrustPolicy::InheritOnce,
180 weights: ExperimentWeights::default(),
181 })
182 }
183
184 pub fn create(
185 &mut self,
186 id: &str,
187 branch: &str,
188 port: Option<u16>,
189 ) -> DevelopmentResult<ExperimentSnapshot> {
190 validate_name(id, "experiment")?;
191 validate_name(branch, "experiment branch")?;
192 if self.experiments.contains_key(id) {
193 return Err(DevelopmentError::Conflict(format!(
194 "experiment {id} already exists"
195 )));
196 }
197 if self.experiments.len() >= MAX_EXPERIMENTS {
198 return Err(DevelopmentError::Conflict(format!(
199 "experiment limit is {MAX_EXPERIMENTS}"
200 )));
201 }
202 if let Some(port) = port
203 && (port == 0 || !self.ports.insert(port))
204 {
205 return Err(DevelopmentError::Conflict(
206 "experiment ports must be non-zero and unique".into(),
207 ));
208 }
209 let worktree = self.worktree_root.join(id);
210 if worktree.exists() {
211 if let Some(port) = port {
212 self.ports.remove(&port);
213 }
214 return Err(DevelopmentError::Conflict(format!(
215 "experiment path already exists: {}",
216 worktree.display()
217 )));
218 }
219 if let Err(error) = self.repository.create_worktree(&worktree, branch, true) {
220 if let Some(port) = port {
221 self.ports.remove(&port);
222 }
223 return Err(DevelopmentError::Process(error.to_string()));
224 }
225 let workspace = match DevelopmentWorkspace::open(&worktree) {
226 Ok(mut workspace) => {
227 if self.trust_policy == ExperimentTrustPolicy::InheritOnce
228 && let Err(error) =
229 workspace.apply_local_trust_decision(LocalTrustDecision::TrustOnce)
230 {
231 drop(workspace);
232 let _ = self.repository.remove_worktree(&worktree, true);
233 if let Some(port) = port {
234 self.ports.remove(&port);
235 }
236 return Err(error);
237 }
238 workspace
239 }
240 Err(error) => {
241 let _ = self.repository.remove_worktree(&worktree, true);
242 if let Some(port) = port {
243 self.ports.remove(&port);
244 }
245 return Err(error);
246 }
247 };
248 let snapshot = ExperimentSnapshot {
249 id: id.into(),
250 branch: branch.into(),
251 worktree,
252 port,
253 state: ExperimentState::Ready,
254 agent_id: None,
255 evidence: ExperimentEvidence::default(),
256 };
257 self.experiments.insert(
258 id.into(),
259 Experiment {
260 snapshot: snapshot.clone(),
261 workspace,
262 },
263 );
264 Ok(snapshot)
265 }
266
267 pub fn assign_agent(
268 &mut self,
269 id: &str,
270 agents: &mut AgentRegistry,
271 mut spec: AgentSpec,
272 ) -> DevelopmentResult<AgentId> {
273 let experiment = self.experiment_mut(id)?;
274 if experiment.snapshot.agent_id.is_some() {
275 return Err(DevelopmentError::Conflict(format!(
276 "experiment {id} already has an agent"
277 )));
278 }
279 spec.worktree = Some(experiment.snapshot.worktree.clone());
280 let agent = agents.create(spec)?;
281 experiment.snapshot.agent_id = Some(agent.clone());
282 experiment.snapshot.state = ExperimentState::Running;
283 Ok(agent)
284 }
285
286 pub fn start_process(
287 &mut self,
288 id: &str,
289 name: &str,
290 command: &str,
291 ) -> DevelopmentResult<serde_json::Value> {
292 let experiment = self.experiment_mut(id)?;
293 experiment.snapshot.state = ExperimentState::Running;
294 Ok(serde_json::to_value(
295 experiment
296 .workspace
297 .project_mut()
298 .start_process(name, command)?,
299 )?)
300 }
301
302 pub fn run_test(
303 &mut self,
304 id: &str,
305 run_id: &str,
306 suite_id: &str,
307 actor_id: &str,
308 timeout: Option<Duration>,
309 ) -> DevelopmentResult<TestRun> {
310 let experiment = self.experiment_mut(id)?;
311 let revision = experiment.workspace.project().revision();
312 experiment.snapshot.state = ExperimentState::Running;
313 experiment
314 .workspace
315 .tests_mut()
316 .start(run_id, suite_id, actor_id, revision, timeout)
317 .map_err(|error| DevelopmentError::Process(error.to_string()))
318 }
319
320 pub fn poll_tests(&mut self, id: &str) -> DevelopmentResult<Vec<TestRun>> {
321 let experiment = self.experiment_mut(id)?;
322 let finished = experiment
323 .workspace
324 .tests_mut()
325 .poll()
326 .map_err(|error| DevelopmentError::Process(error.to_string()))?;
327 for run in &finished {
328 let failed = run.exit_code != Some(0);
329 let count = u64::try_from(run.cases.len().max(1)).unwrap_or(u64::MAX);
330 if failed {
331 experiment.snapshot.evidence.tests_failed = experiment
332 .snapshot
333 .evidence
334 .tests_failed
335 .saturating_add(count);
336 } else {
337 experiment.snapshot.evidence.tests_passed = experiment
338 .snapshot
339 .evidence
340 .tests_passed
341 .saturating_add(count);
342 }
343 }
344 Ok(finished)
345 }
346
347 pub fn collect_automatic(&mut self, id: &str) -> DevelopmentResult<ExperimentEvidence> {
351 let experiment = self.experiment_mut(id)?;
352 experiment.snapshot.state = ExperimentState::Running;
353 let revision = experiment.workspace.project().revision();
354 let browser_state = experiment.workspace.browser().state().ok();
355 let browser_revision = browser_state
356 .as_ref()
357 .and_then(|state| state.get("browserRevision"))
358 .and_then(Value::as_u64);
359 let mut evidence = ExperimentEvidence::default();
360
361 let detection = experiment.workspace.project().detection().clone();
362 if let Some(command) = detection.build_command.as_deref() {
363 let measurement = run_measured_command(&experiment.snapshot.worktree, command, 600)?;
364 evidence.build_passed = Some(measurement.passed);
365 evidence.provenance.insert(
366 "buildPassed".into(),
367 measured_provenance(
368 "project-build",
369 revision,
370 browser_revision,
371 Some(measurement.run_id),
372 serde_json::json!({
373 "command":command,
374 "exitCode":measurement.exit_code,
375 "durationMs":measurement.duration_ms
376 }),
377 ),
378 );
379 } else {
380 evidence.provenance.insert(
381 "buildPassed".into(),
382 unavailable_provenance("project-build", revision, browser_revision),
383 );
384 }
385
386 if let Some(command) = detection.test_command.as_deref() {
387 let measurement = run_measured_command(&experiment.snapshot.worktree, command, 600)?;
388 if measurement.passed {
389 evidence.tests_passed = 1;
390 } else {
391 evidence.tests_failed = 1;
392 }
393 evidence.provenance.insert(
394 "tests".into(),
395 measured_provenance(
396 "project-test",
397 revision,
398 browser_revision,
399 Some(measurement.run_id),
400 serde_json::json!({
401 "command":command,
402 "exitCode":measurement.exit_code,
403 "durationMs":measurement.duration_ms,
404 "aggregateCommandResult":true
405 }),
406 ),
407 );
408 } else {
409 evidence.provenance.insert(
410 "tests".into(),
411 unavailable_provenance("project-test", revision, browser_revision),
412 );
413 }
414
415 if let Some(git) = experiment.workspace.git() {
416 let status = git
417 .status()
418 .map_err(|error| DevelopmentError::Process(error.to_string()))?;
419 evidence.changed_files = u64::try_from(status.entries.len()).unwrap_or(u64::MAX);
420 evidence.provenance.insert(
421 "changedFiles".into(),
422 measured_provenance(
423 "resident-git",
424 revision,
425 browser_revision,
426 None,
427 serde_json::json!({
428 "entries":status.entries,
429 "conflicts":status.conflicts
430 }),
431 ),
432 );
433 }
434
435 let processes = experiment
436 .workspace
437 .project_mut()
438 .processes()
439 .list_checked()?;
440 evidence.process_crashes = processes
441 .iter()
442 .filter(|process| process.health == ProcessHealth::Failed)
443 .count() as u64;
444 evidence.startup_healthy = (!processes.is_empty()).then(|| {
445 evidence.process_crashes == 0
446 && processes.iter().all(|process| {
447 matches!(
448 process.health,
449 ProcessHealth::Healthy | ProcessHealth::Starting
450 )
451 })
452 });
453 evidence.provenance.insert(
454 "startupHealth".into(),
455 EvidenceProvenance {
456 producer: "resident-process-service".into(),
457 timestamp_ms: now_ms(),
458 workspace_revision: revision,
459 browser_revision,
460 run_id: None,
461 measured: true,
462 available: !processes.is_empty(),
463 details: serde_json::to_value(&processes)?,
464 },
465 );
466
467 let diagnostics = experiment
468 .workspace
469 .language()
470 .events(0)
471 .into_iter()
472 .filter(|event| event.operation == "diagnostics")
473 .filter_map(|event| event.result_count)
474 .sum::<usize>();
475 evidence.diagnostics = diagnostics as u64;
476 evidence.provenance.insert(
477 "diagnostics".into(),
478 EvidenceProvenance {
479 producer: "resident-lsp".into(),
480 timestamp_ms: now_ms(),
481 workspace_revision: revision,
482 browser_revision,
483 run_id: None,
484 measured: true,
485 available: experiment.workspace.language().names().next().is_some(),
486 details: serde_json::json!({"observedDiagnostics":diagnostics}),
487 },
488 );
489
490 let debugger_names = experiment
491 .workspace
492 .debugger_names()
493 .map(str::to_string)
494 .collect::<Vec<_>>();
495 let mut debugger_stops = 0_u64;
496 for debugger in &debugger_names {
497 debugger_stops = debugger_stops.saturating_add(
498 experiment
499 .workspace
500 .debugger_mut(debugger)
501 .map_err(|error| DevelopmentError::Process(error.to_string()))?
502 .poll_events()
503 .map_err(|error| DevelopmentError::Process(error.to_string()))?
504 .iter()
505 .filter(|event| event.event == "stopped")
506 .count() as u64,
507 );
508 }
509 evidence.debugger_stops = debugger_stops;
510 evidence.provenance.insert(
511 "debuggerStops".into(),
512 EvidenceProvenance {
513 producer: "resident-dap".into(),
514 timestamp_ms: now_ms(),
515 workspace_revision: revision,
516 browser_revision,
517 run_id: None,
518 measured: true,
519 available: !debugger_names.is_empty(),
520 details: serde_json::json!({"stops":debugger_stops,"sessions":debugger_names}),
521 },
522 );
523
524 if browser_state
525 .as_ref()
526 .and_then(|state| state.get("connected"))
527 .and_then(Value::as_bool)
528 == Some(true)
529 {
530 match experiment.workspace.browser().verify_workflow() {
531 Ok(result) => {
532 evidence.workflow_passed = result.get("verified").and_then(Value::as_bool);
533 evidence.provenance.insert(
534 "workflowPassed".into(),
535 measured_provenance(
536 "resident-browser-workflow",
537 revision,
538 browser_revision,
539 result
540 .pointer("/result/runId")
541 .and_then(Value::as_str)
542 .map(str::to_string),
543 result,
544 ),
545 );
546 }
547 Err(error) => {
548 evidence.provenance.insert(
549 "workflowPassed".into(),
550 EvidenceProvenance {
551 details: serde_json::json!({"error":error.to_string()}),
552 ..unavailable_provenance(
553 "resident-browser-workflow",
554 revision,
555 browser_revision,
556 )
557 },
558 );
559 }
560 }
561 match experiment.workspace.browser().diff() {
562 Ok(result) => {
563 evidence.semantic_regressions = result
564 .get("changes")
565 .and_then(Value::as_array)
566 .map_or(0, |changes| changes.len() as u64);
567 evidence.provenance.insert(
568 "semanticRegressions".into(),
569 measured_provenance(
570 "resident-browser-semantic-diff",
571 revision,
572 browser_revision,
573 None,
574 result,
575 ),
576 );
577 }
578 Err(error) => {
579 evidence.provenance.insert(
580 "semanticRegressions".into(),
581 EvidenceProvenance {
582 details: serde_json::json!({"error":error.to_string()}),
583 ..unavailable_provenance(
584 "resident-browser-semantic-diff",
585 revision,
586 browser_revision,
587 )
588 },
589 );
590 }
591 }
592 } else {
593 for (metric, producer) in [
594 ("workflowPassed", "resident-browser-workflow"),
595 ("semanticRegressions", "resident-browser-semantic-diff"),
596 ("visualDifference", "resident-browser-visual-compare"),
597 ("lcpMs", "resident-browser-performance"),
598 ] {
599 evidence.provenance.insert(
600 metric.into(),
601 unavailable_provenance(producer, revision, browser_revision),
602 );
603 }
604 }
605 experiment.snapshot.evidence = evidence.clone();
606 experiment.snapshot.state = ExperimentState::Completed;
607 Ok(evidence)
608 }
609
610 pub fn record_evidence(
611 &mut self,
612 id: &str,
613 mut evidence: ExperimentEvidence,
614 ) -> DevelopmentResult<()> {
615 if evidence.notes.len() > MAX_NOTES_BYTES || evidence.notes.contains('\0') {
616 return Err(DevelopmentError::InvalidInput(
617 "experiment notes exceed their bound or contain NUL".into(),
618 ));
619 }
620 if evidence
621 .visual_difference
622 .is_some_and(|value| !value.is_finite() || value < 0.0)
623 || evidence
624 .lcp_ms
625 .is_some_and(|value| !value.is_finite() || value < 0.0)
626 {
627 return Err(DevelopmentError::InvalidInput(
628 "experiment metrics must be finite and non-negative".into(),
629 ));
630 }
631 let experiment = self.experiment_mut(id)?;
632 let revision = experiment.workspace.project().revision();
633 let browser_revision = experiment
634 .workspace
635 .browser()
636 .state()
637 .ok()
638 .and_then(|state| state.get("browserRevision").and_then(Value::as_u64));
639 for metric in [
640 "tests",
641 "workflowPassed",
642 "semanticRegressions",
643 "visualDifference",
644 "lcpMs",
645 "diagnostics",
646 "debuggerStops",
647 "changedFiles",
648 "buildPassed",
649 "startupHealth",
650 "processCrashes",
651 ] {
652 evidence.provenance.insert(
653 metric.into(),
654 EvidenceProvenance {
655 producer: "manual-external".into(),
656 timestamp_ms: now_ms(),
657 workspace_revision: revision,
658 browser_revision,
659 run_id: None,
660 measured: false,
661 available: true,
662 details: Value::Null,
663 },
664 );
665 }
666 experiment.snapshot.evidence = evidence;
667 experiment.snapshot.state = ExperimentState::Completed;
668 Ok(())
669 }
670
671 pub fn refresh_changed_files(&mut self, id: &str) -> DevelopmentResult<u64> {
672 let experiment = self.experiment_mut(id)?;
673 let status = experiment
674 .workspace
675 .git()
676 .ok_or_else(|| DevelopmentError::NotFound("experiment Git repository".into()))?
677 .status()
678 .map_err(|error| DevelopmentError::Process(error.to_string()))?;
679 let changed = u64::try_from(status.entries.len()).unwrap_or(u64::MAX);
680 experiment.snapshot.evidence.changed_files = changed;
681 Ok(changed)
682 }
683
684 pub fn compare(&self) -> ExperimentComparison {
685 let mut rankings = self
686 .experiments
687 .values()
688 .map(|experiment| rank(&experiment.snapshot, &self.weights))
689 .collect::<Vec<_>>();
690 rankings.sort_by(|left, right| {
691 right
692 .score
693 .cmp(&left.score)
694 .then_with(|| left.id.cmp(&right.id))
695 });
696 let recommended = rankings
697 .first()
698 .filter(|ranking| {
699 self.experiments.get(&ranking.id).is_some_and(|experiment| {
700 experiment.snapshot.state == ExperimentState::Completed
701 })
702 })
703 .map(|ranking| ranking.id.clone());
704 ExperimentComparison {
705 rankings,
706 recommended,
707 }
708 }
709
710 pub fn set_weights(
711 &mut self,
712 weights: ExperimentWeights,
713 trust: WorkspaceTrust,
714 ) -> DevelopmentResult<()> {
715 if !trust.permits_project_execution() {
716 return Err(DevelopmentError::Conflict(
717 "experiment ranking weights require a trusted workspace".into(),
718 ));
719 }
720 if [
721 weights.test_passed,
722 weights.test_failed,
723 weights.workflow_passed,
724 weights.workflow_failed,
725 weights.semantic_regression,
726 weights.diagnostic,
727 weights.changed_file,
728 weights.process_crash,
729 ]
730 .iter()
731 .any(|weight| weight.unsigned_abs() > 1_000_000)
732 {
733 return Err(DevelopmentError::InvalidInput(
734 "experiment weight magnitude exceeds 1000000".into(),
735 ));
736 }
737 self.weights = weights;
738 Ok(())
739 }
740
741 pub fn select(&mut self, id: &str) -> DevelopmentResult<ExperimentSnapshot> {
742 let comparison = self.compare();
743 if comparison.recommended.as_deref() != Some(id) {
744 return Err(DevelopmentError::Conflict(
745 "selected experiment is not the current evidence-derived recommendation".into(),
746 ));
747 }
748 for experiment in self.experiments.values_mut() {
749 if experiment.snapshot.state == ExperimentState::Selected {
750 experiment.snapshot.state = ExperimentState::Completed;
751 }
752 }
753 let experiment = self.experiment_mut(id)?;
754 experiment.snapshot.state = ExperimentState::Selected;
755 Ok(experiment.snapshot.clone())
756 }
757
758 pub fn snapshots(&self) -> Vec<ExperimentSnapshot> {
759 self.experiments
760 .values()
761 .map(|experiment| experiment.snapshot.clone())
762 .collect()
763 }
764
765 pub fn remove(&mut self, id: &str, force: bool) -> DevelopmentResult<()> {
766 let experiment = self
767 .experiments
768 .remove(id)
769 .ok_or_else(|| DevelopmentError::NotFound(format!("experiment {id}")))?;
770 if let Some(port) = experiment.snapshot.port {
771 self.ports.remove(&port);
772 }
773 drop(experiment.workspace);
774 self.repository
775 .remove_worktree(&experiment.snapshot.worktree, force)
776 .map_err(|error| DevelopmentError::Process(error.to_string()))
777 }
778
779 fn experiment_mut(&mut self, id: &str) -> DevelopmentResult<&mut Experiment> {
780 self.experiments
781 .get_mut(id)
782 .ok_or_else(|| DevelopmentError::NotFound(format!("experiment {id}")))
783 }
784}
785
786struct CommandMeasurement {
787 passed: bool,
788 exit_code: i32,
789 duration_ms: u128,
790 run_id: String,
791}
792
793fn run_measured_command(
794 worktree: &Path,
795 command: &str,
796 timeout_seconds: u64,
797) -> DevelopmentResult<CommandMeasurement> {
798 if command.is_empty() || command.len() > 16 * 1024 || command.contains('\0') {
799 return Err(DevelopmentError::InvalidInput(
800 "experiment command is empty, oversized, or contains NUL".into(),
801 ));
802 }
803 let started = Instant::now();
804 let mut process = if cfg!(windows) {
805 let mut process = Command::new("cmd.exe");
806 process.args(["/d", "/s", "/c", command]);
807 process
808 } else {
809 let mut process = Command::new("sh");
810 process.args(["-lc", command]);
811 process
812 };
813 let mut child = process
814 .current_dir(worktree)
815 .stdin(Stdio::null())
816 .stdout(Stdio::null())
817 .stderr(Stdio::null())
818 .spawn()?;
819 let deadline = Instant::now() + Duration::from_secs(timeout_seconds);
820 let exit_code = loop {
821 if let Some(status) = child.try_wait()? {
822 break status.code().unwrap_or(-1);
823 }
824 if Instant::now() >= deadline {
825 child.kill()?;
826 child.wait()?;
827 break -1;
828 }
829 thread::sleep(Duration::from_millis(10));
830 };
831 let timestamp = now_ms();
832 Ok(CommandMeasurement {
833 passed: exit_code == 0,
834 exit_code,
835 duration_ms: started.elapsed().as_millis(),
836 run_id: format!("experiment-command-{timestamp}"),
837 })
838}
839
840fn measured_provenance(
841 producer: &str,
842 workspace_revision: u64,
843 browser_revision: Option<u64>,
844 run_id: Option<String>,
845 details: Value,
846) -> EvidenceProvenance {
847 EvidenceProvenance {
848 producer: producer.into(),
849 timestamp_ms: now_ms(),
850 workspace_revision,
851 browser_revision,
852 run_id,
853 measured: true,
854 available: true,
855 details,
856 }
857}
858
859fn unavailable_provenance(
860 producer: &str,
861 workspace_revision: u64,
862 browser_revision: Option<u64>,
863) -> EvidenceProvenance {
864 EvidenceProvenance {
865 producer: producer.into(),
866 timestamp_ms: now_ms(),
867 workspace_revision,
868 browser_revision,
869 run_id: None,
870 measured: true,
871 available: false,
872 details: Value::Null,
873 }
874}
875
876fn now_ms() -> u128 {
877 SystemTime::now()
878 .duration_since(UNIX_EPOCH)
879 .unwrap_or_default()
880 .as_millis()
881}
882
883fn rank(snapshot: &ExperimentSnapshot, weights: &ExperimentWeights) -> ExperimentRanking {
884 let evidence = &snapshot.evidence;
885 let mut score = 0_i64;
886 let mut reasons = Vec::new();
887 score = score.saturating_add(
888 (evidence.tests_passed.min(10_000) as i64).saturating_mul(weights.test_passed),
889 );
890 score = score.saturating_add(
891 (evidence.tests_failed.min(10_000) as i64).saturating_mul(weights.test_failed),
892 );
893 match evidence.workflow_passed {
894 Some(true) => {
895 score = score.saturating_add(weights.workflow_passed);
896 reasons.push("workflow passed".into());
897 }
898 Some(false) => {
899 score = score.saturating_add(weights.workflow_failed);
900 reasons.push("workflow failed".into());
901 }
902 None => reasons.push("workflow not measured".into()),
903 }
904 score = score.saturating_add(
905 (evidence.semantic_regressions.min(10_000) as i64)
906 .saturating_mul(weights.semantic_regression),
907 );
908 score = score.saturating_add(
909 (evidence.diagnostics.min(10_000) as i64).saturating_mul(weights.diagnostic),
910 );
911 score = score.saturating_add(
912 (evidence.changed_files.min(10_000) as i64).saturating_mul(weights.changed_file),
913 );
914 score = score.saturating_add(
915 (evidence.process_crashes.min(10_000) as i64).saturating_mul(weights.process_crash),
916 );
917 match evidence.build_passed {
918 Some(true) => score = score.saturating_add(2_000),
919 Some(false) => score = score.saturating_sub(10_000),
920 None => {}
921 }
922 match evidence.startup_healthy {
923 Some(true) => score = score.saturating_add(1_000),
924 Some(false) => score = score.saturating_sub(5_000),
925 None => {}
926 }
927 if let Some(difference) = evidence.visual_difference {
928 score = score.saturating_sub((difference.clamp(0.0, 1.0) * 1_000.0) as i64);
929 }
930 if let Some(lcp) = evidence.lcp_ms {
931 score = score.saturating_sub(lcp.min(i64::MAX as f64) as i64);
932 reasons.push(format!("LCP {lcp:.2} ms"));
933 }
934 if evidence.tests_failed == 0 {
935 reasons.push(format!("{} tests passed", evidence.tests_passed));
936 } else {
937 reasons.push(format!("{} tests failed", evidence.tests_failed));
938 }
939 if evidence.semantic_regressions > 0 {
940 reasons.push(format!(
941 "{} semantic regressions",
942 evidence.semantic_regressions
943 ));
944 }
945 ExperimentRanking {
946 id: snapshot.id.clone(),
947 score,
948 reasons,
949 }
950}
951
952fn validate_name(value: &str, description: &str) -> DevelopmentResult<()> {
953 if value.is_empty()
954 || value.len() > 128
955 || !value
956 .chars()
957 .all(|character| character.is_ascii_alphanumeric() || "-_.".contains(character))
958 {
959 return Err(DevelopmentError::InvalidInput(format!(
960 "{description} must be 1..=128 ASCII letters, digits, '-', '_' or '.'"
961 )));
962 }
963 Ok(())
964}
965
966fn absolutize(path: &Path) -> DevelopmentResult<PathBuf> {
967 if path.is_absolute() {
968 Ok(path.to_path_buf())
969 } else {
970 Ok(std::env::current_dir()?.join(path))
971 }
972}
973
974impl From<GitError> for DevelopmentError {
975 fn from(error: GitError) -> Self {
976 Self::Process(error.to_string())
977 }
978}
979
980#[cfg(test)]
981mod tests {
982 use super::*;
983 use crate::AgentStatus;
984 use std::process::Command;
985 use std::sync::atomic::{AtomicU64, Ordering};
986
987 static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
988
989 fn repository() -> (PathBuf, PathBuf) {
990 let sequence = NEXT_ROOT.fetch_add(1, Ordering::Relaxed);
991 let base = std::env::temp_dir().join(format!(
992 "glass-experiments-{}-{sequence}",
993 std::process::id()
994 ));
995 let root = base.join("repository");
996 std::fs::create_dir_all(&root).unwrap();
997 std::fs::write(
998 root.join("Cargo.toml"),
999 "[package]\nname='fixture'\nversion='0.1.0'\n",
1000 )
1001 .unwrap();
1002 std::fs::write(
1003 root.join("glass.toml"),
1004 "[commands]\nbuild='rustc --version'\ntest='rustc --version'\n",
1005 )
1006 .unwrap();
1007 for arguments in [
1008 vec!["init"],
1009 vec!["config", "user.name", "Glass Test"],
1010 vec!["config", "user.email", "glass@example.invalid"],
1011 vec!["add", "."],
1012 vec!["commit", "-m", "initial"],
1013 ] {
1014 assert!(
1015 Command::new("git")
1016 .args(arguments)
1017 .current_dir(&root)
1018 .status()
1019 .unwrap()
1020 .success()
1021 );
1022 }
1023 (base.clone(), root)
1024 }
1025
1026 #[test]
1027 fn isolated_worktrees_are_ranked_and_selected_from_evidence() {
1028 let (base, root) = repository();
1029 let worktrees = base.join("worktrees");
1030 assert!(
1031 ExperimentManager::new_governed(&root, &worktrees, WorkspaceTrust::Untrusted).is_err()
1032 );
1033 let mut manager =
1034 ExperimentManager::new_governed(&root, &worktrees, WorkspaceTrust::TrustedOnce)
1035 .unwrap();
1036 let first = manager
1037 .create("approach-a", "experiment-a", Some(3101))
1038 .unwrap();
1039 let second = manager
1040 .create("approach-b", "experiment-b", Some(3102))
1041 .unwrap();
1042 let third = manager
1043 .create("approach-c", "experiment-c", Some(3103))
1044 .unwrap();
1045 assert!(first.worktree.exists());
1046 assert!(second.worktree.exists());
1047 assert!(third.worktree.exists());
1048 let pi_available = Command::new("pi")
1049 .arg("--version")
1050 .output()
1051 .is_ok_and(|output| output.status.success());
1052 let mut agents = AgentRegistry::new(&root).unwrap();
1053 let mut assigned = Vec::new();
1054 if pi_available {
1055 for id in ["approach-a", "approach-b", "approach-c"] {
1056 assigned.push(
1057 manager
1058 .assign_agent(
1059 id,
1060 &mut agents,
1061 AgentSpec::new("experiment-implementer", format!("evaluate {id}")),
1062 )
1063 .unwrap(),
1064 );
1065 }
1066 for agent in &assigned {
1067 let mut ready = false;
1068 for _ in 0..200 {
1069 let snapshot = agents.snapshot(agent).unwrap();
1070 if snapshot.status == AgentStatus::Idle {
1071 ready = true;
1072 break;
1073 }
1074 assert_ne!(
1075 snapshot.status,
1076 AgentStatus::Failed,
1077 "native Pi experiment worker failed: {:?}",
1078 snapshot.last_error
1079 );
1080 thread::sleep(Duration::from_millis(25));
1081 }
1082 assert!(ready, "native Pi experiment worker did not become idle");
1083 }
1084 assert!(manager.snapshots().iter().all(|snapshot| {
1085 snapshot
1086 .agent_id
1087 .as_ref()
1088 .is_some_and(|agent| assigned.contains(agent))
1089 }));
1090 }
1091 std::fs::write(first.worktree.join("a.rs"), "fn a() {}\n").unwrap();
1092 std::fs::write(second.worktree.join("b.rs"), "fn b() {}\n").unwrap();
1093 std::fs::write(second.worktree.join("b-extra.rs"), "fn b_extra() {}\n").unwrap();
1094 std::fs::write(third.worktree.join("c.rs"), "fn c() {}\n").unwrap();
1095 std::fs::write(third.worktree.join("c-extra-1.rs"), "fn c1() {}\n").unwrap();
1096 std::fs::write(third.worktree.join("c-extra-2.rs"), "fn c2() {}\n").unwrap();
1097 assert!(
1098 manager
1099 .set_weights(ExperimentWeights::default(), WorkspaceTrust::Untrusted)
1100 .is_err()
1101 );
1102 for (id, changed_files) in [
1103 ("approach-a", 1_u64),
1104 ("approach-b", 2_u64),
1105 ("approach-c", 3_u64),
1106 ] {
1107 let automatic = manager.collect_automatic(id).unwrap();
1108 assert_eq!(automatic.build_passed, Some(true));
1109 assert_eq!(automatic.tests_passed, 1);
1110 assert_eq!(automatic.changed_files, changed_files);
1111 assert!(automatic.provenance["buildPassed"].measured);
1112 assert!(automatic.provenance["buildPassed"].available);
1113 assert!(automatic.provenance["tests"].measured);
1114 assert!(automatic.provenance["tests"].available);
1115 assert!(automatic.provenance["changedFiles"].measured);
1116 assert!(!automatic.provenance["workflowPassed"].available);
1117 assert!(!automatic.provenance["visualDifference"].available);
1118 assert!(!automatic.provenance["lcpMs"].available);
1119 }
1120 let comparison = manager.compare();
1121 assert_eq!(comparison.rankings.len(), 3);
1122 assert_eq!(comparison.recommended.as_deref(), Some("approach-a"));
1123 assert_eq!(
1124 manager.select("approach-a").unwrap().state,
1125 ExperimentState::Selected
1126 );
1127 assert!(manager.select("approach-b").is_err());
1128 manager
1129 .record_evidence("approach-b", ExperimentEvidence::default())
1130 .unwrap();
1131 let manual = manager
1132 .snapshots()
1133 .into_iter()
1134 .find(|snapshot| snapshot.id == "approach-b")
1135 .unwrap();
1136 assert_eq!(
1137 manual.evidence.provenance["tests"].producer,
1138 "manual-external"
1139 );
1140 assert!(!manual.evidence.provenance["tests"].measured);
1141 drop(agents);
1142 manager.remove("approach-a", true).unwrap();
1143 manager.remove("approach-b", true).unwrap();
1144 manager.remove("approach-c", true).unwrap();
1145 drop(manager);
1146 std::fs::remove_dir_all(base).unwrap();
1147 }
1148}