1use crate::agents::{AgentId, AgentRegistry, AgentSpec};
4use crate::development::{DevelopmentError, DevelopmentResult, ProcessHealth};
5use crate::git::{GitError, GitService};
6use crate::testing::TestRun;
7use crate::{
8 DevelopmentWorkspace, LocalTrustDecision, WorkspaceIdentity, WorkspaceTrust,
9 WorkspaceTrustStore,
10};
11use serde::{Deserialize, Serialize};
12use serde_json::Value;
13use std::collections::{BTreeMap, BTreeSet};
14use std::path::{Path, PathBuf};
15use std::process::{Command, Stdio};
16use std::thread;
17use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
18
19const MAX_EXPERIMENTS: usize = 8;
20const MAX_NOTES_BYTES: usize = 16 * 1024;
21
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
23#[serde(rename_all = "camelCase")]
24pub enum ExperimentState {
25 Ready,
26 Running,
27 Completed,
28 Failed,
29 Selected,
30}
31
32impl ExperimentState {
33 pub fn label(&self) -> &'static str {
35 match self {
36 Self::Ready => "ready",
37 Self::Running => "running",
38 Self::Completed => "completed",
39 Self::Failed => "failed",
40 Self::Selected => "selected",
41 }
42 }
43}
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
46#[serde(rename_all = "camelCase")]
47pub enum ExperimentTrustPolicy {
48 Reevaluate,
50 InheritOnce,
52}
53
54#[derive(Debug, Clone, Default, Serialize, Deserialize)]
55#[serde(rename_all = "camelCase")]
56pub struct ExperimentEvidence {
57 pub tests_passed: u64,
58 pub tests_failed: u64,
59 pub workflow_passed: Option<bool>,
60 pub semantic_regressions: u64,
61 pub visual_difference: Option<f64>,
62 pub lcp_ms: Option<f64>,
63 pub diagnostics: u64,
64 pub debugger_stops: u64,
65 pub changed_files: u64,
66 pub build_passed: Option<bool>,
67 pub startup_healthy: Option<bool>,
68 pub process_crashes: u64,
69 pub notes: String,
70 #[serde(default)]
71 pub provenance: BTreeMap<String, EvidenceProvenance>,
72}
73
74#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
75#[serde(rename_all = "camelCase")]
76pub struct EvidenceProvenance {
77 pub producer: String,
78 pub timestamp_ms: u128,
79 pub workspace_revision: u64,
80 pub browser_revision: Option<u64>,
81 pub run_id: Option<String>,
82 pub measured: bool,
83 pub available: bool,
84 pub details: Value,
85}
86
87#[derive(Debug, Clone, Serialize, Deserialize)]
88#[serde(rename_all = "camelCase")]
89pub struct ExperimentSnapshot {
90 pub id: String,
91 pub branch: String,
92 pub worktree: PathBuf,
93 pub port: Option<u16>,
94 pub state: ExperimentState,
95 pub agent_id: Option<AgentId>,
96 pub evidence: ExperimentEvidence,
97}
98
99#[derive(Debug, Clone, Serialize, Deserialize)]
100#[serde(rename_all = "camelCase")]
101pub struct ExperimentRanking {
102 pub id: String,
103 pub score: i64,
104 pub reasons: Vec<String>,
105}
106
107#[derive(Debug, Clone, Serialize, Deserialize)]
108#[serde(rename_all = "camelCase")]
109pub struct ExperimentComparison {
110 pub rankings: Vec<ExperimentRanking>,
111 pub recommended: Option<String>,
112}
113
114#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
115#[serde(rename_all = "camelCase")]
116pub struct ExperimentWeights {
117 pub test_passed: i64,
118 pub test_failed: i64,
119 pub workflow_passed: i64,
120 pub workflow_failed: i64,
121 pub semantic_regression: i64,
122 pub diagnostic: i64,
123 pub changed_file: i64,
124 pub process_crash: i64,
125}
126
127impl Default for ExperimentWeights {
128 fn default() -> Self {
129 Self {
130 test_passed: 10,
131 test_failed: -1_000,
132 workflow_passed: 5_000,
133 workflow_failed: -20_000,
134 semantic_regression: -2_000,
135 diagnostic: -200,
136 changed_file: -5,
137 process_crash: -5_000,
138 }
139 }
140}
141
142struct Experiment {
143 snapshot: ExperimentSnapshot,
144 workspace: DevelopmentWorkspace,
145}
146
147pub struct ExperimentManager {
149 repository: GitService,
150 worktree_root: PathBuf,
151 experiments: BTreeMap<String, Experiment>,
152 ports: BTreeSet<u16>,
153 trust_policy: ExperimentTrustPolicy,
154 weights: ExperimentWeights,
155}
156
157impl ExperimentManager {
158 pub fn new(
159 repository_root: impl AsRef<Path>,
160 worktree_root: impl AsRef<Path>,
161 ) -> DevelopmentResult<Self> {
162 let identity = WorkspaceIdentity::inspect(repository_root.as_ref())?;
163 let trust = WorkspaceTrustStore::platform_default()?.status(&identity)?;
164 Self::new_governed(repository_root, worktree_root, trust)
165 }
166
167 pub fn new_governed(
168 repository_root: impl AsRef<Path>,
169 worktree_root: impl AsRef<Path>,
170 parent_trust: WorkspaceTrust,
171 ) -> DevelopmentResult<Self> {
172 if !parent_trust.permits_project_execution() {
173 return Err(DevelopmentError::Conflict(
174 "experiments cannot create executable worktrees before workspace trust".into(),
175 ));
176 }
177 let repository = GitService::open(repository_root)
178 .map_err(|error| DevelopmentError::Process(error.to_string()))?;
179 let worktree_root = absolutize(worktree_root.as_ref())?;
180 if worktree_root == Path::new("/") || worktree_root == repository.root() {
181 return Err(DevelopmentError::InvalidInput(
182 "experiment worktree root must be an explicit directory outside the repository"
183 .into(),
184 ));
185 }
186 std::fs::create_dir_all(&worktree_root)?;
187 Ok(Self {
188 repository,
189 worktree_root,
190 experiments: BTreeMap::new(),
191 ports: BTreeSet::new(),
192 trust_policy: ExperimentTrustPolicy::InheritOnce,
193 weights: ExperimentWeights::default(),
194 })
195 }
196
197 pub fn create(
198 &mut self,
199 id: &str,
200 branch: &str,
201 port: Option<u16>,
202 ) -> DevelopmentResult<ExperimentSnapshot> {
203 validate_name(id, "experiment")?;
204 validate_name(branch, "experiment branch")?;
205 if self.experiments.contains_key(id) {
206 return Err(DevelopmentError::Conflict(format!(
207 "experiment {id} already exists"
208 )));
209 }
210 if self.experiments.len() >= MAX_EXPERIMENTS {
211 return Err(DevelopmentError::Conflict(format!(
212 "experiment limit is {MAX_EXPERIMENTS}"
213 )));
214 }
215 if let Some(port) = port
216 && (port == 0 || !self.ports.insert(port))
217 {
218 return Err(DevelopmentError::Conflict(
219 "experiment ports must be non-zero and unique".into(),
220 ));
221 }
222 let worktree = self.worktree_root.join(id);
223 if worktree.exists() {
224 if let Some(port) = port {
225 self.ports.remove(&port);
226 }
227 return Err(DevelopmentError::Conflict(format!(
228 "experiment path already exists: {}",
229 worktree.display()
230 )));
231 }
232 if let Err(error) = self.repository.create_worktree(&worktree, branch, true) {
233 if let Some(port) = port {
234 self.ports.remove(&port);
235 }
236 return Err(DevelopmentError::Process(error.to_string()));
237 }
238 let workspace = match DevelopmentWorkspace::open(&worktree) {
239 Ok(mut workspace) => {
240 if self.trust_policy == ExperimentTrustPolicy::InheritOnce
241 && let Err(error) =
242 workspace.apply_local_trust_decision(LocalTrustDecision::TrustOnce)
243 {
244 drop(workspace);
245 let _ = self.repository.remove_worktree(&worktree, true);
246 if let Some(port) = port {
247 self.ports.remove(&port);
248 }
249 return Err(error);
250 }
251 workspace
252 }
253 Err(error) => {
254 let _ = self.repository.remove_worktree(&worktree, true);
255 if let Some(port) = port {
256 self.ports.remove(&port);
257 }
258 return Err(error);
259 }
260 };
261 let snapshot = ExperimentSnapshot {
262 id: id.into(),
263 branch: branch.into(),
264 worktree,
265 port,
266 state: ExperimentState::Ready,
267 agent_id: None,
268 evidence: ExperimentEvidence::default(),
269 };
270 self.experiments.insert(
271 id.into(),
272 Experiment {
273 snapshot: snapshot.clone(),
274 workspace,
275 },
276 );
277 Ok(snapshot)
278 }
279
280 pub fn assign_agent(
281 &mut self,
282 id: &str,
283 agents: &mut AgentRegistry,
284 mut spec: AgentSpec,
285 ) -> DevelopmentResult<AgentId> {
286 let experiment = self.experiment_mut(id)?;
287 if experiment.snapshot.agent_id.is_some() {
288 return Err(DevelopmentError::Conflict(format!(
289 "experiment {id} already has an agent"
290 )));
291 }
292 spec.worktree = Some(experiment.snapshot.worktree.clone());
293 let agent = agents.create(spec)?;
294 experiment.snapshot.agent_id = Some(agent.clone());
295 experiment.snapshot.state = ExperimentState::Running;
296 Ok(agent)
297 }
298
299 pub fn start_process(
300 &mut self,
301 id: &str,
302 name: &str,
303 command: &str,
304 ) -> DevelopmentResult<serde_json::Value> {
305 let experiment = self.experiment_mut(id)?;
306 experiment.snapshot.state = ExperimentState::Running;
307 Ok(serde_json::to_value(
308 experiment
309 .workspace
310 .project_mut()
311 .start_process(name, command)?,
312 )?)
313 }
314
315 pub fn run_test(
316 &mut self,
317 id: &str,
318 run_id: &str,
319 suite_id: &str,
320 actor_id: &str,
321 timeout: Option<Duration>,
322 ) -> DevelopmentResult<TestRun> {
323 let experiment = self.experiment_mut(id)?;
324 let revision = experiment.workspace.project().revision();
325 experiment.snapshot.state = ExperimentState::Running;
326 experiment
327 .workspace
328 .tests_mut()
329 .start(run_id, suite_id, actor_id, revision, timeout)
330 .map_err(|error| DevelopmentError::Process(error.to_string()))
331 }
332
333 pub fn poll_tests(&mut self, id: &str) -> DevelopmentResult<Vec<TestRun>> {
334 let experiment = self.experiment_mut(id)?;
335 let finished = experiment
336 .workspace
337 .tests_mut()
338 .poll()
339 .map_err(|error| DevelopmentError::Process(error.to_string()))?;
340 for run in &finished {
341 let failed = run.exit_code != Some(0);
342 let count = u64::try_from(run.cases.len().max(1)).unwrap_or(u64::MAX);
343 if failed {
344 experiment.snapshot.evidence.tests_failed = experiment
345 .snapshot
346 .evidence
347 .tests_failed
348 .saturating_add(count);
349 } else {
350 experiment.snapshot.evidence.tests_passed = experiment
351 .snapshot
352 .evidence
353 .tests_passed
354 .saturating_add(count);
355 }
356 }
357 Ok(finished)
358 }
359
360 pub fn collect_automatic(&mut self, id: &str) -> DevelopmentResult<ExperimentEvidence> {
364 let experiment = self.experiment_mut(id)?;
365 experiment.snapshot.state = ExperimentState::Running;
366 let revision = experiment.workspace.project().revision();
367 let browser_state = experiment.workspace.browser().state().ok();
368 let browser_revision = browser_state
369 .as_ref()
370 .and_then(|state| state.get("browserRevision"))
371 .and_then(Value::as_u64);
372 let mut evidence = ExperimentEvidence::default();
373
374 let detection = experiment.workspace.project().detection().clone();
375 if let Some(command) = detection.build_command.as_deref() {
376 let measurement = run_measured_command(&experiment.snapshot.worktree, command, 600)?;
377 evidence.build_passed = Some(measurement.passed);
378 evidence.provenance.insert(
379 "buildPassed".into(),
380 measured_provenance(
381 "project-build",
382 revision,
383 browser_revision,
384 Some(measurement.run_id),
385 serde_json::json!({
386 "command":command,
387 "exitCode":measurement.exit_code,
388 "durationMs":measurement.duration_ms
389 }),
390 ),
391 );
392 } else {
393 evidence.provenance.insert(
394 "buildPassed".into(),
395 unavailable_provenance("project-build", revision, browser_revision),
396 );
397 }
398
399 if let Some(command) = detection.test_command.as_deref() {
400 let measurement = run_measured_command(&experiment.snapshot.worktree, command, 600)?;
401 if measurement.passed {
402 evidence.tests_passed = 1;
403 } else {
404 evidence.tests_failed = 1;
405 }
406 evidence.provenance.insert(
407 "tests".into(),
408 measured_provenance(
409 "project-test",
410 revision,
411 browser_revision,
412 Some(measurement.run_id),
413 serde_json::json!({
414 "command":command,
415 "exitCode":measurement.exit_code,
416 "durationMs":measurement.duration_ms,
417 "aggregateCommandResult":true
418 }),
419 ),
420 );
421 } else {
422 evidence.provenance.insert(
423 "tests".into(),
424 unavailable_provenance("project-test", revision, browser_revision),
425 );
426 }
427
428 if let Some(git) = experiment.workspace.git() {
429 let status = git
430 .status()
431 .map_err(|error| DevelopmentError::Process(error.to_string()))?;
432 evidence.changed_files = u64::try_from(status.entries.len()).unwrap_or(u64::MAX);
433 evidence.provenance.insert(
434 "changedFiles".into(),
435 measured_provenance(
436 "resident-git",
437 revision,
438 browser_revision,
439 None,
440 serde_json::json!({
441 "entries":status.entries,
442 "conflicts":status.conflicts
443 }),
444 ),
445 );
446 }
447
448 let processes = experiment
449 .workspace
450 .project_mut()
451 .processes()
452 .list_checked()?;
453 evidence.process_crashes = processes
454 .iter()
455 .filter(|process| process.health == ProcessHealth::Failed)
456 .count() as u64;
457 evidence.startup_healthy = (!processes.is_empty()).then(|| {
458 evidence.process_crashes == 0
459 && processes.iter().all(|process| {
460 matches!(
461 process.health,
462 ProcessHealth::Healthy | ProcessHealth::Starting
463 )
464 })
465 });
466 evidence.provenance.insert(
467 "startupHealth".into(),
468 EvidenceProvenance {
469 producer: "resident-process-service".into(),
470 timestamp_ms: now_ms(),
471 workspace_revision: revision,
472 browser_revision,
473 run_id: None,
474 measured: true,
475 available: !processes.is_empty(),
476 details: serde_json::to_value(&processes)?,
477 },
478 );
479
480 let diagnostics = experiment
481 .workspace
482 .language()
483 .events(0)
484 .into_iter()
485 .filter(|event| event.operation == "diagnostics")
486 .filter_map(|event| event.result_count)
487 .sum::<usize>();
488 evidence.diagnostics = diagnostics as u64;
489 evidence.provenance.insert(
490 "diagnostics".into(),
491 EvidenceProvenance {
492 producer: "resident-lsp".into(),
493 timestamp_ms: now_ms(),
494 workspace_revision: revision,
495 browser_revision,
496 run_id: None,
497 measured: true,
498 available: experiment.workspace.language().names().next().is_some(),
499 details: serde_json::json!({"observedDiagnostics":diagnostics}),
500 },
501 );
502
503 let debugger_names = experiment
504 .workspace
505 .debugger_names()
506 .map(str::to_string)
507 .collect::<Vec<_>>();
508 let mut debugger_stops = 0_u64;
509 for debugger in &debugger_names {
510 debugger_stops = debugger_stops.saturating_add(
511 experiment
512 .workspace
513 .debugger_mut(debugger)
514 .map_err(|error| DevelopmentError::Process(error.to_string()))?
515 .poll_events()
516 .map_err(|error| DevelopmentError::Process(error.to_string()))?
517 .iter()
518 .filter(|event| event.event == "stopped")
519 .count() as u64,
520 );
521 }
522 evidence.debugger_stops = debugger_stops;
523 evidence.provenance.insert(
524 "debuggerStops".into(),
525 EvidenceProvenance {
526 producer: "resident-dap".into(),
527 timestamp_ms: now_ms(),
528 workspace_revision: revision,
529 browser_revision,
530 run_id: None,
531 measured: true,
532 available: !debugger_names.is_empty(),
533 details: serde_json::json!({"stops":debugger_stops,"sessions":debugger_names}),
534 },
535 );
536
537 if browser_state
538 .as_ref()
539 .and_then(|state| state.get("connected"))
540 .and_then(Value::as_bool)
541 == Some(true)
542 {
543 match experiment.workspace.browser().verify_workflow() {
544 Ok(result) => {
545 evidence.workflow_passed = result.get("verified").and_then(Value::as_bool);
546 evidence.provenance.insert(
547 "workflowPassed".into(),
548 measured_provenance(
549 "resident-browser-workflow",
550 revision,
551 browser_revision,
552 result
553 .pointer("/result/runId")
554 .and_then(Value::as_str)
555 .map(str::to_string),
556 result,
557 ),
558 );
559 }
560 Err(error) => {
561 evidence.provenance.insert(
562 "workflowPassed".into(),
563 EvidenceProvenance {
564 details: serde_json::json!({"error":error.to_string()}),
565 ..unavailable_provenance(
566 "resident-browser-workflow",
567 revision,
568 browser_revision,
569 )
570 },
571 );
572 }
573 }
574 match experiment.workspace.browser().diff() {
575 Ok(result) => {
576 evidence.semantic_regressions = result
577 .get("changes")
578 .and_then(Value::as_array)
579 .map_or(0, |changes| changes.len() as u64);
580 evidence.provenance.insert(
581 "semanticRegressions".into(),
582 measured_provenance(
583 "resident-browser-semantic-diff",
584 revision,
585 browser_revision,
586 None,
587 result,
588 ),
589 );
590 }
591 Err(error) => {
592 evidence.provenance.insert(
593 "semanticRegressions".into(),
594 EvidenceProvenance {
595 details: serde_json::json!({"error":error.to_string()}),
596 ..unavailable_provenance(
597 "resident-browser-semantic-diff",
598 revision,
599 browser_revision,
600 )
601 },
602 );
603 }
604 }
605 } else {
606 for (metric, producer) in [
607 ("workflowPassed", "resident-browser-workflow"),
608 ("semanticRegressions", "resident-browser-semantic-diff"),
609 ("visualDifference", "resident-browser-visual-compare"),
610 ("lcpMs", "resident-browser-performance"),
611 ] {
612 evidence.provenance.insert(
613 metric.into(),
614 unavailable_provenance(producer, revision, browser_revision),
615 );
616 }
617 }
618 experiment.snapshot.evidence = evidence.clone();
619 experiment.snapshot.state = ExperimentState::Completed;
620 Ok(evidence)
621 }
622
623 pub fn record_evidence(
624 &mut self,
625 id: &str,
626 mut evidence: ExperimentEvidence,
627 ) -> DevelopmentResult<()> {
628 if evidence.notes.len() > MAX_NOTES_BYTES || evidence.notes.contains('\0') {
629 return Err(DevelopmentError::InvalidInput(
630 "experiment notes exceed their bound or contain NUL".into(),
631 ));
632 }
633 if evidence
634 .visual_difference
635 .is_some_and(|value| !value.is_finite() || value < 0.0)
636 || evidence
637 .lcp_ms
638 .is_some_and(|value| !value.is_finite() || value < 0.0)
639 {
640 return Err(DevelopmentError::InvalidInput(
641 "experiment metrics must be finite and non-negative".into(),
642 ));
643 }
644 let experiment = self.experiment_mut(id)?;
645 let revision = experiment.workspace.project().revision();
646 let browser_revision = experiment
647 .workspace
648 .browser()
649 .state()
650 .ok()
651 .and_then(|state| state.get("browserRevision").and_then(Value::as_u64));
652 for metric in [
653 "tests",
654 "workflowPassed",
655 "semanticRegressions",
656 "visualDifference",
657 "lcpMs",
658 "diagnostics",
659 "debuggerStops",
660 "changedFiles",
661 "buildPassed",
662 "startupHealth",
663 "processCrashes",
664 ] {
665 evidence.provenance.insert(
666 metric.into(),
667 EvidenceProvenance {
668 producer: "manual-external".into(),
669 timestamp_ms: now_ms(),
670 workspace_revision: revision,
671 browser_revision,
672 run_id: None,
673 measured: false,
674 available: true,
675 details: Value::Null,
676 },
677 );
678 }
679 experiment.snapshot.evidence = evidence;
680 experiment.snapshot.state = ExperimentState::Completed;
681 Ok(())
682 }
683
684 pub fn refresh_changed_files(&mut self, id: &str) -> DevelopmentResult<u64> {
685 let experiment = self.experiment_mut(id)?;
686 let status = experiment
687 .workspace
688 .git()
689 .ok_or_else(|| DevelopmentError::NotFound("experiment Git repository".into()))?
690 .status()
691 .map_err(|error| DevelopmentError::Process(error.to_string()))?;
692 let changed = u64::try_from(status.entries.len()).unwrap_or(u64::MAX);
693 experiment.snapshot.evidence.changed_files = changed;
694 Ok(changed)
695 }
696
697 pub fn compare(&self) -> ExperimentComparison {
698 let mut rankings = self
699 .experiments
700 .values()
701 .map(|experiment| rank(&experiment.snapshot, &self.weights))
702 .collect::<Vec<_>>();
703 rankings.sort_by(|left, right| {
704 right
705 .score
706 .cmp(&left.score)
707 .then_with(|| left.id.cmp(&right.id))
708 });
709 let recommended = rankings
710 .first()
711 .filter(|ranking| {
712 self.experiments.get(&ranking.id).is_some_and(|experiment| {
713 experiment.snapshot.state == ExperimentState::Completed
714 })
715 })
716 .map(|ranking| ranking.id.clone());
717 ExperimentComparison {
718 rankings,
719 recommended,
720 }
721 }
722
723 pub fn set_weights(
724 &mut self,
725 weights: ExperimentWeights,
726 trust: WorkspaceTrust,
727 ) -> DevelopmentResult<()> {
728 if !trust.permits_project_execution() {
729 return Err(DevelopmentError::Conflict(
730 "experiment ranking weights require a trusted workspace".into(),
731 ));
732 }
733 if [
734 weights.test_passed,
735 weights.test_failed,
736 weights.workflow_passed,
737 weights.workflow_failed,
738 weights.semantic_regression,
739 weights.diagnostic,
740 weights.changed_file,
741 weights.process_crash,
742 ]
743 .iter()
744 .any(|weight| weight.unsigned_abs() > 1_000_000)
745 {
746 return Err(DevelopmentError::InvalidInput(
747 "experiment weight magnitude exceeds 1000000".into(),
748 ));
749 }
750 self.weights = weights;
751 Ok(())
752 }
753
754 pub fn select(&mut self, id: &str) -> DevelopmentResult<ExperimentSnapshot> {
755 let comparison = self.compare();
756 if comparison.recommended.as_deref() != Some(id) {
757 return Err(DevelopmentError::Conflict(
758 "selected experiment is not the current evidence-derived recommendation".into(),
759 ));
760 }
761 for experiment in self.experiments.values_mut() {
762 if experiment.snapshot.state == ExperimentState::Selected {
763 experiment.snapshot.state = ExperimentState::Completed;
764 }
765 }
766 let experiment = self.experiment_mut(id)?;
767 experiment.snapshot.state = ExperimentState::Selected;
768 Ok(experiment.snapshot.clone())
769 }
770
771 pub fn snapshots(&self) -> Vec<ExperimentSnapshot> {
772 self.experiments
773 .values()
774 .map(|experiment| experiment.snapshot.clone())
775 .collect()
776 }
777
778 pub fn remove(&mut self, id: &str, force: bool) -> DevelopmentResult<()> {
779 let experiment = self
780 .experiments
781 .remove(id)
782 .ok_or_else(|| DevelopmentError::NotFound(format!("experiment {id}")))?;
783 if let Some(port) = experiment.snapshot.port {
784 self.ports.remove(&port);
785 }
786 drop(experiment.workspace);
787 self.repository
788 .remove_worktree(&experiment.snapshot.worktree, force)
789 .map_err(|error| DevelopmentError::Process(error.to_string()))
790 }
791
792 fn experiment_mut(&mut self, id: &str) -> DevelopmentResult<&mut Experiment> {
793 self.experiments
794 .get_mut(id)
795 .ok_or_else(|| DevelopmentError::NotFound(format!("experiment {id}")))
796 }
797}
798
799struct CommandMeasurement {
800 passed: bool,
801 exit_code: i32,
802 duration_ms: u128,
803 run_id: String,
804}
805
806fn run_measured_command(
807 worktree: &Path,
808 command: &str,
809 timeout_seconds: u64,
810) -> DevelopmentResult<CommandMeasurement> {
811 if command.is_empty() || command.len() > 16 * 1024 || command.contains('\0') {
812 return Err(DevelopmentError::InvalidInput(
813 "experiment command is empty, oversized, or contains NUL".into(),
814 ));
815 }
816 let started = Instant::now();
817 let mut process = if cfg!(windows) {
818 let mut process = Command::new("cmd.exe");
819 process.args(["/d", "/s", "/c", command]);
820 process
821 } else {
822 let mut process = Command::new("sh");
823 process.args(["-lc", command]);
824 process
825 };
826 let mut child = process
827 .current_dir(worktree)
828 .stdin(Stdio::null())
829 .stdout(Stdio::null())
830 .stderr(Stdio::null())
831 .spawn()?;
832 let deadline = Instant::now() + Duration::from_secs(timeout_seconds);
833 let exit_code = loop {
834 if let Some(status) = child.try_wait()? {
835 break status.code().unwrap_or(-1);
836 }
837 if Instant::now() >= deadline {
838 child.kill()?;
839 child.wait()?;
840 break -1;
841 }
842 thread::sleep(Duration::from_millis(10));
843 };
844 let timestamp = now_ms();
845 Ok(CommandMeasurement {
846 passed: exit_code == 0,
847 exit_code,
848 duration_ms: started.elapsed().as_millis(),
849 run_id: format!("experiment-command-{timestamp}"),
850 })
851}
852
853fn measured_provenance(
854 producer: &str,
855 workspace_revision: u64,
856 browser_revision: Option<u64>,
857 run_id: Option<String>,
858 details: Value,
859) -> EvidenceProvenance {
860 EvidenceProvenance {
861 producer: producer.into(),
862 timestamp_ms: now_ms(),
863 workspace_revision,
864 browser_revision,
865 run_id,
866 measured: true,
867 available: true,
868 details,
869 }
870}
871
872fn unavailable_provenance(
873 producer: &str,
874 workspace_revision: u64,
875 browser_revision: Option<u64>,
876) -> EvidenceProvenance {
877 EvidenceProvenance {
878 producer: producer.into(),
879 timestamp_ms: now_ms(),
880 workspace_revision,
881 browser_revision,
882 run_id: None,
883 measured: true,
884 available: false,
885 details: Value::Null,
886 }
887}
888
889fn now_ms() -> u128 {
890 SystemTime::now()
891 .duration_since(UNIX_EPOCH)
892 .unwrap_or_default()
893 .as_millis()
894}
895
896fn rank(snapshot: &ExperimentSnapshot, weights: &ExperimentWeights) -> ExperimentRanking {
897 let evidence = &snapshot.evidence;
898 let mut score = 0_i64;
899 let mut reasons = Vec::new();
900 score = score.saturating_add(
901 (evidence.tests_passed.min(10_000) as i64).saturating_mul(weights.test_passed),
902 );
903 score = score.saturating_add(
904 (evidence.tests_failed.min(10_000) as i64).saturating_mul(weights.test_failed),
905 );
906 match evidence.workflow_passed {
907 Some(true) => {
908 score = score.saturating_add(weights.workflow_passed);
909 reasons.push("workflow passed".into());
910 }
911 Some(false) => {
912 score = score.saturating_add(weights.workflow_failed);
913 reasons.push("workflow failed".into());
914 }
915 None => reasons.push("workflow not measured".into()),
916 }
917 score = score.saturating_add(
918 (evidence.semantic_regressions.min(10_000) as i64)
919 .saturating_mul(weights.semantic_regression),
920 );
921 score = score.saturating_add(
922 (evidence.diagnostics.min(10_000) as i64).saturating_mul(weights.diagnostic),
923 );
924 score = score.saturating_add(
925 (evidence.changed_files.min(10_000) as i64).saturating_mul(weights.changed_file),
926 );
927 score = score.saturating_add(
928 (evidence.process_crashes.min(10_000) as i64).saturating_mul(weights.process_crash),
929 );
930 match evidence.build_passed {
931 Some(true) => score = score.saturating_add(2_000),
932 Some(false) => score = score.saturating_sub(10_000),
933 None => {}
934 }
935 match evidence.startup_healthy {
936 Some(true) => score = score.saturating_add(1_000),
937 Some(false) => score = score.saturating_sub(5_000),
938 None => {}
939 }
940 if let Some(difference) = evidence.visual_difference {
941 score = score.saturating_sub((difference.clamp(0.0, 1.0) * 1_000.0) as i64);
942 }
943 if let Some(lcp) = evidence.lcp_ms {
944 score = score.saturating_sub(lcp.min(i64::MAX as f64) as i64);
945 reasons.push(format!("LCP {lcp:.2} ms"));
946 }
947 if evidence.tests_failed == 0 {
948 reasons.push(format!("{} tests passed", evidence.tests_passed));
949 } else {
950 reasons.push(format!("{} tests failed", evidence.tests_failed));
951 }
952 if evidence.semantic_regressions > 0 {
953 reasons.push(format!(
954 "{} semantic regressions",
955 evidence.semantic_regressions
956 ));
957 }
958 ExperimentRanking {
959 id: snapshot.id.clone(),
960 score,
961 reasons,
962 }
963}
964
965fn validate_name(value: &str, description: &str) -> DevelopmentResult<()> {
966 if value.is_empty()
967 || value.len() > 128
968 || !value
969 .chars()
970 .all(|character| character.is_ascii_alphanumeric() || "-_.".contains(character))
971 {
972 return Err(DevelopmentError::InvalidInput(format!(
973 "{description} must be 1..=128 ASCII letters, digits, '-', '_' or '.'"
974 )));
975 }
976 Ok(())
977}
978
979fn absolutize(path: &Path) -> DevelopmentResult<PathBuf> {
980 if path.is_absolute() {
981 Ok(path.to_path_buf())
982 } else {
983 Ok(std::env::current_dir()?.join(path))
984 }
985}
986
987impl From<GitError> for DevelopmentError {
988 fn from(error: GitError) -> Self {
989 Self::Process(error.to_string())
990 }
991}
992
993#[cfg(test)]
994mod tests {
995 use super::*;
996 use crate::AgentStatus;
997 use std::process::Command;
998 use std::sync::atomic::{AtomicU64, Ordering};
999
1000 static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
1001
1002 fn repository() -> (PathBuf, PathBuf) {
1003 let sequence = NEXT_ROOT.fetch_add(1, Ordering::Relaxed);
1004 let base = std::env::temp_dir().join(format!(
1005 "glass-experiments-{}-{sequence}",
1006 std::process::id()
1007 ));
1008 let root = base.join("repository");
1009 std::fs::create_dir_all(&root).unwrap();
1010 std::fs::write(
1011 root.join("Cargo.toml"),
1012 "[package]\nname='fixture'\nversion='0.1.0'\n",
1013 )
1014 .unwrap();
1015 std::fs::write(
1016 root.join("glass.toml"),
1017 "[commands]\nbuild='rustc --version'\ntest='rustc --version'\n",
1018 )
1019 .unwrap();
1020 for arguments in [
1021 vec!["init"],
1022 vec!["config", "user.name", "Glass Test"],
1023 vec!["config", "user.email", "glass@example.invalid"],
1024 vec!["add", "."],
1025 vec!["commit", "-m", "initial"],
1026 ] {
1027 assert!(
1028 Command::new("git")
1029 .args(arguments)
1030 .current_dir(&root)
1031 .status()
1032 .unwrap()
1033 .success()
1034 );
1035 }
1036 (base.clone(), root)
1037 }
1038
1039 #[test]
1040 fn isolated_worktrees_are_ranked_and_selected_from_evidence() {
1041 let (base, root) = repository();
1042 let worktrees = base.join("worktrees");
1043 assert!(
1044 ExperimentManager::new_governed(&root, &worktrees, WorkspaceTrust::Untrusted).is_err()
1045 );
1046 let mut manager =
1047 ExperimentManager::new_governed(&root, &worktrees, WorkspaceTrust::TrustedOnce)
1048 .unwrap();
1049 let first = manager
1050 .create("approach-a", "experiment-a", Some(3101))
1051 .unwrap();
1052 let second = manager
1053 .create("approach-b", "experiment-b", Some(3102))
1054 .unwrap();
1055 let third = manager
1056 .create("approach-c", "experiment-c", Some(3103))
1057 .unwrap();
1058 assert!(first.worktree.exists());
1059 assert!(second.worktree.exists());
1060 assert!(third.worktree.exists());
1061 let pi_available = Command::new("pi")
1062 .arg("--version")
1063 .output()
1064 .is_ok_and(|output| output.status.success());
1065 let mut agents = AgentRegistry::new(&root).unwrap();
1066 let mut assigned = Vec::new();
1067 if pi_available {
1068 for id in ["approach-a", "approach-b", "approach-c"] {
1069 assigned.push(
1070 manager
1071 .assign_agent(
1072 id,
1073 &mut agents,
1074 AgentSpec::new("experiment-implementer", format!("evaluate {id}")),
1075 )
1076 .unwrap(),
1077 );
1078 }
1079 for agent in &assigned {
1080 let deadline = Instant::now() + Duration::from_secs(20);
1081 loop {
1082 let snapshot = agents.snapshot(agent).unwrap();
1083 if snapshot.status == AgentStatus::Idle {
1084 break;
1085 }
1086 assert_ne!(
1087 snapshot.status,
1088 AgentStatus::Failed,
1089 "native Pi experiment worker failed: {:?}",
1090 snapshot.last_error
1091 );
1092 if Instant::now() >= deadline {
1093 panic!(
1094 "native Pi experiment worker did not become idle: {:?}",
1095 snapshot
1096 );
1097 }
1098 thread::sleep(Duration::from_millis(25));
1099 }
1100 }
1101 assert!(manager.snapshots().iter().all(|snapshot| {
1102 snapshot
1103 .agent_id
1104 .as_ref()
1105 .is_some_and(|agent| assigned.contains(agent))
1106 }));
1107 }
1108 std::fs::write(first.worktree.join("a.rs"), "fn a() {}\n").unwrap();
1109 std::fs::write(second.worktree.join("b.rs"), "fn b() {}\n").unwrap();
1110 std::fs::write(second.worktree.join("b-extra.rs"), "fn b_extra() {}\n").unwrap();
1111 std::fs::write(third.worktree.join("c.rs"), "fn c() {}\n").unwrap();
1112 std::fs::write(third.worktree.join("c-extra-1.rs"), "fn c1() {}\n").unwrap();
1113 std::fs::write(third.worktree.join("c-extra-2.rs"), "fn c2() {}\n").unwrap();
1114 assert!(
1115 manager
1116 .set_weights(ExperimentWeights::default(), WorkspaceTrust::Untrusted)
1117 .is_err()
1118 );
1119 for (id, changed_files) in [
1120 ("approach-a", 1_u64),
1121 ("approach-b", 2_u64),
1122 ("approach-c", 3_u64),
1123 ] {
1124 let automatic = manager.collect_automatic(id).unwrap();
1125 assert_eq!(automatic.build_passed, Some(true));
1126 assert_eq!(automatic.tests_passed, 1);
1127 assert_eq!(automatic.changed_files, changed_files);
1128 assert!(automatic.provenance["buildPassed"].measured);
1129 assert!(automatic.provenance["buildPassed"].available);
1130 assert!(automatic.provenance["tests"].measured);
1131 assert!(automatic.provenance["tests"].available);
1132 assert!(automatic.provenance["changedFiles"].measured);
1133 assert!(!automatic.provenance["workflowPassed"].available);
1134 assert!(!automatic.provenance["visualDifference"].available);
1135 assert!(!automatic.provenance["lcpMs"].available);
1136 }
1137 let comparison = manager.compare();
1138 assert_eq!(comparison.rankings.len(), 3);
1139 assert_eq!(comparison.recommended.as_deref(), Some("approach-a"));
1140 assert_eq!(
1141 manager.select("approach-a").unwrap().state,
1142 ExperimentState::Selected
1143 );
1144 assert!(manager.select("approach-b").is_err());
1145 manager
1146 .record_evidence("approach-b", ExperimentEvidence::default())
1147 .unwrap();
1148 let manual = manager
1149 .snapshots()
1150 .into_iter()
1151 .find(|snapshot| snapshot.id == "approach-b")
1152 .unwrap();
1153 assert_eq!(
1154 manual.evidence.provenance["tests"].producer,
1155 "manual-external"
1156 );
1157 assert!(!manual.evidence.provenance["tests"].measured);
1158 drop(agents);
1159 manager.remove("approach-a", true).unwrap();
1160 manager.remove("approach-b", true).unwrap();
1161 manager.remove("approach-c", true).unwrap();
1162 drop(manager);
1163 std::fs::remove_dir_all(base).unwrap();
1164 }
1165}