agentos_execution/
benchmark.rs

1use crate::{
2    CreateJavascriptContextRequest, JavascriptExecutionEngine, JavascriptExecutionError,
3    StartJavascriptExecutionRequest,
4};
5use agentos_runtime::RuntimeContext;
6use serde::{Deserialize, Serialize};
7use std::collections::BTreeMap;
8use std::env;
9use std::fmt;
10use std::fmt::Write as _;
11use std::fs;
12use std::path::{Path, PathBuf};
13use std::process::Command;
14use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
15
16const BENCHMARK_MARKER_PREFIX: &str = "__AGENTOS_BENCH__:";
17const LOCAL_GRAPH_MODULE_COUNT: usize = 24;
18const BENCHMARK_ARTIFACT_VERSION: u32 = 5;
19const BENCHMARK_ARTIFACT_DIR: &str = "target/benchmark-reports/node-import-bench";
20const BENCHMARK_RUN_STATE_FILE: &str = "run-state.json";
21const TRANSPORT_RTT_CHANNEL: &str = "execution-stdio-echo";
22const TRANSPORT_RTT_PAYLOAD_BYTES: [usize; 3] = [32, 4 * 1024, 64 * 1024];
23const TRANSPORT_POLL_TIMEOUT: Duration = Duration::from_secs(5);
24const MAX_BENCHMARK_ITERATIONS: usize = 1_000;
25const MAX_BENCHMARK_WARMUP_ITERATIONS: usize = 1_000;
26
27#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
28pub struct JavascriptBenchmarkConfig {
29    pub iterations: usize,
30    pub warmup_iterations: usize,
31}
32
33impl Default for JavascriptBenchmarkConfig {
34    fn default() -> Self {
35        Self {
36            iterations: 5,
37            warmup_iterations: 1,
38        }
39    }
40}
41
42#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
43pub struct BenchmarkHost {
44    pub node_binary: String,
45    pub node_version: String,
46    pub os: &'static str,
47    pub arch: &'static str,
48    pub logical_cpus: usize,
49}
50
51#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
52pub struct BenchmarkScenarioPhases<T> {
53    pub context_setup_ms: T,
54    pub startup_ms: T,
55    #[serde(skip_serializing_if = "Option::is_none", default)]
56    pub guest_execution_ms: Option<T>,
57    pub completion_ms: T,
58}
59
60#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
61pub struct BenchmarkStats {
62    pub mean_ms: f64,
63    pub p50_ms: f64,
64    pub p95_ms: f64,
65    pub min_ms: f64,
66    pub max_ms: f64,
67    pub stddev_ms: f64,
68}
69
70#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
71pub struct BenchmarkDistributionStats {
72    pub mean: f64,
73    pub p50: f64,
74    pub p95: f64,
75    pub min: f64,
76    pub max: f64,
77    pub stddev: f64,
78}
79
80#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
81pub struct BenchmarkResourceUsage<T> {
82    #[serde(skip_serializing_if = "Option::is_none", default)]
83    pub rss_bytes: Option<T>,
84    #[serde(skip_serializing_if = "Option::is_none", default)]
85    pub heap_used_bytes: Option<T>,
86    #[serde(skip_serializing_if = "Option::is_none", default)]
87    pub cpu_user_us: Option<T>,
88    #[serde(skip_serializing_if = "Option::is_none", default)]
89    pub cpu_system_us: Option<T>,
90    #[serde(skip_serializing_if = "Option::is_none", default)]
91    pub cpu_total_us: Option<T>,
92}
93
94#[derive(Debug, Clone, PartialEq, Serialize)]
95pub struct BenchmarkTransportRttReport {
96    pub channel: &'static str,
97    pub payload_bytes: usize,
98    pub samples_ms: Vec<f64>,
99    pub stats: BenchmarkStats,
100}
101
102#[derive(Debug, Clone, PartialEq, Serialize)]
103pub struct BenchmarkScenarioReport {
104    pub id: &'static str,
105    pub workload: &'static str,
106    pub runtime: &'static str,
107    pub mode: &'static str,
108    pub description: &'static str,
109    pub fixture: &'static str,
110    pub compile_cache: &'static str,
111    pub wall_samples_ms: Vec<f64>,
112    pub wall_stats: BenchmarkStats,
113    pub guest_import_samples_ms: Option<Vec<f64>>,
114    pub guest_import_stats: Option<BenchmarkStats>,
115    pub startup_overhead_samples_ms: Option<Vec<f64>>,
116    pub startup_overhead_stats: Option<BenchmarkStats>,
117    pub phase_samples_ms: BenchmarkScenarioPhases<Vec<f64>>,
118    pub phase_stats: BenchmarkScenarioPhases<BenchmarkStats>,
119    #[serde(skip_serializing_if = "Option::is_none", default)]
120    pub resource_usage_samples: Option<BenchmarkResourceUsage<Vec<f64>>>,
121    #[serde(skip_serializing_if = "Option::is_none", default)]
122    pub resource_usage_stats: Option<BenchmarkResourceUsage<BenchmarkDistributionStats>>,
123}
124
125#[derive(Debug, Clone, PartialEq, Serialize)]
126pub struct JavascriptBenchmarkReport {
127    pub generated_at_unix_ms: u128,
128    pub config: JavascriptBenchmarkConfig,
129    pub host: BenchmarkHost,
130    pub repo_root: PathBuf,
131    pub transport_rtt: Vec<BenchmarkTransportRttReport>,
132    pub scenarios: Vec<BenchmarkScenarioReport>,
133}
134
135#[derive(Debug, Clone, PartialEq, Serialize)]
136pub struct BenchmarkComparison {
137    pub baseline: BenchmarkComparisonBaseline,
138    pub summary: BenchmarkComparisonSummary,
139    pub scenario_deltas: Vec<BenchmarkScenarioDelta>,
140    pub scenarios_missing_from_baseline: Vec<String>,
141    pub baseline_only_scenarios: Vec<String>,
142}
143
144#[derive(Debug, Clone, PartialEq, Serialize)]
145pub struct BenchmarkComparisonBaseline {
146    pub artifact_version: u32,
147    pub generated_at_unix_ms: u128,
148    pub path: PathBuf,
149}
150
151#[derive(Debug, Clone, PartialEq, Serialize)]
152pub struct BenchmarkComparisonSummary {
153    pub compared_scenario_count: usize,
154    #[serde(skip_serializing_if = "Option::is_none")]
155    pub largest_wall_improvement: Option<BenchmarkDeltaHighlight>,
156    #[serde(skip_serializing_if = "Option::is_none")]
157    pub largest_wall_regression: Option<BenchmarkDeltaHighlight>,
158}
159
160#[derive(Debug, Clone, PartialEq, Serialize)]
161pub struct BenchmarkDeltaHighlight {
162    pub id: String,
163    pub delta_ms: f64,
164    pub delta_pct: f64,
165}
166
167#[derive(Debug, Clone, PartialEq, Serialize)]
168pub struct BenchmarkScenarioDelta {
169    pub id: String,
170    pub description: String,
171    pub wall_mean_ms: BenchmarkMetricDelta,
172    #[serde(skip_serializing_if = "Option::is_none")]
173    pub guest_import_mean_ms: Option<BenchmarkMetricDelta>,
174    #[serde(skip_serializing_if = "Option::is_none")]
175    pub startup_overhead_mean_ms: Option<BenchmarkMetricDelta>,
176    #[serde(skip_serializing_if = "Option::is_none")]
177    pub phase_mean_ms: Option<BenchmarkScenarioPhases<BenchmarkMetricDelta>>,
178}
179
180#[derive(Debug, Clone, PartialEq, Serialize)]
181pub struct BenchmarkMetricDelta {
182    pub baseline_ms: f64,
183    pub current_ms: f64,
184    pub delta_ms: f64,
185    pub delta_pct: f64,
186}
187
188impl JavascriptBenchmarkReport {
189    pub fn render_markdown(&self) -> String {
190        self.render_markdown_with_comparison(None)
191    }
192
193    pub fn render_markdown_with_comparison(
194        &self,
195        comparison: Option<&BenchmarkComparison>,
196    ) -> String {
197        let mut markdown = String::new();
198        let _ = writeln!(&mut markdown, "# secure-exec Node Import Benchmark");
199        let _ = writeln!(&mut markdown);
200        let _ = writeln!(
201            &mut markdown,
202            "- Generated at unix ms: `{}`",
203            self.generated_at_unix_ms
204        );
205        let _ = writeln!(&mut markdown, "- Node binary: `{}`", self.host.node_binary);
206        let _ = writeln!(
207            &mut markdown,
208            "- Node version: `{}`",
209            self.host.node_version.trim()
210        );
211        let _ = writeln!(
212            &mut markdown,
213            "- Host: `{}` / `{}` / `{}` logical CPUs",
214            self.host.os, self.host.arch, self.host.logical_cpus
215        );
216        let _ = writeln!(&mut markdown, "- Repo root: `{}`", self.repo_root.display());
217        let _ = writeln!(
218            &mut markdown,
219            "- Iterations: `{}` recorded, `{}` warmup",
220            self.config.iterations, self.config.warmup_iterations
221        );
222        let _ = writeln!(
223            &mut markdown,
224            "- Reproduce: `cargo run -p agentos-execution --bin node-import-bench -- --iterations {} --warmup-iterations {}`",
225            self.config.iterations, self.config.warmup_iterations
226        );
227        let _ = writeln!(&mut markdown);
228        let _ = writeln!(&mut markdown, "## Transport RTT");
229        let _ = writeln!(&mut markdown);
230        let _ = writeln!(
231            &mut markdown,
232            "| Channel | Payload (bytes) | Mean RTT (ms) | P50 | P95 |"
233        );
234        let _ = writeln!(&mut markdown, "| --- | ---: | ---: | ---: | ---: |");
235
236        for transport in &self.transport_rtt {
237            let _ = writeln!(
238                &mut markdown,
239                "| `{}` | {} | {} | {} | {} |",
240                transport.channel,
241                transport.payload_bytes,
242                format_ms(transport.stats.mean_ms),
243                format_ms(transport.stats.p50_ms),
244                format_ms(transport.stats.p95_ms),
245            );
246        }
247
248        let _ = writeln!(&mut markdown, "## Control Matrix");
249        let _ = writeln!(&mut markdown);
250
251        for row in self.control_matrix() {
252            let _ = writeln!(
253                &mut markdown,
254                "- Workload `{}`: runtimes {}, modes {}, scenarios {}",
255                row.workload,
256                format_label_list(&row.runtimes),
257                format_label_list(&row.modes),
258                format_label_list(&row.scenario_ids),
259            );
260        }
261
262        let _ = writeln!(&mut markdown);
263        let _ = writeln!(&mut markdown, "## Scenario Summary");
264        let _ = writeln!(&mut markdown);
265        let _ = writeln!(
266            &mut markdown,
267            "| Scenario | Workload | Runtime | Mode | Fixture | Cache | Mean wall (ms) | Mean context (ms) | Mean startup (ms) | Mean guest exec (ms) | Mean completion (ms) | Mean startup overhead (ms) |"
268        );
269        let _ = writeln!(
270            &mut markdown,
271            "| --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |"
272        );
273
274        for scenario in &self.scenarios {
275            let guest_execution_mean = scenario
276                .phase_stats
277                .guest_execution_ms
278                .as_ref()
279                .map(|stats| format_ms(stats.mean_ms))
280                .unwrap_or_else(|| String::from("n/a"));
281            let startup_overhead_mean = scenario
282                .startup_overhead_stats
283                .as_ref()
284                .map(|stats| format_ms(stats.mean_ms))
285                .unwrap_or_else(|| String::from("n/a"));
286
287            let _ = writeln!(
288                &mut markdown,
289                "| `{}` | `{}` | `{}` | `{}` | {} | {} | {} | {} | {} | {} | {} | {} |",
290                scenario.id,
291                scenario.workload,
292                scenario.runtime,
293                scenario.mode,
294                scenario.fixture,
295                scenario.compile_cache,
296                format_ms(scenario.wall_stats.mean_ms),
297                format_ms(scenario.phase_stats.context_setup_ms.mean_ms),
298                format_ms(scenario.phase_stats.startup_ms.mean_ms),
299                guest_execution_mean,
300                format_ms(scenario.phase_stats.completion_ms.mean_ms),
301                startup_overhead_mean,
302            );
303        }
304
305        let _ = writeln!(&mut markdown);
306        let _ = writeln!(&mut markdown, "## Stability And Resource Summary");
307        let _ = writeln!(&mut markdown);
308        let _ = writeln!(
309            &mut markdown,
310            "| Scenario | Wall P50 (ms) | Wall min-max (ms) | Wall stddev (ms) | Mean RSS (MiB) | Mean heap (MiB) | Mean total CPU (ms) |"
311        );
312        let _ = writeln!(
313            &mut markdown,
314            "| --- | ---: | --- | ---: | ---: | ---: | ---: |"
315        );
316
317        for scenario in &self.scenarios {
318            let _ = writeln!(
319                &mut markdown,
320                "| `{}` | {} | {}-{} | {} | {} | {} | {} |",
321                scenario.id,
322                format_ms(scenario.wall_stats.p50_ms),
323                format_ms(scenario.wall_stats.min_ms),
324                format_ms(scenario.wall_stats.max_ms),
325                format_ms(scenario.wall_stats.stddev_ms),
326                scenario
327                    .resource_usage_stats
328                    .as_ref()
329                    .and_then(|stats| stats.rss_bytes.as_ref())
330                    .map(|stats| format_mib(bytes_to_mib(stats.mean)))
331                    .unwrap_or_else(|| String::from("n/a")),
332                scenario
333                    .resource_usage_stats
334                    .as_ref()
335                    .and_then(|stats| stats.heap_used_bytes.as_ref())
336                    .map(|stats| format_mib(bytes_to_mib(stats.mean)))
337                    .unwrap_or_else(|| String::from("n/a")),
338                scenario
339                    .resource_usage_stats
340                    .as_ref()
341                    .and_then(|stats| stats.cpu_total_us.as_ref())
342                    .map(|stats| format_ms(micros_to_ms(stats.mean)))
343                    .unwrap_or_else(|| String::from("n/a")),
344            );
345        }
346
347        let _ = writeln!(&mut markdown);
348        let _ = writeln!(&mut markdown, "## Ranked Hotspots");
349        let _ = writeln!(&mut markdown);
350
351        for ranking in self.hotspot_rankings() {
352            let _ = writeln!(
353                &mut markdown,
354                "### {} (`{}`, `{}`)",
355                ranking.label, ranking.dimension, ranking.unit
356            );
357            let _ = writeln!(&mut markdown);
358            let _ = writeln!(
359                &mut markdown,
360                "| Rank | Scenario | Workload | Runtime | Mode | Value |"
361            );
362            let _ = writeln!(&mut markdown, "| ---: | --- | --- | --- | --- | ---: |");
363
364            for scenario in &ranking.ranked_scenarios {
365                let _ = writeln!(
366                    &mut markdown,
367                    "| {} | `{}` | `{}` | `{}` | `{}` | {} |",
368                    scenario.rank,
369                    scenario.id,
370                    scenario.workload,
371                    scenario.runtime,
372                    scenario.mode,
373                    format_hotspot_value(ranking.unit, scenario.value),
374                );
375            }
376
377            if !ranking.scenarios_without_metric.is_empty() {
378                let _ = writeln!(&mut markdown);
379                let _ = writeln!(
380                    &mut markdown,
381                    "Missing metric for: {}",
382                    format_string_label_list(&ranking.scenarios_without_metric),
383                );
384            }
385
386            let _ = writeln!(&mut markdown);
387        }
388
389        let _ = writeln!(&mut markdown, "## Hotspot Guidance");
390        let _ = writeln!(&mut markdown);
391
392        for line in self.guidance_lines() {
393            let _ = writeln!(&mut markdown, "- {line}");
394        }
395
396        if let Some(comparison) = comparison {
397            let _ = writeln!(&mut markdown);
398            let _ = writeln!(&mut markdown, "## Baseline Comparison");
399            let _ = writeln!(&mut markdown);
400            let _ = writeln!(
401                &mut markdown,
402                "- Baseline artifact: `{}`",
403                comparison.baseline.path.display()
404            );
405            let _ = writeln!(
406                &mut markdown,
407                "- Baseline generated at unix ms: `{}`",
408                comparison.baseline.generated_at_unix_ms
409            );
410            let _ = writeln!(
411                &mut markdown,
412                "- Compared scenarios: `{}`",
413                comparison.summary.compared_scenario_count
414            );
415            if let Some(improvement) = &comparison.summary.largest_wall_improvement {
416                let _ = writeln!(
417                    &mut markdown,
418                    "- Largest wall-time improvement: `{}` at {} ({})",
419                    improvement.id,
420                    format_delta_ms(improvement.delta_ms),
421                    format_delta_pct(improvement.delta_pct),
422                );
423            }
424            if let Some(regression) = &comparison.summary.largest_wall_regression {
425                let _ = writeln!(
426                    &mut markdown,
427                    "- Largest wall-time regression: `{}` at {} ({})",
428                    regression.id,
429                    format_delta_ms(regression.delta_ms),
430                    format_delta_pct(regression.delta_pct),
431                );
432            }
433            if !comparison.scenarios_missing_from_baseline.is_empty() {
434                let _ = writeln!(
435                    &mut markdown,
436                    "- Scenarios missing from baseline: {}",
437                    comparison.scenarios_missing_from_baseline.join(", ")
438                );
439            }
440            if !comparison.baseline_only_scenarios.is_empty() {
441                let _ = writeln!(
442                    &mut markdown,
443                    "- Baseline-only scenarios: {}",
444                    comparison.baseline_only_scenarios.join(", ")
445                );
446            }
447            let _ = writeln!(&mut markdown);
448            let _ = writeln!(
449                &mut markdown,
450                "| Scenario | Wall delta (ms) | Wall delta % | Import delta (ms) | Startup delta (ms) | Context delta (ms) | Completion delta (ms) |"
451            );
452            let _ = writeln!(
453                &mut markdown,
454                "| --- | ---: | ---: | ---: | ---: | ---: | ---: |"
455            );
456
457            for scenario in &comparison.scenario_deltas {
458                let import_delta = scenario
459                    .guest_import_mean_ms
460                    .as_ref()
461                    .map(|delta| format_delta_ms(delta.delta_ms))
462                    .unwrap_or_else(|| String::from("n/a"));
463                let startup_delta = scenario
464                    .startup_overhead_mean_ms
465                    .as_ref()
466                    .map(|delta| format_delta_ms(delta.delta_ms))
467                    .unwrap_or_else(|| String::from("n/a"));
468                let context_delta = scenario
469                    .phase_mean_ms
470                    .as_ref()
471                    .map(|delta| format_delta_ms(delta.context_setup_ms.delta_ms))
472                    .unwrap_or_else(|| String::from("n/a"));
473                let completion_delta = scenario
474                    .phase_mean_ms
475                    .as_ref()
476                    .map(|delta| format_delta_ms(delta.completion_ms.delta_ms))
477                    .unwrap_or_else(|| String::from("n/a"));
478
479                let _ = writeln!(
480                    &mut markdown,
481                    "| `{}` | {} | {} | {} | {} | {} | {} |",
482                    scenario.id,
483                    format_delta_ms(scenario.wall_mean_ms.delta_ms),
484                    format_delta_pct(scenario.wall_mean_ms.delta_pct),
485                    import_delta,
486                    startup_delta,
487                    context_delta,
488                    completion_delta,
489                );
490            }
491        }
492
493        let _ = writeln!(&mut markdown);
494        let _ = writeln!(&mut markdown, "## Raw Samples");
495        let _ = writeln!(&mut markdown);
496
497        for scenario in &self.scenarios {
498            let _ = writeln!(&mut markdown, "### `{}`", scenario.id);
499            let _ = writeln!(&mut markdown, "- Workload: `{}`", scenario.workload);
500            let _ = writeln!(&mut markdown, "- Runtime: `{}`", scenario.runtime);
501            let _ = writeln!(&mut markdown, "- Mode: `{}`", scenario.mode);
502            let _ = writeln!(&mut markdown, "- Description: {}", scenario.description);
503            let _ = writeln!(
504                &mut markdown,
505                "- Wall samples (ms): {}",
506                format_sample_list(&scenario.wall_samples_ms)
507            );
508            if let Some(samples) = &scenario.guest_import_samples_ms {
509                let _ = writeln!(
510                    &mut markdown,
511                    "- Guest import samples (ms): {}",
512                    format_sample_list(samples)
513                );
514            }
515            if let Some(samples) = &scenario.startup_overhead_samples_ms {
516                let _ = writeln!(
517                    &mut markdown,
518                    "- Startup overhead samples (ms): {}",
519                    format_sample_list(samples)
520                );
521            }
522            let _ = writeln!(
523                &mut markdown,
524                "- Context setup samples (ms): {}",
525                format_sample_list(&scenario.phase_samples_ms.context_setup_ms)
526            );
527            let _ = writeln!(
528                &mut markdown,
529                "- Startup samples (ms): {}",
530                format_sample_list(&scenario.phase_samples_ms.startup_ms)
531            );
532            if let Some(samples) = &scenario.phase_samples_ms.guest_execution_ms {
533                let _ = writeln!(
534                    &mut markdown,
535                    "- Guest execution samples (ms): {}",
536                    format_sample_list(samples)
537                );
538            }
539            let _ = writeln!(
540                &mut markdown,
541                "- Completion samples (ms): {}",
542                format_sample_list(&scenario.phase_samples_ms.completion_ms)
543            );
544            if let Some(samples) = &scenario.resource_usage_samples {
545                if let Some(rss_samples) = &samples.rss_bytes {
546                    let _ = writeln!(
547                        &mut markdown,
548                        "- RSS samples (MiB): {}",
549                        format_scaled_sample_list(rss_samples, bytes_to_mib)
550                    );
551                }
552                if let Some(heap_samples) = &samples.heap_used_bytes {
553                    let _ = writeln!(
554                        &mut markdown,
555                        "- Heap samples (MiB): {}",
556                        format_scaled_sample_list(heap_samples, bytes_to_mib)
557                    );
558                }
559                if let Some(cpu_samples) = &samples.cpu_total_us {
560                    let _ = writeln!(
561                        &mut markdown,
562                        "- Total CPU samples (ms): {}",
563                        format_scaled_sample_list(cpu_samples, micros_to_ms)
564                    );
565                }
566            }
567            let _ = writeln!(&mut markdown);
568        }
569
570        markdown
571    }
572
573    pub fn render_json(&self) -> Result<String, serde_json::Error> {
574        self.render_json_with_comparison(None)
575    }
576
577    pub fn render_json_with_comparison(
578        &self,
579        comparison: Option<&BenchmarkComparison>,
580    ) -> Result<String, serde_json::Error> {
581        serde_json::to_string_pretty(&self.json_artifact(comparison))
582    }
583
584    pub fn write_artifacts(
585        &self,
586        output_dir: &Path,
587    ) -> Result<JavascriptBenchmarkArtifactPaths, JavascriptBenchmarkError> {
588        self.write_artifacts_with_comparison(output_dir, None)
589    }
590
591    pub fn write_artifacts_with_comparison(
592        &self,
593        output_dir: &Path,
594        comparison: Option<&BenchmarkComparison>,
595    ) -> Result<JavascriptBenchmarkArtifactPaths, JavascriptBenchmarkError> {
596        fs::create_dir_all(output_dir)?;
597
598        let markdown_path = output_dir.join("report.md");
599        let json_path = output_dir.join("report.json");
600        write_string_atomic(
601            &markdown_path,
602            &self.render_markdown_with_comparison(comparison),
603        )?;
604        write_string_atomic(&json_path, &self.render_json_with_comparison(comparison)?)?;
605
606        Ok(JavascriptBenchmarkArtifactPaths {
607            markdown_path,
608            json_path,
609        })
610    }
611
612    pub fn compare_to_baseline_path(
613        &self,
614        baseline_path: &Path,
615    ) -> Result<BenchmarkComparison, JavascriptBenchmarkError> {
616        let baseline = load_benchmark_artifact(baseline_path)?;
617        Ok(BenchmarkComparison::from_reports(
618            self,
619            baseline_path,
620            &baseline,
621        ))
622    }
623
624    fn guidance_lines(&self) -> Vec<String> {
625        let isolate = self.scenario("isolate-startup");
626        let cold_local = self.scenario("cold-local-import");
627        let warm_local = self.scenario("warm-local-import");
628        let prewarmed_local = self.scenario("prewarmed-local-import");
629        let builtin = self.scenario("builtin-import");
630        let large = self.scenario("large-package-import");
631
632        let mut guidance = Vec::new();
633
634        if let (
635            Some(cold_import),
636            Some(warm_import),
637            Some(warm_context),
638            Some(warm_startup_phase),
639            Some(warm_completion),
640            Some(warm_startup_overhead),
641            Some(warm_wall),
642            Some(isolate_wall),
643        ) = (
644            cold_local
645                .and_then(|scenario| scenario.guest_import_stats.as_ref())
646                .map(|stats| stats.mean_ms),
647            warm_local
648                .and_then(|scenario| scenario.guest_import_stats.as_ref())
649                .map(|stats| stats.mean_ms),
650            warm_local.map(|scenario| scenario.phase_stats.context_setup_ms.mean_ms),
651            warm_local.map(|scenario| scenario.phase_stats.startup_ms.mean_ms),
652            warm_local.map(|scenario| scenario.phase_stats.completion_ms.mean_ms),
653            warm_local
654                .and_then(|scenario| scenario.startup_overhead_stats.as_ref())
655                .map(|stats| stats.mean_ms),
656            warm_local.map(|scenario| scenario.wall_stats.mean_ms),
657            isolate.map(|scenario| scenario.wall_stats.mean_ms),
658        ) {
659            guidance.push(format!(
660                "Compile-cache reuse cuts the local import graph from {} to {} on average ({:.1}% faster), but the warm path still spends {} outside guest module evaluation. That keeps startup prewarm work in `ARC-021D` and sidecar warm-pool/snapshot work in `ARC-022` on the critical path above the `{}` empty-isolate floor.",
661                format_ms(cold_import),
662                format_ms(warm_import),
663                percentage_reduction(cold_import, warm_import),
664                format_ms(warm_startup_overhead),
665                format_ms(isolate_wall),
666            ));
667            if warm_wall > 0.0 {
668                guidance.push(format!(
669                    "Warm local imports still spend {:.1}% of wall time in process startup, wrapper evaluation, and stdio handling instead of guest import work. Optimizations that only touch module compilation will not remove that floor.",
670                    percentage_share(warm_startup_overhead, warm_wall),
671                ));
672            }
673            let warm_guest = warm_local
674                .and_then(|scenario| scenario.phase_stats.guest_execution_ms.as_ref())
675                .map(|stats| stats.mean_ms)
676                .unwrap_or(0.0);
677            guidance.push(format!(
678                "The warm path phase split is {} context setup, {} runtime startup, {} guest execution, and {} completion/stdio work. Future attribution can now separate bootstrap wins from pure transport/collection wins instead of treating them as one startup bucket.",
679                format_ms(warm_context),
680                format_ms(warm_startup_phase),
681                format_ms(warm_guest),
682                format_ms(warm_completion),
683            ));
684        }
685
686        if let (Some(warm_startup_overhead), Some(prewarmed_startup_overhead), Some(isolate_wall)) = (
687            warm_local
688                .and_then(|scenario| scenario.startup_overhead_stats.as_ref())
689                .map(|stats| stats.mean_ms),
690            prewarmed_local
691                .and_then(|scenario| scenario.startup_overhead_stats.as_ref())
692                .map(|stats| stats.mean_ms),
693            isolate.map(|scenario| scenario.wall_stats.mean_ms),
694        ) {
695            guidance.push(format!(
696                "Keeping the current import-cache materialization and builtin/polyfill prewarm alive inside one execution engine cuts warm local startup overhead from {} to {} ({:.1}% faster). The remaining {} of non-import work is the post-prewarm floor that broader warm-pool/snapshot work would still need to attack above the `{}` empty-isolate baseline.",
697                format_ms(warm_startup_overhead),
698                format_ms(prewarmed_startup_overhead),
699                percentage_reduction(warm_startup_overhead, prewarmed_startup_overhead),
700                format_ms(prewarmed_startup_overhead),
701                format_ms(isolate_wall),
702            ));
703        }
704
705        if let (Some(builtin_import), Some(large_import)) = (
706            builtin
707                .and_then(|scenario| scenario.guest_import_stats.as_ref())
708                .map(|stats| stats.mean_ms),
709            large
710                .and_then(|scenario| scenario.guest_import_stats.as_ref())
711                .map(|stats| stats.mean_ms),
712        ) {
713            guidance.push(format!(
714                "The large real-world package import (`typescript`) is {:.1}x the builtin path ({} versus {}). That makes `ARC-021C` the right next import-path optimization story: cache sidecar-scoped resolution results, package-type lookups, and module-format classification before attempting deeper structural rewrites.",
715                safe_ratio(large_import, builtin_import),
716                format_ms(large_import),
717                format_ms(builtin_import),
718            ));
719        }
720
721        if let (Some(smallest), Some(largest)) =
722            (self.transport_rtt.first(), self.transport_rtt.last())
723        {
724            guidance.push(format!(
725                "Execution-transport RTT over the stdio bridge rises from {} at {} bytes to {} at {} bytes. That gives later work a direct transport floor to compare against the larger startup and import phases.",
726                format_ms(smallest.stats.mean_ms),
727                smallest.payload_bytes,
728                format_ms(largest.stats.mean_ms),
729                largest.payload_bytes,
730            ));
731        }
732
733        if let Some(noisiest) = self.scenarios.iter().max_by(|lhs, rhs| {
734            lhs.wall_stats
735                .stddev_ms
736                .total_cmp(&rhs.wall_stats.stddev_ms)
737        }) {
738            guidance.push(format!(
739                "Wall-time noise is now surfaced directly in the same artifact set: `{}` currently shows the largest spread at {} stddev over a {}-{} wall range, so future deltas on that path should be judged against stability as well as mean time.",
740                noisiest.id,
741                format_ms(noisiest.wall_stats.stddev_ms),
742                format_ms(noisiest.wall_stats.min_ms),
743                format_ms(noisiest.wall_stats.max_ms),
744            ));
745        }
746
747        if let Some(heaviest) = self.scenarios.iter().max_by(|lhs, rhs| {
748            lhs.resource_usage_stats
749                .as_ref()
750                .and_then(|stats| stats.rss_bytes.as_ref())
751                .map(|stats| stats.mean)
752                .unwrap_or(f64::NEG_INFINITY)
753                .total_cmp(
754                    &rhs.resource_usage_stats
755                        .as_ref()
756                        .and_then(|stats| stats.rss_bytes.as_ref())
757                        .map(|stats| stats.mean)
758                        .unwrap_or(f64::NEG_INFINITY),
759                )
760        }) {
761            if let Some(rss_mean) = heaviest
762                .resource_usage_stats
763                .as_ref()
764                .and_then(|stats| stats.rss_bytes.as_ref())
765            {
766                guidance.push(format!(
767                    "Per-scenario resource reporting is now attached to the benchmark rows themselves: `{}` currently has the highest mean RSS at {} MiB, so import-path changes can now be judged for memory regressions without a separate memory-only pass.",
768                    heaviest.id,
769                    format_mib(bytes_to_mib(rss_mean.mean)),
770                ));
771            }
772        }
773
774        guidance.push(String::from(
775            "No new PRD stories were added from this run. The measured hotspots already map cleanly onto existing follow-ons: `ARC-021C` for safe resolution and metadata caches, `ARC-021D` for builtin/polyfill prewarm, and `ARC-022` for broader warm-pool and timing-mitigation execution work.",
776        ));
777
778        guidance
779    }
780
781    fn scenario(&self, id: &str) -> Option<&BenchmarkScenarioReport> {
782        self.scenarios.iter().find(|scenario| scenario.id == id)
783    }
784
785    fn json_artifact<'a>(
786        &'a self,
787        comparison: Option<&'a BenchmarkComparison>,
788    ) -> JavascriptBenchmarkArtifact<'a> {
789        JavascriptBenchmarkArtifact {
790            artifact_version: BENCHMARK_ARTIFACT_VERSION,
791            generated_at_unix_ms: self.generated_at_unix_ms,
792            command: format!(
793                "cargo run -p agentos-execution --bin node-import-bench -- --iterations {} --warmup-iterations {}",
794                self.config.iterations, self.config.warmup_iterations
795            ),
796            config: &self.config,
797            host: &self.host,
798            repo_root: &self.repo_root,
799            summary: self.summary(),
800            comparison,
801            transport_rtt: self
802                .transport_rtt
803                .iter()
804                .map(|transport| BenchmarkTransportRttArtifact {
805                    channel: transport.channel,
806                    payload_bytes: transport.payload_bytes,
807                    samples_ms: &transport.samples_ms,
808                    stats: &transport.stats,
809                })
810                .collect(),
811            scenarios: self
812                .scenarios
813                .iter()
814                .map(|scenario| BenchmarkScenarioArtifact {
815                    id: scenario.id,
816                    workload: scenario.workload,
817                    runtime: scenario.runtime,
818                    mode: scenario.mode,
819                    description: scenario.description,
820                    fixture: scenario.fixture,
821                    compile_cache: scenario.compile_cache,
822                    wall_samples_ms: &scenario.wall_samples_ms,
823                    wall_stats: &scenario.wall_stats,
824                    guest_import_samples_ms: scenario.guest_import_samples_ms.as_deref(),
825                    guest_import_stats: scenario.guest_import_stats.as_ref(),
826                    startup_overhead_samples_ms: scenario.startup_overhead_samples_ms.as_deref(),
827                    startup_overhead_stats: scenario.startup_overhead_stats.as_ref(),
828                    mean_startup_share_pct: scenario.mean_startup_share_pct(),
829                    phase_samples_ms: &scenario.phase_samples_ms,
830                    phase_stats: &scenario.phase_stats,
831                    resource_usage_samples: scenario.resource_usage_samples.as_ref(),
832                    resource_usage_stats: scenario.resource_usage_stats.as_ref(),
833                })
834                .collect(),
835        }
836    }
837
838    fn summary(&self) -> BenchmarkSummaryArtifact<'_> {
839        BenchmarkSummaryArtifact {
840            scenario_count: self.scenarios.len(),
841            recorded_samples_per_scenario: self.config.iterations,
842            warmup_iterations: self.config.warmup_iterations,
843            control_matrix: self.control_matrix(),
844            slowest_wall_scenario: self.slowest_scenario_by(|scenario| scenario.wall_stats.mean_ms),
845            slowest_guest_import_scenario: self.slowest_scenario_by(|scenario| {
846                scenario
847                    .guest_import_stats
848                    .as_ref()
849                    .map(|stats| stats.mean_ms)
850                    .unwrap_or(f64::NEG_INFINITY)
851            }),
852            highest_startup_share_scenario: self.scenarios.iter().max_by(|lhs, rhs| {
853                lhs.mean_startup_share_pct()
854                    .unwrap_or(f64::NEG_INFINITY)
855                    .total_cmp(&rhs.mean_startup_share_pct().unwrap_or(f64::NEG_INFINITY))
856            }),
857            hotspot_rankings: self.hotspot_rankings(),
858            guidance_lines: self.guidance_lines(),
859        }
860    }
861
862    fn control_matrix(&self) -> Vec<BenchmarkControlMatrixArtifact<'_>> {
863        let mut rows = Vec::new();
864        let mut row_indexes = BTreeMap::new();
865
866        for scenario in &self.scenarios {
867            let row_index = *row_indexes.entry(scenario.workload).or_insert_with(|| {
868                rows.push(BenchmarkControlMatrixArtifact {
869                    workload: scenario.workload,
870                    runtimes: Vec::new(),
871                    modes: Vec::new(),
872                    scenario_ids: Vec::new(),
873                });
874                rows.len() - 1
875            });
876            let row = &mut rows[row_index];
877            push_unique_label(&mut row.runtimes, scenario.runtime);
878            push_unique_label(&mut row.modes, scenario.mode);
879            row.scenario_ids.push(scenario.id);
880        }
881
882        rows
883    }
884
885    fn slowest_scenario_by(
886        &self,
887        value: impl Fn(&BenchmarkScenarioReport) -> f64,
888    ) -> Option<&BenchmarkScenarioReport> {
889        self.scenarios
890            .iter()
891            .max_by(|lhs, rhs| value(lhs).total_cmp(&value(rhs)))
892    }
893
894    fn hotspot_rankings(&self) -> Vec<BenchmarkHotspotRankingArtifact<'_>> {
895        HOTSPOT_METRICS
896            .iter()
897            .map(|metric| {
898                let mut ranked_scenarios = self
899                    .scenarios
900                    .iter()
901                    .filter_map(|scenario| {
902                        (metric.value)(scenario).map(|value| BenchmarkHotspotScenarioArtifact {
903                            rank: 0,
904                            id: scenario.id,
905                            workload: scenario.workload,
906                            runtime: scenario.runtime,
907                            mode: scenario.mode,
908                            value,
909                        })
910                    })
911                    .collect::<Vec<_>>();
912                ranked_scenarios.sort_by(|lhs, rhs| {
913                    rhs.value
914                        .total_cmp(&lhs.value)
915                        .then_with(|| lhs.id.cmp(rhs.id))
916                });
917                for (index, scenario) in ranked_scenarios.iter_mut().enumerate() {
918                    scenario.rank = index + 1;
919                }
920
921                BenchmarkHotspotRankingArtifact {
922                    metric: metric.metric,
923                    label: metric.label,
924                    dimension: metric.dimension,
925                    unit: metric.unit,
926                    ranked_scenarios,
927                    scenarios_without_metric: self
928                        .scenarios
929                        .iter()
930                        .filter(|scenario| (metric.value)(scenario).is_none())
931                        .map(|scenario| scenario.id)
932                        .collect(),
933                }
934            })
935            .collect()
936    }
937}
938
939impl BenchmarkScenarioReport {
940    fn mean_startup_share_pct(&self) -> Option<f64> {
941        let startup_mean = self.startup_overhead_stats.as_ref()?.mean_ms;
942        let wall_mean = self.wall_stats.mean_ms;
943        if wall_mean <= 0.0 {
944            Some(0.0)
945        } else {
946            Some((startup_mean / wall_mean) * 100.0)
947        }
948    }
949
950    fn wall_range_ms(&self) -> f64 {
951        self.wall_stats.max_ms - self.wall_stats.min_ms
952    }
953}
954
955impl BenchmarkResourceUsage<Vec<f64>> {
956    fn push_sample(&mut self, sample: &BenchmarkResourceUsage<f64>) {
957        push_optional_sample(&mut self.rss_bytes, sample.rss_bytes);
958        push_optional_sample(&mut self.heap_used_bytes, sample.heap_used_bytes);
959        push_optional_sample(&mut self.cpu_user_us, sample.cpu_user_us);
960        push_optional_sample(&mut self.cpu_system_us, sample.cpu_system_us);
961        push_optional_sample(&mut self.cpu_total_us, sample.cpu_total_us);
962    }
963
964    fn into_populated(self) -> Option<Self> {
965        (!self.is_empty()).then_some(self)
966    }
967}
968
969impl<T> BenchmarkResourceUsage<T> {
970    fn is_empty(&self) -> bool {
971        self.rss_bytes.is_none()
972            && self.heap_used_bytes.is_none()
973            && self.cpu_user_us.is_none()
974            && self.cpu_system_us.is_none()
975            && self.cpu_total_us.is_none()
976    }
977}
978
979impl BenchmarkComparison {
980    fn from_reports(
981        current: &JavascriptBenchmarkReport,
982        baseline_path: &Path,
983        baseline: &StoredBenchmarkArtifact,
984    ) -> Self {
985        let baseline_path =
986            fs::canonicalize(baseline_path).unwrap_or_else(|_| baseline_path.to_path_buf());
987        let baseline_by_id = baseline
988            .scenarios
989            .iter()
990            .map(|scenario| (scenario.id.as_str(), scenario))
991            .collect::<BTreeMap<_, _>>();
992
993        let mut scenario_deltas = Vec::new();
994        let mut scenarios_missing_from_baseline = Vec::new();
995
996        for scenario in &current.scenarios {
997            if let Some(baseline_scenario) = baseline_by_id.get(scenario.id) {
998                scenario_deltas.push(BenchmarkScenarioDelta {
999                    id: scenario.id.to_owned(),
1000                    description: scenario.description.to_owned(),
1001                    wall_mean_ms: BenchmarkMetricDelta::from_means(
1002                        baseline_scenario.wall_stats.mean_ms,
1003                        scenario.wall_stats.mean_ms,
1004                    ),
1005                    guest_import_mean_ms: match (
1006                        baseline_scenario.guest_import_stats.as_ref(),
1007                        scenario.guest_import_stats.as_ref(),
1008                    ) {
1009                        (Some(baseline_stats), Some(current_stats)) => {
1010                            Some(BenchmarkMetricDelta::from_means(
1011                                baseline_stats.mean_ms,
1012                                current_stats.mean_ms,
1013                            ))
1014                        }
1015                        _ => None,
1016                    },
1017                    startup_overhead_mean_ms: match (
1018                        baseline_scenario.startup_overhead_stats.as_ref(),
1019                        scenario.startup_overhead_stats.as_ref(),
1020                    ) {
1021                        (Some(baseline_stats), Some(current_stats)) => {
1022                            Some(BenchmarkMetricDelta::from_means(
1023                                baseline_stats.mean_ms,
1024                                current_stats.mean_ms,
1025                            ))
1026                        }
1027                        _ => None,
1028                    },
1029                    phase_mean_ms: match (
1030                        baseline_scenario.phase_stats.as_ref(),
1031                        Some(&scenario.phase_stats),
1032                    ) {
1033                        (Some(baseline_phase), Some(current_phase)) => {
1034                            Some(BenchmarkScenarioPhases {
1035                                context_setup_ms: BenchmarkMetricDelta::from_means(
1036                                    baseline_phase.context_setup_ms.mean_ms,
1037                                    current_phase.context_setup_ms.mean_ms,
1038                                ),
1039                                startup_ms: BenchmarkMetricDelta::from_means(
1040                                    baseline_phase.startup_ms.mean_ms,
1041                                    current_phase.startup_ms.mean_ms,
1042                                ),
1043                                guest_execution_ms: match (
1044                                    baseline_phase.guest_execution_ms.as_ref(),
1045                                    current_phase.guest_execution_ms.as_ref(),
1046                                ) {
1047                                    (Some(baseline_stats), Some(current_stats)) => {
1048                                        Some(BenchmarkMetricDelta::from_means(
1049                                            baseline_stats.mean_ms,
1050                                            current_stats.mean_ms,
1051                                        ))
1052                                    }
1053                                    _ => None,
1054                                },
1055                                completion_ms: BenchmarkMetricDelta::from_means(
1056                                    baseline_phase.completion_ms.mean_ms,
1057                                    current_phase.completion_ms.mean_ms,
1058                                ),
1059                            })
1060                        }
1061                        _ => None,
1062                    },
1063                });
1064            } else {
1065                scenarios_missing_from_baseline.push(scenario.id.to_owned());
1066            }
1067        }
1068
1069        let current_ids = current
1070            .scenarios
1071            .iter()
1072            .map(|scenario| (scenario.id, ()))
1073            .collect::<BTreeMap<_, _>>();
1074        let baseline_only_scenarios = baseline
1075            .scenarios
1076            .iter()
1077            .filter(|scenario| !current_ids.contains_key(scenario.id.as_str()))
1078            .map(|scenario| scenario.id.clone())
1079            .collect::<Vec<_>>();
1080
1081        let largest_wall_improvement = scenario_deltas
1082            .iter()
1083            .filter(|scenario| scenario.wall_mean_ms.delta_ms < 0.0)
1084            .min_by(|lhs, rhs| {
1085                lhs.wall_mean_ms
1086                    .delta_ms
1087                    .total_cmp(&rhs.wall_mean_ms.delta_ms)
1088            })
1089            .map(BenchmarkDeltaHighlight::from_wall_delta);
1090        let largest_wall_regression = scenario_deltas
1091            .iter()
1092            .filter(|scenario| scenario.wall_mean_ms.delta_ms > 0.0)
1093            .max_by(|lhs, rhs| {
1094                lhs.wall_mean_ms
1095                    .delta_ms
1096                    .total_cmp(&rhs.wall_mean_ms.delta_ms)
1097            })
1098            .map(BenchmarkDeltaHighlight::from_wall_delta);
1099
1100        Self {
1101            baseline: BenchmarkComparisonBaseline {
1102                artifact_version: baseline.artifact_version,
1103                generated_at_unix_ms: baseline.generated_at_unix_ms,
1104                path: baseline_path,
1105            },
1106            summary: BenchmarkComparisonSummary {
1107                compared_scenario_count: scenario_deltas.len(),
1108                largest_wall_improvement,
1109                largest_wall_regression,
1110            },
1111            scenario_deltas,
1112            scenarios_missing_from_baseline,
1113            baseline_only_scenarios,
1114        }
1115    }
1116}
1117
1118impl BenchmarkDeltaHighlight {
1119    fn from_wall_delta(delta: &BenchmarkScenarioDelta) -> Self {
1120        Self {
1121            id: delta.id.clone(),
1122            delta_ms: delta.wall_mean_ms.delta_ms,
1123            delta_pct: delta.wall_mean_ms.delta_pct,
1124        }
1125    }
1126}
1127
1128impl BenchmarkMetricDelta {
1129    fn from_means(baseline_ms: f64, current_ms: f64) -> Self {
1130        let delta_ms = current_ms - baseline_ms;
1131        let delta_pct = if baseline_ms <= 0.0 {
1132            0.0
1133        } else {
1134            (delta_ms / baseline_ms) * 100.0
1135        };
1136
1137        Self {
1138            baseline_ms,
1139            current_ms,
1140            delta_ms,
1141            delta_pct,
1142        }
1143    }
1144}
1145
1146#[derive(Debug, Clone, PartialEq, Eq)]
1147pub struct JavascriptBenchmarkArtifactPaths {
1148    pub markdown_path: PathBuf,
1149    pub json_path: PathBuf,
1150}
1151
1152#[derive(Debug, Clone, PartialEq, Eq)]
1153pub struct JavascriptBenchmarkRunOutput {
1154    pub artifact_paths: JavascriptBenchmarkArtifactPaths,
1155    pub resumed_stage_count: usize,
1156}
1157
1158#[derive(Debug, Serialize)]
1159struct JavascriptBenchmarkArtifact<'a> {
1160    artifact_version: u32,
1161    generated_at_unix_ms: u128,
1162    command: String,
1163    config: &'a JavascriptBenchmarkConfig,
1164    host: &'a BenchmarkHost,
1165    repo_root: &'a Path,
1166    summary: BenchmarkSummaryArtifact<'a>,
1167    #[serde(skip_serializing_if = "Option::is_none")]
1168    comparison: Option<&'a BenchmarkComparison>,
1169    transport_rtt: Vec<BenchmarkTransportRttArtifact<'a>>,
1170    scenarios: Vec<BenchmarkScenarioArtifact<'a>>,
1171}
1172
1173#[derive(Debug, Serialize)]
1174struct BenchmarkSummaryArtifact<'a> {
1175    scenario_count: usize,
1176    recorded_samples_per_scenario: usize,
1177    warmup_iterations: usize,
1178    control_matrix: Vec<BenchmarkControlMatrixArtifact<'a>>,
1179    #[serde(skip_serializing_if = "Option::is_none")]
1180    slowest_wall_scenario: Option<&'a BenchmarkScenarioReport>,
1181    #[serde(skip_serializing_if = "Option::is_none")]
1182    slowest_guest_import_scenario: Option<&'a BenchmarkScenarioReport>,
1183    #[serde(skip_serializing_if = "Option::is_none")]
1184    highest_startup_share_scenario: Option<&'a BenchmarkScenarioReport>,
1185    hotspot_rankings: Vec<BenchmarkHotspotRankingArtifact<'a>>,
1186    guidance_lines: Vec<String>,
1187}
1188
1189#[derive(Debug, Serialize)]
1190struct BenchmarkScenarioArtifact<'a> {
1191    id: &'static str,
1192    workload: &'static str,
1193    runtime: &'static str,
1194    mode: &'static str,
1195    description: &'static str,
1196    fixture: &'static str,
1197    compile_cache: &'static str,
1198    wall_samples_ms: &'a [f64],
1199    wall_stats: &'a BenchmarkStats,
1200    #[serde(skip_serializing_if = "Option::is_none")]
1201    guest_import_samples_ms: Option<&'a [f64]>,
1202    #[serde(skip_serializing_if = "Option::is_none")]
1203    guest_import_stats: Option<&'a BenchmarkStats>,
1204    #[serde(skip_serializing_if = "Option::is_none")]
1205    startup_overhead_samples_ms: Option<&'a [f64]>,
1206    #[serde(skip_serializing_if = "Option::is_none")]
1207    startup_overhead_stats: Option<&'a BenchmarkStats>,
1208    #[serde(skip_serializing_if = "Option::is_none")]
1209    mean_startup_share_pct: Option<f64>,
1210    phase_samples_ms: &'a BenchmarkScenarioPhases<Vec<f64>>,
1211    phase_stats: &'a BenchmarkScenarioPhases<BenchmarkStats>,
1212    #[serde(skip_serializing_if = "Option::is_none")]
1213    resource_usage_samples: Option<&'a BenchmarkResourceUsage<Vec<f64>>>,
1214    #[serde(skip_serializing_if = "Option::is_none")]
1215    resource_usage_stats: Option<&'a BenchmarkResourceUsage<BenchmarkDistributionStats>>,
1216}
1217
1218#[derive(Debug, Serialize)]
1219struct BenchmarkControlMatrixArtifact<'a> {
1220    workload: &'a str,
1221    runtimes: Vec<&'a str>,
1222    modes: Vec<&'a str>,
1223    scenario_ids: Vec<&'a str>,
1224}
1225
1226#[derive(Debug, Serialize)]
1227struct BenchmarkTransportRttArtifact<'a> {
1228    channel: &'static str,
1229    payload_bytes: usize,
1230    samples_ms: &'a [f64],
1231    stats: &'a BenchmarkStats,
1232}
1233
1234#[derive(Debug, Serialize)]
1235struct BenchmarkHotspotRankingArtifact<'a> {
1236    metric: &'static str,
1237    label: &'static str,
1238    dimension: &'static str,
1239    unit: &'static str,
1240    ranked_scenarios: Vec<BenchmarkHotspotScenarioArtifact<'a>>,
1241    #[serde(skip_serializing_if = "Vec::is_empty")]
1242    scenarios_without_metric: Vec<&'a str>,
1243}
1244
1245#[derive(Debug, Serialize)]
1246struct BenchmarkHotspotScenarioArtifact<'a> {
1247    rank: usize,
1248    id: &'a str,
1249    workload: &'a str,
1250    runtime: &'a str,
1251    mode: &'a str,
1252    value: f64,
1253}
1254
1255struct HotspotMetricDefinition {
1256    metric: &'static str,
1257    label: &'static str,
1258    dimension: &'static str,
1259    unit: &'static str,
1260    value: fn(&BenchmarkScenarioReport) -> Option<f64>,
1261}
1262
1263const HOTSPOT_METRICS: [HotspotMetricDefinition; 13] = [
1264    HotspotMetricDefinition {
1265        metric: "wall_mean_ms",
1266        label: "Wall Time",
1267        dimension: "time",
1268        unit: "ms",
1269        value: hotspot_wall_mean_ms,
1270    },
1271    HotspotMetricDefinition {
1272        metric: "wall_stddev_ms",
1273        label: "Wall Time Stddev",
1274        dimension: "stability",
1275        unit: "ms",
1276        value: hotspot_wall_stddev_ms,
1277    },
1278    HotspotMetricDefinition {
1279        metric: "wall_range_ms",
1280        label: "Wall Time Range",
1281        dimension: "stability",
1282        unit: "ms",
1283        value: hotspot_wall_range_ms,
1284    },
1285    HotspotMetricDefinition {
1286        metric: "guest_import_mean_ms",
1287        label: "Guest Import Time",
1288        dimension: "time",
1289        unit: "ms",
1290        value: hotspot_guest_import_mean_ms,
1291    },
1292    HotspotMetricDefinition {
1293        metric: "startup_overhead_mean_ms",
1294        label: "Startup Overhead",
1295        dimension: "time",
1296        unit: "ms",
1297        value: hotspot_startup_overhead_mean_ms,
1298    },
1299    HotspotMetricDefinition {
1300        metric: "context_setup_mean_ms",
1301        label: "Context Setup Phase",
1302        dimension: "time",
1303        unit: "ms",
1304        value: hotspot_context_setup_mean_ms,
1305    },
1306    HotspotMetricDefinition {
1307        metric: "startup_phase_mean_ms",
1308        label: "Runtime Startup Phase",
1309        dimension: "time",
1310        unit: "ms",
1311        value: hotspot_startup_phase_mean_ms,
1312    },
1313    HotspotMetricDefinition {
1314        metric: "guest_execution_mean_ms",
1315        label: "Guest Execution Phase",
1316        dimension: "time",
1317        unit: "ms",
1318        value: hotspot_guest_execution_mean_ms,
1319    },
1320    HotspotMetricDefinition {
1321        metric: "completion_mean_ms",
1322        label: "Completion/Stdio Phase",
1323        dimension: "time",
1324        unit: "ms",
1325        value: hotspot_completion_mean_ms,
1326    },
1327    HotspotMetricDefinition {
1328        metric: "startup_share_pct",
1329        label: "Startup Share Of Wall",
1330        dimension: "share",
1331        unit: "pct",
1332        value: hotspot_startup_share_pct,
1333    },
1334    HotspotMetricDefinition {
1335        metric: "rss_mean_mib",
1336        label: "RSS",
1337        dimension: "memory",
1338        unit: "MiB",
1339        value: hotspot_rss_mean_mib,
1340    },
1341    HotspotMetricDefinition {
1342        metric: "heap_mean_mib",
1343        label: "Heap Used",
1344        dimension: "memory",
1345        unit: "MiB",
1346        value: hotspot_heap_mean_mib,
1347    },
1348    HotspotMetricDefinition {
1349        metric: "cpu_total_mean_ms",
1350        label: "Total CPU",
1351        dimension: "cpu",
1352        unit: "ms",
1353        value: hotspot_total_cpu_mean_ms,
1354    },
1355];
1356
1357#[derive(Debug)]
1358pub enum JavascriptBenchmarkError {
1359    InvalidConfig(&'static str),
1360    InvalidWorkspaceRoot(PathBuf),
1361    InvalidBaselineReport {
1362        path: PathBuf,
1363        message: String,
1364    },
1365    Io(std::io::Error),
1366    Utf8(std::string::FromUtf8Error),
1367    Execution(JavascriptExecutionError),
1368    NodeVersion(std::io::Error),
1369    MissingBenchmarkMetric(&'static str),
1370    InvalidBenchmarkMetric {
1371        scenario: &'static str,
1372        raw_value: String,
1373    },
1374    TransportProbeTimeout {
1375        payload_bytes: usize,
1376    },
1377    TransportProbeExited {
1378        exit_code: i32,
1379        stderr: String,
1380    },
1381    InvalidTransportProbeResponse {
1382        payload_bytes: usize,
1383        expected: String,
1384        actual: String,
1385    },
1386    NonZeroExit {
1387        scenario: &'static str,
1388        exit_code: i32,
1389        stderr: String,
1390    },
1391}
1392
1393impl fmt::Display for JavascriptBenchmarkError {
1394    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1395        match self {
1396            Self::InvalidConfig(message) => write!(f, "invalid benchmark config: {message}"),
1397            Self::InvalidWorkspaceRoot(path) => {
1398                write!(
1399                    f,
1400                    "failed to resolve workspace root from execution crate path: {}",
1401                    path.display()
1402                )
1403            }
1404            Self::InvalidBaselineReport { path, message } => {
1405                write!(
1406                    f,
1407                    "failed to parse benchmark baseline artifact {}: {message}",
1408                    path.display()
1409                )
1410            }
1411            Self::Io(err) => write!(f, "benchmark I/O failure: {err}"),
1412            Self::Utf8(err) => write!(f, "benchmark output was not valid UTF-8: {err}"),
1413            Self::Execution(err) => write!(f, "benchmark execution failed: {err}"),
1414            Self::NodeVersion(err) => write!(f, "failed to query node version: {err}"),
1415            Self::MissingBenchmarkMetric(scenario) => {
1416                write!(
1417                    f,
1418                    "benchmark scenario `{scenario}` did not emit a metric marker"
1419                )
1420            }
1421            Self::InvalidBenchmarkMetric {
1422                scenario,
1423                raw_value,
1424            } => write!(
1425                f,
1426                "benchmark scenario `{scenario}` emitted an invalid metric: {raw_value}"
1427            ),
1428            Self::TransportProbeTimeout { payload_bytes } => {
1429                write!(
1430                    f,
1431                    "transport probe timed out waiting for {payload_bytes}-byte round-trip"
1432                )
1433            }
1434            Self::TransportProbeExited { exit_code, stderr } => {
1435                write!(f, "transport probe exited with code {exit_code}: {stderr}")
1436            }
1437            Self::InvalidTransportProbeResponse {
1438                payload_bytes,
1439                expected,
1440                actual,
1441            } => write!(
1442                f,
1443                "transport probe returned unexpected payload for {payload_bytes}-byte round-trip: expected {expected:?}, got {actual:?}"
1444            ),
1445            Self::NonZeroExit {
1446                scenario,
1447                exit_code,
1448                stderr,
1449            } => write!(
1450                f,
1451                "benchmark scenario `{scenario}` exited with code {exit_code}: {stderr}"
1452            ),
1453        }
1454    }
1455}
1456
1457impl std::error::Error for JavascriptBenchmarkError {}
1458
1459impl From<std::io::Error> for JavascriptBenchmarkError {
1460    fn from(err: std::io::Error) -> Self {
1461        Self::Io(err)
1462    }
1463}
1464
1465impl From<std::string::FromUtf8Error> for JavascriptBenchmarkError {
1466    fn from(err: std::string::FromUtf8Error) -> Self {
1467        Self::Utf8(err)
1468    }
1469}
1470
1471impl From<serde_json::Error> for JavascriptBenchmarkError {
1472    fn from(err: serde_json::Error) -> Self {
1473        Self::Io(std::io::Error::new(std::io::ErrorKind::InvalidData, err))
1474    }
1475}
1476
1477impl From<JavascriptExecutionError> for JavascriptBenchmarkError {
1478    fn from(err: JavascriptExecutionError) -> Self {
1479        Self::Execution(err)
1480    }
1481}
1482
1483pub fn run_javascript_benchmarks(
1484    runtime: &RuntimeContext,
1485    config: &JavascriptBenchmarkConfig,
1486) -> Result<JavascriptBenchmarkReport, JavascriptBenchmarkError> {
1487    validate_benchmark_config(config)?;
1488
1489    let repo_root = workspace_root()?;
1490    let host = benchmark_host()?;
1491    let workspace = BenchmarkWorkspace::create(&repo_root)?;
1492    let transport_rtt = measure_transport_rtt(runtime, &workspace, config)?;
1493
1494    let mut scenarios = Vec::new();
1495
1496    for scenario in benchmark_scenarios() {
1497        scenarios.push(run_scenario(runtime, &workspace, config, scenario)?);
1498    }
1499
1500    Ok(JavascriptBenchmarkReport {
1501        generated_at_unix_ms: SystemTime::now()
1502            .duration_since(UNIX_EPOCH)
1503            .unwrap_or_default()
1504            .as_millis(),
1505        config: config.clone(),
1506        host,
1507        repo_root,
1508        transport_rtt,
1509        scenarios,
1510    })
1511}
1512
1513fn benchmark_artifact_dir(repo_root: &Path) -> PathBuf {
1514    repo_root.join(BENCHMARK_ARTIFACT_DIR)
1515}
1516
1517fn benchmark_run_state_path(artifact_dir: &Path) -> PathBuf {
1518    artifact_dir.join(BENCHMARK_RUN_STATE_FILE)
1519}
1520
1521fn load_benchmark_run_state(
1522    state_path: &Path,
1523    config: &JavascriptBenchmarkConfig,
1524    host: &BenchmarkHost,
1525    repo_root: &Path,
1526    definitions: &[ScenarioDefinition],
1527) -> Result<StoredBenchmarkRunState, JavascriptBenchmarkError> {
1528    match fs::read_to_string(state_path) {
1529        Ok(raw) => match serde_json::from_str::<StoredBenchmarkRunState>(&raw) {
1530            Ok(state) if state.is_compatible(config, host, repo_root) => {
1531                Ok(state.sanitized(definitions))
1532            }
1533            Ok(_) | Err(_) => Ok(StoredBenchmarkRunState::new(config, host, repo_root)),
1534        },
1535        Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
1536            Ok(StoredBenchmarkRunState::new(config, host, repo_root))
1537        }
1538        Err(err) => Err(JavascriptBenchmarkError::Io(err)),
1539    }
1540}
1541
1542fn persist_benchmark_run_state(
1543    state_path: &Path,
1544    state: &StoredBenchmarkRunState,
1545) -> Result<(), JavascriptBenchmarkError> {
1546    write_string_atomic(state_path, &serde_json::to_string_pretty(state)?)
1547}
1548
1549fn write_string_atomic(path: &Path, contents: &str) -> Result<(), JavascriptBenchmarkError> {
1550    if let Some(parent) = path.parent() {
1551        fs::create_dir_all(parent)?;
1552    }
1553
1554    let temp_path = path.with_file_name(format!(
1555        ".{}.tmp-{}-{}",
1556        path.file_name()
1557            .and_then(|name| name.to_str())
1558            .unwrap_or("artifact"),
1559        std::process::id(),
1560        SystemTime::now()
1561            .duration_since(UNIX_EPOCH)
1562            .unwrap_or_default()
1563            .as_nanos()
1564    ));
1565    fs::write(&temp_path, contents)?;
1566    if let Err(err) = fs::rename(&temp_path, path) {
1567        let _ = fs::remove_file(&temp_path);
1568        return Err(JavascriptBenchmarkError::Io(err));
1569    }
1570
1571    Ok(())
1572}
1573
1574fn remove_file_if_exists(path: &Path) -> Result<(), JavascriptBenchmarkError> {
1575    match fs::remove_file(path) {
1576        Ok(()) => Ok(()),
1577        Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
1578        Err(err) => Err(JavascriptBenchmarkError::Io(err)),
1579    }
1580}
1581
1582fn current_unix_ms() -> u128 {
1583    SystemTime::now()
1584        .duration_since(UNIX_EPOCH)
1585        .unwrap_or_default()
1586        .as_millis()
1587}
1588
1589#[derive(Debug, Clone, Copy)]
1590struct ScenarioDefinition {
1591    id: &'static str,
1592    workload: &'static str,
1593    runtime: ScenarioRuntime,
1594    mode: ScenarioMode,
1595    description: &'static str,
1596    fixture: &'static str,
1597    entrypoint: &'static str,
1598    compile_cache: CompileCacheStrategy,
1599    engine_reuse: EngineReuseStrategy,
1600    expect_import_metric: bool,
1601    env: ScenarioEnvironment,
1602}
1603
1604#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1605enum CompileCacheStrategy {
1606    Disabled,
1607    Primed,
1608}
1609
1610impl CompileCacheStrategy {
1611    fn label(self) -> &'static str {
1612        match self {
1613            Self::Disabled => "disabled",
1614            Self::Primed => "primed",
1615        }
1616    }
1617}
1618
1619#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1620enum EngineReuseStrategy {
1621    FreshPerSample,
1622    SharedAcrossScenario,
1623    SharedContextAcrossScenario,
1624}
1625
1626#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1627enum ScenarioEnvironment {
1628    None,
1629    ProjectedWorkspaceNodeModules,
1630}
1631
1632#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1633enum ScenarioRuntime {
1634    NativeExecution,
1635    HostNode,
1636}
1637
1638impl ScenarioRuntime {
1639    fn label(self) -> &'static str {
1640        match self {
1641            Self::NativeExecution => "native-execution",
1642            Self::HostNode => "host-node",
1643        }
1644    }
1645}
1646
1647#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1648enum ScenarioMode {
1649    BaselineControl,
1650    TrueColdStart,
1651    NewSessionReplay,
1652    SameSessionReplay,
1653    SameEngineReplay,
1654    HostControl,
1655}
1656
1657impl ScenarioMode {
1658    fn label(self) -> &'static str {
1659        match self {
1660            Self::BaselineControl => "baseline-control",
1661            Self::TrueColdStart => "true-cold-start",
1662            Self::NewSessionReplay => "new-session-replay",
1663            Self::SameSessionReplay => "same-session-replay",
1664            Self::SameEngineReplay => "same-engine-replay",
1665            Self::HostControl => "host-control",
1666        }
1667    }
1668}
1669
1670#[derive(Debug)]
1671struct SampleMeasurement {
1672    wall_ms: f64,
1673    guest_import_ms: Option<f64>,
1674    context_setup_ms: f64,
1675    startup_ms: f64,
1676    completion_ms: f64,
1677    resource_usage: Option<BenchmarkResourceUsage<f64>>,
1678}
1679
1680#[derive(Debug)]
1681struct BenchmarkWorkspace {
1682    root: PathBuf,
1683    repo_root: PathBuf,
1684}
1685
1686#[derive(Debug, Deserialize)]
1687struct StoredBenchmarkArtifact {
1688    artifact_version: u32,
1689    generated_at_unix_ms: u128,
1690    scenarios: Vec<StoredBenchmarkScenario>,
1691}
1692
1693#[derive(Debug, Deserialize)]
1694struct StoredBenchmarkScenario {
1695    id: String,
1696    wall_stats: BenchmarkStats,
1697    #[serde(default)]
1698    guest_import_stats: Option<BenchmarkStats>,
1699    #[serde(default)]
1700    startup_overhead_stats: Option<BenchmarkStats>,
1701    #[serde(default)]
1702    phase_stats: Option<BenchmarkScenarioPhases<BenchmarkStats>>,
1703}
1704
1705#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1706struct StoredBenchmarkRunHost {
1707    node_binary: String,
1708    node_version: String,
1709    os: String,
1710    arch: String,
1711    logical_cpus: usize,
1712}
1713
1714#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1715struct StoredBenchmarkRunState {
1716    artifact_version: u32,
1717    config: JavascriptBenchmarkConfig,
1718    host: StoredBenchmarkRunHost,
1719    repo_root: PathBuf,
1720    #[serde(default)]
1721    transport_rtt: Option<Vec<StoredBenchmarkTransportRttReport>>,
1722    #[serde(default)]
1723    scenarios: Vec<StoredBenchmarkScenarioReport>,
1724}
1725
1726#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1727struct StoredBenchmarkTransportRttReport {
1728    payload_bytes: usize,
1729    samples_ms: Vec<f64>,
1730    stats: BenchmarkStats,
1731}
1732
1733#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1734struct StoredBenchmarkScenarioReport {
1735    id: String,
1736    wall_samples_ms: Vec<f64>,
1737    wall_stats: BenchmarkStats,
1738    #[serde(default)]
1739    guest_import_samples_ms: Option<Vec<f64>>,
1740    #[serde(default)]
1741    guest_import_stats: Option<BenchmarkStats>,
1742    #[serde(default)]
1743    startup_overhead_samples_ms: Option<Vec<f64>>,
1744    #[serde(default)]
1745    startup_overhead_stats: Option<BenchmarkStats>,
1746    phase_samples_ms: BenchmarkScenarioPhases<Vec<f64>>,
1747    phase_stats: BenchmarkScenarioPhases<BenchmarkStats>,
1748    #[serde(default)]
1749    resource_usage_samples: Option<BenchmarkResourceUsage<Vec<f64>>>,
1750    #[serde(default)]
1751    resource_usage_stats: Option<BenchmarkResourceUsage<BenchmarkDistributionStats>>,
1752}
1753
1754impl BenchmarkWorkspace {
1755    fn create(repo_root: &Path) -> Result<Self, JavascriptBenchmarkError> {
1756        let root = repo_root.join(format!(
1757            ".tmp-agentos-execution-bench-{}-{}",
1758            std::process::id(),
1759            SystemTime::now()
1760                .duration_since(UNIX_EPOCH)
1761                .unwrap_or_default()
1762                .as_nanos()
1763        ));
1764        fs::create_dir_all(&root)?;
1765        write_benchmark_workspace(&root, repo_root)?;
1766        Ok(Self {
1767            root,
1768            repo_root: repo_root.to_path_buf(),
1769        })
1770    }
1771}
1772
1773impl Drop for BenchmarkWorkspace {
1774    fn drop(&mut self) {
1775        let _ = fs::remove_dir_all(&self.root);
1776    }
1777}
1778
1779impl StoredBenchmarkRunHost {
1780    fn from_host(host: &BenchmarkHost) -> Self {
1781        Self {
1782            node_binary: host.node_binary.clone(),
1783            node_version: host.node_version.clone(),
1784            os: host.os.to_owned(),
1785            arch: host.arch.to_owned(),
1786            logical_cpus: host.logical_cpus,
1787        }
1788    }
1789
1790    fn matches_host(&self, host: &BenchmarkHost) -> bool {
1791        self.node_binary == host.node_binary
1792            && self.node_version == host.node_version
1793            && self.os == host.os
1794            && self.arch == host.arch
1795            && self.logical_cpus == host.logical_cpus
1796    }
1797}
1798
1799impl StoredBenchmarkRunState {
1800    fn new(config: &JavascriptBenchmarkConfig, host: &BenchmarkHost, repo_root: &Path) -> Self {
1801        Self {
1802            artifact_version: BENCHMARK_ARTIFACT_VERSION,
1803            config: config.clone(),
1804            host: StoredBenchmarkRunHost::from_host(host),
1805            repo_root: repo_root.to_path_buf(),
1806            transport_rtt: None,
1807            scenarios: Vec::new(),
1808        }
1809    }
1810
1811    fn is_compatible(
1812        &self,
1813        config: &JavascriptBenchmarkConfig,
1814        host: &BenchmarkHost,
1815        repo_root: &Path,
1816    ) -> bool {
1817        self.artifact_version == BENCHMARK_ARTIFACT_VERSION
1818            && self.config == *config
1819            && self.host.matches_host(host)
1820            && self.repo_root == repo_root
1821    }
1822
1823    fn sanitized(mut self, definitions: &[ScenarioDefinition]) -> Self {
1824        if let Some(transport_rtt) = &self.transport_rtt {
1825            let payloads = transport_rtt
1826                .iter()
1827                .map(|report| report.payload_bytes)
1828                .collect::<Vec<_>>();
1829            if payloads != TRANSPORT_RTT_PAYLOAD_BYTES {
1830                self.transport_rtt = None;
1831            }
1832        }
1833
1834        let mut scenarios_by_id = self
1835            .scenarios
1836            .into_iter()
1837            .map(|scenario| (scenario.id.clone(), scenario))
1838            .collect::<BTreeMap<_, _>>();
1839        self.scenarios = definitions
1840            .iter()
1841            .filter_map(|definition| scenarios_by_id.remove(definition.id))
1842            .collect();
1843        self
1844    }
1845
1846    fn resumed_stage_count(&self, definitions: &[ScenarioDefinition]) -> usize {
1847        usize::from(self.transport_rtt.is_some())
1848            + definitions
1849                .iter()
1850                .filter(|definition| self.has_scenario(definition.id))
1851                .count()
1852    }
1853
1854    fn has_scenario(&self, id: &str) -> bool {
1855        self.scenarios.iter().any(|scenario| scenario.id == id)
1856    }
1857
1858    fn record_transport_rtt(&mut self, transport_rtt: &[BenchmarkTransportRttReport]) {
1859        self.transport_rtt = Some(
1860            transport_rtt
1861                .iter()
1862                .map(StoredBenchmarkTransportRttReport::from_report)
1863                .collect(),
1864        );
1865    }
1866
1867    fn record_scenario(&mut self, scenario: &BenchmarkScenarioReport) {
1868        self.scenarios.retain(|stored| stored.id != scenario.id);
1869        self.scenarios
1870            .push(StoredBenchmarkScenarioReport::from_report(scenario));
1871    }
1872
1873    fn to_report(
1874        &self,
1875        config: &JavascriptBenchmarkConfig,
1876        host: &BenchmarkHost,
1877        repo_root: &Path,
1878        definitions: &[ScenarioDefinition],
1879    ) -> JavascriptBenchmarkReport {
1880        let scenarios_by_id = self
1881            .scenarios
1882            .iter()
1883            .map(|scenario| (scenario.id.as_str(), scenario))
1884            .collect::<BTreeMap<_, _>>();
1885
1886        JavascriptBenchmarkReport {
1887            generated_at_unix_ms: current_unix_ms(),
1888            config: config.clone(),
1889            host: host.clone(),
1890            repo_root: repo_root.to_path_buf(),
1891            transport_rtt: self
1892                .transport_rtt
1893                .clone()
1894                .unwrap_or_default()
1895                .into_iter()
1896                .map(StoredBenchmarkTransportRttReport::into_report)
1897                .collect(),
1898            scenarios: definitions
1899                .iter()
1900                .filter_map(|definition| {
1901                    scenarios_by_id
1902                        .get(definition.id)
1903                        .map(|scenario| scenario.to_report(*definition))
1904                })
1905                .collect(),
1906        }
1907    }
1908}
1909
1910impl StoredBenchmarkTransportRttReport {
1911    fn from_report(report: &BenchmarkTransportRttReport) -> Self {
1912        Self {
1913            payload_bytes: report.payload_bytes,
1914            samples_ms: report.samples_ms.clone(),
1915            stats: report.stats.clone(),
1916        }
1917    }
1918
1919    fn into_report(self) -> BenchmarkTransportRttReport {
1920        BenchmarkTransportRttReport {
1921            channel: TRANSPORT_RTT_CHANNEL,
1922            payload_bytes: self.payload_bytes,
1923            samples_ms: self.samples_ms,
1924            stats: self.stats,
1925        }
1926    }
1927}
1928
1929impl StoredBenchmarkScenarioReport {
1930    fn from_report(report: &BenchmarkScenarioReport) -> Self {
1931        Self {
1932            id: report.id.to_owned(),
1933            wall_samples_ms: report.wall_samples_ms.clone(),
1934            wall_stats: report.wall_stats.clone(),
1935            guest_import_samples_ms: report.guest_import_samples_ms.clone(),
1936            guest_import_stats: report.guest_import_stats.clone(),
1937            startup_overhead_samples_ms: report.startup_overhead_samples_ms.clone(),
1938            startup_overhead_stats: report.startup_overhead_stats.clone(),
1939            phase_samples_ms: report.phase_samples_ms.clone(),
1940            phase_stats: report.phase_stats.clone(),
1941            resource_usage_samples: report.resource_usage_samples.clone(),
1942            resource_usage_stats: report.resource_usage_stats.clone(),
1943        }
1944    }
1945
1946    fn to_report(&self, definition: ScenarioDefinition) -> BenchmarkScenarioReport {
1947        BenchmarkScenarioReport {
1948            id: definition.id,
1949            workload: definition.workload,
1950            runtime: definition.runtime.label(),
1951            mode: definition.mode.label(),
1952            description: definition.description,
1953            fixture: definition.fixture,
1954            compile_cache: definition.compile_cache.label(),
1955            wall_samples_ms: self.wall_samples_ms.clone(),
1956            wall_stats: self.wall_stats.clone(),
1957            guest_import_samples_ms: self.guest_import_samples_ms.clone(),
1958            guest_import_stats: self.guest_import_stats.clone(),
1959            startup_overhead_samples_ms: self.startup_overhead_samples_ms.clone(),
1960            startup_overhead_stats: self.startup_overhead_stats.clone(),
1961            phase_samples_ms: self.phase_samples_ms.clone(),
1962            phase_stats: self.phase_stats.clone(),
1963            resource_usage_samples: self.resource_usage_samples.clone(),
1964            resource_usage_stats: self.resource_usage_stats.clone(),
1965        }
1966    }
1967}
1968
1969pub fn run_javascript_benchmarks_with_recovery(
1970    runtime: &RuntimeContext,
1971    config: &JavascriptBenchmarkConfig,
1972    baseline_path: Option<&Path>,
1973) -> Result<JavascriptBenchmarkRunOutput, JavascriptBenchmarkError> {
1974    validate_benchmark_config(config)?;
1975
1976    let repo_root = workspace_root()?;
1977    let host = benchmark_host()?;
1978    let artifact_dir = benchmark_artifact_dir(&repo_root);
1979    let workspace = BenchmarkWorkspace::create(&repo_root)?;
1980    let (report, resumed_stage_count, state_path) = orchestrate_javascript_benchmark_report(
1981        config,
1982        &repo_root,
1983        &host,
1984        &artifact_dir,
1985        || measure_transport_rtt(runtime, &workspace, config),
1986        |scenario| run_scenario(runtime, &workspace, config, scenario),
1987    )?;
1988    let comparison = baseline_path
1989        .map(|path| report.compare_to_baseline_path(path))
1990        .transpose()?;
1991    let artifact_paths =
1992        report.write_artifacts_with_comparison(&artifact_dir, comparison.as_ref())?;
1993    remove_file_if_exists(&state_path)?;
1994
1995    Ok(JavascriptBenchmarkRunOutput {
1996        artifact_paths,
1997        resumed_stage_count,
1998    })
1999}
2000
2001fn orchestrate_javascript_benchmark_report<MeasureTransport, RunScenario>(
2002    config: &JavascriptBenchmarkConfig,
2003    repo_root: &Path,
2004    host: &BenchmarkHost,
2005    artifact_dir: &Path,
2006    mut measure_transport: MeasureTransport,
2007    mut run_scenario: RunScenario,
2008) -> Result<(JavascriptBenchmarkReport, usize, PathBuf), JavascriptBenchmarkError>
2009where
2010    MeasureTransport: FnMut() -> Result<Vec<BenchmarkTransportRttReport>, JavascriptBenchmarkError>,
2011    RunScenario:
2012        FnMut(ScenarioDefinition) -> Result<BenchmarkScenarioReport, JavascriptBenchmarkError>,
2013{
2014    validate_benchmark_config(config)?;
2015
2016    fs::create_dir_all(artifact_dir)?;
2017
2018    let definitions = benchmark_scenarios();
2019    let state_path = benchmark_run_state_path(artifact_dir);
2020    let mut state = load_benchmark_run_state(&state_path, config, host, repo_root, &definitions)?;
2021    let resumed_stage_count = state.resumed_stage_count(&definitions);
2022
2023    if state.transport_rtt.is_none() {
2024        let transport_rtt = measure_transport()?;
2025        state.record_transport_rtt(&transport_rtt);
2026        persist_benchmark_run_state(&state_path, &state)?;
2027    }
2028
2029    for definition in definitions {
2030        if state.has_scenario(definition.id) {
2031            continue;
2032        }
2033
2034        let scenario = run_scenario(definition)?;
2035        state.record_scenario(&scenario);
2036        persist_benchmark_run_state(&state_path, &state)?;
2037    }
2038
2039    Ok((
2040        state.to_report(config, host, repo_root, &benchmark_scenarios()),
2041        resumed_stage_count,
2042        state_path,
2043    ))
2044}
2045
2046fn validate_benchmark_config(
2047    config: &JavascriptBenchmarkConfig,
2048) -> Result<(), JavascriptBenchmarkError> {
2049    if config.iterations == 0 {
2050        return Err(JavascriptBenchmarkError::InvalidConfig(
2051            "iterations must be greater than zero",
2052        ));
2053    }
2054    if config.iterations > MAX_BENCHMARK_ITERATIONS {
2055        return Err(JavascriptBenchmarkError::InvalidConfig(
2056            "iterations must be less than or equal to 1000",
2057        ));
2058    }
2059    if config.warmup_iterations > MAX_BENCHMARK_WARMUP_ITERATIONS {
2060        return Err(JavascriptBenchmarkError::InvalidConfig(
2061            "warmup iterations must be less than or equal to 1000",
2062        ));
2063    }
2064
2065    Ok(())
2066}
2067
2068fn benchmark_scenarios() -> [ScenarioDefinition; 21] {
2069    [
2070        ScenarioDefinition {
2071            id: "isolate-startup",
2072            workload: "startup-floor",
2073            runtime: ScenarioRuntime::NativeExecution,
2074            mode: ScenarioMode::BaselineControl,
2075            description: "Minimal guest with no extra imports. Measures the current startup floor for create-context plus node process bootstrap.",
2076            fixture: "empty entrypoint",
2077            entrypoint: "./bench/isolate-startup.mjs",
2078            compile_cache: CompileCacheStrategy::Disabled,
2079            engine_reuse: EngineReuseStrategy::FreshPerSample,
2080            expect_import_metric: false,
2081            env: ScenarioEnvironment::None,
2082        },
2083        ScenarioDefinition {
2084            id: "prewarmed-isolate-startup",
2085            workload: "startup-floor",
2086            runtime: ScenarioRuntime::NativeExecution,
2087            mode: ScenarioMode::SameEngineReplay,
2088            description: "Minimal guest after a priming pass while one execution engine keeps materialized assets and builtin/polyfill prewarm state alive, isolating the hot startup floor from import work.",
2089            fixture: "empty entrypoint",
2090            entrypoint: "./bench/isolate-startup.mjs",
2091            compile_cache: CompileCacheStrategy::Primed,
2092            engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2093            expect_import_metric: false,
2094            env: ScenarioEnvironment::None,
2095        },
2096        ScenarioDefinition {
2097            id: "cold-local-import",
2098            workload: "local-import",
2099            runtime: ScenarioRuntime::NativeExecution,
2100            mode: ScenarioMode::TrueColdStart,
2101            description: "Cold import of a repo-local ESM graph that simulates layered application modules without compile-cache reuse.",
2102            fixture: "24-module local ESM graph",
2103            entrypoint: "./bench/cold-local-import.mjs",
2104            compile_cache: CompileCacheStrategy::Disabled,
2105            engine_reuse: EngineReuseStrategy::FreshPerSample,
2106            expect_import_metric: true,
2107            env: ScenarioEnvironment::None,
2108        },
2109        ScenarioDefinition {
2110            id: "warm-local-import",
2111            workload: "local-import",
2112            runtime: ScenarioRuntime::NativeExecution,
2113            mode: ScenarioMode::NewSessionReplay,
2114            description: "Warm import of the same local ESM graph after a compile-cache priming pass in an earlier isolate.",
2115            fixture: "24-module local ESM graph",
2116            entrypoint: "./bench/warm-local-import.mjs",
2117            compile_cache: CompileCacheStrategy::Primed,
2118            engine_reuse: EngineReuseStrategy::FreshPerSample,
2119            expect_import_metric: true,
2120            env: ScenarioEnvironment::None,
2121        },
2122        ScenarioDefinition {
2123            id: "same-context-local-import",
2124            workload: "local-import",
2125            runtime: ScenarioRuntime::NativeExecution,
2126            mode: ScenarioMode::SameSessionReplay,
2127            description: "Warm import of the same local ESM graph by replaying executions against one reused JavaScript context after a compile-cache priming pass.",
2128            fixture: "24-module local ESM graph",
2129            entrypoint: "./bench/warm-local-import.mjs",
2130            compile_cache: CompileCacheStrategy::Primed,
2131            engine_reuse: EngineReuseStrategy::SharedContextAcrossScenario,
2132            expect_import_metric: true,
2133            env: ScenarioEnvironment::None,
2134        },
2135        ScenarioDefinition {
2136            id: "prewarmed-local-import",
2137            workload: "local-import",
2138            runtime: ScenarioRuntime::NativeExecution,
2139            mode: ScenarioMode::SameEngineReplay,
2140            description: "Warm import of the same local ESM graph after compile-cache priming while one execution engine keeps materialized assets and builtin/polyfill prewarm state alive.",
2141            fixture: "24-module local ESM graph",
2142            entrypoint: "./bench/warm-local-import.mjs",
2143            compile_cache: CompileCacheStrategy::Primed,
2144            engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2145            expect_import_metric: true,
2146            env: ScenarioEnvironment::None,
2147        },
2148        ScenarioDefinition {
2149            id: "host-local-import",
2150            workload: "local-import",
2151            runtime: ScenarioRuntime::HostNode,
2152            mode: ScenarioMode::HostControl,
2153            description: "Direct host-Node control for the same local ESM graph so later runs can separate native executor overhead from guest import work.",
2154            fixture: "24-module local ESM graph",
2155            entrypoint: "./bench/cold-local-import.mjs",
2156            compile_cache: CompileCacheStrategy::Disabled,
2157            engine_reuse: EngineReuseStrategy::FreshPerSample,
2158            expect_import_metric: true,
2159            env: ScenarioEnvironment::None,
2160        },
2161        ScenarioDefinition {
2162            id: "builtin-import",
2163            workload: "builtin-import",
2164            runtime: ScenarioRuntime::NativeExecution,
2165            mode: ScenarioMode::TrueColdStart,
2166            description: "Import of the common builtin path used by the wrappers and polyfill-adjacent bootstrap code.",
2167            fixture: "node:path + node:url + node:fs/promises",
2168            entrypoint: "./bench/builtin-import.mjs",
2169            compile_cache: CompileCacheStrategy::Disabled,
2170            engine_reuse: EngineReuseStrategy::FreshPerSample,
2171            expect_import_metric: true,
2172            env: ScenarioEnvironment::None,
2173        },
2174        ScenarioDefinition {
2175            id: "hot-builtin-stream-import",
2176            workload: "builtin-hot-import",
2177            runtime: ScenarioRuntime::NativeExecution,
2178            mode: ScenarioMode::SameEngineReplay,
2179            description: "Hot single-import microbench for `node:stream` after a priming pass inside one reused execution engine.",
2180            fixture: "node:stream",
2181            entrypoint: "./bench/hot-builtin-stream-import.mjs",
2182            compile_cache: CompileCacheStrategy::Primed,
2183            engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2184            expect_import_metric: true,
2185            env: ScenarioEnvironment::None,
2186        },
2187        ScenarioDefinition {
2188            id: "hot-builtin-stream-web-import",
2189            workload: "builtin-hot-import",
2190            runtime: ScenarioRuntime::NativeExecution,
2191            mode: ScenarioMode::SameEngineReplay,
2192            description: "Hot single-import microbench for `node:stream/web` after a priming pass inside one reused execution engine.",
2193            fixture: "node:stream/web",
2194            entrypoint: "./bench/hot-builtin-stream-web-import.mjs",
2195            compile_cache: CompileCacheStrategy::Primed,
2196            engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2197            expect_import_metric: true,
2198            env: ScenarioEnvironment::None,
2199        },
2200        ScenarioDefinition {
2201            id: "hot-builtin-crypto-import",
2202            workload: "builtin-hot-import",
2203            runtime: ScenarioRuntime::NativeExecution,
2204            mode: ScenarioMode::SameEngineReplay,
2205            description: "Hot single-import microbench for `node:crypto` after a priming pass inside one reused execution engine.",
2206            fixture: "node:crypto",
2207            entrypoint: "./bench/hot-builtin-crypto-import.mjs",
2208            compile_cache: CompileCacheStrategy::Primed,
2209            engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2210            expect_import_metric: true,
2211            env: ScenarioEnvironment::None,
2212        },
2213        ScenarioDefinition {
2214            id: "hot-builtin-zlib-import",
2215            workload: "builtin-hot-import",
2216            runtime: ScenarioRuntime::NativeExecution,
2217            mode: ScenarioMode::SameEngineReplay,
2218            description: "Hot single-import microbench for `node:zlib` after a priming pass inside one reused execution engine.",
2219            fixture: "node:zlib",
2220            entrypoint: "./bench/hot-builtin-zlib-import.mjs",
2221            compile_cache: CompileCacheStrategy::Primed,
2222            engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2223            expect_import_metric: true,
2224            env: ScenarioEnvironment::None,
2225        },
2226        ScenarioDefinition {
2227            id: "hot-builtin-assert-import",
2228            workload: "builtin-hot-import",
2229            runtime: ScenarioRuntime::NativeExecution,
2230            mode: ScenarioMode::SameEngineReplay,
2231            description: "Hot single-import microbench for `node:assert/strict` after a priming pass inside one reused execution engine.",
2232            fixture: "node:assert/strict",
2233            entrypoint: "./bench/hot-builtin-assert-import.mjs",
2234            compile_cache: CompileCacheStrategy::Primed,
2235            engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2236            expect_import_metric: true,
2237            env: ScenarioEnvironment::None,
2238        },
2239        ScenarioDefinition {
2240            id: "hot-builtin-url-import",
2241            workload: "builtin-hot-import",
2242            runtime: ScenarioRuntime::NativeExecution,
2243            mode: ScenarioMode::SameEngineReplay,
2244            description: "Hot single-import microbench for `node:url` after a priming pass inside one reused execution engine.",
2245            fixture: "node:url",
2246            entrypoint: "./bench/hot-builtin-url-import.mjs",
2247            compile_cache: CompileCacheStrategy::Primed,
2248            engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2249            expect_import_metric: true,
2250            env: ScenarioEnvironment::None,
2251        },
2252        ScenarioDefinition {
2253            id: "hot-projected-package-file-import",
2254            workload: "projected-package-hot-import",
2255            runtime: ScenarioRuntime::HostNode,
2256            mode: ScenarioMode::SameEngineReplay,
2257            description: "Hot projected-package single-import microbench for the TypeScript compiler file with compile cache and projected-source manifest reuse enabled across repeated contexts.",
2258            fixture: "projected TypeScript compiler file",
2259            entrypoint: "./bench/hot-projected-package-file-import.mjs",
2260            compile_cache: CompileCacheStrategy::Primed,
2261            engine_reuse: EngineReuseStrategy::FreshPerSample,
2262            expect_import_metric: true,
2263            env: ScenarioEnvironment::ProjectedWorkspaceNodeModules,
2264        },
2265        ScenarioDefinition {
2266            id: "large-package-import",
2267            workload: "large-package-import",
2268            runtime: ScenarioRuntime::HostNode,
2269            mode: ScenarioMode::TrueColdStart,
2270            description: "Cold import of the real-world `typescript` package from the workspace root `node_modules` tree.",
2271            fixture: "typescript",
2272            entrypoint: "./bench/large-package-import.mjs",
2273            compile_cache: CompileCacheStrategy::Disabled,
2274            engine_reuse: EngineReuseStrategy::FreshPerSample,
2275            expect_import_metric: true,
2276            env: ScenarioEnvironment::None,
2277        },
2278        ScenarioDefinition {
2279            id: "projected-package-import",
2280            workload: "projected-package-import",
2281            runtime: ScenarioRuntime::HostNode,
2282            mode: ScenarioMode::HostControl,
2283            description: "Projected-package guest-path import of TypeScript with compile cache and projected-source manifest reuse enabled across repeated contexts.",
2284            fixture: "projected TypeScript guest-path import",
2285            entrypoint: "./bench/projected-package-import.mjs",
2286            compile_cache: CompileCacheStrategy::Primed,
2287            engine_reuse: EngineReuseStrategy::FreshPerSample,
2288            expect_import_metric: true,
2289            env: ScenarioEnvironment::ProjectedWorkspaceNodeModules,
2290        },
2291        ScenarioDefinition {
2292            id: "pdf-lib-startup",
2293            workload: "pdf-lib-startup",
2294            runtime: ScenarioRuntime::HostNode,
2295            mode: ScenarioMode::HostControl,
2296            description: "Cold import of `pdf-lib` plus representative document setup that creates a PDF page and embeds a standard font.",
2297            fixture: "pdf-lib document creation",
2298            entrypoint: "./bench/pdf-lib-startup.mjs",
2299            compile_cache: CompileCacheStrategy::Disabled,
2300            engine_reuse: EngineReuseStrategy::FreshPerSample,
2301            expect_import_metric: true,
2302            env: ScenarioEnvironment::None,
2303        },
2304        ScenarioDefinition {
2305            id: "jszip-startup",
2306            workload: "jszip-startup",
2307            runtime: ScenarioRuntime::HostNode,
2308            mode: ScenarioMode::HostControl,
2309            description: "Cold import of `jszip` plus representative archive staging that builds a nested archive structure.",
2310            fixture: "jszip archive staging",
2311            entrypoint: "./bench/jszip-startup.mjs",
2312            compile_cache: CompileCacheStrategy::Disabled,
2313            engine_reuse: EngineReuseStrategy::FreshPerSample,
2314            expect_import_metric: true,
2315            env: ScenarioEnvironment::None,
2316        },
2317        ScenarioDefinition {
2318            id: "jszip-end-to-end",
2319            workload: "jszip-end-to-end",
2320            runtime: ScenarioRuntime::HostNode,
2321            mode: ScenarioMode::HostControl,
2322            description: "Cold import of `jszip` plus a full compressed archive roundtrip that writes, compresses, reloads, and validates nested archive contents.",
2323            fixture: "jszip end-to-end archive roundtrip",
2324            entrypoint: "./bench/jszip-end-to-end.mjs",
2325            compile_cache: CompileCacheStrategy::Disabled,
2326            engine_reuse: EngineReuseStrategy::FreshPerSample,
2327            expect_import_metric: true,
2328            env: ScenarioEnvironment::None,
2329        },
2330        ScenarioDefinition {
2331            id: "jszip-repeated-session-compressed",
2332            workload: "jszip-repeated-session-compressed",
2333            runtime: ScenarioRuntime::HostNode,
2334            mode: ScenarioMode::HostControl,
2335            description: "Repeated-session `jszip` workload after a compile-cache priming pass that compresses and reloads a nested archive in each fresh isolate.",
2336            fixture: "jszip compressed archive roundtrip",
2337            entrypoint: "./bench/jszip-repeated-session-compressed.mjs",
2338            compile_cache: CompileCacheStrategy::Primed,
2339            engine_reuse: EngineReuseStrategy::FreshPerSample,
2340            expect_import_metric: true,
2341            env: ScenarioEnvironment::None,
2342        },
2343    ]
2344}
2345
2346fn run_scenario(
2347    runtime: &RuntimeContext,
2348    workspace: &BenchmarkWorkspace,
2349    config: &JavascriptBenchmarkConfig,
2350    scenario: ScenarioDefinition,
2351) -> Result<BenchmarkScenarioReport, JavascriptBenchmarkError> {
2352    let compile_cache_root = workspace
2353        .root
2354        .join("compile-cache")
2355        .join(scenario.id.replace('-', "_"));
2356    let mut shared_engine = match scenario.engine_reuse {
2357        EngineReuseStrategy::FreshPerSample => None,
2358        EngineReuseStrategy::SharedAcrossScenario
2359        | EngineReuseStrategy::SharedContextAcrossScenario => {
2360            Some(JavascriptExecutionEngine::new(runtime.clone()))
2361        }
2362    };
2363    let mut shared_context = None;
2364
2365    if scenario.compile_cache == CompileCacheStrategy::Primed {
2366        run_sample(
2367            runtime,
2368            workspace,
2369            &scenario,
2370            Some(compile_cache_root.clone()),
2371            shared_engine.as_mut(),
2372            &mut shared_context,
2373        )?;
2374    }
2375
2376    for _ in 0..config.warmup_iterations {
2377        run_sample(
2378            runtime,
2379            workspace,
2380            &scenario,
2381            compile_cache_root_for_strategy(scenario.compile_cache, &compile_cache_root),
2382            shared_engine.as_mut(),
2383            &mut shared_context,
2384        )?;
2385    }
2386
2387    let mut wall_samples_ms = Vec::with_capacity(config.iterations);
2388    let mut guest_import_samples_ms = if scenario.expect_import_metric {
2389        Some(Vec::with_capacity(config.iterations))
2390    } else {
2391        None
2392    };
2393    let mut context_setup_samples_ms = Vec::with_capacity(config.iterations);
2394    let mut startup_samples_ms = Vec::with_capacity(config.iterations);
2395    let mut completion_samples_ms = Vec::with_capacity(config.iterations);
2396    let mut resource_usage_samples = BenchmarkResourceUsage::<Vec<f64>>::default();
2397
2398    for _ in 0..config.iterations {
2399        let sample = run_sample(
2400            runtime,
2401            workspace,
2402            &scenario,
2403            compile_cache_root_for_strategy(scenario.compile_cache, &compile_cache_root),
2404            shared_engine.as_mut(),
2405            &mut shared_context,
2406        )?;
2407        wall_samples_ms.push(sample.wall_ms);
2408        context_setup_samples_ms.push(sample.context_setup_ms);
2409        startup_samples_ms.push(sample.startup_ms);
2410        completion_samples_ms.push(sample.completion_ms);
2411
2412        if let (Some(import_ms), Some(samples)) =
2413            (sample.guest_import_ms, guest_import_samples_ms.as_mut())
2414        {
2415            samples.push(import_ms);
2416        }
2417        if let Some(resource_usage) = sample.resource_usage.as_ref() {
2418            resource_usage_samples.push_sample(resource_usage);
2419        }
2420    }
2421
2422    let startup_overhead_samples_ms = guest_import_samples_ms.as_ref().map(|guest_samples| {
2423        context_setup_samples_ms
2424            .iter()
2425            .zip(startup_samples_ms.iter())
2426            .zip(completion_samples_ms.iter())
2427            .zip(guest_samples.iter())
2428            .map(|(((context_ms, startup_ms), completion_ms), _guest_ms)| {
2429                context_ms + startup_ms + completion_ms
2430            })
2431            .collect::<Vec<_>>()
2432    });
2433
2434    let phase_samples_ms = BenchmarkScenarioPhases {
2435        context_setup_ms: context_setup_samples_ms,
2436        startup_ms: startup_samples_ms,
2437        guest_execution_ms: guest_import_samples_ms.clone(),
2438        completion_ms: completion_samples_ms,
2439    };
2440    let resource_usage_samples = resource_usage_samples.into_populated();
2441
2442    Ok(BenchmarkScenarioReport {
2443        id: scenario.id,
2444        workload: scenario.workload,
2445        runtime: scenario.runtime.label(),
2446        mode: scenario.mode.label(),
2447        description: scenario.description,
2448        fixture: scenario.fixture,
2449        compile_cache: scenario.compile_cache.label(),
2450        wall_stats: compute_stats(&wall_samples_ms),
2451        guest_import_stats: guest_import_samples_ms
2452            .as_ref()
2453            .map(|samples| compute_stats(samples)),
2454        startup_overhead_stats: startup_overhead_samples_ms
2455            .as_ref()
2456            .map(|samples| compute_stats(samples)),
2457        phase_stats: BenchmarkScenarioPhases {
2458            context_setup_ms: compute_stats(&phase_samples_ms.context_setup_ms),
2459            startup_ms: compute_stats(&phase_samples_ms.startup_ms),
2460            guest_execution_ms: phase_samples_ms
2461                .guest_execution_ms
2462                .as_ref()
2463                .map(|samples| compute_stats(samples)),
2464            completion_ms: compute_stats(&phase_samples_ms.completion_ms),
2465        },
2466        resource_usage_stats: resource_usage_samples
2467            .as_ref()
2468            .and_then(compute_resource_usage_stats),
2469        wall_samples_ms,
2470        guest_import_samples_ms,
2471        startup_overhead_samples_ms,
2472        phase_samples_ms,
2473        resource_usage_samples,
2474    })
2475}
2476
2477fn compile_cache_root_for_strategy(strategy: CompileCacheStrategy, root: &Path) -> Option<PathBuf> {
2478    match strategy {
2479        CompileCacheStrategy::Disabled => None,
2480        CompileCacheStrategy::Primed => Some(root.to_path_buf()),
2481    }
2482}
2483
2484fn run_sample(
2485    runtime: &RuntimeContext,
2486    workspace: &BenchmarkWorkspace,
2487    scenario: &ScenarioDefinition,
2488    compile_cache_root: Option<PathBuf>,
2489    shared_engine: Option<&mut JavascriptExecutionEngine>,
2490    shared_context: &mut Option<crate::JavascriptContext>,
2491) -> Result<SampleMeasurement, JavascriptBenchmarkError> {
2492    match scenario.runtime {
2493        ScenarioRuntime::NativeExecution => run_native_sample(
2494            runtime,
2495            workspace,
2496            scenario,
2497            compile_cache_root,
2498            shared_engine,
2499            shared_context,
2500        ),
2501        ScenarioRuntime::HostNode => run_host_node_sample(workspace, scenario),
2502    }
2503}
2504
2505fn run_native_sample(
2506    runtime: &RuntimeContext,
2507    workspace: &BenchmarkWorkspace,
2508    scenario: &ScenarioDefinition,
2509    compile_cache_root: Option<PathBuf>,
2510    shared_engine: Option<&mut JavascriptExecutionEngine>,
2511    shared_context: &mut Option<crate::JavascriptContext>,
2512) -> Result<SampleMeasurement, JavascriptBenchmarkError> {
2513    let mut fresh_engine = JavascriptExecutionEngine::new(runtime.clone());
2514    let engine = shared_engine.unwrap_or(&mut fresh_engine);
2515    let context_started_at = Instant::now();
2516    let (context, context_setup_ms) = match scenario.engine_reuse {
2517        EngineReuseStrategy::SharedContextAcrossScenario => {
2518            if let Some(context) = shared_context.as_ref() {
2519                (context.clone(), 0.0)
2520            } else {
2521                let context = engine.create_context(CreateJavascriptContextRequest {
2522                    vm_id: String::from("vm-bench"),
2523                    bootstrap_module: None,
2524                    compile_cache_root,
2525                });
2526                let context_setup_ms = context_started_at.elapsed().as_secs_f64() * 1000.0;
2527                *shared_context = Some(context.clone());
2528                (context, context_setup_ms)
2529            }
2530        }
2531        _ => {
2532            let context = engine.create_context(CreateJavascriptContextRequest {
2533                vm_id: String::from("vm-bench"),
2534                bootstrap_module: None,
2535                compile_cache_root,
2536            });
2537            let context_setup_ms = context_started_at.elapsed().as_secs_f64() * 1000.0;
2538            (context, context_setup_ms)
2539        }
2540    };
2541
2542    let startup_started_at = Instant::now();
2543    let execution = engine.start_execution(StartJavascriptExecutionRequest {
2544        limits: Default::default(),
2545        argv0: None,
2546        guest_runtime: Default::default(),
2547        vm_id: String::from("vm-bench"),
2548        context_id: context.context_id,
2549        argv: vec![String::from(scenario.entrypoint)],
2550        env: scenario_env(workspace, scenario),
2551        cwd: workspace.root.clone(),
2552        wasm_module_bytes: None,
2553        inline_code: None,
2554    })?;
2555    let startup_ms = startup_started_at.elapsed().as_secs_f64() * 1000.0;
2556
2557    let completion_started_at = Instant::now();
2558    let result = execution.wait()?;
2559    let completion_total_ms = completion_started_at.elapsed().as_secs_f64() * 1000.0;
2560    let stdout = String::from_utf8(result.stdout)?;
2561    let stderr = String::from_utf8(result.stderr)?;
2562
2563    if result.exit_code != 0 {
2564        return Err(JavascriptBenchmarkError::NonZeroExit {
2565            scenario: scenario.id,
2566            exit_code: result.exit_code,
2567            stderr,
2568        });
2569    }
2570
2571    let parsed_metrics =
2572        parse_benchmark_metrics(scenario.id, &stdout, scenario.expect_import_metric)?;
2573    let guest_import_ms = parsed_metrics.import_ms;
2574    let completion_ms = guest_import_ms
2575        .map(|guest_ms| saturating_delta_ms(completion_total_ms, guest_ms))
2576        .unwrap_or(completion_total_ms);
2577    let wall_ms = context_setup_ms + startup_ms + completion_total_ms;
2578
2579    Ok(SampleMeasurement {
2580        wall_ms,
2581        guest_import_ms,
2582        context_setup_ms,
2583        startup_ms,
2584        completion_ms,
2585        resource_usage: parsed_metrics.resource_usage,
2586    })
2587}
2588
2589fn run_host_node_sample(
2590    workspace: &BenchmarkWorkspace,
2591    scenario: &ScenarioDefinition,
2592) -> Result<SampleMeasurement, JavascriptBenchmarkError> {
2593    let started_at = Instant::now();
2594    let output = Command::new(crate::host_node::node_binary())
2595        .arg(scenario.entrypoint)
2596        .current_dir(&workspace.root)
2597        .envs(scenario_env(workspace, scenario))
2598        .output()?;
2599    let wall_ms = started_at.elapsed().as_secs_f64() * 1000.0;
2600    let stdout = String::from_utf8(output.stdout)?;
2601    let stderr = String::from_utf8(output.stderr)?;
2602
2603    if !output.status.success() {
2604        return Err(JavascriptBenchmarkError::NonZeroExit {
2605            scenario: scenario.id,
2606            exit_code: output.status.code().unwrap_or(-1),
2607            stderr,
2608        });
2609    }
2610
2611    let parsed_metrics =
2612        parse_benchmark_metrics(scenario.id, &stdout, scenario.expect_import_metric)?;
2613    let guest_import_ms = parsed_metrics.import_ms;
2614    let startup_ms = guest_import_ms
2615        .map(|guest_ms| saturating_delta_ms(wall_ms, guest_ms))
2616        .unwrap_or(wall_ms);
2617
2618    Ok(SampleMeasurement {
2619        wall_ms,
2620        guest_import_ms,
2621        context_setup_ms: 0.0,
2622        startup_ms,
2623        completion_ms: 0.0,
2624        resource_usage: parsed_metrics.resource_usage,
2625    })
2626}
2627
2628fn scenario_env(
2629    workspace: &BenchmarkWorkspace,
2630    scenario: &ScenarioDefinition,
2631) -> BTreeMap<String, String> {
2632    match scenario.env {
2633        ScenarioEnvironment::None => BTreeMap::new(),
2634        ScenarioEnvironment::ProjectedWorkspaceNodeModules => {
2635            let projected_node_modules = workspace.repo_root.join("node_modules");
2636            let projected_node_modules_json =
2637                serde_json::to_string(&vec![projected_node_modules.display().to_string()])
2638                    .expect("serialize projected node_modules read path");
2639            let guest_path_mappings = serde_json::json!([{
2640                "guestPath": "/root/node_modules",
2641                "hostPath": projected_node_modules.display().to_string(),
2642            }])
2643            .to_string();
2644
2645            BTreeMap::from([
2646                (
2647                    String::from("AGENTOS_EXTRA_FS_READ_PATHS"),
2648                    projected_node_modules_json,
2649                ),
2650                (
2651                    String::from("AGENTOS_GUEST_PATH_MAPPINGS"),
2652                    guest_path_mappings,
2653                ),
2654            ])
2655        }
2656    }
2657}
2658
2659fn measure_transport_rtt(
2660    runtime: &RuntimeContext,
2661    workspace: &BenchmarkWorkspace,
2662    config: &JavascriptBenchmarkConfig,
2663) -> Result<Vec<BenchmarkTransportRttReport>, JavascriptBenchmarkError> {
2664    let mut engine = JavascriptExecutionEngine::new(runtime.clone());
2665    let context = engine.create_context(CreateJavascriptContextRequest {
2666        vm_id: String::from("vm-transport"),
2667        bootstrap_module: None,
2668        compile_cache_root: None,
2669    });
2670    let mut execution = engine.start_execution(StartJavascriptExecutionRequest {
2671        limits: Default::default(),
2672        argv0: None,
2673        guest_runtime: Default::default(),
2674        vm_id: String::from("vm-transport"),
2675        context_id: context.context_id,
2676        argv: vec![String::from("./bench/transport-echo.mjs")],
2677        env: BTreeMap::from([(String::from("AGENTOS_KEEP_STDIN_OPEN"), String::from("1"))]),
2678        cwd: workspace.root.clone(),
2679        wasm_module_bytes: None,
2680        inline_code: None,
2681    })?;
2682
2683    let mut stdout_buffer = String::new();
2684    let mut stderr_buffer = String::new();
2685    let mut reports = Vec::with_capacity(TRANSPORT_RTT_PAYLOAD_BYTES.len());
2686
2687    for payload_bytes in TRANSPORT_RTT_PAYLOAD_BYTES {
2688        for warmup_index in 0..config.warmup_iterations {
2689            let label = format!("warmup-{}-{warmup_index}", payload_bytes);
2690            measure_transport_roundtrip(
2691                &mut execution,
2692                payload_bytes,
2693                &label,
2694                &mut stdout_buffer,
2695                &mut stderr_buffer,
2696            )?;
2697        }
2698
2699        let mut samples_ms = Vec::with_capacity(config.iterations);
2700        for iteration in 0..config.iterations {
2701            let label = format!("measure-{}-{iteration}", payload_bytes);
2702            samples_ms.push(measure_transport_roundtrip(
2703                &mut execution,
2704                payload_bytes,
2705                &label,
2706                &mut stdout_buffer,
2707                &mut stderr_buffer,
2708            )?);
2709        }
2710
2711        reports.push(BenchmarkTransportRttReport {
2712            channel: TRANSPORT_RTT_CHANNEL,
2713            payload_bytes,
2714            stats: compute_stats(&samples_ms),
2715            samples_ms,
2716        });
2717    }
2718
2719    execution.close_stdin()?;
2720    let result = execution.wait()?;
2721    if result.exit_code != 0 {
2722        stderr_buffer.push_str(&String::from_utf8(result.stderr)?);
2723        return Err(JavascriptBenchmarkError::TransportProbeExited {
2724            exit_code: result.exit_code,
2725            stderr: stderr_buffer,
2726        });
2727    }
2728
2729    Ok(reports)
2730}
2731
2732fn measure_transport_roundtrip(
2733    execution: &mut crate::JavascriptExecution,
2734    payload_bytes: usize,
2735    label: &str,
2736    stdout_buffer: &mut String,
2737    stderr_buffer: &mut String,
2738) -> Result<f64, JavascriptBenchmarkError> {
2739    let payload = transport_probe_payload(payload_bytes, label);
2740    let expected_line = format!("{payload}\n");
2741    let started_at = Instant::now();
2742    execution.write_stdin(expected_line.as_bytes())?;
2743
2744    loop {
2745        if let Some(line) = take_complete_line(stdout_buffer) {
2746            if line == payload {
2747                return Ok(started_at.elapsed().as_secs_f64() * 1000.0);
2748            }
2749            return Err(JavascriptBenchmarkError::InvalidTransportProbeResponse {
2750                payload_bytes,
2751                expected: payload,
2752                actual: line,
2753            });
2754        }
2755
2756        match execution.poll_event_blocking(TRANSPORT_POLL_TIMEOUT)? {
2757            Some(crate::JavascriptExecutionEvent::Stdout(chunk)) => {
2758                stdout_buffer.push_str(&String::from_utf8(chunk)?);
2759            }
2760            Some(crate::JavascriptExecutionEvent::Stderr(chunk)) => {
2761                stderr_buffer.push_str(&String::from_utf8(chunk)?);
2762            }
2763            Some(crate::JavascriptExecutionEvent::SyncRpcRequest(request)) => {
2764                return Err(JavascriptBenchmarkError::Execution(
2765                    JavascriptExecutionError::PendingSyncRpcRequest(request.id),
2766                ));
2767            }
2768            Some(crate::JavascriptExecutionEvent::SignalState { .. }) => {}
2769            Some(crate::JavascriptExecutionEvent::Exited(exit_code)) => {
2770                return Err(JavascriptBenchmarkError::TransportProbeExited {
2771                    exit_code,
2772                    stderr: stderr_buffer.clone(),
2773                });
2774            }
2775            None => {
2776                return Err(JavascriptBenchmarkError::TransportProbeTimeout { payload_bytes });
2777            }
2778        }
2779    }
2780}
2781
2782fn transport_probe_payload(payload_bytes: usize, label: &str) -> String {
2783    if payload_bytes == 0 {
2784        return format!("transport:{label}:");
2785    }
2786
2787    let header = format!("transport:{label}:");
2788    let fill_len = payload_bytes.saturating_sub(header.len());
2789    format!("{header}{}", "x".repeat(fill_len))
2790}
2791
2792fn take_complete_line(buffer: &mut String) -> Option<String> {
2793    let newline_index = buffer.find('\n')?;
2794    let line = buffer[..newline_index].trim_end_matches('\r').to_owned();
2795    buffer.drain(..=newline_index);
2796    Some(line)
2797}
2798
2799#[derive(Debug, Default, Deserialize)]
2800struct ParsedBenchmarkMetrics {
2801    #[serde(default)]
2802    import_ms: Option<f64>,
2803    #[serde(default)]
2804    resource_usage: Option<BenchmarkResourceUsage<f64>>,
2805}
2806
2807fn parse_benchmark_metrics(
2808    scenario_id: &'static str,
2809    stdout: &str,
2810    expect_import_metric: bool,
2811) -> Result<ParsedBenchmarkMetrics, JavascriptBenchmarkError> {
2812    let raw_value = stdout
2813        .lines()
2814        .rev()
2815        .find_map(|line| line.strip_prefix(BENCHMARK_MARKER_PREFIX))
2816        .ok_or(JavascriptBenchmarkError::MissingBenchmarkMetric(
2817            scenario_id,
2818        ))?
2819        .trim();
2820
2821    if let Ok(parsed) = serde_json::from_str::<ParsedBenchmarkMetrics>(raw_value) {
2822        let has_resource_usage = match parsed.resource_usage.as_ref() {
2823            Some(resource_usage) => !resource_usage.is_empty(),
2824            None => false,
2825        };
2826        if parsed.import_ms.is_some() || has_resource_usage {
2827            if expect_import_metric && parsed.import_ms.is_none() {
2828                return Err(JavascriptBenchmarkError::MissingBenchmarkMetric(
2829                    scenario_id,
2830                ));
2831            }
2832            return Ok(parsed);
2833        }
2834    }
2835
2836    raw_value
2837        .parse::<f64>()
2838        .map(|import_ms| ParsedBenchmarkMetrics {
2839            import_ms: Some(import_ms),
2840            resource_usage: None,
2841        })
2842        .map_err(|_| JavascriptBenchmarkError::InvalidBenchmarkMetric {
2843            scenario: scenario_id,
2844            raw_value: raw_value.to_owned(),
2845        })
2846}
2847
2848fn workspace_root() -> Result<PathBuf, JavascriptBenchmarkError> {
2849    let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
2850    manifest_dir
2851        .parent()
2852        .and_then(Path::parent)
2853        .map(Path::to_path_buf)
2854        .ok_or(JavascriptBenchmarkError::InvalidWorkspaceRoot(manifest_dir))
2855}
2856
2857fn load_benchmark_artifact(
2858    baseline_path: &Path,
2859) -> Result<StoredBenchmarkArtifact, JavascriptBenchmarkError> {
2860    let raw = fs::read_to_string(baseline_path)?;
2861    serde_json::from_str(&raw).map_err(|err| JavascriptBenchmarkError::InvalidBaselineReport {
2862        path: baseline_path.to_path_buf(),
2863        message: err.to_string(),
2864    })
2865}
2866
2867fn benchmark_host() -> Result<BenchmarkHost, JavascriptBenchmarkError> {
2868    let node_binary = crate::host_node::node_binary();
2869    let output = Command::new(&node_binary)
2870        .arg("--version")
2871        .output()
2872        .map_err(JavascriptBenchmarkError::NodeVersion)?;
2873    let node_version = String::from_utf8(output.stdout)?;
2874
2875    Ok(BenchmarkHost {
2876        node_binary,
2877        node_version,
2878        os: env::consts::OS,
2879        arch: env::consts::ARCH,
2880        logical_cpus: std::thread::available_parallelism()
2881            .map(usize::from)
2882            .unwrap_or(1),
2883    })
2884}
2885
2886fn write_benchmark_workspace(
2887    root: &Path,
2888    repo_root: &Path,
2889) -> Result<(), JavascriptBenchmarkError> {
2890    fs::create_dir_all(root.join("bench"))?;
2891    fs::create_dir_all(root.join("bench/local-graph"))?;
2892    let host_node_modules = repo_root.join("node_modules");
2893    let workspace_node_modules = root.join("node_modules");
2894    if host_node_modules.exists() && !workspace_node_modules.exists() {
2895        std::os::unix::fs::symlink(&host_node_modules, &workspace_node_modules)?;
2896    }
2897    fs::write(
2898        root.join("package.json"),
2899        "{\n  \"name\": \"agentos-execution-bench\",\n  \"private\": true,\n  \"type\": \"module\"\n}\n",
2900    )?;
2901
2902    for index in 0..LOCAL_GRAPH_MODULE_COUNT {
2903        let path = root
2904            .join("bench/local-graph")
2905            .join(format!("mod-{index:02}.mjs"));
2906        let source = if index == 0 {
2907            String::from("export const value = 1;\n")
2908        } else {
2909            format!(
2910                "import {{ value as previous }} from './mod-{previous:02}.mjs';\nexport const value = previous + {index};\n",
2911                previous = index - 1
2912            )
2913        };
2914        fs::write(path, source)?;
2915    }
2916
2917    let final_value = local_graph_terminal_value();
2918    fs::write(
2919        root.join("bench/local-graph/root.mjs"),
2920        format!(
2921            "import {{ value }} from './mod-{last:02}.mjs';\nexport {{ value }};\nexport const expected = {final_value};\n",
2922            last = LOCAL_GRAPH_MODULE_COUNT - 1
2923        ),
2924    )?;
2925    fs::write(
2926        root.join("bench/benchmark-metrics.mjs"),
2927        benchmark_metrics_module_source(),
2928    )?;
2929
2930    fs::write(
2931        root.join("bench/isolate-startup.mjs"),
2932        resource_only_entrypoint_source("console.log('isolate-ready');"),
2933    )?;
2934    fs::write(
2935        root.join("bench/cold-local-import.mjs"),
2936        local_import_entrypoint_source(final_value),
2937    )?;
2938    fs::write(
2939        root.join("bench/warm-local-import.mjs"),
2940        local_import_entrypoint_source(final_value),
2941    )?;
2942    fs::write(
2943        root.join("bench/builtin-import.mjs"),
2944        timed_entrypoint_source(
2945            "const [pathMod, fsMod, urlMod] = await Promise.all([\n  import('node:path'),\n  import('node:fs/promises'),\n  import('node:url'),\n]);\nif (typeof pathMod.basename !== 'function' || typeof fsMod.readFile !== 'function' || typeof urlMod.pathToFileURL !== 'function') {\n  throw new Error('builtin import fixture did not load expected exports');\n}",
2946        ),
2947    )?;
2948    fs::write(
2949        root.join("bench/hot-builtin-stream-import.mjs"),
2950        single_import_entrypoint_source(
2951            "node:stream",
2952            "typeof imported.Readable === 'function'",
2953            "node:stream import did not expose Readable",
2954        ),
2955    )?;
2956    fs::write(
2957        root.join("bench/hot-builtin-stream-web-import.mjs"),
2958        single_import_entrypoint_source(
2959            "node:stream/web",
2960            "typeof imported.ReadableStream === 'function'",
2961            "node:stream/web import did not expose ReadableStream",
2962        ),
2963    )?;
2964    fs::write(
2965        root.join("bench/hot-builtin-crypto-import.mjs"),
2966        single_import_entrypoint_source(
2967            "node:crypto",
2968            "typeof imported.createHash === 'function'",
2969            "node:crypto import did not expose createHash",
2970        ),
2971    )?;
2972    fs::write(
2973        root.join("bench/hot-builtin-zlib-import.mjs"),
2974        single_import_entrypoint_source(
2975            "node:zlib",
2976            "typeof imported.gzipSync === 'function'",
2977            "node:zlib import did not expose gzipSync",
2978        ),
2979    )?;
2980    fs::write(
2981        root.join("bench/hot-builtin-assert-import.mjs"),
2982        single_import_entrypoint_source(
2983            "node:assert/strict",
2984            "typeof imported.strictEqual === 'function'",
2985            "node:assert/strict import did not expose strictEqual",
2986        ),
2987    )?;
2988    fs::write(
2989        root.join("bench/hot-builtin-url-import.mjs"),
2990        single_import_entrypoint_source(
2991            "node:url",
2992            "typeof imported.pathToFileURL === 'function'",
2993            "node:url import did not expose pathToFileURL",
2994        ),
2995    )?;
2996    fs::write(
2997        root.join("bench/large-package-import.mjs"),
2998        timed_entrypoint_source(
2999            "const typescript = await import('typescript');\nif (typeof typescript.transpileModule !== 'function') {\n  throw new Error('typescript import did not expose transpileModule');\n}",
3000        ),
3001    )?;
3002    fs::write(
3003        root.join("bench/hot-projected-package-file-import.mjs"),
3004        projected_package_file_import_entrypoint_source(),
3005    )?;
3006    fs::write(
3007        root.join("bench/projected-package-import.mjs"),
3008        projected_package_import_entrypoint_source(),
3009    )?;
3010    fs::write(
3011        root.join("bench/pdf-lib-startup.mjs"),
3012        pdf_lib_startup_entrypoint_source(),
3013    )?;
3014    fs::write(
3015        root.join("bench/jszip-startup.mjs"),
3016        jszip_startup_entrypoint_source(),
3017    )?;
3018    fs::write(
3019        root.join("bench/jszip-end-to-end.mjs"),
3020        jszip_end_to_end_entrypoint_source(),
3021    )?;
3022    fs::write(
3023        root.join("bench/jszip-repeated-session-compressed.mjs"),
3024        jszip_repeated_session_compressed_entrypoint_source(),
3025    )?;
3026    fs::write(
3027        root.join("bench/transport-echo.mjs"),
3028        "process.stdin.setEncoding('utf8');\nlet buffered = '';\nconst flushLines = () => {\n  let newlineIndex = buffered.indexOf('\\n');\n  while (newlineIndex >= 0) {\n    const line = buffered.slice(0, newlineIndex).replace(/\\r$/, '');\n    buffered = buffered.slice(newlineIndex + 1);\n    process.stdout.write(line);\n    newlineIndex = buffered.indexOf('\\n');\n  }\n};\nprocess.stdin.on('data', (chunk) => {\n  buffered += chunk;\n  flushLines();\n});\nprocess.stdin.on('end', () => {\n  if (buffered.length > 0) {\n    process.stdout.write(buffered.replace(/\\r$/, ''));\n  }\n});\n",
3029    )?;
3030
3031    Ok(())
3032}
3033
3034fn local_import_entrypoint_source(final_value: usize) -> String {
3035    timed_entrypoint_source(&format!(
3036        "const graph = await import('./local-graph/root.mjs');\nif (graph.value !== {final_value} || graph.expected !== {final_value}) {{\n  throw new Error(`local graph import returned ${{\n    graph.value\n  }} instead of {final_value}`);\n}}"
3037    ))
3038}
3039
3040fn single_import_entrypoint_source(
3041    specifier: &str,
3042    validation_expression: &str,
3043    error_message: &str,
3044) -> String {
3045    timed_entrypoint_source(&format!(
3046        "const imported = await import('{specifier}');\nif (!({validation_expression})) {{\n  throw new Error('{error_message}');\n}}"
3047    ))
3048}
3049
3050fn projected_package_file_import_entrypoint_source() -> String {
3051    timed_entrypoint_source(
3052        "const typescriptModule = await import('../node_modules/typescript/lib/typescript.js');\nconst typescript = typescriptModule.default ?? typescriptModule;\nif (typeof typescript.transpileModule !== 'function') {\n  throw new Error('projected package file import did not expose transpileModule');\n}",
3053    )
3054}
3055
3056fn projected_package_import_entrypoint_source() -> String {
3057    timed_entrypoint_source(
3058        "const typescriptModule = await import('../node_modules/typescript/lib/typescript.js');\nconst typescript = typescriptModule.default ?? typescriptModule;\nconst sourceFile = typescript.createSourceFile(\n  'bench.ts',\n  'const answer: number = 42;',\n  typescript.ScriptTarget.ES2022,\n  true,\n);\nif (\n  typeof typescript.transpileModule !== 'function' ||\n  typeof typescript.createSourceFile !== 'function' ||\n  !sourceFile ||\n  sourceFile.statements.length !== 1\n) {\n  throw new Error('projected package import did not expose TypeScript compiler APIs');\n}",
3059    )
3060}
3061
3062fn pdf_lib_startup_entrypoint_source() -> String {
3063    timed_entrypoint_source(
3064        "const pdfLib = await import('pdf-lib');\nconst pdfDoc = await pdfLib.PDFDocument.create();\nconst page = pdfDoc.addPage([612, 792]);\nconst font = await pdfDoc.embedFont(pdfLib.StandardFonts.Helvetica);\npage.drawText('secure-exec pdf-lib benchmark', {\n  x: 50,\n  y: 750,\n  font,\n  size: 18,\n});\nif (pdfDoc.getPageCount() !== 1 || page.getSize().width !== 612) {\n  throw new Error('pdf-lib fixture did not create the expected document');\n}",
3065    )
3066}
3067
3068fn jszip_startup_entrypoint_source() -> String {
3069    timed_entrypoint_source(
3070        "const jszipModule = await import('jszip');\nconst JSZip = jszipModule.default ?? jszipModule;\nconst zip = new JSZip();\nzip.file('README.txt', 'secure-exec benchmark archive');\nconst notes = zip.folder('notes');\nif (!notes) {\n  throw new Error('jszip fixture failed to create nested folder');\n}\nnotes.file('todo.txt', 'benchmark staging payload');\nconst fileCount = Object.values(zip.files).filter((entry) => !entry.dir).length;\nif (typeof zip.generateAsync !== 'function' || fileCount !== 2) {\n  throw new Error('jszip fixture did not stage the expected archive');\n}",
3071    )
3072}
3073
3074fn jszip_end_to_end_entrypoint_source() -> String {
3075    timed_entrypoint_source(
3076        "const jszipModule = await import('jszip');\nconst JSZip = jszipModule.default ?? jszipModule;\nconst zip = new JSZip();\nconst repeatedPayload = 'secure-exec benchmark payload '.repeat(512);\nzip.file('README.txt', repeatedPayload);\nconst notes = zip.folder('notes');\nif (!notes) {\n  throw new Error('jszip end-to-end fixture failed to create notes folder');\n}\nnotes.file('todo.txt', 'complete the archive roundtrip');\nconst data = zip.folder('data');\nif (!data) {\n  throw new Error('jszip end-to-end fixture failed to create data folder');\n}\ndata.file('payload.json', JSON.stringify({\n  repeatedPayloadLength: repeatedPayload.length,\n  mode: 'cold-end-to-end',\n}));\nconst archiveBytes = await zip.generateAsync({\n  type: 'uint8array',\n  compression: 'DEFLATE',\n  compressionOptions: { level: 6 },\n});\nconst restored = await JSZip.loadAsync(archiveBytes);\nconst restoredFileCount = Object.values(restored.files).filter((entry) => !entry.dir).length;\nconst restoredReadme = await restored.file('README.txt')?.async('string');\nconst restoredTodo = await restored.file('notes/todo.txt')?.async('string');\nconst restoredPayload = await restored.file('data/payload.json')?.async('string');\nif (\n  archiveBytes.byteLength >= repeatedPayload.length ||\n  restoredFileCount !== 3 ||\n  restoredReadme !== repeatedPayload ||\n  restoredTodo !== 'complete the archive roundtrip' ||\n  !restoredPayload?.includes('cold-end-to-end')\n) {\n  throw new Error('jszip end-to-end fixture did not complete the compressed archive roundtrip');\n}",
3077    )
3078}
3079
3080fn jszip_repeated_session_compressed_entrypoint_source() -> String {
3081    timed_entrypoint_source(
3082        "const jszipModule = await import('jszip');\nconst JSZip = jszipModule.default ?? jszipModule;\nconst zip = new JSZip();\nconst repeatedPayload = 'secure-exec benchmark payload '.repeat(512);\nzip.file('README.txt', repeatedPayload);\nconst notes = zip.folder('notes');\nif (!notes) {\n  throw new Error('jszip repeated-session fixture failed to create notes folder');\n}\nnotes.file('todo.txt', 'repeat this session workload');\nconst data = zip.folder('data');\nif (!data) {\n  throw new Error('jszip repeated-session fixture failed to create data folder');\n}\ndata.file('payload.json', JSON.stringify({\n  repeatedPayloadLength: repeatedPayload.length,\n  repeatedSessions: true,\n}));\nconst archiveBytes = await zip.generateAsync({\n  type: 'uint8array',\n  compression: 'DEFLATE',\n  compressionOptions: { level: 6 },\n});\nconst restored = await JSZip.loadAsync(archiveBytes);\nconst restoredFileCount = Object.values(restored.files).filter((entry) => !entry.dir).length;\nconst restoredReadme = await restored.file('README.txt')?.async('string');\nconst restoredTodo = await restored.file('notes/todo.txt')?.async('string');\nif (\n  archiveBytes.byteLength >= repeatedPayload.length ||\n  restoredFileCount !== 3 ||\n  restoredReadme !== repeatedPayload ||\n  restoredTodo !== 'repeat this session workload'\n) {\n  throw new Error('jszip repeated-session fixture did not complete the compressed archive roundtrip');\n}",
3083    )
3084}
3085
3086fn benchmark_metrics_module_source() -> String {
3087    format!(
3088        "const BENCHMARK_MARKER_PREFIX = '{BENCHMARK_MARKER_PREFIX}';\n\nexport function emitBenchmarkMetrics(importMs) {{\n  const memoryUsage = process.memoryUsage();\n  const resourceUsage = typeof process.resourceUsage === 'function'\n    ? process.resourceUsage()\n    : null;\n  const payload = {{\n    resource_usage: {{\n      rss_bytes: memoryUsage.rss,\n      heap_used_bytes: memoryUsage.heapUsed,\n      ...(resourceUsage\n        ? {{\n            cpu_user_us: resourceUsage.userCPUTime,\n            cpu_system_us: resourceUsage.systemCPUTime,\n            cpu_total_us: resourceUsage.userCPUTime + resourceUsage.systemCPUTime,\n          }}\n        : {{}}),\n    }},\n  }};\n\n  if (typeof importMs === 'number') {{\n    payload.import_ms = importMs;\n  }}\n\n  console.log(BENCHMARK_MARKER_PREFIX + JSON.stringify(payload));\n}}\n"
3089    )
3090}
3091
3092fn resource_only_entrypoint_source(body: &str) -> String {
3093    format!(
3094        "import {{ emitBenchmarkMetrics }} from './benchmark-metrics.mjs';\n{body}\nemitBenchmarkMetrics();\n"
3095    )
3096}
3097
3098fn timed_entrypoint_source(body: &str) -> String {
3099    format!(
3100        "import {{ performance }} from 'node:perf_hooks';\nimport {{ emitBenchmarkMetrics }} from './benchmark-metrics.mjs';\nconst started = performance.now();\n{body}\nemitBenchmarkMetrics(performance.now() - started);\n"
3101    )
3102}
3103
3104fn local_graph_terminal_value() -> usize {
3105    let mut value = 1;
3106
3107    for index in 1..LOCAL_GRAPH_MODULE_COUNT {
3108        value += index;
3109    }
3110
3111    value
3112}
3113
3114fn compute_distribution_stats(samples: &[f64]) -> BenchmarkDistributionStats {
3115    let mut sorted = samples.to_vec();
3116    sorted.sort_by(|a, b| a.total_cmp(b));
3117    let mean = sorted.iter().sum::<f64>() / sorted.len() as f64;
3118
3119    BenchmarkDistributionStats {
3120        mean,
3121        p50: percentile(&sorted, 50.0),
3122        p95: percentile(&sorted, 95.0),
3123        min: *sorted.first().unwrap_or(&0.0),
3124        max: *sorted.last().unwrap_or(&0.0),
3125        stddev: standard_deviation(&sorted, mean),
3126    }
3127}
3128
3129fn compute_stats(samples: &[f64]) -> BenchmarkStats {
3130    let stats = compute_distribution_stats(samples);
3131
3132    BenchmarkStats {
3133        mean_ms: stats.mean,
3134        p50_ms: stats.p50,
3135        p95_ms: stats.p95,
3136        min_ms: stats.min,
3137        max_ms: stats.max,
3138        stddev_ms: stats.stddev,
3139    }
3140}
3141
3142fn compute_resource_usage_stats(
3143    samples: &BenchmarkResourceUsage<Vec<f64>>,
3144) -> Option<BenchmarkResourceUsage<BenchmarkDistributionStats>> {
3145    let stats = BenchmarkResourceUsage {
3146        rss_bytes: samples
3147            .rss_bytes
3148            .as_ref()
3149            .map(|samples| compute_distribution_stats(samples)),
3150        heap_used_bytes: samples
3151            .heap_used_bytes
3152            .as_ref()
3153            .map(|samples| compute_distribution_stats(samples)),
3154        cpu_user_us: samples
3155            .cpu_user_us
3156            .as_ref()
3157            .map(|samples| compute_distribution_stats(samples)),
3158        cpu_system_us: samples
3159            .cpu_system_us
3160            .as_ref()
3161            .map(|samples| compute_distribution_stats(samples)),
3162        cpu_total_us: samples
3163            .cpu_total_us
3164            .as_ref()
3165            .map(|samples| compute_distribution_stats(samples)),
3166    };
3167
3168    (!stats.is_empty()).then_some(stats)
3169}
3170
3171fn standard_deviation(samples: &[f64], mean: f64) -> f64 {
3172    if samples.is_empty() {
3173        return 0.0;
3174    }
3175
3176    let variance = samples
3177        .iter()
3178        .map(|sample| {
3179            let delta = sample - mean;
3180            delta * delta
3181        })
3182        .sum::<f64>()
3183        / samples.len() as f64;
3184
3185    variance.sqrt()
3186}
3187
3188fn percentile(sorted: &[f64], p: f64) -> f64 {
3189    if sorted.is_empty() {
3190        return 0.0;
3191    }
3192
3193    let rank = ((p / 100.0) * sorted.len() as f64).ceil() as usize;
3194    let index = rank.saturating_sub(1).min(sorted.len() - 1);
3195    sorted[index]
3196}
3197
3198fn percentage_reduction(original: f64, current: f64) -> f64 {
3199    if original <= 0.0 {
3200        0.0
3201    } else {
3202        ((original - current) / original) * 100.0
3203    }
3204}
3205
3206fn percentage_share(part: f64, total: f64) -> f64 {
3207    if total <= 0.0 {
3208        0.0
3209    } else {
3210        (part / total) * 100.0
3211    }
3212}
3213
3214fn safe_ratio(lhs: f64, rhs: f64) -> f64 {
3215    if rhs <= 0.0 {
3216        0.0
3217    } else {
3218        lhs / rhs
3219    }
3220}
3221
3222fn saturating_delta_ms(total_ms: f64, subtracted_ms: f64) -> f64 {
3223    (total_ms - subtracted_ms).max(0.0)
3224}
3225
3226fn format_ms(value: f64) -> String {
3227    format!("{value:.2}")
3228}
3229
3230fn format_hotspot_value(unit: &str, value: f64) -> String {
3231    match unit {
3232        "pct" => format!("{value:.1}%"),
3233        "MiB" => format_mib(value),
3234        _ => format_ms(value),
3235    }
3236}
3237
3238fn format_sample_list(samples: &[f64]) -> String {
3239    format_scaled_sample_list(samples, std::convert::identity)
3240}
3241
3242fn format_scaled_sample_list(samples: &[f64], scale: impl Fn(f64) -> f64) -> String {
3243    let mut formatted = String::from("[");
3244
3245    for (index, sample) in samples.iter().enumerate() {
3246        if index > 0 {
3247            formatted.push_str(", ");
3248        }
3249        let _ = write!(&mut formatted, "{:.2}", scale(*sample));
3250    }
3251
3252    formatted.push(']');
3253    formatted
3254}
3255
3256fn format_mib(value: f64) -> String {
3257    format!("{value:.2}")
3258}
3259
3260fn format_label_list(labels: &[&str]) -> String {
3261    labels
3262        .iter()
3263        .map(|label| format!("`{label}`"))
3264        .collect::<Vec<_>>()
3265        .join(", ")
3266}
3267
3268fn format_string_label_list(labels: &[&str]) -> String {
3269    labels
3270        .iter()
3271        .map(|label| format!("`{label}`"))
3272        .collect::<Vec<_>>()
3273        .join(", ")
3274}
3275
3276fn push_unique_label<'a>(labels: &mut Vec<&'a str>, value: &'a str) {
3277    if !labels.contains(&value) {
3278        labels.push(value);
3279    }
3280}
3281
3282fn format_delta_ms(value: f64) -> String {
3283    format!("{value:+.2}")
3284}
3285
3286fn format_delta_pct(value: f64) -> String {
3287    format!("{value:+.1}%")
3288}
3289
3290fn push_optional_sample(samples: &mut Option<Vec<f64>>, value: Option<f64>) {
3291    if let Some(value) = value {
3292        samples.get_or_insert_with(Vec::new).push(value);
3293    }
3294}
3295
3296fn bytes_to_mib(value: f64) -> f64 {
3297    value / (1024.0 * 1024.0)
3298}
3299
3300fn micros_to_ms(value: f64) -> f64 {
3301    value / 1000.0
3302}
3303
3304fn hotspot_wall_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3305    Some(scenario.wall_stats.mean_ms)
3306}
3307
3308fn hotspot_wall_stddev_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3309    Some(scenario.wall_stats.stddev_ms)
3310}
3311
3312fn hotspot_wall_range_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3313    Some(scenario.wall_range_ms())
3314}
3315
3316fn hotspot_guest_import_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3317    scenario
3318        .guest_import_stats
3319        .as_ref()
3320        .map(|stats| stats.mean_ms)
3321}
3322
3323fn hotspot_startup_overhead_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3324    scenario
3325        .startup_overhead_stats
3326        .as_ref()
3327        .map(|stats| stats.mean_ms)
3328}
3329
3330fn hotspot_context_setup_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3331    Some(scenario.phase_stats.context_setup_ms.mean_ms)
3332}
3333
3334fn hotspot_startup_phase_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3335    Some(scenario.phase_stats.startup_ms.mean_ms)
3336}
3337
3338fn hotspot_guest_execution_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3339    scenario
3340        .phase_stats
3341        .guest_execution_ms
3342        .as_ref()
3343        .map(|stats| stats.mean_ms)
3344}
3345
3346fn hotspot_completion_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3347    Some(scenario.phase_stats.completion_ms.mean_ms)
3348}
3349
3350fn hotspot_startup_share_pct(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3351    scenario.mean_startup_share_pct()
3352}
3353
3354fn hotspot_rss_mean_mib(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3355    scenario
3356        .resource_usage_stats
3357        .as_ref()?
3358        .rss_bytes
3359        .as_ref()
3360        .map(|stats| bytes_to_mib(stats.mean))
3361}
3362
3363fn hotspot_heap_mean_mib(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3364    scenario
3365        .resource_usage_stats
3366        .as_ref()?
3367        .heap_used_bytes
3368        .as_ref()
3369        .map(|stats| bytes_to_mib(stats.mean))
3370}
3371
3372fn hotspot_total_cpu_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3373    scenario
3374        .resource_usage_stats
3375        .as_ref()?
3376        .cpu_total_us
3377        .as_ref()
3378        .map(|stats| micros_to_ms(stats.mean))
3379}
3380
3381#[cfg(test)]
3382mod tests {
3383    use super::*;
3384    use std::cell::RefCell;
3385    use tempfile::tempdir;
3386
3387    fn synthetic_transport_reports() -> Vec<BenchmarkTransportRttReport> {
3388        TRANSPORT_RTT_PAYLOAD_BYTES
3389            .iter()
3390            .enumerate()
3391            .map(|(index, payload_bytes)| {
3392                let sample = index as f64 + 1.0;
3393                BenchmarkTransportRttReport {
3394                    channel: TRANSPORT_RTT_CHANNEL,
3395                    payload_bytes: *payload_bytes,
3396                    samples_ms: vec![sample],
3397                    stats: compute_stats(&[sample]),
3398                }
3399            })
3400            .collect()
3401    }
3402
3403    fn synthetic_scenario_report(
3404        definition: ScenarioDefinition,
3405        wall_sample_ms: f64,
3406    ) -> BenchmarkScenarioReport {
3407        let context_setup_ms = wall_sample_ms / 5.0;
3408        let startup_ms = wall_sample_ms / 4.0;
3409        let guest_execution_ms = definition
3410            .expect_import_metric
3411            .then_some(wall_sample_ms / 3.0);
3412        let completion_ms =
3413            wall_sample_ms - context_setup_ms - startup_ms - guest_execution_ms.unwrap_or(0.0);
3414        let startup_overhead_ms = definition
3415            .expect_import_metric
3416            .then_some(context_setup_ms + startup_ms + completion_ms);
3417        let resource_usage_samples = BenchmarkResourceUsage {
3418            rss_bytes: Some(vec![64.0 * 1024.0 * 1024.0]),
3419            heap_used_bytes: Some(vec![12.0 * 1024.0 * 1024.0]),
3420            cpu_user_us: None,
3421            cpu_system_us: None,
3422            cpu_total_us: Some(vec![wall_sample_ms * 1000.0]),
3423        };
3424
3425        BenchmarkScenarioReport {
3426            id: definition.id,
3427            workload: definition.workload,
3428            runtime: definition.runtime.label(),
3429            mode: definition.mode.label(),
3430            description: definition.description,
3431            fixture: definition.fixture,
3432            compile_cache: definition.compile_cache.label(),
3433            wall_samples_ms: vec![wall_sample_ms],
3434            wall_stats: compute_stats(&[wall_sample_ms]),
3435            guest_import_samples_ms: guest_execution_ms.map(|sample| vec![sample]),
3436            guest_import_stats: guest_execution_ms.map(|sample| compute_stats(&[sample])),
3437            startup_overhead_samples_ms: startup_overhead_ms.map(|sample| vec![sample]),
3438            startup_overhead_stats: startup_overhead_ms.map(|sample| compute_stats(&[sample])),
3439            phase_samples_ms: BenchmarkScenarioPhases {
3440                context_setup_ms: vec![context_setup_ms],
3441                startup_ms: vec![startup_ms],
3442                guest_execution_ms: guest_execution_ms.map(|sample| vec![sample]),
3443                completion_ms: vec![completion_ms],
3444            },
3445            phase_stats: BenchmarkScenarioPhases {
3446                context_setup_ms: compute_stats(&[context_setup_ms]),
3447                startup_ms: compute_stats(&[startup_ms]),
3448                guest_execution_ms: guest_execution_ms.map(|sample| compute_stats(&[sample])),
3449                completion_ms: compute_stats(&[completion_ms]),
3450            },
3451            resource_usage_stats: compute_resource_usage_stats(&resource_usage_samples),
3452            resource_usage_samples: Some(resource_usage_samples),
3453        }
3454    }
3455
3456    fn synthetic_host() -> BenchmarkHost {
3457        BenchmarkHost {
3458            node_binary: String::from("node"),
3459            node_version: String::from("v22.0.0"),
3460            os: "linux",
3461            arch: "x86_64",
3462            logical_cpus: 8,
3463        }
3464    }
3465
3466    #[test]
3467    fn javascript_benchmark_config_rejects_unbounded_iteration_counts() {
3468        assert!(matches!(
3469            validate_benchmark_config(&JavascriptBenchmarkConfig {
3470                iterations: 0,
3471                warmup_iterations: 0,
3472            }),
3473            Err(JavascriptBenchmarkError::InvalidConfig(
3474                "iterations must be greater than zero"
3475            ))
3476        ));
3477        assert!(matches!(
3478            validate_benchmark_config(&JavascriptBenchmarkConfig {
3479                iterations: MAX_BENCHMARK_ITERATIONS + 1,
3480                warmup_iterations: 0,
3481            }),
3482            Err(JavascriptBenchmarkError::InvalidConfig(
3483                "iterations must be less than or equal to 1000"
3484            ))
3485        ));
3486        assert!(matches!(
3487            validate_benchmark_config(&JavascriptBenchmarkConfig {
3488                iterations: 1,
3489                warmup_iterations: MAX_BENCHMARK_WARMUP_ITERATIONS + 1,
3490            }),
3491            Err(JavascriptBenchmarkError::InvalidConfig(
3492                "warmup iterations must be less than or equal to 1000"
3493            ))
3494        ));
3495    }
3496
3497    #[test]
3498    fn javascript_benchmark_orchestration_resumes_completed_stages_from_run_state() {
3499        let tempdir = tempdir().expect("create tempdir");
3500        let repo_root = tempdir.path().join("repo");
3501        let artifact_dir = tempdir.path().join("artifacts");
3502        fs::create_dir_all(&repo_root).expect("create repo root");
3503
3504        let config = JavascriptBenchmarkConfig {
3505            iterations: 1,
3506            warmup_iterations: 0,
3507        };
3508        let host = synthetic_host();
3509        let definitions = benchmark_scenarios();
3510        let mut state = StoredBenchmarkRunState::new(&config, &host, &repo_root);
3511        state.record_transport_rtt(&synthetic_transport_reports());
3512        state.record_scenario(&synthetic_scenario_report(definitions[0], 10.0));
3513        persist_benchmark_run_state(&benchmark_run_state_path(&artifact_dir), &state)
3514            .expect("persist initial run state");
3515
3516        let transport_calls = RefCell::new(0usize);
3517        let scenario_calls = RefCell::new(Vec::new());
3518        let (report, resumed_stage_count, _) = orchestrate_javascript_benchmark_report(
3519            &config,
3520            &repo_root,
3521            &host,
3522            &artifact_dir,
3523            || {
3524                *transport_calls.borrow_mut() += 1;
3525                Ok(synthetic_transport_reports())
3526            },
3527            |definition| {
3528                scenario_calls.borrow_mut().push(definition.id.to_owned());
3529                Ok(synthetic_scenario_report(definition, 20.0))
3530            },
3531        )
3532        .expect("resume benchmark orchestration");
3533
3534        assert_eq!(resumed_stage_count, 2);
3535        assert_eq!(*transport_calls.borrow(), 0);
3536        assert_eq!(
3537            scenario_calls.borrow().as_slice(),
3538            &definitions[1..]
3539                .iter()
3540                .map(|definition| definition.id.to_owned())
3541                .collect::<Vec<_>>()
3542        );
3543        assert_eq!(
3544            report.transport_rtt.len(),
3545            TRANSPORT_RTT_PAYLOAD_BYTES.len()
3546        );
3547        assert_eq!(report.scenarios.len(), definitions.len());
3548        assert_eq!(report.scenarios[0].id, definitions[0].id);
3549        assert_eq!(report.scenarios[1].id, definitions[1].id);
3550    }
3551
3552    #[test]
3553    fn javascript_benchmark_orchestration_persists_completed_stages_before_failure() {
3554        let tempdir = tempdir().expect("create tempdir");
3555        let repo_root = tempdir.path().join("repo");
3556        let artifact_dir = tempdir.path().join("artifacts");
3557        fs::create_dir_all(&repo_root).expect("create repo root");
3558
3559        let config = JavascriptBenchmarkConfig {
3560            iterations: 1,
3561            warmup_iterations: 0,
3562        };
3563        let host = synthetic_host();
3564        let state_path = benchmark_run_state_path(&artifact_dir);
3565        let failure = orchestrate_javascript_benchmark_report(
3566            &config,
3567            &repo_root,
3568            &host,
3569            &artifact_dir,
3570            || Ok(synthetic_transport_reports()),
3571            |definition| {
3572                if definition.id == "cold-local-import" {
3573                    Err(JavascriptBenchmarkError::InvalidConfig("synthetic failure"))
3574                } else {
3575                    Ok(synthetic_scenario_report(definition, 15.0))
3576                }
3577            },
3578        )
3579        .expect_err("expected synthetic orchestration failure");
3580
3581        assert!(matches!(
3582            failure,
3583            JavascriptBenchmarkError::InvalidConfig("synthetic failure")
3584        ));
3585
3586        let stored_state = serde_json::from_str::<StoredBenchmarkRunState>(
3587            &fs::read_to_string(&state_path).expect("read persisted run state"),
3588        )
3589        .expect("parse persisted run state");
3590        assert!(stored_state.transport_rtt.is_some());
3591        assert_eq!(
3592            stored_state
3593                .scenarios
3594                .iter()
3595                .map(|scenario| scenario.id.as_str())
3596                .collect::<Vec<_>>(),
3597            vec!["isolate-startup", "prewarmed-isolate-startup"]
3598        );
3599    }
3600}
agentos_execution/benchmark.rs

agentos_execution/
benchmark.rs
