1use crate::{
2 CreateJavascriptContextRequest, JavascriptExecutionEngine, JavascriptExecutionError,
3 StartJavascriptExecutionRequest,
4};
5use serde::{Deserialize, Serialize};
6use std::collections::BTreeMap;
7use std::env;
8use std::fmt;
9use std::fmt::Write as _;
10use std::fs;
11use std::path::{Path, PathBuf};
12use std::process::Command;
13use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
14
15const BENCHMARK_MARKER_PREFIX: &str = "__AGENTOS_BENCH__:";
16const LOCAL_GRAPH_MODULE_COUNT: usize = 24;
17const BENCHMARK_ARTIFACT_VERSION: u32 = 5;
18const BENCHMARK_ARTIFACT_DIR: &str = "target/benchmark-reports/node-import-bench";
19const BENCHMARK_RUN_STATE_FILE: &str = "run-state.json";
20const TRANSPORT_RTT_CHANNEL: &str = "execution-stdio-echo";
21const TRANSPORT_RTT_PAYLOAD_BYTES: [usize; 3] = [32, 4 * 1024, 64 * 1024];
22const TRANSPORT_POLL_TIMEOUT: Duration = Duration::from_secs(5);
23const MAX_BENCHMARK_ITERATIONS: usize = 1_000;
24const MAX_BENCHMARK_WARMUP_ITERATIONS: usize = 1_000;
25
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
27pub struct JavascriptBenchmarkConfig {
28 pub iterations: usize,
29 pub warmup_iterations: usize,
30}
31
32impl Default for JavascriptBenchmarkConfig {
33 fn default() -> Self {
34 Self {
35 iterations: 5,
36 warmup_iterations: 1,
37 }
38 }
39}
40
41#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
42pub struct BenchmarkHost {
43 pub node_binary: String,
44 pub node_version: String,
45 pub os: &'static str,
46 pub arch: &'static str,
47 pub logical_cpus: usize,
48}
49
50#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
51pub struct BenchmarkScenarioPhases<T> {
52 pub context_setup_ms: T,
53 pub startup_ms: T,
54 #[serde(skip_serializing_if = "Option::is_none", default)]
55 pub guest_execution_ms: Option<T>,
56 pub completion_ms: T,
57}
58
59#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
60pub struct BenchmarkStats {
61 pub mean_ms: f64,
62 pub p50_ms: f64,
63 pub p95_ms: f64,
64 pub min_ms: f64,
65 pub max_ms: f64,
66 pub stddev_ms: f64,
67}
68
69#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
70pub struct BenchmarkDistributionStats {
71 pub mean: f64,
72 pub p50: f64,
73 pub p95: f64,
74 pub min: f64,
75 pub max: f64,
76 pub stddev: f64,
77}
78
79#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
80pub struct BenchmarkResourceUsage<T> {
81 #[serde(skip_serializing_if = "Option::is_none", default)]
82 pub rss_bytes: Option<T>,
83 #[serde(skip_serializing_if = "Option::is_none", default)]
84 pub heap_used_bytes: Option<T>,
85 #[serde(skip_serializing_if = "Option::is_none", default)]
86 pub cpu_user_us: Option<T>,
87 #[serde(skip_serializing_if = "Option::is_none", default)]
88 pub cpu_system_us: Option<T>,
89 #[serde(skip_serializing_if = "Option::is_none", default)]
90 pub cpu_total_us: Option<T>,
91}
92
93#[derive(Debug, Clone, PartialEq, Serialize)]
94pub struct BenchmarkTransportRttReport {
95 pub channel: &'static str,
96 pub payload_bytes: usize,
97 pub samples_ms: Vec<f64>,
98 pub stats: BenchmarkStats,
99}
100
101#[derive(Debug, Clone, PartialEq, Serialize)]
102pub struct BenchmarkScenarioReport {
103 pub id: &'static str,
104 pub workload: &'static str,
105 pub runtime: &'static str,
106 pub mode: &'static str,
107 pub description: &'static str,
108 pub fixture: &'static str,
109 pub compile_cache: &'static str,
110 pub wall_samples_ms: Vec<f64>,
111 pub wall_stats: BenchmarkStats,
112 pub guest_import_samples_ms: Option<Vec<f64>>,
113 pub guest_import_stats: Option<BenchmarkStats>,
114 pub startup_overhead_samples_ms: Option<Vec<f64>>,
115 pub startup_overhead_stats: Option<BenchmarkStats>,
116 pub phase_samples_ms: BenchmarkScenarioPhases<Vec<f64>>,
117 pub phase_stats: BenchmarkScenarioPhases<BenchmarkStats>,
118 #[serde(skip_serializing_if = "Option::is_none", default)]
119 pub resource_usage_samples: Option<BenchmarkResourceUsage<Vec<f64>>>,
120 #[serde(skip_serializing_if = "Option::is_none", default)]
121 pub resource_usage_stats: Option<BenchmarkResourceUsage<BenchmarkDistributionStats>>,
122}
123
124#[derive(Debug, Clone, PartialEq, Serialize)]
125pub struct JavascriptBenchmarkReport {
126 pub generated_at_unix_ms: u128,
127 pub config: JavascriptBenchmarkConfig,
128 pub host: BenchmarkHost,
129 pub repo_root: PathBuf,
130 pub transport_rtt: Vec<BenchmarkTransportRttReport>,
131 pub scenarios: Vec<BenchmarkScenarioReport>,
132}
133
134#[derive(Debug, Clone, PartialEq, Serialize)]
135pub struct BenchmarkComparison {
136 pub baseline: BenchmarkComparisonBaseline,
137 pub summary: BenchmarkComparisonSummary,
138 pub scenario_deltas: Vec<BenchmarkScenarioDelta>,
139 pub scenarios_missing_from_baseline: Vec<String>,
140 pub baseline_only_scenarios: Vec<String>,
141}
142
143#[derive(Debug, Clone, PartialEq, Serialize)]
144pub struct BenchmarkComparisonBaseline {
145 pub artifact_version: u32,
146 pub generated_at_unix_ms: u128,
147 pub path: PathBuf,
148}
149
150#[derive(Debug, Clone, PartialEq, Serialize)]
151pub struct BenchmarkComparisonSummary {
152 pub compared_scenario_count: usize,
153 #[serde(skip_serializing_if = "Option::is_none")]
154 pub largest_wall_improvement: Option<BenchmarkDeltaHighlight>,
155 #[serde(skip_serializing_if = "Option::is_none")]
156 pub largest_wall_regression: Option<BenchmarkDeltaHighlight>,
157}
158
159#[derive(Debug, Clone, PartialEq, Serialize)]
160pub struct BenchmarkDeltaHighlight {
161 pub id: String,
162 pub delta_ms: f64,
163 pub delta_pct: f64,
164}
165
166#[derive(Debug, Clone, PartialEq, Serialize)]
167pub struct BenchmarkScenarioDelta {
168 pub id: String,
169 pub description: String,
170 pub wall_mean_ms: BenchmarkMetricDelta,
171 #[serde(skip_serializing_if = "Option::is_none")]
172 pub guest_import_mean_ms: Option<BenchmarkMetricDelta>,
173 #[serde(skip_serializing_if = "Option::is_none")]
174 pub startup_overhead_mean_ms: Option<BenchmarkMetricDelta>,
175 #[serde(skip_serializing_if = "Option::is_none")]
176 pub phase_mean_ms: Option<BenchmarkScenarioPhases<BenchmarkMetricDelta>>,
177}
178
179#[derive(Debug, Clone, PartialEq, Serialize)]
180pub struct BenchmarkMetricDelta {
181 pub baseline_ms: f64,
182 pub current_ms: f64,
183 pub delta_ms: f64,
184 pub delta_pct: f64,
185}
186
187impl JavascriptBenchmarkReport {
188 pub fn render_markdown(&self) -> String {
189 self.render_markdown_with_comparison(None)
190 }
191
192 pub fn render_markdown_with_comparison(
193 &self,
194 comparison: Option<&BenchmarkComparison>,
195 ) -> String {
196 let mut markdown = String::new();
197 let _ = writeln!(&mut markdown, "# secure-exec Node Import Benchmark");
198 let _ = writeln!(&mut markdown);
199 let _ = writeln!(
200 &mut markdown,
201 "- Generated at unix ms: `{}`",
202 self.generated_at_unix_ms
203 );
204 let _ = writeln!(&mut markdown, "- Node binary: `{}`", self.host.node_binary);
205 let _ = writeln!(
206 &mut markdown,
207 "- Node version: `{}`",
208 self.host.node_version.trim()
209 );
210 let _ = writeln!(
211 &mut markdown,
212 "- Host: `{}` / `{}` / `{}` logical CPUs",
213 self.host.os, self.host.arch, self.host.logical_cpus
214 );
215 let _ = writeln!(&mut markdown, "- Repo root: `{}`", self.repo_root.display());
216 let _ = writeln!(
217 &mut markdown,
218 "- Iterations: `{}` recorded, `{}` warmup",
219 self.config.iterations, self.config.warmup_iterations
220 );
221 let _ = writeln!(
222 &mut markdown,
223 "- Reproduce: `cargo run -p agentos-execution --bin node-import-bench -- --iterations {} --warmup-iterations {}`",
224 self.config.iterations, self.config.warmup_iterations
225 );
226 let _ = writeln!(&mut markdown);
227 let _ = writeln!(&mut markdown, "## Transport RTT");
228 let _ = writeln!(&mut markdown);
229 let _ = writeln!(
230 &mut markdown,
231 "| Channel | Payload (bytes) | Mean RTT (ms) | P50 | P95 |"
232 );
233 let _ = writeln!(&mut markdown, "| --- | ---: | ---: | ---: | ---: |");
234
235 for transport in &self.transport_rtt {
236 let _ = writeln!(
237 &mut markdown,
238 "| `{}` | {} | {} | {} | {} |",
239 transport.channel,
240 transport.payload_bytes,
241 format_ms(transport.stats.mean_ms),
242 format_ms(transport.stats.p50_ms),
243 format_ms(transport.stats.p95_ms),
244 );
245 }
246
247 let _ = writeln!(&mut markdown, "## Control Matrix");
248 let _ = writeln!(&mut markdown);
249
250 for row in self.control_matrix() {
251 let _ = writeln!(
252 &mut markdown,
253 "- Workload `{}`: runtimes {}, modes {}, scenarios {}",
254 row.workload,
255 format_label_list(&row.runtimes),
256 format_label_list(&row.modes),
257 format_label_list(&row.scenario_ids),
258 );
259 }
260
261 let _ = writeln!(&mut markdown);
262 let _ = writeln!(&mut markdown, "## Scenario Summary");
263 let _ = writeln!(&mut markdown);
264 let _ = writeln!(
265 &mut markdown,
266 "| Scenario | Workload | Runtime | Mode | Fixture | Cache | Mean wall (ms) | Mean context (ms) | Mean startup (ms) | Mean guest exec (ms) | Mean completion (ms) | Mean startup overhead (ms) |"
267 );
268 let _ = writeln!(
269 &mut markdown,
270 "| --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |"
271 );
272
273 for scenario in &self.scenarios {
274 let guest_execution_mean = scenario
275 .phase_stats
276 .guest_execution_ms
277 .as_ref()
278 .map(|stats| format_ms(stats.mean_ms))
279 .unwrap_or_else(|| String::from("n/a"));
280 let startup_overhead_mean = scenario
281 .startup_overhead_stats
282 .as_ref()
283 .map(|stats| format_ms(stats.mean_ms))
284 .unwrap_or_else(|| String::from("n/a"));
285
286 let _ = writeln!(
287 &mut markdown,
288 "| `{}` | `{}` | `{}` | `{}` | {} | {} | {} | {} | {} | {} | {} | {} |",
289 scenario.id,
290 scenario.workload,
291 scenario.runtime,
292 scenario.mode,
293 scenario.fixture,
294 scenario.compile_cache,
295 format_ms(scenario.wall_stats.mean_ms),
296 format_ms(scenario.phase_stats.context_setup_ms.mean_ms),
297 format_ms(scenario.phase_stats.startup_ms.mean_ms),
298 guest_execution_mean,
299 format_ms(scenario.phase_stats.completion_ms.mean_ms),
300 startup_overhead_mean,
301 );
302 }
303
304 let _ = writeln!(&mut markdown);
305 let _ = writeln!(&mut markdown, "## Stability And Resource Summary");
306 let _ = writeln!(&mut markdown);
307 let _ = writeln!(
308 &mut markdown,
309 "| Scenario | Wall P50 (ms) | Wall min-max (ms) | Wall stddev (ms) | Mean RSS (MiB) | Mean heap (MiB) | Mean total CPU (ms) |"
310 );
311 let _ = writeln!(
312 &mut markdown,
313 "| --- | ---: | --- | ---: | ---: | ---: | ---: |"
314 );
315
316 for scenario in &self.scenarios {
317 let _ = writeln!(
318 &mut markdown,
319 "| `{}` | {} | {}-{} | {} | {} | {} | {} |",
320 scenario.id,
321 format_ms(scenario.wall_stats.p50_ms),
322 format_ms(scenario.wall_stats.min_ms),
323 format_ms(scenario.wall_stats.max_ms),
324 format_ms(scenario.wall_stats.stddev_ms),
325 scenario
326 .resource_usage_stats
327 .as_ref()
328 .and_then(|stats| stats.rss_bytes.as_ref())
329 .map(|stats| format_mib(bytes_to_mib(stats.mean)))
330 .unwrap_or_else(|| String::from("n/a")),
331 scenario
332 .resource_usage_stats
333 .as_ref()
334 .and_then(|stats| stats.heap_used_bytes.as_ref())
335 .map(|stats| format_mib(bytes_to_mib(stats.mean)))
336 .unwrap_or_else(|| String::from("n/a")),
337 scenario
338 .resource_usage_stats
339 .as_ref()
340 .and_then(|stats| stats.cpu_total_us.as_ref())
341 .map(|stats| format_ms(micros_to_ms(stats.mean)))
342 .unwrap_or_else(|| String::from("n/a")),
343 );
344 }
345
346 let _ = writeln!(&mut markdown);
347 let _ = writeln!(&mut markdown, "## Ranked Hotspots");
348 let _ = writeln!(&mut markdown);
349
350 for ranking in self.hotspot_rankings() {
351 let _ = writeln!(
352 &mut markdown,
353 "### {} (`{}`, `{}`)",
354 ranking.label, ranking.dimension, ranking.unit
355 );
356 let _ = writeln!(&mut markdown);
357 let _ = writeln!(
358 &mut markdown,
359 "| Rank | Scenario | Workload | Runtime | Mode | Value |"
360 );
361 let _ = writeln!(&mut markdown, "| ---: | --- | --- | --- | --- | ---: |");
362
363 for scenario in &ranking.ranked_scenarios {
364 let _ = writeln!(
365 &mut markdown,
366 "| {} | `{}` | `{}` | `{}` | `{}` | {} |",
367 scenario.rank,
368 scenario.id,
369 scenario.workload,
370 scenario.runtime,
371 scenario.mode,
372 format_hotspot_value(ranking.unit, scenario.value),
373 );
374 }
375
376 if !ranking.scenarios_without_metric.is_empty() {
377 let _ = writeln!(&mut markdown);
378 let _ = writeln!(
379 &mut markdown,
380 "Missing metric for: {}",
381 format_string_label_list(&ranking.scenarios_without_metric),
382 );
383 }
384
385 let _ = writeln!(&mut markdown);
386 }
387
388 let _ = writeln!(&mut markdown, "## Hotspot Guidance");
389 let _ = writeln!(&mut markdown);
390
391 for line in self.guidance_lines() {
392 let _ = writeln!(&mut markdown, "- {line}");
393 }
394
395 if let Some(comparison) = comparison {
396 let _ = writeln!(&mut markdown);
397 let _ = writeln!(&mut markdown, "## Baseline Comparison");
398 let _ = writeln!(&mut markdown);
399 let _ = writeln!(
400 &mut markdown,
401 "- Baseline artifact: `{}`",
402 comparison.baseline.path.display()
403 );
404 let _ = writeln!(
405 &mut markdown,
406 "- Baseline generated at unix ms: `{}`",
407 comparison.baseline.generated_at_unix_ms
408 );
409 let _ = writeln!(
410 &mut markdown,
411 "- Compared scenarios: `{}`",
412 comparison.summary.compared_scenario_count
413 );
414 if let Some(improvement) = &comparison.summary.largest_wall_improvement {
415 let _ = writeln!(
416 &mut markdown,
417 "- Largest wall-time improvement: `{}` at {} ({})",
418 improvement.id,
419 format_delta_ms(improvement.delta_ms),
420 format_delta_pct(improvement.delta_pct),
421 );
422 }
423 if let Some(regression) = &comparison.summary.largest_wall_regression {
424 let _ = writeln!(
425 &mut markdown,
426 "- Largest wall-time regression: `{}` at {} ({})",
427 regression.id,
428 format_delta_ms(regression.delta_ms),
429 format_delta_pct(regression.delta_pct),
430 );
431 }
432 if !comparison.scenarios_missing_from_baseline.is_empty() {
433 let _ = writeln!(
434 &mut markdown,
435 "- Scenarios missing from baseline: {}",
436 comparison.scenarios_missing_from_baseline.join(", ")
437 );
438 }
439 if !comparison.baseline_only_scenarios.is_empty() {
440 let _ = writeln!(
441 &mut markdown,
442 "- Baseline-only scenarios: {}",
443 comparison.baseline_only_scenarios.join(", ")
444 );
445 }
446 let _ = writeln!(&mut markdown);
447 let _ = writeln!(
448 &mut markdown,
449 "| Scenario | Wall delta (ms) | Wall delta % | Import delta (ms) | Startup delta (ms) | Context delta (ms) | Completion delta (ms) |"
450 );
451 let _ = writeln!(
452 &mut markdown,
453 "| --- | ---: | ---: | ---: | ---: | ---: | ---: |"
454 );
455
456 for scenario in &comparison.scenario_deltas {
457 let import_delta = scenario
458 .guest_import_mean_ms
459 .as_ref()
460 .map(|delta| format_delta_ms(delta.delta_ms))
461 .unwrap_or_else(|| String::from("n/a"));
462 let startup_delta = scenario
463 .startup_overhead_mean_ms
464 .as_ref()
465 .map(|delta| format_delta_ms(delta.delta_ms))
466 .unwrap_or_else(|| String::from("n/a"));
467 let context_delta = scenario
468 .phase_mean_ms
469 .as_ref()
470 .map(|delta| format_delta_ms(delta.context_setup_ms.delta_ms))
471 .unwrap_or_else(|| String::from("n/a"));
472 let completion_delta = scenario
473 .phase_mean_ms
474 .as_ref()
475 .map(|delta| format_delta_ms(delta.completion_ms.delta_ms))
476 .unwrap_or_else(|| String::from("n/a"));
477
478 let _ = writeln!(
479 &mut markdown,
480 "| `{}` | {} | {} | {} | {} | {} | {} |",
481 scenario.id,
482 format_delta_ms(scenario.wall_mean_ms.delta_ms),
483 format_delta_pct(scenario.wall_mean_ms.delta_pct),
484 import_delta,
485 startup_delta,
486 context_delta,
487 completion_delta,
488 );
489 }
490 }
491
492 let _ = writeln!(&mut markdown);
493 let _ = writeln!(&mut markdown, "## Raw Samples");
494 let _ = writeln!(&mut markdown);
495
496 for scenario in &self.scenarios {
497 let _ = writeln!(&mut markdown, "### `{}`", scenario.id);
498 let _ = writeln!(&mut markdown, "- Workload: `{}`", scenario.workload);
499 let _ = writeln!(&mut markdown, "- Runtime: `{}`", scenario.runtime);
500 let _ = writeln!(&mut markdown, "- Mode: `{}`", scenario.mode);
501 let _ = writeln!(&mut markdown, "- Description: {}", scenario.description);
502 let _ = writeln!(
503 &mut markdown,
504 "- Wall samples (ms): {}",
505 format_sample_list(&scenario.wall_samples_ms)
506 );
507 if let Some(samples) = &scenario.guest_import_samples_ms {
508 let _ = writeln!(
509 &mut markdown,
510 "- Guest import samples (ms): {}",
511 format_sample_list(samples)
512 );
513 }
514 if let Some(samples) = &scenario.startup_overhead_samples_ms {
515 let _ = writeln!(
516 &mut markdown,
517 "- Startup overhead samples (ms): {}",
518 format_sample_list(samples)
519 );
520 }
521 let _ = writeln!(
522 &mut markdown,
523 "- Context setup samples (ms): {}",
524 format_sample_list(&scenario.phase_samples_ms.context_setup_ms)
525 );
526 let _ = writeln!(
527 &mut markdown,
528 "- Startup samples (ms): {}",
529 format_sample_list(&scenario.phase_samples_ms.startup_ms)
530 );
531 if let Some(samples) = &scenario.phase_samples_ms.guest_execution_ms {
532 let _ = writeln!(
533 &mut markdown,
534 "- Guest execution samples (ms): {}",
535 format_sample_list(samples)
536 );
537 }
538 let _ = writeln!(
539 &mut markdown,
540 "- Completion samples (ms): {}",
541 format_sample_list(&scenario.phase_samples_ms.completion_ms)
542 );
543 if let Some(samples) = &scenario.resource_usage_samples {
544 if let Some(rss_samples) = &samples.rss_bytes {
545 let _ = writeln!(
546 &mut markdown,
547 "- RSS samples (MiB): {}",
548 format_scaled_sample_list(rss_samples, bytes_to_mib)
549 );
550 }
551 if let Some(heap_samples) = &samples.heap_used_bytes {
552 let _ = writeln!(
553 &mut markdown,
554 "- Heap samples (MiB): {}",
555 format_scaled_sample_list(heap_samples, bytes_to_mib)
556 );
557 }
558 if let Some(cpu_samples) = &samples.cpu_total_us {
559 let _ = writeln!(
560 &mut markdown,
561 "- Total CPU samples (ms): {}",
562 format_scaled_sample_list(cpu_samples, micros_to_ms)
563 );
564 }
565 }
566 let _ = writeln!(&mut markdown);
567 }
568
569 markdown
570 }
571
572 pub fn render_json(&self) -> Result<String, serde_json::Error> {
573 self.render_json_with_comparison(None)
574 }
575
576 pub fn render_json_with_comparison(
577 &self,
578 comparison: Option<&BenchmarkComparison>,
579 ) -> Result<String, serde_json::Error> {
580 serde_json::to_string_pretty(&self.json_artifact(comparison))
581 }
582
583 pub fn write_artifacts(
584 &self,
585 output_dir: &Path,
586 ) -> Result<JavascriptBenchmarkArtifactPaths, JavascriptBenchmarkError> {
587 self.write_artifacts_with_comparison(output_dir, None)
588 }
589
590 pub fn write_artifacts_with_comparison(
591 &self,
592 output_dir: &Path,
593 comparison: Option<&BenchmarkComparison>,
594 ) -> Result<JavascriptBenchmarkArtifactPaths, JavascriptBenchmarkError> {
595 fs::create_dir_all(output_dir)?;
596
597 let markdown_path = output_dir.join("report.md");
598 let json_path = output_dir.join("report.json");
599 write_string_atomic(
600 &markdown_path,
601 &self.render_markdown_with_comparison(comparison),
602 )?;
603 write_string_atomic(&json_path, &self.render_json_with_comparison(comparison)?)?;
604
605 Ok(JavascriptBenchmarkArtifactPaths {
606 markdown_path,
607 json_path,
608 })
609 }
610
611 pub fn compare_to_baseline_path(
612 &self,
613 baseline_path: &Path,
614 ) -> Result<BenchmarkComparison, JavascriptBenchmarkError> {
615 let baseline = load_benchmark_artifact(baseline_path)?;
616 Ok(BenchmarkComparison::from_reports(
617 self,
618 baseline_path,
619 &baseline,
620 ))
621 }
622
623 fn guidance_lines(&self) -> Vec<String> {
624 let isolate = self.scenario("isolate-startup");
625 let cold_local = self.scenario("cold-local-import");
626 let warm_local = self.scenario("warm-local-import");
627 let prewarmed_local = self.scenario("prewarmed-local-import");
628 let builtin = self.scenario("builtin-import");
629 let large = self.scenario("large-package-import");
630
631 let mut guidance = Vec::new();
632
633 if let (
634 Some(cold_import),
635 Some(warm_import),
636 Some(warm_context),
637 Some(warm_startup_phase),
638 Some(warm_completion),
639 Some(warm_startup_overhead),
640 Some(warm_wall),
641 Some(isolate_wall),
642 ) = (
643 cold_local
644 .and_then(|scenario| scenario.guest_import_stats.as_ref())
645 .map(|stats| stats.mean_ms),
646 warm_local
647 .and_then(|scenario| scenario.guest_import_stats.as_ref())
648 .map(|stats| stats.mean_ms),
649 warm_local.map(|scenario| scenario.phase_stats.context_setup_ms.mean_ms),
650 warm_local.map(|scenario| scenario.phase_stats.startup_ms.mean_ms),
651 warm_local.map(|scenario| scenario.phase_stats.completion_ms.mean_ms),
652 warm_local
653 .and_then(|scenario| scenario.startup_overhead_stats.as_ref())
654 .map(|stats| stats.mean_ms),
655 warm_local.map(|scenario| scenario.wall_stats.mean_ms),
656 isolate.map(|scenario| scenario.wall_stats.mean_ms),
657 ) {
658 guidance.push(format!(
659 "Compile-cache reuse cuts the local import graph from {} to {} on average ({:.1}% faster), but the warm path still spends {} outside guest module evaluation. That keeps startup prewarm work in `ARC-021D` and sidecar warm-pool/snapshot work in `ARC-022` on the critical path above the `{}` empty-isolate floor.",
660 format_ms(cold_import),
661 format_ms(warm_import),
662 percentage_reduction(cold_import, warm_import),
663 format_ms(warm_startup_overhead),
664 format_ms(isolate_wall),
665 ));
666 if warm_wall > 0.0 {
667 guidance.push(format!(
668 "Warm local imports still spend {:.1}% of wall time in process startup, wrapper evaluation, and stdio handling instead of guest import work. Optimizations that only touch module compilation will not remove that floor.",
669 percentage_share(warm_startup_overhead, warm_wall),
670 ));
671 }
672 let warm_guest = warm_local
673 .and_then(|scenario| scenario.phase_stats.guest_execution_ms.as_ref())
674 .map(|stats| stats.mean_ms)
675 .unwrap_or(0.0);
676 guidance.push(format!(
677 "The warm path phase split is {} context setup, {} runtime startup, {} guest execution, and {} completion/stdio work. Future attribution can now separate bootstrap wins from pure transport/collection wins instead of treating them as one startup bucket.",
678 format_ms(warm_context),
679 format_ms(warm_startup_phase),
680 format_ms(warm_guest),
681 format_ms(warm_completion),
682 ));
683 }
684
685 if let (Some(warm_startup_overhead), Some(prewarmed_startup_overhead), Some(isolate_wall)) = (
686 warm_local
687 .and_then(|scenario| scenario.startup_overhead_stats.as_ref())
688 .map(|stats| stats.mean_ms),
689 prewarmed_local
690 .and_then(|scenario| scenario.startup_overhead_stats.as_ref())
691 .map(|stats| stats.mean_ms),
692 isolate.map(|scenario| scenario.wall_stats.mean_ms),
693 ) {
694 guidance.push(format!(
695 "Keeping the current import-cache materialization and builtin/polyfill prewarm alive inside one execution engine cuts warm local startup overhead from {} to {} ({:.1}% faster). The remaining {} of non-import work is the post-prewarm floor that broader warm-pool/snapshot work would still need to attack above the `{}` empty-isolate baseline.",
696 format_ms(warm_startup_overhead),
697 format_ms(prewarmed_startup_overhead),
698 percentage_reduction(warm_startup_overhead, prewarmed_startup_overhead),
699 format_ms(prewarmed_startup_overhead),
700 format_ms(isolate_wall),
701 ));
702 }
703
704 if let (Some(builtin_import), Some(large_import)) = (
705 builtin
706 .and_then(|scenario| scenario.guest_import_stats.as_ref())
707 .map(|stats| stats.mean_ms),
708 large
709 .and_then(|scenario| scenario.guest_import_stats.as_ref())
710 .map(|stats| stats.mean_ms),
711 ) {
712 guidance.push(format!(
713 "The large real-world package import (`typescript`) is {:.1}x the builtin path ({} versus {}). That makes `ARC-021C` the right next import-path optimization story: cache sidecar-scoped resolution results, package-type lookups, and module-format classification before attempting deeper structural rewrites.",
714 safe_ratio(large_import, builtin_import),
715 format_ms(large_import),
716 format_ms(builtin_import),
717 ));
718 }
719
720 if let (Some(smallest), Some(largest)) =
721 (self.transport_rtt.first(), self.transport_rtt.last())
722 {
723 guidance.push(format!(
724 "Execution-transport RTT over the stdio bridge rises from {} at {} bytes to {} at {} bytes. That gives later work a direct transport floor to compare against the larger startup and import phases.",
725 format_ms(smallest.stats.mean_ms),
726 smallest.payload_bytes,
727 format_ms(largest.stats.mean_ms),
728 largest.payload_bytes,
729 ));
730 }
731
732 if let Some(noisiest) = self.scenarios.iter().max_by(|lhs, rhs| {
733 lhs.wall_stats
734 .stddev_ms
735 .total_cmp(&rhs.wall_stats.stddev_ms)
736 }) {
737 guidance.push(format!(
738 "Wall-time noise is now surfaced directly in the same artifact set: `{}` currently shows the largest spread at {} stddev over a {}-{} wall range, so future deltas on that path should be judged against stability as well as mean time.",
739 noisiest.id,
740 format_ms(noisiest.wall_stats.stddev_ms),
741 format_ms(noisiest.wall_stats.min_ms),
742 format_ms(noisiest.wall_stats.max_ms),
743 ));
744 }
745
746 if let Some(heaviest) = self.scenarios.iter().max_by(|lhs, rhs| {
747 lhs.resource_usage_stats
748 .as_ref()
749 .and_then(|stats| stats.rss_bytes.as_ref())
750 .map(|stats| stats.mean)
751 .unwrap_or(f64::NEG_INFINITY)
752 .total_cmp(
753 &rhs.resource_usage_stats
754 .as_ref()
755 .and_then(|stats| stats.rss_bytes.as_ref())
756 .map(|stats| stats.mean)
757 .unwrap_or(f64::NEG_INFINITY),
758 )
759 }) {
760 if let Some(rss_mean) = heaviest
761 .resource_usage_stats
762 .as_ref()
763 .and_then(|stats| stats.rss_bytes.as_ref())
764 {
765 guidance.push(format!(
766 "Per-scenario resource reporting is now attached to the benchmark rows themselves: `{}` currently has the highest mean RSS at {} MiB, so import-path changes can now be judged for memory regressions without a separate memory-only pass.",
767 heaviest.id,
768 format_mib(bytes_to_mib(rss_mean.mean)),
769 ));
770 }
771 }
772
773 guidance.push(String::from(
774 "No new PRD stories were added from this run. The measured hotspots already map cleanly onto existing follow-ons: `ARC-021C` for safe resolution and metadata caches, `ARC-021D` for builtin/polyfill prewarm, and `ARC-022` for broader warm-pool and timing-mitigation execution work.",
775 ));
776
777 guidance
778 }
779
780 fn scenario(&self, id: &str) -> Option<&BenchmarkScenarioReport> {
781 self.scenarios.iter().find(|scenario| scenario.id == id)
782 }
783
784 fn json_artifact<'a>(
785 &'a self,
786 comparison: Option<&'a BenchmarkComparison>,
787 ) -> JavascriptBenchmarkArtifact<'a> {
788 JavascriptBenchmarkArtifact {
789 artifact_version: BENCHMARK_ARTIFACT_VERSION,
790 generated_at_unix_ms: self.generated_at_unix_ms,
791 command: format!(
792 "cargo run -p agentos-execution --bin node-import-bench -- --iterations {} --warmup-iterations {}",
793 self.config.iterations, self.config.warmup_iterations
794 ),
795 config: &self.config,
796 host: &self.host,
797 repo_root: &self.repo_root,
798 summary: self.summary(),
799 comparison,
800 transport_rtt: self
801 .transport_rtt
802 .iter()
803 .map(|transport| BenchmarkTransportRttArtifact {
804 channel: transport.channel,
805 payload_bytes: transport.payload_bytes,
806 samples_ms: &transport.samples_ms,
807 stats: &transport.stats,
808 })
809 .collect(),
810 scenarios: self
811 .scenarios
812 .iter()
813 .map(|scenario| BenchmarkScenarioArtifact {
814 id: scenario.id,
815 workload: scenario.workload,
816 runtime: scenario.runtime,
817 mode: scenario.mode,
818 description: scenario.description,
819 fixture: scenario.fixture,
820 compile_cache: scenario.compile_cache,
821 wall_samples_ms: &scenario.wall_samples_ms,
822 wall_stats: &scenario.wall_stats,
823 guest_import_samples_ms: scenario.guest_import_samples_ms.as_deref(),
824 guest_import_stats: scenario.guest_import_stats.as_ref(),
825 startup_overhead_samples_ms: scenario.startup_overhead_samples_ms.as_deref(),
826 startup_overhead_stats: scenario.startup_overhead_stats.as_ref(),
827 mean_startup_share_pct: scenario.mean_startup_share_pct(),
828 phase_samples_ms: &scenario.phase_samples_ms,
829 phase_stats: &scenario.phase_stats,
830 resource_usage_samples: scenario.resource_usage_samples.as_ref(),
831 resource_usage_stats: scenario.resource_usage_stats.as_ref(),
832 })
833 .collect(),
834 }
835 }
836
837 fn summary(&self) -> BenchmarkSummaryArtifact<'_> {
838 BenchmarkSummaryArtifact {
839 scenario_count: self.scenarios.len(),
840 recorded_samples_per_scenario: self.config.iterations,
841 warmup_iterations: self.config.warmup_iterations,
842 control_matrix: self.control_matrix(),
843 slowest_wall_scenario: self.slowest_scenario_by(|scenario| scenario.wall_stats.mean_ms),
844 slowest_guest_import_scenario: self.slowest_scenario_by(|scenario| {
845 scenario
846 .guest_import_stats
847 .as_ref()
848 .map(|stats| stats.mean_ms)
849 .unwrap_or(f64::NEG_INFINITY)
850 }),
851 highest_startup_share_scenario: self.scenarios.iter().max_by(|lhs, rhs| {
852 lhs.mean_startup_share_pct()
853 .unwrap_or(f64::NEG_INFINITY)
854 .total_cmp(&rhs.mean_startup_share_pct().unwrap_or(f64::NEG_INFINITY))
855 }),
856 hotspot_rankings: self.hotspot_rankings(),
857 guidance_lines: self.guidance_lines(),
858 }
859 }
860
861 fn control_matrix(&self) -> Vec<BenchmarkControlMatrixArtifact<'_>> {
862 let mut rows = Vec::new();
863 let mut row_indexes = BTreeMap::new();
864
865 for scenario in &self.scenarios {
866 let row_index = *row_indexes.entry(scenario.workload).or_insert_with(|| {
867 rows.push(BenchmarkControlMatrixArtifact {
868 workload: scenario.workload,
869 runtimes: Vec::new(),
870 modes: Vec::new(),
871 scenario_ids: Vec::new(),
872 });
873 rows.len() - 1
874 });
875 let row = &mut rows[row_index];
876 push_unique_label(&mut row.runtimes, scenario.runtime);
877 push_unique_label(&mut row.modes, scenario.mode);
878 row.scenario_ids.push(scenario.id);
879 }
880
881 rows
882 }
883
884 fn slowest_scenario_by(
885 &self,
886 value: impl Fn(&BenchmarkScenarioReport) -> f64,
887 ) -> Option<&BenchmarkScenarioReport> {
888 self.scenarios
889 .iter()
890 .max_by(|lhs, rhs| value(lhs).total_cmp(&value(rhs)))
891 }
892
893 fn hotspot_rankings(&self) -> Vec<BenchmarkHotspotRankingArtifact<'_>> {
894 HOTSPOT_METRICS
895 .iter()
896 .map(|metric| {
897 let mut ranked_scenarios = self
898 .scenarios
899 .iter()
900 .filter_map(|scenario| {
901 (metric.value)(scenario).map(|value| BenchmarkHotspotScenarioArtifact {
902 rank: 0,
903 id: scenario.id,
904 workload: scenario.workload,
905 runtime: scenario.runtime,
906 mode: scenario.mode,
907 value,
908 })
909 })
910 .collect::<Vec<_>>();
911 ranked_scenarios.sort_by(|lhs, rhs| {
912 rhs.value
913 .total_cmp(&lhs.value)
914 .then_with(|| lhs.id.cmp(rhs.id))
915 });
916 for (index, scenario) in ranked_scenarios.iter_mut().enumerate() {
917 scenario.rank = index + 1;
918 }
919
920 BenchmarkHotspotRankingArtifact {
921 metric: metric.metric,
922 label: metric.label,
923 dimension: metric.dimension,
924 unit: metric.unit,
925 ranked_scenarios,
926 scenarios_without_metric: self
927 .scenarios
928 .iter()
929 .filter(|scenario| (metric.value)(scenario).is_none())
930 .map(|scenario| scenario.id)
931 .collect(),
932 }
933 })
934 .collect()
935 }
936}
937
938impl BenchmarkScenarioReport {
939 fn mean_startup_share_pct(&self) -> Option<f64> {
940 let startup_mean = self.startup_overhead_stats.as_ref()?.mean_ms;
941 let wall_mean = self.wall_stats.mean_ms;
942 if wall_mean <= 0.0 {
943 Some(0.0)
944 } else {
945 Some((startup_mean / wall_mean) * 100.0)
946 }
947 }
948
949 fn wall_range_ms(&self) -> f64 {
950 self.wall_stats.max_ms - self.wall_stats.min_ms
951 }
952}
953
954impl BenchmarkResourceUsage<Vec<f64>> {
955 fn push_sample(&mut self, sample: &BenchmarkResourceUsage<f64>) {
956 push_optional_sample(&mut self.rss_bytes, sample.rss_bytes);
957 push_optional_sample(&mut self.heap_used_bytes, sample.heap_used_bytes);
958 push_optional_sample(&mut self.cpu_user_us, sample.cpu_user_us);
959 push_optional_sample(&mut self.cpu_system_us, sample.cpu_system_us);
960 push_optional_sample(&mut self.cpu_total_us, sample.cpu_total_us);
961 }
962
963 fn into_populated(self) -> Option<Self> {
964 (!self.is_empty()).then_some(self)
965 }
966}
967
968impl<T> BenchmarkResourceUsage<T> {
969 fn is_empty(&self) -> bool {
970 self.rss_bytes.is_none()
971 && self.heap_used_bytes.is_none()
972 && self.cpu_user_us.is_none()
973 && self.cpu_system_us.is_none()
974 && self.cpu_total_us.is_none()
975 }
976}
977
978impl BenchmarkComparison {
979 fn from_reports(
980 current: &JavascriptBenchmarkReport,
981 baseline_path: &Path,
982 baseline: &StoredBenchmarkArtifact,
983 ) -> Self {
984 let baseline_path =
985 fs::canonicalize(baseline_path).unwrap_or_else(|_| baseline_path.to_path_buf());
986 let baseline_by_id = baseline
987 .scenarios
988 .iter()
989 .map(|scenario| (scenario.id.as_str(), scenario))
990 .collect::<BTreeMap<_, _>>();
991
992 let mut scenario_deltas = Vec::new();
993 let mut scenarios_missing_from_baseline = Vec::new();
994
995 for scenario in ¤t.scenarios {
996 if let Some(baseline_scenario) = baseline_by_id.get(scenario.id) {
997 scenario_deltas.push(BenchmarkScenarioDelta {
998 id: scenario.id.to_owned(),
999 description: scenario.description.to_owned(),
1000 wall_mean_ms: BenchmarkMetricDelta::from_means(
1001 baseline_scenario.wall_stats.mean_ms,
1002 scenario.wall_stats.mean_ms,
1003 ),
1004 guest_import_mean_ms: match (
1005 baseline_scenario.guest_import_stats.as_ref(),
1006 scenario.guest_import_stats.as_ref(),
1007 ) {
1008 (Some(baseline_stats), Some(current_stats)) => {
1009 Some(BenchmarkMetricDelta::from_means(
1010 baseline_stats.mean_ms,
1011 current_stats.mean_ms,
1012 ))
1013 }
1014 _ => None,
1015 },
1016 startup_overhead_mean_ms: match (
1017 baseline_scenario.startup_overhead_stats.as_ref(),
1018 scenario.startup_overhead_stats.as_ref(),
1019 ) {
1020 (Some(baseline_stats), Some(current_stats)) => {
1021 Some(BenchmarkMetricDelta::from_means(
1022 baseline_stats.mean_ms,
1023 current_stats.mean_ms,
1024 ))
1025 }
1026 _ => None,
1027 },
1028 phase_mean_ms: match (
1029 baseline_scenario.phase_stats.as_ref(),
1030 Some(&scenario.phase_stats),
1031 ) {
1032 (Some(baseline_phase), Some(current_phase)) => {
1033 Some(BenchmarkScenarioPhases {
1034 context_setup_ms: BenchmarkMetricDelta::from_means(
1035 baseline_phase.context_setup_ms.mean_ms,
1036 current_phase.context_setup_ms.mean_ms,
1037 ),
1038 startup_ms: BenchmarkMetricDelta::from_means(
1039 baseline_phase.startup_ms.mean_ms,
1040 current_phase.startup_ms.mean_ms,
1041 ),
1042 guest_execution_ms: match (
1043 baseline_phase.guest_execution_ms.as_ref(),
1044 current_phase.guest_execution_ms.as_ref(),
1045 ) {
1046 (Some(baseline_stats), Some(current_stats)) => {
1047 Some(BenchmarkMetricDelta::from_means(
1048 baseline_stats.mean_ms,
1049 current_stats.mean_ms,
1050 ))
1051 }
1052 _ => None,
1053 },
1054 completion_ms: BenchmarkMetricDelta::from_means(
1055 baseline_phase.completion_ms.mean_ms,
1056 current_phase.completion_ms.mean_ms,
1057 ),
1058 })
1059 }
1060 _ => None,
1061 },
1062 });
1063 } else {
1064 scenarios_missing_from_baseline.push(scenario.id.to_owned());
1065 }
1066 }
1067
1068 let current_ids = current
1069 .scenarios
1070 .iter()
1071 .map(|scenario| (scenario.id, ()))
1072 .collect::<BTreeMap<_, _>>();
1073 let baseline_only_scenarios = baseline
1074 .scenarios
1075 .iter()
1076 .filter(|scenario| !current_ids.contains_key(scenario.id.as_str()))
1077 .map(|scenario| scenario.id.clone())
1078 .collect::<Vec<_>>();
1079
1080 let largest_wall_improvement = scenario_deltas
1081 .iter()
1082 .filter(|scenario| scenario.wall_mean_ms.delta_ms < 0.0)
1083 .min_by(|lhs, rhs| {
1084 lhs.wall_mean_ms
1085 .delta_ms
1086 .total_cmp(&rhs.wall_mean_ms.delta_ms)
1087 })
1088 .map(BenchmarkDeltaHighlight::from_wall_delta);
1089 let largest_wall_regression = scenario_deltas
1090 .iter()
1091 .filter(|scenario| scenario.wall_mean_ms.delta_ms > 0.0)
1092 .max_by(|lhs, rhs| {
1093 lhs.wall_mean_ms
1094 .delta_ms
1095 .total_cmp(&rhs.wall_mean_ms.delta_ms)
1096 })
1097 .map(BenchmarkDeltaHighlight::from_wall_delta);
1098
1099 Self {
1100 baseline: BenchmarkComparisonBaseline {
1101 artifact_version: baseline.artifact_version,
1102 generated_at_unix_ms: baseline.generated_at_unix_ms,
1103 path: baseline_path,
1104 },
1105 summary: BenchmarkComparisonSummary {
1106 compared_scenario_count: scenario_deltas.len(),
1107 largest_wall_improvement,
1108 largest_wall_regression,
1109 },
1110 scenario_deltas,
1111 scenarios_missing_from_baseline,
1112 baseline_only_scenarios,
1113 }
1114 }
1115}
1116
1117impl BenchmarkDeltaHighlight {
1118 fn from_wall_delta(delta: &BenchmarkScenarioDelta) -> Self {
1119 Self {
1120 id: delta.id.clone(),
1121 delta_ms: delta.wall_mean_ms.delta_ms,
1122 delta_pct: delta.wall_mean_ms.delta_pct,
1123 }
1124 }
1125}
1126
1127impl BenchmarkMetricDelta {
1128 fn from_means(baseline_ms: f64, current_ms: f64) -> Self {
1129 let delta_ms = current_ms - baseline_ms;
1130 let delta_pct = if baseline_ms <= 0.0 {
1131 0.0
1132 } else {
1133 (delta_ms / baseline_ms) * 100.0
1134 };
1135
1136 Self {
1137 baseline_ms,
1138 current_ms,
1139 delta_ms,
1140 delta_pct,
1141 }
1142 }
1143}
1144
1145#[derive(Debug, Clone, PartialEq, Eq)]
1146pub struct JavascriptBenchmarkArtifactPaths {
1147 pub markdown_path: PathBuf,
1148 pub json_path: PathBuf,
1149}
1150
1151#[derive(Debug, Clone, PartialEq, Eq)]
1152pub struct JavascriptBenchmarkRunOutput {
1153 pub artifact_paths: JavascriptBenchmarkArtifactPaths,
1154 pub resumed_stage_count: usize,
1155}
1156
1157#[derive(Debug, Serialize)]
1158struct JavascriptBenchmarkArtifact<'a> {
1159 artifact_version: u32,
1160 generated_at_unix_ms: u128,
1161 command: String,
1162 config: &'a JavascriptBenchmarkConfig,
1163 host: &'a BenchmarkHost,
1164 repo_root: &'a Path,
1165 summary: BenchmarkSummaryArtifact<'a>,
1166 #[serde(skip_serializing_if = "Option::is_none")]
1167 comparison: Option<&'a BenchmarkComparison>,
1168 transport_rtt: Vec<BenchmarkTransportRttArtifact<'a>>,
1169 scenarios: Vec<BenchmarkScenarioArtifact<'a>>,
1170}
1171
1172#[derive(Debug, Serialize)]
1173struct BenchmarkSummaryArtifact<'a> {
1174 scenario_count: usize,
1175 recorded_samples_per_scenario: usize,
1176 warmup_iterations: usize,
1177 control_matrix: Vec<BenchmarkControlMatrixArtifact<'a>>,
1178 #[serde(skip_serializing_if = "Option::is_none")]
1179 slowest_wall_scenario: Option<&'a BenchmarkScenarioReport>,
1180 #[serde(skip_serializing_if = "Option::is_none")]
1181 slowest_guest_import_scenario: Option<&'a BenchmarkScenarioReport>,
1182 #[serde(skip_serializing_if = "Option::is_none")]
1183 highest_startup_share_scenario: Option<&'a BenchmarkScenarioReport>,
1184 hotspot_rankings: Vec<BenchmarkHotspotRankingArtifact<'a>>,
1185 guidance_lines: Vec<String>,
1186}
1187
1188#[derive(Debug, Serialize)]
1189struct BenchmarkScenarioArtifact<'a> {
1190 id: &'static str,
1191 workload: &'static str,
1192 runtime: &'static str,
1193 mode: &'static str,
1194 description: &'static str,
1195 fixture: &'static str,
1196 compile_cache: &'static str,
1197 wall_samples_ms: &'a [f64],
1198 wall_stats: &'a BenchmarkStats,
1199 #[serde(skip_serializing_if = "Option::is_none")]
1200 guest_import_samples_ms: Option<&'a [f64]>,
1201 #[serde(skip_serializing_if = "Option::is_none")]
1202 guest_import_stats: Option<&'a BenchmarkStats>,
1203 #[serde(skip_serializing_if = "Option::is_none")]
1204 startup_overhead_samples_ms: Option<&'a [f64]>,
1205 #[serde(skip_serializing_if = "Option::is_none")]
1206 startup_overhead_stats: Option<&'a BenchmarkStats>,
1207 #[serde(skip_serializing_if = "Option::is_none")]
1208 mean_startup_share_pct: Option<f64>,
1209 phase_samples_ms: &'a BenchmarkScenarioPhases<Vec<f64>>,
1210 phase_stats: &'a BenchmarkScenarioPhases<BenchmarkStats>,
1211 #[serde(skip_serializing_if = "Option::is_none")]
1212 resource_usage_samples: Option<&'a BenchmarkResourceUsage<Vec<f64>>>,
1213 #[serde(skip_serializing_if = "Option::is_none")]
1214 resource_usage_stats: Option<&'a BenchmarkResourceUsage<BenchmarkDistributionStats>>,
1215}
1216
1217#[derive(Debug, Serialize)]
1218struct BenchmarkControlMatrixArtifact<'a> {
1219 workload: &'a str,
1220 runtimes: Vec<&'a str>,
1221 modes: Vec<&'a str>,
1222 scenario_ids: Vec<&'a str>,
1223}
1224
1225#[derive(Debug, Serialize)]
1226struct BenchmarkTransportRttArtifact<'a> {
1227 channel: &'static str,
1228 payload_bytes: usize,
1229 samples_ms: &'a [f64],
1230 stats: &'a BenchmarkStats,
1231}
1232
1233#[derive(Debug, Serialize)]
1234struct BenchmarkHotspotRankingArtifact<'a> {
1235 metric: &'static str,
1236 label: &'static str,
1237 dimension: &'static str,
1238 unit: &'static str,
1239 ranked_scenarios: Vec<BenchmarkHotspotScenarioArtifact<'a>>,
1240 #[serde(skip_serializing_if = "Vec::is_empty")]
1241 scenarios_without_metric: Vec<&'a str>,
1242}
1243
1244#[derive(Debug, Serialize)]
1245struct BenchmarkHotspotScenarioArtifact<'a> {
1246 rank: usize,
1247 id: &'a str,
1248 workload: &'a str,
1249 runtime: &'a str,
1250 mode: &'a str,
1251 value: f64,
1252}
1253
1254struct HotspotMetricDefinition {
1255 metric: &'static str,
1256 label: &'static str,
1257 dimension: &'static str,
1258 unit: &'static str,
1259 value: fn(&BenchmarkScenarioReport) -> Option<f64>,
1260}
1261
1262const HOTSPOT_METRICS: [HotspotMetricDefinition; 13] = [
1263 HotspotMetricDefinition {
1264 metric: "wall_mean_ms",
1265 label: "Wall Time",
1266 dimension: "time",
1267 unit: "ms",
1268 value: hotspot_wall_mean_ms,
1269 },
1270 HotspotMetricDefinition {
1271 metric: "wall_stddev_ms",
1272 label: "Wall Time Stddev",
1273 dimension: "stability",
1274 unit: "ms",
1275 value: hotspot_wall_stddev_ms,
1276 },
1277 HotspotMetricDefinition {
1278 metric: "wall_range_ms",
1279 label: "Wall Time Range",
1280 dimension: "stability",
1281 unit: "ms",
1282 value: hotspot_wall_range_ms,
1283 },
1284 HotspotMetricDefinition {
1285 metric: "guest_import_mean_ms",
1286 label: "Guest Import Time",
1287 dimension: "time",
1288 unit: "ms",
1289 value: hotspot_guest_import_mean_ms,
1290 },
1291 HotspotMetricDefinition {
1292 metric: "startup_overhead_mean_ms",
1293 label: "Startup Overhead",
1294 dimension: "time",
1295 unit: "ms",
1296 value: hotspot_startup_overhead_mean_ms,
1297 },
1298 HotspotMetricDefinition {
1299 metric: "context_setup_mean_ms",
1300 label: "Context Setup Phase",
1301 dimension: "time",
1302 unit: "ms",
1303 value: hotspot_context_setup_mean_ms,
1304 },
1305 HotspotMetricDefinition {
1306 metric: "startup_phase_mean_ms",
1307 label: "Runtime Startup Phase",
1308 dimension: "time",
1309 unit: "ms",
1310 value: hotspot_startup_phase_mean_ms,
1311 },
1312 HotspotMetricDefinition {
1313 metric: "guest_execution_mean_ms",
1314 label: "Guest Execution Phase",
1315 dimension: "time",
1316 unit: "ms",
1317 value: hotspot_guest_execution_mean_ms,
1318 },
1319 HotspotMetricDefinition {
1320 metric: "completion_mean_ms",
1321 label: "Completion/Stdio Phase",
1322 dimension: "time",
1323 unit: "ms",
1324 value: hotspot_completion_mean_ms,
1325 },
1326 HotspotMetricDefinition {
1327 metric: "startup_share_pct",
1328 label: "Startup Share Of Wall",
1329 dimension: "share",
1330 unit: "pct",
1331 value: hotspot_startup_share_pct,
1332 },
1333 HotspotMetricDefinition {
1334 metric: "rss_mean_mib",
1335 label: "RSS",
1336 dimension: "memory",
1337 unit: "MiB",
1338 value: hotspot_rss_mean_mib,
1339 },
1340 HotspotMetricDefinition {
1341 metric: "heap_mean_mib",
1342 label: "Heap Used",
1343 dimension: "memory",
1344 unit: "MiB",
1345 value: hotspot_heap_mean_mib,
1346 },
1347 HotspotMetricDefinition {
1348 metric: "cpu_total_mean_ms",
1349 label: "Total CPU",
1350 dimension: "cpu",
1351 unit: "ms",
1352 value: hotspot_total_cpu_mean_ms,
1353 },
1354];
1355
1356#[derive(Debug)]
1357pub enum JavascriptBenchmarkError {
1358 InvalidConfig(&'static str),
1359 InvalidWorkspaceRoot(PathBuf),
1360 InvalidBaselineReport {
1361 path: PathBuf,
1362 message: String,
1363 },
1364 Io(std::io::Error),
1365 Utf8(std::string::FromUtf8Error),
1366 Execution(JavascriptExecutionError),
1367 NodeVersion(std::io::Error),
1368 MissingBenchmarkMetric(&'static str),
1369 InvalidBenchmarkMetric {
1370 scenario: &'static str,
1371 raw_value: String,
1372 },
1373 TransportProbeTimeout {
1374 payload_bytes: usize,
1375 },
1376 TransportProbeExited {
1377 exit_code: i32,
1378 stderr: String,
1379 },
1380 InvalidTransportProbeResponse {
1381 payload_bytes: usize,
1382 expected: String,
1383 actual: String,
1384 },
1385 NonZeroExit {
1386 scenario: &'static str,
1387 exit_code: i32,
1388 stderr: String,
1389 },
1390}
1391
1392impl fmt::Display for JavascriptBenchmarkError {
1393 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1394 match self {
1395 Self::InvalidConfig(message) => write!(f, "invalid benchmark config: {message}"),
1396 Self::InvalidWorkspaceRoot(path) => {
1397 write!(
1398 f,
1399 "failed to resolve workspace root from execution crate path: {}",
1400 path.display()
1401 )
1402 }
1403 Self::InvalidBaselineReport { path, message } => {
1404 write!(
1405 f,
1406 "failed to parse benchmark baseline artifact {}: {message}",
1407 path.display()
1408 )
1409 }
1410 Self::Io(err) => write!(f, "benchmark I/O failure: {err}"),
1411 Self::Utf8(err) => write!(f, "benchmark output was not valid UTF-8: {err}"),
1412 Self::Execution(err) => write!(f, "benchmark execution failed: {err}"),
1413 Self::NodeVersion(err) => write!(f, "failed to query node version: {err}"),
1414 Self::MissingBenchmarkMetric(scenario) => {
1415 write!(
1416 f,
1417 "benchmark scenario `{scenario}` did not emit a metric marker"
1418 )
1419 }
1420 Self::InvalidBenchmarkMetric {
1421 scenario,
1422 raw_value,
1423 } => write!(
1424 f,
1425 "benchmark scenario `{scenario}` emitted an invalid metric: {raw_value}"
1426 ),
1427 Self::TransportProbeTimeout { payload_bytes } => {
1428 write!(
1429 f,
1430 "transport probe timed out waiting for {payload_bytes}-byte round-trip"
1431 )
1432 }
1433 Self::TransportProbeExited { exit_code, stderr } => {
1434 write!(f, "transport probe exited with code {exit_code}: {stderr}")
1435 }
1436 Self::InvalidTransportProbeResponse {
1437 payload_bytes,
1438 expected,
1439 actual,
1440 } => write!(
1441 f,
1442 "transport probe returned unexpected payload for {payload_bytes}-byte round-trip: expected {expected:?}, got {actual:?}"
1443 ),
1444 Self::NonZeroExit {
1445 scenario,
1446 exit_code,
1447 stderr,
1448 } => write!(
1449 f,
1450 "benchmark scenario `{scenario}` exited with code {exit_code}: {stderr}"
1451 ),
1452 }
1453 }
1454}
1455
1456impl std::error::Error for JavascriptBenchmarkError {}
1457
1458impl From<std::io::Error> for JavascriptBenchmarkError {
1459 fn from(err: std::io::Error) -> Self {
1460 Self::Io(err)
1461 }
1462}
1463
1464impl From<std::string::FromUtf8Error> for JavascriptBenchmarkError {
1465 fn from(err: std::string::FromUtf8Error) -> Self {
1466 Self::Utf8(err)
1467 }
1468}
1469
1470impl From<serde_json::Error> for JavascriptBenchmarkError {
1471 fn from(err: serde_json::Error) -> Self {
1472 Self::Io(std::io::Error::new(std::io::ErrorKind::InvalidData, err))
1473 }
1474}
1475
1476impl From<JavascriptExecutionError> for JavascriptBenchmarkError {
1477 fn from(err: JavascriptExecutionError) -> Self {
1478 Self::Execution(err)
1479 }
1480}
1481
1482pub fn run_javascript_benchmarks(
1483 config: &JavascriptBenchmarkConfig,
1484) -> Result<JavascriptBenchmarkReport, JavascriptBenchmarkError> {
1485 validate_benchmark_config(config)?;
1486
1487 let repo_root = workspace_root()?;
1488 let host = benchmark_host()?;
1489 let workspace = BenchmarkWorkspace::create(&repo_root)?;
1490 let transport_rtt = measure_transport_rtt(&workspace, config)?;
1491
1492 let mut scenarios = Vec::new();
1493
1494 for scenario in benchmark_scenarios() {
1495 scenarios.push(run_scenario(&workspace, config, scenario)?);
1496 }
1497
1498 Ok(JavascriptBenchmarkReport {
1499 generated_at_unix_ms: SystemTime::now()
1500 .duration_since(UNIX_EPOCH)
1501 .unwrap_or_default()
1502 .as_millis(),
1503 config: config.clone(),
1504 host,
1505 repo_root,
1506 transport_rtt,
1507 scenarios,
1508 })
1509}
1510
1511fn benchmark_artifact_dir(repo_root: &Path) -> PathBuf {
1512 repo_root.join(BENCHMARK_ARTIFACT_DIR)
1513}
1514
1515fn benchmark_run_state_path(artifact_dir: &Path) -> PathBuf {
1516 artifact_dir.join(BENCHMARK_RUN_STATE_FILE)
1517}
1518
1519fn load_benchmark_run_state(
1520 state_path: &Path,
1521 config: &JavascriptBenchmarkConfig,
1522 host: &BenchmarkHost,
1523 repo_root: &Path,
1524 definitions: &[ScenarioDefinition],
1525) -> Result<StoredBenchmarkRunState, JavascriptBenchmarkError> {
1526 match fs::read_to_string(state_path) {
1527 Ok(raw) => match serde_json::from_str::<StoredBenchmarkRunState>(&raw) {
1528 Ok(state) if state.is_compatible(config, host, repo_root) => {
1529 Ok(state.sanitized(definitions))
1530 }
1531 Ok(_) | Err(_) => Ok(StoredBenchmarkRunState::new(config, host, repo_root)),
1532 },
1533 Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
1534 Ok(StoredBenchmarkRunState::new(config, host, repo_root))
1535 }
1536 Err(err) => Err(JavascriptBenchmarkError::Io(err)),
1537 }
1538}
1539
1540fn persist_benchmark_run_state(
1541 state_path: &Path,
1542 state: &StoredBenchmarkRunState,
1543) -> Result<(), JavascriptBenchmarkError> {
1544 write_string_atomic(state_path, &serde_json::to_string_pretty(state)?)
1545}
1546
1547fn write_string_atomic(path: &Path, contents: &str) -> Result<(), JavascriptBenchmarkError> {
1548 if let Some(parent) = path.parent() {
1549 fs::create_dir_all(parent)?;
1550 }
1551
1552 let temp_path = path.with_file_name(format!(
1553 ".{}.tmp-{}-{}",
1554 path.file_name()
1555 .and_then(|name| name.to_str())
1556 .unwrap_or("artifact"),
1557 std::process::id(),
1558 SystemTime::now()
1559 .duration_since(UNIX_EPOCH)
1560 .unwrap_or_default()
1561 .as_nanos()
1562 ));
1563 fs::write(&temp_path, contents)?;
1564 if let Err(err) = fs::rename(&temp_path, path) {
1565 let _ = fs::remove_file(&temp_path);
1566 return Err(JavascriptBenchmarkError::Io(err));
1567 }
1568
1569 Ok(())
1570}
1571
1572fn remove_file_if_exists(path: &Path) -> Result<(), JavascriptBenchmarkError> {
1573 match fs::remove_file(path) {
1574 Ok(()) => Ok(()),
1575 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
1576 Err(err) => Err(JavascriptBenchmarkError::Io(err)),
1577 }
1578}
1579
1580fn current_unix_ms() -> u128 {
1581 SystemTime::now()
1582 .duration_since(UNIX_EPOCH)
1583 .unwrap_or_default()
1584 .as_millis()
1585}
1586
1587#[derive(Debug, Clone, Copy)]
1588struct ScenarioDefinition {
1589 id: &'static str,
1590 workload: &'static str,
1591 runtime: ScenarioRuntime,
1592 mode: ScenarioMode,
1593 description: &'static str,
1594 fixture: &'static str,
1595 entrypoint: &'static str,
1596 compile_cache: CompileCacheStrategy,
1597 engine_reuse: EngineReuseStrategy,
1598 expect_import_metric: bool,
1599 env: ScenarioEnvironment,
1600}
1601
1602#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1603enum CompileCacheStrategy {
1604 Disabled,
1605 Primed,
1606}
1607
1608impl CompileCacheStrategy {
1609 fn label(self) -> &'static str {
1610 match self {
1611 Self::Disabled => "disabled",
1612 Self::Primed => "primed",
1613 }
1614 }
1615}
1616
1617#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1618enum EngineReuseStrategy {
1619 FreshPerSample,
1620 SharedAcrossScenario,
1621 SharedContextAcrossScenario,
1622}
1623
1624#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1625enum ScenarioEnvironment {
1626 None,
1627 ProjectedWorkspaceNodeModules,
1628}
1629
1630#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1631enum ScenarioRuntime {
1632 NativeExecution,
1633 HostNode,
1634}
1635
1636impl ScenarioRuntime {
1637 fn label(self) -> &'static str {
1638 match self {
1639 Self::NativeExecution => "native-execution",
1640 Self::HostNode => "host-node",
1641 }
1642 }
1643}
1644
1645#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1646enum ScenarioMode {
1647 BaselineControl,
1648 TrueColdStart,
1649 NewSessionReplay,
1650 SameSessionReplay,
1651 SameEngineReplay,
1652 HostControl,
1653}
1654
1655impl ScenarioMode {
1656 fn label(self) -> &'static str {
1657 match self {
1658 Self::BaselineControl => "baseline-control",
1659 Self::TrueColdStart => "true-cold-start",
1660 Self::NewSessionReplay => "new-session-replay",
1661 Self::SameSessionReplay => "same-session-replay",
1662 Self::SameEngineReplay => "same-engine-replay",
1663 Self::HostControl => "host-control",
1664 }
1665 }
1666}
1667
1668#[derive(Debug)]
1669struct SampleMeasurement {
1670 wall_ms: f64,
1671 guest_import_ms: Option<f64>,
1672 context_setup_ms: f64,
1673 startup_ms: f64,
1674 completion_ms: f64,
1675 resource_usage: Option<BenchmarkResourceUsage<f64>>,
1676}
1677
1678#[derive(Debug)]
1679struct BenchmarkWorkspace {
1680 root: PathBuf,
1681 repo_root: PathBuf,
1682}
1683
1684#[derive(Debug, Deserialize)]
1685struct StoredBenchmarkArtifact {
1686 artifact_version: u32,
1687 generated_at_unix_ms: u128,
1688 scenarios: Vec<StoredBenchmarkScenario>,
1689}
1690
1691#[derive(Debug, Deserialize)]
1692struct StoredBenchmarkScenario {
1693 id: String,
1694 wall_stats: BenchmarkStats,
1695 #[serde(default)]
1696 guest_import_stats: Option<BenchmarkStats>,
1697 #[serde(default)]
1698 startup_overhead_stats: Option<BenchmarkStats>,
1699 #[serde(default)]
1700 phase_stats: Option<BenchmarkScenarioPhases<BenchmarkStats>>,
1701}
1702
1703#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1704struct StoredBenchmarkRunHost {
1705 node_binary: String,
1706 node_version: String,
1707 os: String,
1708 arch: String,
1709 logical_cpus: usize,
1710}
1711
1712#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1713struct StoredBenchmarkRunState {
1714 artifact_version: u32,
1715 config: JavascriptBenchmarkConfig,
1716 host: StoredBenchmarkRunHost,
1717 repo_root: PathBuf,
1718 #[serde(default)]
1719 transport_rtt: Option<Vec<StoredBenchmarkTransportRttReport>>,
1720 #[serde(default)]
1721 scenarios: Vec<StoredBenchmarkScenarioReport>,
1722}
1723
1724#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1725struct StoredBenchmarkTransportRttReport {
1726 payload_bytes: usize,
1727 samples_ms: Vec<f64>,
1728 stats: BenchmarkStats,
1729}
1730
1731#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1732struct StoredBenchmarkScenarioReport {
1733 id: String,
1734 wall_samples_ms: Vec<f64>,
1735 wall_stats: BenchmarkStats,
1736 #[serde(default)]
1737 guest_import_samples_ms: Option<Vec<f64>>,
1738 #[serde(default)]
1739 guest_import_stats: Option<BenchmarkStats>,
1740 #[serde(default)]
1741 startup_overhead_samples_ms: Option<Vec<f64>>,
1742 #[serde(default)]
1743 startup_overhead_stats: Option<BenchmarkStats>,
1744 phase_samples_ms: BenchmarkScenarioPhases<Vec<f64>>,
1745 phase_stats: BenchmarkScenarioPhases<BenchmarkStats>,
1746 #[serde(default)]
1747 resource_usage_samples: Option<BenchmarkResourceUsage<Vec<f64>>>,
1748 #[serde(default)]
1749 resource_usage_stats: Option<BenchmarkResourceUsage<BenchmarkDistributionStats>>,
1750}
1751
1752impl BenchmarkWorkspace {
1753 fn create(repo_root: &Path) -> Result<Self, JavascriptBenchmarkError> {
1754 let root = repo_root.join(format!(
1755 ".tmp-agentos-execution-bench-{}-{}",
1756 std::process::id(),
1757 SystemTime::now()
1758 .duration_since(UNIX_EPOCH)
1759 .unwrap_or_default()
1760 .as_nanos()
1761 ));
1762 fs::create_dir_all(&root)?;
1763 write_benchmark_workspace(&root, repo_root)?;
1764 Ok(Self {
1765 root,
1766 repo_root: repo_root.to_path_buf(),
1767 })
1768 }
1769}
1770
1771impl Drop for BenchmarkWorkspace {
1772 fn drop(&mut self) {
1773 let _ = fs::remove_dir_all(&self.root);
1774 }
1775}
1776
1777impl StoredBenchmarkRunHost {
1778 fn from_host(host: &BenchmarkHost) -> Self {
1779 Self {
1780 node_binary: host.node_binary.clone(),
1781 node_version: host.node_version.clone(),
1782 os: host.os.to_owned(),
1783 arch: host.arch.to_owned(),
1784 logical_cpus: host.logical_cpus,
1785 }
1786 }
1787
1788 fn matches_host(&self, host: &BenchmarkHost) -> bool {
1789 self.node_binary == host.node_binary
1790 && self.node_version == host.node_version
1791 && self.os == host.os
1792 && self.arch == host.arch
1793 && self.logical_cpus == host.logical_cpus
1794 }
1795}
1796
1797impl StoredBenchmarkRunState {
1798 fn new(config: &JavascriptBenchmarkConfig, host: &BenchmarkHost, repo_root: &Path) -> Self {
1799 Self {
1800 artifact_version: BENCHMARK_ARTIFACT_VERSION,
1801 config: config.clone(),
1802 host: StoredBenchmarkRunHost::from_host(host),
1803 repo_root: repo_root.to_path_buf(),
1804 transport_rtt: None,
1805 scenarios: Vec::new(),
1806 }
1807 }
1808
1809 fn is_compatible(
1810 &self,
1811 config: &JavascriptBenchmarkConfig,
1812 host: &BenchmarkHost,
1813 repo_root: &Path,
1814 ) -> bool {
1815 self.artifact_version == BENCHMARK_ARTIFACT_VERSION
1816 && self.config == *config
1817 && self.host.matches_host(host)
1818 && self.repo_root == repo_root
1819 }
1820
1821 fn sanitized(mut self, definitions: &[ScenarioDefinition]) -> Self {
1822 if let Some(transport_rtt) = &self.transport_rtt {
1823 let payloads = transport_rtt
1824 .iter()
1825 .map(|report| report.payload_bytes)
1826 .collect::<Vec<_>>();
1827 if payloads != TRANSPORT_RTT_PAYLOAD_BYTES {
1828 self.transport_rtt = None;
1829 }
1830 }
1831
1832 let mut scenarios_by_id = self
1833 .scenarios
1834 .into_iter()
1835 .map(|scenario| (scenario.id.clone(), scenario))
1836 .collect::<BTreeMap<_, _>>();
1837 self.scenarios = definitions
1838 .iter()
1839 .filter_map(|definition| scenarios_by_id.remove(definition.id))
1840 .collect();
1841 self
1842 }
1843
1844 fn resumed_stage_count(&self, definitions: &[ScenarioDefinition]) -> usize {
1845 usize::from(self.transport_rtt.is_some())
1846 + definitions
1847 .iter()
1848 .filter(|definition| self.has_scenario(definition.id))
1849 .count()
1850 }
1851
1852 fn has_scenario(&self, id: &str) -> bool {
1853 self.scenarios.iter().any(|scenario| scenario.id == id)
1854 }
1855
1856 fn record_transport_rtt(&mut self, transport_rtt: &[BenchmarkTransportRttReport]) {
1857 self.transport_rtt = Some(
1858 transport_rtt
1859 .iter()
1860 .map(StoredBenchmarkTransportRttReport::from_report)
1861 .collect(),
1862 );
1863 }
1864
1865 fn record_scenario(&mut self, scenario: &BenchmarkScenarioReport) {
1866 self.scenarios.retain(|stored| stored.id != scenario.id);
1867 self.scenarios
1868 .push(StoredBenchmarkScenarioReport::from_report(scenario));
1869 }
1870
1871 fn to_report(
1872 &self,
1873 config: &JavascriptBenchmarkConfig,
1874 host: &BenchmarkHost,
1875 repo_root: &Path,
1876 definitions: &[ScenarioDefinition],
1877 ) -> JavascriptBenchmarkReport {
1878 let scenarios_by_id = self
1879 .scenarios
1880 .iter()
1881 .map(|scenario| (scenario.id.as_str(), scenario))
1882 .collect::<BTreeMap<_, _>>();
1883
1884 JavascriptBenchmarkReport {
1885 generated_at_unix_ms: current_unix_ms(),
1886 config: config.clone(),
1887 host: host.clone(),
1888 repo_root: repo_root.to_path_buf(),
1889 transport_rtt: self
1890 .transport_rtt
1891 .clone()
1892 .unwrap_or_default()
1893 .into_iter()
1894 .map(StoredBenchmarkTransportRttReport::into_report)
1895 .collect(),
1896 scenarios: definitions
1897 .iter()
1898 .filter_map(|definition| {
1899 scenarios_by_id
1900 .get(definition.id)
1901 .map(|scenario| scenario.to_report(*definition))
1902 })
1903 .collect(),
1904 }
1905 }
1906}
1907
1908impl StoredBenchmarkTransportRttReport {
1909 fn from_report(report: &BenchmarkTransportRttReport) -> Self {
1910 Self {
1911 payload_bytes: report.payload_bytes,
1912 samples_ms: report.samples_ms.clone(),
1913 stats: report.stats.clone(),
1914 }
1915 }
1916
1917 fn into_report(self) -> BenchmarkTransportRttReport {
1918 BenchmarkTransportRttReport {
1919 channel: TRANSPORT_RTT_CHANNEL,
1920 payload_bytes: self.payload_bytes,
1921 samples_ms: self.samples_ms,
1922 stats: self.stats,
1923 }
1924 }
1925}
1926
1927impl StoredBenchmarkScenarioReport {
1928 fn from_report(report: &BenchmarkScenarioReport) -> Self {
1929 Self {
1930 id: report.id.to_owned(),
1931 wall_samples_ms: report.wall_samples_ms.clone(),
1932 wall_stats: report.wall_stats.clone(),
1933 guest_import_samples_ms: report.guest_import_samples_ms.clone(),
1934 guest_import_stats: report.guest_import_stats.clone(),
1935 startup_overhead_samples_ms: report.startup_overhead_samples_ms.clone(),
1936 startup_overhead_stats: report.startup_overhead_stats.clone(),
1937 phase_samples_ms: report.phase_samples_ms.clone(),
1938 phase_stats: report.phase_stats.clone(),
1939 resource_usage_samples: report.resource_usage_samples.clone(),
1940 resource_usage_stats: report.resource_usage_stats.clone(),
1941 }
1942 }
1943
1944 fn to_report(&self, definition: ScenarioDefinition) -> BenchmarkScenarioReport {
1945 BenchmarkScenarioReport {
1946 id: definition.id,
1947 workload: definition.workload,
1948 runtime: definition.runtime.label(),
1949 mode: definition.mode.label(),
1950 description: definition.description,
1951 fixture: definition.fixture,
1952 compile_cache: definition.compile_cache.label(),
1953 wall_samples_ms: self.wall_samples_ms.clone(),
1954 wall_stats: self.wall_stats.clone(),
1955 guest_import_samples_ms: self.guest_import_samples_ms.clone(),
1956 guest_import_stats: self.guest_import_stats.clone(),
1957 startup_overhead_samples_ms: self.startup_overhead_samples_ms.clone(),
1958 startup_overhead_stats: self.startup_overhead_stats.clone(),
1959 phase_samples_ms: self.phase_samples_ms.clone(),
1960 phase_stats: self.phase_stats.clone(),
1961 resource_usage_samples: self.resource_usage_samples.clone(),
1962 resource_usage_stats: self.resource_usage_stats.clone(),
1963 }
1964 }
1965}
1966
1967pub fn run_javascript_benchmarks_with_recovery(
1968 config: &JavascriptBenchmarkConfig,
1969 baseline_path: Option<&Path>,
1970) -> Result<JavascriptBenchmarkRunOutput, JavascriptBenchmarkError> {
1971 validate_benchmark_config(config)?;
1972
1973 let repo_root = workspace_root()?;
1974 let host = benchmark_host()?;
1975 let artifact_dir = benchmark_artifact_dir(&repo_root);
1976 let workspace = BenchmarkWorkspace::create(&repo_root)?;
1977 let (report, resumed_stage_count, state_path) = orchestrate_javascript_benchmark_report(
1978 config,
1979 &repo_root,
1980 &host,
1981 &artifact_dir,
1982 || measure_transport_rtt(&workspace, config),
1983 |scenario| run_scenario(&workspace, config, scenario),
1984 )?;
1985 let comparison = baseline_path
1986 .map(|path| report.compare_to_baseline_path(path))
1987 .transpose()?;
1988 let artifact_paths =
1989 report.write_artifacts_with_comparison(&artifact_dir, comparison.as_ref())?;
1990 remove_file_if_exists(&state_path)?;
1991
1992 Ok(JavascriptBenchmarkRunOutput {
1993 artifact_paths,
1994 resumed_stage_count,
1995 })
1996}
1997
1998fn orchestrate_javascript_benchmark_report<MeasureTransport, RunScenario>(
1999 config: &JavascriptBenchmarkConfig,
2000 repo_root: &Path,
2001 host: &BenchmarkHost,
2002 artifact_dir: &Path,
2003 mut measure_transport: MeasureTransport,
2004 mut run_scenario: RunScenario,
2005) -> Result<(JavascriptBenchmarkReport, usize, PathBuf), JavascriptBenchmarkError>
2006where
2007 MeasureTransport: FnMut() -> Result<Vec<BenchmarkTransportRttReport>, JavascriptBenchmarkError>,
2008 RunScenario:
2009 FnMut(ScenarioDefinition) -> Result<BenchmarkScenarioReport, JavascriptBenchmarkError>,
2010{
2011 validate_benchmark_config(config)?;
2012
2013 fs::create_dir_all(artifact_dir)?;
2014
2015 let definitions = benchmark_scenarios();
2016 let state_path = benchmark_run_state_path(artifact_dir);
2017 let mut state = load_benchmark_run_state(&state_path, config, host, repo_root, &definitions)?;
2018 let resumed_stage_count = state.resumed_stage_count(&definitions);
2019
2020 if state.transport_rtt.is_none() {
2021 let transport_rtt = measure_transport()?;
2022 state.record_transport_rtt(&transport_rtt);
2023 persist_benchmark_run_state(&state_path, &state)?;
2024 }
2025
2026 for definition in definitions {
2027 if state.has_scenario(definition.id) {
2028 continue;
2029 }
2030
2031 let scenario = run_scenario(definition)?;
2032 state.record_scenario(&scenario);
2033 persist_benchmark_run_state(&state_path, &state)?;
2034 }
2035
2036 Ok((
2037 state.to_report(config, host, repo_root, &benchmark_scenarios()),
2038 resumed_stage_count,
2039 state_path,
2040 ))
2041}
2042
2043fn validate_benchmark_config(
2044 config: &JavascriptBenchmarkConfig,
2045) -> Result<(), JavascriptBenchmarkError> {
2046 if config.iterations == 0 {
2047 return Err(JavascriptBenchmarkError::InvalidConfig(
2048 "iterations must be greater than zero",
2049 ));
2050 }
2051 if config.iterations > MAX_BENCHMARK_ITERATIONS {
2052 return Err(JavascriptBenchmarkError::InvalidConfig(
2053 "iterations must be less than or equal to 1000",
2054 ));
2055 }
2056 if config.warmup_iterations > MAX_BENCHMARK_WARMUP_ITERATIONS {
2057 return Err(JavascriptBenchmarkError::InvalidConfig(
2058 "warmup iterations must be less than or equal to 1000",
2059 ));
2060 }
2061
2062 Ok(())
2063}
2064
2065fn benchmark_scenarios() -> [ScenarioDefinition; 21] {
2066 [
2067 ScenarioDefinition {
2068 id: "isolate-startup",
2069 workload: "startup-floor",
2070 runtime: ScenarioRuntime::NativeExecution,
2071 mode: ScenarioMode::BaselineControl,
2072 description: "Minimal guest with no extra imports. Measures the current startup floor for create-context plus node process bootstrap.",
2073 fixture: "empty entrypoint",
2074 entrypoint: "./bench/isolate-startup.mjs",
2075 compile_cache: CompileCacheStrategy::Disabled,
2076 engine_reuse: EngineReuseStrategy::FreshPerSample,
2077 expect_import_metric: false,
2078 env: ScenarioEnvironment::None,
2079 },
2080 ScenarioDefinition {
2081 id: "prewarmed-isolate-startup",
2082 workload: "startup-floor",
2083 runtime: ScenarioRuntime::NativeExecution,
2084 mode: ScenarioMode::SameEngineReplay,
2085 description: "Minimal guest after a priming pass while one execution engine keeps materialized assets and builtin/polyfill prewarm state alive, isolating the hot startup floor from import work.",
2086 fixture: "empty entrypoint",
2087 entrypoint: "./bench/isolate-startup.mjs",
2088 compile_cache: CompileCacheStrategy::Primed,
2089 engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2090 expect_import_metric: false,
2091 env: ScenarioEnvironment::None,
2092 },
2093 ScenarioDefinition {
2094 id: "cold-local-import",
2095 workload: "local-import",
2096 runtime: ScenarioRuntime::NativeExecution,
2097 mode: ScenarioMode::TrueColdStart,
2098 description: "Cold import of a repo-local ESM graph that simulates layered application modules without compile-cache reuse.",
2099 fixture: "24-module local ESM graph",
2100 entrypoint: "./bench/cold-local-import.mjs",
2101 compile_cache: CompileCacheStrategy::Disabled,
2102 engine_reuse: EngineReuseStrategy::FreshPerSample,
2103 expect_import_metric: true,
2104 env: ScenarioEnvironment::None,
2105 },
2106 ScenarioDefinition {
2107 id: "warm-local-import",
2108 workload: "local-import",
2109 runtime: ScenarioRuntime::NativeExecution,
2110 mode: ScenarioMode::NewSessionReplay,
2111 description: "Warm import of the same local ESM graph after a compile-cache priming pass in an earlier isolate.",
2112 fixture: "24-module local ESM graph",
2113 entrypoint: "./bench/warm-local-import.mjs",
2114 compile_cache: CompileCacheStrategy::Primed,
2115 engine_reuse: EngineReuseStrategy::FreshPerSample,
2116 expect_import_metric: true,
2117 env: ScenarioEnvironment::None,
2118 },
2119 ScenarioDefinition {
2120 id: "same-context-local-import",
2121 workload: "local-import",
2122 runtime: ScenarioRuntime::NativeExecution,
2123 mode: ScenarioMode::SameSessionReplay,
2124 description: "Warm import of the same local ESM graph by replaying executions against one reused JavaScript context after a compile-cache priming pass.",
2125 fixture: "24-module local ESM graph",
2126 entrypoint: "./bench/warm-local-import.mjs",
2127 compile_cache: CompileCacheStrategy::Primed,
2128 engine_reuse: EngineReuseStrategy::SharedContextAcrossScenario,
2129 expect_import_metric: true,
2130 env: ScenarioEnvironment::None,
2131 },
2132 ScenarioDefinition {
2133 id: "prewarmed-local-import",
2134 workload: "local-import",
2135 runtime: ScenarioRuntime::NativeExecution,
2136 mode: ScenarioMode::SameEngineReplay,
2137 description: "Warm import of the same local ESM graph after compile-cache priming while one execution engine keeps materialized assets and builtin/polyfill prewarm state alive.",
2138 fixture: "24-module local ESM graph",
2139 entrypoint: "./bench/warm-local-import.mjs",
2140 compile_cache: CompileCacheStrategy::Primed,
2141 engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2142 expect_import_metric: true,
2143 env: ScenarioEnvironment::None,
2144 },
2145 ScenarioDefinition {
2146 id: "host-local-import",
2147 workload: "local-import",
2148 runtime: ScenarioRuntime::HostNode,
2149 mode: ScenarioMode::HostControl,
2150 description: "Direct host-Node control for the same local ESM graph so later runs can separate native executor overhead from guest import work.",
2151 fixture: "24-module local ESM graph",
2152 entrypoint: "./bench/cold-local-import.mjs",
2153 compile_cache: CompileCacheStrategy::Disabled,
2154 engine_reuse: EngineReuseStrategy::FreshPerSample,
2155 expect_import_metric: true,
2156 env: ScenarioEnvironment::None,
2157 },
2158 ScenarioDefinition {
2159 id: "builtin-import",
2160 workload: "builtin-import",
2161 runtime: ScenarioRuntime::NativeExecution,
2162 mode: ScenarioMode::TrueColdStart,
2163 description: "Import of the common builtin path used by the wrappers and polyfill-adjacent bootstrap code.",
2164 fixture: "node:path + node:url + node:fs/promises",
2165 entrypoint: "./bench/builtin-import.mjs",
2166 compile_cache: CompileCacheStrategy::Disabled,
2167 engine_reuse: EngineReuseStrategy::FreshPerSample,
2168 expect_import_metric: true,
2169 env: ScenarioEnvironment::None,
2170 },
2171 ScenarioDefinition {
2172 id: "hot-builtin-stream-import",
2173 workload: "builtin-hot-import",
2174 runtime: ScenarioRuntime::NativeExecution,
2175 mode: ScenarioMode::SameEngineReplay,
2176 description: "Hot single-import microbench for `node:stream` after a priming pass inside one reused execution engine.",
2177 fixture: "node:stream",
2178 entrypoint: "./bench/hot-builtin-stream-import.mjs",
2179 compile_cache: CompileCacheStrategy::Primed,
2180 engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2181 expect_import_metric: true,
2182 env: ScenarioEnvironment::None,
2183 },
2184 ScenarioDefinition {
2185 id: "hot-builtin-stream-web-import",
2186 workload: "builtin-hot-import",
2187 runtime: ScenarioRuntime::NativeExecution,
2188 mode: ScenarioMode::SameEngineReplay,
2189 description: "Hot single-import microbench for `node:stream/web` after a priming pass inside one reused execution engine.",
2190 fixture: "node:stream/web",
2191 entrypoint: "./bench/hot-builtin-stream-web-import.mjs",
2192 compile_cache: CompileCacheStrategy::Primed,
2193 engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2194 expect_import_metric: true,
2195 env: ScenarioEnvironment::None,
2196 },
2197 ScenarioDefinition {
2198 id: "hot-builtin-crypto-import",
2199 workload: "builtin-hot-import",
2200 runtime: ScenarioRuntime::NativeExecution,
2201 mode: ScenarioMode::SameEngineReplay,
2202 description: "Hot single-import microbench for `node:crypto` after a priming pass inside one reused execution engine.",
2203 fixture: "node:crypto",
2204 entrypoint: "./bench/hot-builtin-crypto-import.mjs",
2205 compile_cache: CompileCacheStrategy::Primed,
2206 engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2207 expect_import_metric: true,
2208 env: ScenarioEnvironment::None,
2209 },
2210 ScenarioDefinition {
2211 id: "hot-builtin-zlib-import",
2212 workload: "builtin-hot-import",
2213 runtime: ScenarioRuntime::NativeExecution,
2214 mode: ScenarioMode::SameEngineReplay,
2215 description: "Hot single-import microbench for `node:zlib` after a priming pass inside one reused execution engine.",
2216 fixture: "node:zlib",
2217 entrypoint: "./bench/hot-builtin-zlib-import.mjs",
2218 compile_cache: CompileCacheStrategy::Primed,
2219 engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2220 expect_import_metric: true,
2221 env: ScenarioEnvironment::None,
2222 },
2223 ScenarioDefinition {
2224 id: "hot-builtin-assert-import",
2225 workload: "builtin-hot-import",
2226 runtime: ScenarioRuntime::NativeExecution,
2227 mode: ScenarioMode::SameEngineReplay,
2228 description: "Hot single-import microbench for `node:assert/strict` after a priming pass inside one reused execution engine.",
2229 fixture: "node:assert/strict",
2230 entrypoint: "./bench/hot-builtin-assert-import.mjs",
2231 compile_cache: CompileCacheStrategy::Primed,
2232 engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2233 expect_import_metric: true,
2234 env: ScenarioEnvironment::None,
2235 },
2236 ScenarioDefinition {
2237 id: "hot-builtin-url-import",
2238 workload: "builtin-hot-import",
2239 runtime: ScenarioRuntime::NativeExecution,
2240 mode: ScenarioMode::SameEngineReplay,
2241 description: "Hot single-import microbench for `node:url` after a priming pass inside one reused execution engine.",
2242 fixture: "node:url",
2243 entrypoint: "./bench/hot-builtin-url-import.mjs",
2244 compile_cache: CompileCacheStrategy::Primed,
2245 engine_reuse: EngineReuseStrategy::SharedAcrossScenario,
2246 expect_import_metric: true,
2247 env: ScenarioEnvironment::None,
2248 },
2249 ScenarioDefinition {
2250 id: "hot-projected-package-file-import",
2251 workload: "projected-package-hot-import",
2252 runtime: ScenarioRuntime::HostNode,
2253 mode: ScenarioMode::SameEngineReplay,
2254 description: "Hot projected-package single-import microbench for the TypeScript compiler file with compile cache and projected-source manifest reuse enabled across repeated contexts.",
2255 fixture: "projected TypeScript compiler file",
2256 entrypoint: "./bench/hot-projected-package-file-import.mjs",
2257 compile_cache: CompileCacheStrategy::Primed,
2258 engine_reuse: EngineReuseStrategy::FreshPerSample,
2259 expect_import_metric: true,
2260 env: ScenarioEnvironment::ProjectedWorkspaceNodeModules,
2261 },
2262 ScenarioDefinition {
2263 id: "large-package-import",
2264 workload: "large-package-import",
2265 runtime: ScenarioRuntime::HostNode,
2266 mode: ScenarioMode::TrueColdStart,
2267 description: "Cold import of the real-world `typescript` package from the workspace root `node_modules` tree.",
2268 fixture: "typescript",
2269 entrypoint: "./bench/large-package-import.mjs",
2270 compile_cache: CompileCacheStrategy::Disabled,
2271 engine_reuse: EngineReuseStrategy::FreshPerSample,
2272 expect_import_metric: true,
2273 env: ScenarioEnvironment::None,
2274 },
2275 ScenarioDefinition {
2276 id: "projected-package-import",
2277 workload: "projected-package-import",
2278 runtime: ScenarioRuntime::HostNode,
2279 mode: ScenarioMode::HostControl,
2280 description: "Projected-package guest-path import of TypeScript with compile cache and projected-source manifest reuse enabled across repeated contexts.",
2281 fixture: "projected TypeScript guest-path import",
2282 entrypoint: "./bench/projected-package-import.mjs",
2283 compile_cache: CompileCacheStrategy::Primed,
2284 engine_reuse: EngineReuseStrategy::FreshPerSample,
2285 expect_import_metric: true,
2286 env: ScenarioEnvironment::ProjectedWorkspaceNodeModules,
2287 },
2288 ScenarioDefinition {
2289 id: "pdf-lib-startup",
2290 workload: "pdf-lib-startup",
2291 runtime: ScenarioRuntime::HostNode,
2292 mode: ScenarioMode::HostControl,
2293 description: "Cold import of `pdf-lib` plus representative document setup that creates a PDF page and embeds a standard font.",
2294 fixture: "pdf-lib document creation",
2295 entrypoint: "./bench/pdf-lib-startup.mjs",
2296 compile_cache: CompileCacheStrategy::Disabled,
2297 engine_reuse: EngineReuseStrategy::FreshPerSample,
2298 expect_import_metric: true,
2299 env: ScenarioEnvironment::None,
2300 },
2301 ScenarioDefinition {
2302 id: "jszip-startup",
2303 workload: "jszip-startup",
2304 runtime: ScenarioRuntime::HostNode,
2305 mode: ScenarioMode::HostControl,
2306 description: "Cold import of `jszip` plus representative archive staging that builds a nested archive structure.",
2307 fixture: "jszip archive staging",
2308 entrypoint: "./bench/jszip-startup.mjs",
2309 compile_cache: CompileCacheStrategy::Disabled,
2310 engine_reuse: EngineReuseStrategy::FreshPerSample,
2311 expect_import_metric: true,
2312 env: ScenarioEnvironment::None,
2313 },
2314 ScenarioDefinition {
2315 id: "jszip-end-to-end",
2316 workload: "jszip-end-to-end",
2317 runtime: ScenarioRuntime::HostNode,
2318 mode: ScenarioMode::HostControl,
2319 description: "Cold import of `jszip` plus a full compressed archive roundtrip that writes, compresses, reloads, and validates nested archive contents.",
2320 fixture: "jszip end-to-end archive roundtrip",
2321 entrypoint: "./bench/jszip-end-to-end.mjs",
2322 compile_cache: CompileCacheStrategy::Disabled,
2323 engine_reuse: EngineReuseStrategy::FreshPerSample,
2324 expect_import_metric: true,
2325 env: ScenarioEnvironment::None,
2326 },
2327 ScenarioDefinition {
2328 id: "jszip-repeated-session-compressed",
2329 workload: "jszip-repeated-session-compressed",
2330 runtime: ScenarioRuntime::HostNode,
2331 mode: ScenarioMode::HostControl,
2332 description: "Repeated-session `jszip` workload after a compile-cache priming pass that compresses and reloads a nested archive in each fresh isolate.",
2333 fixture: "jszip compressed archive roundtrip",
2334 entrypoint: "./bench/jszip-repeated-session-compressed.mjs",
2335 compile_cache: CompileCacheStrategy::Primed,
2336 engine_reuse: EngineReuseStrategy::FreshPerSample,
2337 expect_import_metric: true,
2338 env: ScenarioEnvironment::None,
2339 },
2340 ]
2341}
2342
2343fn run_scenario(
2344 workspace: &BenchmarkWorkspace,
2345 config: &JavascriptBenchmarkConfig,
2346 scenario: ScenarioDefinition,
2347) -> Result<BenchmarkScenarioReport, JavascriptBenchmarkError> {
2348 let compile_cache_root = workspace
2349 .root
2350 .join("compile-cache")
2351 .join(scenario.id.replace('-', "_"));
2352 let mut shared_engine = match scenario.engine_reuse {
2353 EngineReuseStrategy::FreshPerSample => None,
2354 EngineReuseStrategy::SharedAcrossScenario
2355 | EngineReuseStrategy::SharedContextAcrossScenario => {
2356 Some(JavascriptExecutionEngine::default())
2357 }
2358 };
2359 let mut shared_context = None;
2360
2361 if scenario.compile_cache == CompileCacheStrategy::Primed {
2362 run_sample(
2363 workspace,
2364 &scenario,
2365 Some(compile_cache_root.clone()),
2366 shared_engine.as_mut(),
2367 &mut shared_context,
2368 )?;
2369 }
2370
2371 for _ in 0..config.warmup_iterations {
2372 run_sample(
2373 workspace,
2374 &scenario,
2375 compile_cache_root_for_strategy(scenario.compile_cache, &compile_cache_root),
2376 shared_engine.as_mut(),
2377 &mut shared_context,
2378 )?;
2379 }
2380
2381 let mut wall_samples_ms = Vec::with_capacity(config.iterations);
2382 let mut guest_import_samples_ms = if scenario.expect_import_metric {
2383 Some(Vec::with_capacity(config.iterations))
2384 } else {
2385 None
2386 };
2387 let mut context_setup_samples_ms = Vec::with_capacity(config.iterations);
2388 let mut startup_samples_ms = Vec::with_capacity(config.iterations);
2389 let mut completion_samples_ms = Vec::with_capacity(config.iterations);
2390 let mut resource_usage_samples = BenchmarkResourceUsage::<Vec<f64>>::default();
2391
2392 for _ in 0..config.iterations {
2393 let sample = run_sample(
2394 workspace,
2395 &scenario,
2396 compile_cache_root_for_strategy(scenario.compile_cache, &compile_cache_root),
2397 shared_engine.as_mut(),
2398 &mut shared_context,
2399 )?;
2400 wall_samples_ms.push(sample.wall_ms);
2401 context_setup_samples_ms.push(sample.context_setup_ms);
2402 startup_samples_ms.push(sample.startup_ms);
2403 completion_samples_ms.push(sample.completion_ms);
2404
2405 if let (Some(import_ms), Some(samples)) =
2406 (sample.guest_import_ms, guest_import_samples_ms.as_mut())
2407 {
2408 samples.push(import_ms);
2409 }
2410 if let Some(resource_usage) = sample.resource_usage.as_ref() {
2411 resource_usage_samples.push_sample(resource_usage);
2412 }
2413 }
2414
2415 let startup_overhead_samples_ms = guest_import_samples_ms.as_ref().map(|guest_samples| {
2416 context_setup_samples_ms
2417 .iter()
2418 .zip(startup_samples_ms.iter())
2419 .zip(completion_samples_ms.iter())
2420 .zip(guest_samples.iter())
2421 .map(|(((context_ms, startup_ms), completion_ms), _guest_ms)| {
2422 context_ms + startup_ms + completion_ms
2423 })
2424 .collect::<Vec<_>>()
2425 });
2426
2427 let phase_samples_ms = BenchmarkScenarioPhases {
2428 context_setup_ms: context_setup_samples_ms,
2429 startup_ms: startup_samples_ms,
2430 guest_execution_ms: guest_import_samples_ms.clone(),
2431 completion_ms: completion_samples_ms,
2432 };
2433 let resource_usage_samples = resource_usage_samples.into_populated();
2434
2435 Ok(BenchmarkScenarioReport {
2436 id: scenario.id,
2437 workload: scenario.workload,
2438 runtime: scenario.runtime.label(),
2439 mode: scenario.mode.label(),
2440 description: scenario.description,
2441 fixture: scenario.fixture,
2442 compile_cache: scenario.compile_cache.label(),
2443 wall_stats: compute_stats(&wall_samples_ms),
2444 guest_import_stats: guest_import_samples_ms
2445 .as_ref()
2446 .map(|samples| compute_stats(samples)),
2447 startup_overhead_stats: startup_overhead_samples_ms
2448 .as_ref()
2449 .map(|samples| compute_stats(samples)),
2450 phase_stats: BenchmarkScenarioPhases {
2451 context_setup_ms: compute_stats(&phase_samples_ms.context_setup_ms),
2452 startup_ms: compute_stats(&phase_samples_ms.startup_ms),
2453 guest_execution_ms: phase_samples_ms
2454 .guest_execution_ms
2455 .as_ref()
2456 .map(|samples| compute_stats(samples)),
2457 completion_ms: compute_stats(&phase_samples_ms.completion_ms),
2458 },
2459 resource_usage_stats: resource_usage_samples
2460 .as_ref()
2461 .and_then(compute_resource_usage_stats),
2462 wall_samples_ms,
2463 guest_import_samples_ms,
2464 startup_overhead_samples_ms,
2465 phase_samples_ms,
2466 resource_usage_samples,
2467 })
2468}
2469
2470fn compile_cache_root_for_strategy(strategy: CompileCacheStrategy, root: &Path) -> Option<PathBuf> {
2471 match strategy {
2472 CompileCacheStrategy::Disabled => None,
2473 CompileCacheStrategy::Primed => Some(root.to_path_buf()),
2474 }
2475}
2476
2477fn run_sample(
2478 workspace: &BenchmarkWorkspace,
2479 scenario: &ScenarioDefinition,
2480 compile_cache_root: Option<PathBuf>,
2481 shared_engine: Option<&mut JavascriptExecutionEngine>,
2482 shared_context: &mut Option<crate::JavascriptContext>,
2483) -> Result<SampleMeasurement, JavascriptBenchmarkError> {
2484 match scenario.runtime {
2485 ScenarioRuntime::NativeExecution => run_native_sample(
2486 workspace,
2487 scenario,
2488 compile_cache_root,
2489 shared_engine,
2490 shared_context,
2491 ),
2492 ScenarioRuntime::HostNode => run_host_node_sample(workspace, scenario),
2493 }
2494}
2495
2496fn run_native_sample(
2497 workspace: &BenchmarkWorkspace,
2498 scenario: &ScenarioDefinition,
2499 compile_cache_root: Option<PathBuf>,
2500 shared_engine: Option<&mut JavascriptExecutionEngine>,
2501 shared_context: &mut Option<crate::JavascriptContext>,
2502) -> Result<SampleMeasurement, JavascriptBenchmarkError> {
2503 let mut fresh_engine = JavascriptExecutionEngine::default();
2504 let engine = shared_engine.unwrap_or(&mut fresh_engine);
2505 let context_started_at = Instant::now();
2506 let (context, context_setup_ms) = match scenario.engine_reuse {
2507 EngineReuseStrategy::SharedContextAcrossScenario => {
2508 if let Some(context) = shared_context.as_ref() {
2509 (context.clone(), 0.0)
2510 } else {
2511 let context = engine.create_context(CreateJavascriptContextRequest {
2512 vm_id: String::from("vm-bench"),
2513 bootstrap_module: None,
2514 compile_cache_root,
2515 });
2516 let context_setup_ms = context_started_at.elapsed().as_secs_f64() * 1000.0;
2517 *shared_context = Some(context.clone());
2518 (context, context_setup_ms)
2519 }
2520 }
2521 _ => {
2522 let context = engine.create_context(CreateJavascriptContextRequest {
2523 vm_id: String::from("vm-bench"),
2524 bootstrap_module: None,
2525 compile_cache_root,
2526 });
2527 let context_setup_ms = context_started_at.elapsed().as_secs_f64() * 1000.0;
2528 (context, context_setup_ms)
2529 }
2530 };
2531
2532 let startup_started_at = Instant::now();
2533 let execution = engine.start_execution(StartJavascriptExecutionRequest {
2534 limits: Default::default(),
2535 guest_runtime: Default::default(),
2536 vm_id: String::from("vm-bench"),
2537 context_id: context.context_id,
2538 argv: vec![String::from(scenario.entrypoint)],
2539 env: scenario_env(workspace, scenario),
2540 cwd: workspace.root.clone(),
2541 wasm_module_bytes: None,
2542 inline_code: None,
2543 })?;
2544 let startup_ms = startup_started_at.elapsed().as_secs_f64() * 1000.0;
2545
2546 let completion_started_at = Instant::now();
2547 let result = execution.wait()?;
2548 let completion_total_ms = completion_started_at.elapsed().as_secs_f64() * 1000.0;
2549 let stdout = String::from_utf8(result.stdout)?;
2550 let stderr = String::from_utf8(result.stderr)?;
2551
2552 if result.exit_code != 0 {
2553 return Err(JavascriptBenchmarkError::NonZeroExit {
2554 scenario: scenario.id,
2555 exit_code: result.exit_code,
2556 stderr,
2557 });
2558 }
2559
2560 let parsed_metrics =
2561 parse_benchmark_metrics(scenario.id, &stdout, scenario.expect_import_metric)?;
2562 let guest_import_ms = parsed_metrics.import_ms;
2563 let completion_ms = guest_import_ms
2564 .map(|guest_ms| saturating_delta_ms(completion_total_ms, guest_ms))
2565 .unwrap_or(completion_total_ms);
2566 let wall_ms = context_setup_ms + startup_ms + completion_total_ms;
2567
2568 Ok(SampleMeasurement {
2569 wall_ms,
2570 guest_import_ms,
2571 context_setup_ms,
2572 startup_ms,
2573 completion_ms,
2574 resource_usage: parsed_metrics.resource_usage,
2575 })
2576}
2577
2578fn run_host_node_sample(
2579 workspace: &BenchmarkWorkspace,
2580 scenario: &ScenarioDefinition,
2581) -> Result<SampleMeasurement, JavascriptBenchmarkError> {
2582 let started_at = Instant::now();
2583 let output = Command::new(crate::host_node::node_binary())
2584 .arg(scenario.entrypoint)
2585 .current_dir(&workspace.root)
2586 .envs(scenario_env(workspace, scenario))
2587 .output()?;
2588 let wall_ms = started_at.elapsed().as_secs_f64() * 1000.0;
2589 let stdout = String::from_utf8(output.stdout)?;
2590 let stderr = String::from_utf8(output.stderr)?;
2591
2592 if !output.status.success() {
2593 return Err(JavascriptBenchmarkError::NonZeroExit {
2594 scenario: scenario.id,
2595 exit_code: output.status.code().unwrap_or(-1),
2596 stderr,
2597 });
2598 }
2599
2600 let parsed_metrics =
2601 parse_benchmark_metrics(scenario.id, &stdout, scenario.expect_import_metric)?;
2602 let guest_import_ms = parsed_metrics.import_ms;
2603 let startup_ms = guest_import_ms
2604 .map(|guest_ms| saturating_delta_ms(wall_ms, guest_ms))
2605 .unwrap_or(wall_ms);
2606
2607 Ok(SampleMeasurement {
2608 wall_ms,
2609 guest_import_ms,
2610 context_setup_ms: 0.0,
2611 startup_ms,
2612 completion_ms: 0.0,
2613 resource_usage: parsed_metrics.resource_usage,
2614 })
2615}
2616
2617fn scenario_env(
2618 workspace: &BenchmarkWorkspace,
2619 scenario: &ScenarioDefinition,
2620) -> BTreeMap<String, String> {
2621 match scenario.env {
2622 ScenarioEnvironment::None => BTreeMap::new(),
2623 ScenarioEnvironment::ProjectedWorkspaceNodeModules => {
2624 let projected_node_modules = workspace.repo_root.join("node_modules");
2625 let projected_node_modules_json =
2626 serde_json::to_string(&vec![projected_node_modules.display().to_string()])
2627 .expect("serialize projected node_modules read path");
2628 let guest_path_mappings = serde_json::json!([{
2629 "guestPath": "/root/node_modules",
2630 "hostPath": projected_node_modules.display().to_string(),
2631 }])
2632 .to_string();
2633
2634 BTreeMap::from([
2635 (
2636 String::from("AGENTOS_EXTRA_FS_READ_PATHS"),
2637 projected_node_modules_json,
2638 ),
2639 (
2640 String::from("AGENTOS_GUEST_PATH_MAPPINGS"),
2641 guest_path_mappings,
2642 ),
2643 ])
2644 }
2645 }
2646}
2647
2648fn measure_transport_rtt(
2649 workspace: &BenchmarkWorkspace,
2650 config: &JavascriptBenchmarkConfig,
2651) -> Result<Vec<BenchmarkTransportRttReport>, JavascriptBenchmarkError> {
2652 let mut engine = JavascriptExecutionEngine::default();
2653 let context = engine.create_context(CreateJavascriptContextRequest {
2654 vm_id: String::from("vm-transport"),
2655 bootstrap_module: None,
2656 compile_cache_root: None,
2657 });
2658 let mut execution = engine.start_execution(StartJavascriptExecutionRequest {
2659 limits: Default::default(),
2660 guest_runtime: Default::default(),
2661 vm_id: String::from("vm-transport"),
2662 context_id: context.context_id,
2663 argv: vec![String::from("./bench/transport-echo.mjs")],
2664 env: BTreeMap::from([(String::from("AGENTOS_KEEP_STDIN_OPEN"), String::from("1"))]),
2665 cwd: workspace.root.clone(),
2666 wasm_module_bytes: None,
2667 inline_code: None,
2668 })?;
2669
2670 let mut stdout_buffer = String::new();
2671 let mut stderr_buffer = String::new();
2672 let mut reports = Vec::with_capacity(TRANSPORT_RTT_PAYLOAD_BYTES.len());
2673
2674 for payload_bytes in TRANSPORT_RTT_PAYLOAD_BYTES {
2675 for warmup_index in 0..config.warmup_iterations {
2676 let label = format!("warmup-{}-{warmup_index}", payload_bytes);
2677 measure_transport_roundtrip(
2678 &mut execution,
2679 payload_bytes,
2680 &label,
2681 &mut stdout_buffer,
2682 &mut stderr_buffer,
2683 )?;
2684 }
2685
2686 let mut samples_ms = Vec::with_capacity(config.iterations);
2687 for iteration in 0..config.iterations {
2688 let label = format!("measure-{}-{iteration}", payload_bytes);
2689 samples_ms.push(measure_transport_roundtrip(
2690 &mut execution,
2691 payload_bytes,
2692 &label,
2693 &mut stdout_buffer,
2694 &mut stderr_buffer,
2695 )?);
2696 }
2697
2698 reports.push(BenchmarkTransportRttReport {
2699 channel: TRANSPORT_RTT_CHANNEL,
2700 payload_bytes,
2701 stats: compute_stats(&samples_ms),
2702 samples_ms,
2703 });
2704 }
2705
2706 execution.close_stdin()?;
2707 let result = execution.wait()?;
2708 if result.exit_code != 0 {
2709 stderr_buffer.push_str(&String::from_utf8(result.stderr)?);
2710 return Err(JavascriptBenchmarkError::TransportProbeExited {
2711 exit_code: result.exit_code,
2712 stderr: stderr_buffer,
2713 });
2714 }
2715
2716 Ok(reports)
2717}
2718
2719fn measure_transport_roundtrip(
2720 execution: &mut crate::JavascriptExecution,
2721 payload_bytes: usize,
2722 label: &str,
2723 stdout_buffer: &mut String,
2724 stderr_buffer: &mut String,
2725) -> Result<f64, JavascriptBenchmarkError> {
2726 let payload = transport_probe_payload(payload_bytes, label);
2727 let expected_line = format!("{payload}\n");
2728 let started_at = Instant::now();
2729 execution.write_stdin(expected_line.as_bytes())?;
2730
2731 loop {
2732 if let Some(line) = take_complete_line(stdout_buffer) {
2733 if line == payload {
2734 return Ok(started_at.elapsed().as_secs_f64() * 1000.0);
2735 }
2736 return Err(JavascriptBenchmarkError::InvalidTransportProbeResponse {
2737 payload_bytes,
2738 expected: payload,
2739 actual: line,
2740 });
2741 }
2742
2743 match execution.poll_event_blocking(TRANSPORT_POLL_TIMEOUT)? {
2744 Some(crate::JavascriptExecutionEvent::Stdout(chunk)) => {
2745 stdout_buffer.push_str(&String::from_utf8(chunk)?);
2746 }
2747 Some(crate::JavascriptExecutionEvent::Stderr(chunk)) => {
2748 stderr_buffer.push_str(&String::from_utf8(chunk)?);
2749 }
2750 Some(crate::JavascriptExecutionEvent::SyncRpcRequest(request)) => {
2751 return Err(JavascriptBenchmarkError::Execution(
2752 JavascriptExecutionError::PendingSyncRpcRequest(request.id),
2753 ));
2754 }
2755 Some(crate::JavascriptExecutionEvent::SignalState { .. }) => {}
2756 Some(crate::JavascriptExecutionEvent::Exited(exit_code)) => {
2757 return Err(JavascriptBenchmarkError::TransportProbeExited {
2758 exit_code,
2759 stderr: stderr_buffer.clone(),
2760 });
2761 }
2762 None => {
2763 return Err(JavascriptBenchmarkError::TransportProbeTimeout { payload_bytes });
2764 }
2765 }
2766 }
2767}
2768
2769fn transport_probe_payload(payload_bytes: usize, label: &str) -> String {
2770 if payload_bytes == 0 {
2771 return format!("transport:{label}:");
2772 }
2773
2774 let header = format!("transport:{label}:");
2775 let fill_len = payload_bytes.saturating_sub(header.len());
2776 format!("{header}{}", "x".repeat(fill_len))
2777}
2778
2779fn take_complete_line(buffer: &mut String) -> Option<String> {
2780 let newline_index = buffer.find('\n')?;
2781 let line = buffer[..newline_index].trim_end_matches('\r').to_owned();
2782 buffer.drain(..=newline_index);
2783 Some(line)
2784}
2785
2786#[derive(Debug, Default, Deserialize)]
2787struct ParsedBenchmarkMetrics {
2788 #[serde(default)]
2789 import_ms: Option<f64>,
2790 #[serde(default)]
2791 resource_usage: Option<BenchmarkResourceUsage<f64>>,
2792}
2793
2794fn parse_benchmark_metrics(
2795 scenario_id: &'static str,
2796 stdout: &str,
2797 expect_import_metric: bool,
2798) -> Result<ParsedBenchmarkMetrics, JavascriptBenchmarkError> {
2799 let raw_value = stdout
2800 .lines()
2801 .rev()
2802 .find_map(|line| line.strip_prefix(BENCHMARK_MARKER_PREFIX))
2803 .ok_or(JavascriptBenchmarkError::MissingBenchmarkMetric(
2804 scenario_id,
2805 ))?
2806 .trim();
2807
2808 if let Ok(parsed) = serde_json::from_str::<ParsedBenchmarkMetrics>(raw_value) {
2809 let has_resource_usage = match parsed.resource_usage.as_ref() {
2810 Some(resource_usage) => !resource_usage.is_empty(),
2811 None => false,
2812 };
2813 if parsed.import_ms.is_some() || has_resource_usage {
2814 if expect_import_metric && parsed.import_ms.is_none() {
2815 return Err(JavascriptBenchmarkError::MissingBenchmarkMetric(
2816 scenario_id,
2817 ));
2818 }
2819 return Ok(parsed);
2820 }
2821 }
2822
2823 raw_value
2824 .parse::<f64>()
2825 .map(|import_ms| ParsedBenchmarkMetrics {
2826 import_ms: Some(import_ms),
2827 resource_usage: None,
2828 })
2829 .map_err(|_| JavascriptBenchmarkError::InvalidBenchmarkMetric {
2830 scenario: scenario_id,
2831 raw_value: raw_value.to_owned(),
2832 })
2833}
2834
2835fn workspace_root() -> Result<PathBuf, JavascriptBenchmarkError> {
2836 let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
2837 manifest_dir
2838 .parent()
2839 .and_then(Path::parent)
2840 .map(Path::to_path_buf)
2841 .ok_or(JavascriptBenchmarkError::InvalidWorkspaceRoot(manifest_dir))
2842}
2843
2844fn load_benchmark_artifact(
2845 baseline_path: &Path,
2846) -> Result<StoredBenchmarkArtifact, JavascriptBenchmarkError> {
2847 let raw = fs::read_to_string(baseline_path)?;
2848 serde_json::from_str(&raw).map_err(|err| JavascriptBenchmarkError::InvalidBaselineReport {
2849 path: baseline_path.to_path_buf(),
2850 message: err.to_string(),
2851 })
2852}
2853
2854fn benchmark_host() -> Result<BenchmarkHost, JavascriptBenchmarkError> {
2855 let node_binary = crate::host_node::node_binary();
2856 let output = Command::new(&node_binary)
2857 .arg("--version")
2858 .output()
2859 .map_err(JavascriptBenchmarkError::NodeVersion)?;
2860 let node_version = String::from_utf8(output.stdout)?;
2861
2862 Ok(BenchmarkHost {
2863 node_binary,
2864 node_version,
2865 os: env::consts::OS,
2866 arch: env::consts::ARCH,
2867 logical_cpus: std::thread::available_parallelism()
2868 .map(usize::from)
2869 .unwrap_or(1),
2870 })
2871}
2872
2873fn write_benchmark_workspace(
2874 root: &Path,
2875 repo_root: &Path,
2876) -> Result<(), JavascriptBenchmarkError> {
2877 fs::create_dir_all(root.join("bench"))?;
2878 fs::create_dir_all(root.join("bench/local-graph"))?;
2879 let host_node_modules = repo_root.join("node_modules");
2880 let workspace_node_modules = root.join("node_modules");
2881 if host_node_modules.exists() && !workspace_node_modules.exists() {
2882 std::os::unix::fs::symlink(&host_node_modules, &workspace_node_modules)?;
2883 }
2884 fs::write(
2885 root.join("package.json"),
2886 "{\n \"name\": \"agentos-execution-bench\",\n \"private\": true,\n \"type\": \"module\"\n}\n",
2887 )?;
2888
2889 for index in 0..LOCAL_GRAPH_MODULE_COUNT {
2890 let path = root
2891 .join("bench/local-graph")
2892 .join(format!("mod-{index:02}.mjs"));
2893 let source = if index == 0 {
2894 String::from("export const value = 1;\n")
2895 } else {
2896 format!(
2897 "import {{ value as previous }} from './mod-{previous:02}.mjs';\nexport const value = previous + {index};\n",
2898 previous = index - 1
2899 )
2900 };
2901 fs::write(path, source)?;
2902 }
2903
2904 let final_value = local_graph_terminal_value();
2905 fs::write(
2906 root.join("bench/local-graph/root.mjs"),
2907 format!(
2908 "import {{ value }} from './mod-{last:02}.mjs';\nexport {{ value }};\nexport const expected = {final_value};\n",
2909 last = LOCAL_GRAPH_MODULE_COUNT - 1
2910 ),
2911 )?;
2912 fs::write(
2913 root.join("bench/benchmark-metrics.mjs"),
2914 benchmark_metrics_module_source(),
2915 )?;
2916
2917 fs::write(
2918 root.join("bench/isolate-startup.mjs"),
2919 resource_only_entrypoint_source("console.log('isolate-ready');"),
2920 )?;
2921 fs::write(
2922 root.join("bench/cold-local-import.mjs"),
2923 local_import_entrypoint_source(final_value),
2924 )?;
2925 fs::write(
2926 root.join("bench/warm-local-import.mjs"),
2927 local_import_entrypoint_source(final_value),
2928 )?;
2929 fs::write(
2930 root.join("bench/builtin-import.mjs"),
2931 timed_entrypoint_source(
2932 "const [pathMod, fsMod, urlMod] = await Promise.all([\n import('node:path'),\n import('node:fs/promises'),\n import('node:url'),\n]);\nif (typeof pathMod.basename !== 'function' || typeof fsMod.readFile !== 'function' || typeof urlMod.pathToFileURL !== 'function') {\n throw new Error('builtin import fixture did not load expected exports');\n}",
2933 ),
2934 )?;
2935 fs::write(
2936 root.join("bench/hot-builtin-stream-import.mjs"),
2937 single_import_entrypoint_source(
2938 "node:stream",
2939 "typeof imported.Readable === 'function'",
2940 "node:stream import did not expose Readable",
2941 ),
2942 )?;
2943 fs::write(
2944 root.join("bench/hot-builtin-stream-web-import.mjs"),
2945 single_import_entrypoint_source(
2946 "node:stream/web",
2947 "typeof imported.ReadableStream === 'function'",
2948 "node:stream/web import did not expose ReadableStream",
2949 ),
2950 )?;
2951 fs::write(
2952 root.join("bench/hot-builtin-crypto-import.mjs"),
2953 single_import_entrypoint_source(
2954 "node:crypto",
2955 "typeof imported.createHash === 'function'",
2956 "node:crypto import did not expose createHash",
2957 ),
2958 )?;
2959 fs::write(
2960 root.join("bench/hot-builtin-zlib-import.mjs"),
2961 single_import_entrypoint_source(
2962 "node:zlib",
2963 "typeof imported.gzipSync === 'function'",
2964 "node:zlib import did not expose gzipSync",
2965 ),
2966 )?;
2967 fs::write(
2968 root.join("bench/hot-builtin-assert-import.mjs"),
2969 single_import_entrypoint_source(
2970 "node:assert/strict",
2971 "typeof imported.strictEqual === 'function'",
2972 "node:assert/strict import did not expose strictEqual",
2973 ),
2974 )?;
2975 fs::write(
2976 root.join("bench/hot-builtin-url-import.mjs"),
2977 single_import_entrypoint_source(
2978 "node:url",
2979 "typeof imported.pathToFileURL === 'function'",
2980 "node:url import did not expose pathToFileURL",
2981 ),
2982 )?;
2983 fs::write(
2984 root.join("bench/large-package-import.mjs"),
2985 timed_entrypoint_source(
2986 "const typescript = await import('typescript');\nif (typeof typescript.transpileModule !== 'function') {\n throw new Error('typescript import did not expose transpileModule');\n}",
2987 ),
2988 )?;
2989 fs::write(
2990 root.join("bench/hot-projected-package-file-import.mjs"),
2991 projected_package_file_import_entrypoint_source(),
2992 )?;
2993 fs::write(
2994 root.join("bench/projected-package-import.mjs"),
2995 projected_package_import_entrypoint_source(),
2996 )?;
2997 fs::write(
2998 root.join("bench/pdf-lib-startup.mjs"),
2999 pdf_lib_startup_entrypoint_source(),
3000 )?;
3001 fs::write(
3002 root.join("bench/jszip-startup.mjs"),
3003 jszip_startup_entrypoint_source(),
3004 )?;
3005 fs::write(
3006 root.join("bench/jszip-end-to-end.mjs"),
3007 jszip_end_to_end_entrypoint_source(),
3008 )?;
3009 fs::write(
3010 root.join("bench/jszip-repeated-session-compressed.mjs"),
3011 jszip_repeated_session_compressed_entrypoint_source(),
3012 )?;
3013 fs::write(
3014 root.join("bench/transport-echo.mjs"),
3015 "process.stdin.setEncoding('utf8');\nlet buffered = '';\nconst flushLines = () => {\n let newlineIndex = buffered.indexOf('\\n');\n while (newlineIndex >= 0) {\n const line = buffered.slice(0, newlineIndex).replace(/\\r$/, '');\n buffered = buffered.slice(newlineIndex + 1);\n process.stdout.write(line);\n newlineIndex = buffered.indexOf('\\n');\n }\n};\nprocess.stdin.on('data', (chunk) => {\n buffered += chunk;\n flushLines();\n});\nprocess.stdin.on('end', () => {\n if (buffered.length > 0) {\n process.stdout.write(buffered.replace(/\\r$/, ''));\n }\n});\n",
3016 )?;
3017
3018 Ok(())
3019}
3020
3021fn local_import_entrypoint_source(final_value: usize) -> String {
3022 timed_entrypoint_source(&format!(
3023 "const graph = await import('./local-graph/root.mjs');\nif (graph.value !== {final_value} || graph.expected !== {final_value}) {{\n throw new Error(`local graph import returned ${{\n graph.value\n }} instead of {final_value}`);\n}}"
3024 ))
3025}
3026
3027fn single_import_entrypoint_source(
3028 specifier: &str,
3029 validation_expression: &str,
3030 error_message: &str,
3031) -> String {
3032 timed_entrypoint_source(&format!(
3033 "const imported = await import('{specifier}');\nif (!({validation_expression})) {{\n throw new Error('{error_message}');\n}}"
3034 ))
3035}
3036
3037fn projected_package_file_import_entrypoint_source() -> String {
3038 timed_entrypoint_source(
3039 "const typescriptModule = await import('../node_modules/typescript/lib/typescript.js');\nconst typescript = typescriptModule.default ?? typescriptModule;\nif (typeof typescript.transpileModule !== 'function') {\n throw new Error('projected package file import did not expose transpileModule');\n}",
3040 )
3041}
3042
3043fn projected_package_import_entrypoint_source() -> String {
3044 timed_entrypoint_source(
3045 "const typescriptModule = await import('../node_modules/typescript/lib/typescript.js');\nconst typescript = typescriptModule.default ?? typescriptModule;\nconst sourceFile = typescript.createSourceFile(\n 'bench.ts',\n 'const answer: number = 42;',\n typescript.ScriptTarget.ES2022,\n true,\n);\nif (\n typeof typescript.transpileModule !== 'function' ||\n typeof typescript.createSourceFile !== 'function' ||\n !sourceFile ||\n sourceFile.statements.length !== 1\n) {\n throw new Error('projected package import did not expose TypeScript compiler APIs');\n}",
3046 )
3047}
3048
3049fn pdf_lib_startup_entrypoint_source() -> String {
3050 timed_entrypoint_source(
3051 "const pdfLib = await import('pdf-lib');\nconst pdfDoc = await pdfLib.PDFDocument.create();\nconst page = pdfDoc.addPage([612, 792]);\nconst font = await pdfDoc.embedFont(pdfLib.StandardFonts.Helvetica);\npage.drawText('secure-exec pdf-lib benchmark', {\n x: 50,\n y: 750,\n font,\n size: 18,\n});\nif (pdfDoc.getPageCount() !== 1 || page.getSize().width !== 612) {\n throw new Error('pdf-lib fixture did not create the expected document');\n}",
3052 )
3053}
3054
3055fn jszip_startup_entrypoint_source() -> String {
3056 timed_entrypoint_source(
3057 "const jszipModule = await import('jszip');\nconst JSZip = jszipModule.default ?? jszipModule;\nconst zip = new JSZip();\nzip.file('README.txt', 'secure-exec benchmark archive');\nconst notes = zip.folder('notes');\nif (!notes) {\n throw new Error('jszip fixture failed to create nested folder');\n}\nnotes.file('todo.txt', 'benchmark staging payload');\nconst fileCount = Object.values(zip.files).filter((entry) => !entry.dir).length;\nif (typeof zip.generateAsync !== 'function' || fileCount !== 2) {\n throw new Error('jszip fixture did not stage the expected archive');\n}",
3058 )
3059}
3060
3061fn jszip_end_to_end_entrypoint_source() -> String {
3062 timed_entrypoint_source(
3063 "const jszipModule = await import('jszip');\nconst JSZip = jszipModule.default ?? jszipModule;\nconst zip = new JSZip();\nconst repeatedPayload = 'secure-exec benchmark payload '.repeat(512);\nzip.file('README.txt', repeatedPayload);\nconst notes = zip.folder('notes');\nif (!notes) {\n throw new Error('jszip end-to-end fixture failed to create notes folder');\n}\nnotes.file('todo.txt', 'complete the archive roundtrip');\nconst data = zip.folder('data');\nif (!data) {\n throw new Error('jszip end-to-end fixture failed to create data folder');\n}\ndata.file('payload.json', JSON.stringify({\n repeatedPayloadLength: repeatedPayload.length,\n mode: 'cold-end-to-end',\n}));\nconst archiveBytes = await zip.generateAsync({\n type: 'uint8array',\n compression: 'DEFLATE',\n compressionOptions: { level: 6 },\n});\nconst restored = await JSZip.loadAsync(archiveBytes);\nconst restoredFileCount = Object.values(restored.files).filter((entry) => !entry.dir).length;\nconst restoredReadme = await restored.file('README.txt')?.async('string');\nconst restoredTodo = await restored.file('notes/todo.txt')?.async('string');\nconst restoredPayload = await restored.file('data/payload.json')?.async('string');\nif (\n archiveBytes.byteLength >= repeatedPayload.length ||\n restoredFileCount !== 3 ||\n restoredReadme !== repeatedPayload ||\n restoredTodo !== 'complete the archive roundtrip' ||\n !restoredPayload?.includes('cold-end-to-end')\n) {\n throw new Error('jszip end-to-end fixture did not complete the compressed archive roundtrip');\n}",
3064 )
3065}
3066
3067fn jszip_repeated_session_compressed_entrypoint_source() -> String {
3068 timed_entrypoint_source(
3069 "const jszipModule = await import('jszip');\nconst JSZip = jszipModule.default ?? jszipModule;\nconst zip = new JSZip();\nconst repeatedPayload = 'secure-exec benchmark payload '.repeat(512);\nzip.file('README.txt', repeatedPayload);\nconst notes = zip.folder('notes');\nif (!notes) {\n throw new Error('jszip repeated-session fixture failed to create notes folder');\n}\nnotes.file('todo.txt', 'repeat this session workload');\nconst data = zip.folder('data');\nif (!data) {\n throw new Error('jszip repeated-session fixture failed to create data folder');\n}\ndata.file('payload.json', JSON.stringify({\n repeatedPayloadLength: repeatedPayload.length,\n repeatedSessions: true,\n}));\nconst archiveBytes = await zip.generateAsync({\n type: 'uint8array',\n compression: 'DEFLATE',\n compressionOptions: { level: 6 },\n});\nconst restored = await JSZip.loadAsync(archiveBytes);\nconst restoredFileCount = Object.values(restored.files).filter((entry) => !entry.dir).length;\nconst restoredReadme = await restored.file('README.txt')?.async('string');\nconst restoredTodo = await restored.file('notes/todo.txt')?.async('string');\nif (\n archiveBytes.byteLength >= repeatedPayload.length ||\n restoredFileCount !== 3 ||\n restoredReadme !== repeatedPayload ||\n restoredTodo !== 'repeat this session workload'\n) {\n throw new Error('jszip repeated-session fixture did not complete the compressed archive roundtrip');\n}",
3070 )
3071}
3072
3073fn benchmark_metrics_module_source() -> String {
3074 format!(
3075 "const BENCHMARK_MARKER_PREFIX = '{BENCHMARK_MARKER_PREFIX}';\n\nexport function emitBenchmarkMetrics(importMs) {{\n const memoryUsage = process.memoryUsage();\n const resourceUsage = typeof process.resourceUsage === 'function'\n ? process.resourceUsage()\n : null;\n const payload = {{\n resource_usage: {{\n rss_bytes: memoryUsage.rss,\n heap_used_bytes: memoryUsage.heapUsed,\n ...(resourceUsage\n ? {{\n cpu_user_us: resourceUsage.userCPUTime,\n cpu_system_us: resourceUsage.systemCPUTime,\n cpu_total_us: resourceUsage.userCPUTime + resourceUsage.systemCPUTime,\n }}\n : {{}}),\n }},\n }};\n\n if (typeof importMs === 'number') {{\n payload.import_ms = importMs;\n }}\n\n console.log(BENCHMARK_MARKER_PREFIX + JSON.stringify(payload));\n}}\n"
3076 )
3077}
3078
3079fn resource_only_entrypoint_source(body: &str) -> String {
3080 format!(
3081 "import {{ emitBenchmarkMetrics }} from './benchmark-metrics.mjs';\n{body}\nemitBenchmarkMetrics();\n"
3082 )
3083}
3084
3085fn timed_entrypoint_source(body: &str) -> String {
3086 format!(
3087 "import {{ performance }} from 'node:perf_hooks';\nimport {{ emitBenchmarkMetrics }} from './benchmark-metrics.mjs';\nconst started = performance.now();\n{body}\nemitBenchmarkMetrics(performance.now() - started);\n"
3088 )
3089}
3090
3091fn local_graph_terminal_value() -> usize {
3092 let mut value = 1;
3093
3094 for index in 1..LOCAL_GRAPH_MODULE_COUNT {
3095 value += index;
3096 }
3097
3098 value
3099}
3100
3101fn compute_distribution_stats(samples: &[f64]) -> BenchmarkDistributionStats {
3102 let mut sorted = samples.to_vec();
3103 sorted.sort_by(|a, b| a.total_cmp(b));
3104 let mean = sorted.iter().sum::<f64>() / sorted.len() as f64;
3105
3106 BenchmarkDistributionStats {
3107 mean,
3108 p50: percentile(&sorted, 50.0),
3109 p95: percentile(&sorted, 95.0),
3110 min: *sorted.first().unwrap_or(&0.0),
3111 max: *sorted.last().unwrap_or(&0.0),
3112 stddev: standard_deviation(&sorted, mean),
3113 }
3114}
3115
3116fn compute_stats(samples: &[f64]) -> BenchmarkStats {
3117 let stats = compute_distribution_stats(samples);
3118
3119 BenchmarkStats {
3120 mean_ms: stats.mean,
3121 p50_ms: stats.p50,
3122 p95_ms: stats.p95,
3123 min_ms: stats.min,
3124 max_ms: stats.max,
3125 stddev_ms: stats.stddev,
3126 }
3127}
3128
3129fn compute_resource_usage_stats(
3130 samples: &BenchmarkResourceUsage<Vec<f64>>,
3131) -> Option<BenchmarkResourceUsage<BenchmarkDistributionStats>> {
3132 let stats = BenchmarkResourceUsage {
3133 rss_bytes: samples
3134 .rss_bytes
3135 .as_ref()
3136 .map(|samples| compute_distribution_stats(samples)),
3137 heap_used_bytes: samples
3138 .heap_used_bytes
3139 .as_ref()
3140 .map(|samples| compute_distribution_stats(samples)),
3141 cpu_user_us: samples
3142 .cpu_user_us
3143 .as_ref()
3144 .map(|samples| compute_distribution_stats(samples)),
3145 cpu_system_us: samples
3146 .cpu_system_us
3147 .as_ref()
3148 .map(|samples| compute_distribution_stats(samples)),
3149 cpu_total_us: samples
3150 .cpu_total_us
3151 .as_ref()
3152 .map(|samples| compute_distribution_stats(samples)),
3153 };
3154
3155 (!stats.is_empty()).then_some(stats)
3156}
3157
3158fn standard_deviation(samples: &[f64], mean: f64) -> f64 {
3159 if samples.is_empty() {
3160 return 0.0;
3161 }
3162
3163 let variance = samples
3164 .iter()
3165 .map(|sample| {
3166 let delta = sample - mean;
3167 delta * delta
3168 })
3169 .sum::<f64>()
3170 / samples.len() as f64;
3171
3172 variance.sqrt()
3173}
3174
3175fn percentile(sorted: &[f64], p: f64) -> f64 {
3176 if sorted.is_empty() {
3177 return 0.0;
3178 }
3179
3180 let rank = ((p / 100.0) * sorted.len() as f64).ceil() as usize;
3181 let index = rank.saturating_sub(1).min(sorted.len() - 1);
3182 sorted[index]
3183}
3184
3185fn percentage_reduction(original: f64, current: f64) -> f64 {
3186 if original <= 0.0 {
3187 0.0
3188 } else {
3189 ((original - current) / original) * 100.0
3190 }
3191}
3192
3193fn percentage_share(part: f64, total: f64) -> f64 {
3194 if total <= 0.0 {
3195 0.0
3196 } else {
3197 (part / total) * 100.0
3198 }
3199}
3200
3201fn safe_ratio(lhs: f64, rhs: f64) -> f64 {
3202 if rhs <= 0.0 {
3203 0.0
3204 } else {
3205 lhs / rhs
3206 }
3207}
3208
3209fn saturating_delta_ms(total_ms: f64, subtracted_ms: f64) -> f64 {
3210 (total_ms - subtracted_ms).max(0.0)
3211}
3212
3213fn format_ms(value: f64) -> String {
3214 format!("{value:.2}")
3215}
3216
3217fn format_hotspot_value(unit: &str, value: f64) -> String {
3218 match unit {
3219 "pct" => format!("{value:.1}%"),
3220 "MiB" => format_mib(value),
3221 _ => format_ms(value),
3222 }
3223}
3224
3225fn format_sample_list(samples: &[f64]) -> String {
3226 format_scaled_sample_list(samples, std::convert::identity)
3227}
3228
3229fn format_scaled_sample_list(samples: &[f64], scale: impl Fn(f64) -> f64) -> String {
3230 let mut formatted = String::from("[");
3231
3232 for (index, sample) in samples.iter().enumerate() {
3233 if index > 0 {
3234 formatted.push_str(", ");
3235 }
3236 let _ = write!(&mut formatted, "{:.2}", scale(*sample));
3237 }
3238
3239 formatted.push(']');
3240 formatted
3241}
3242
3243fn format_mib(value: f64) -> String {
3244 format!("{value:.2}")
3245}
3246
3247fn format_label_list(labels: &[&str]) -> String {
3248 labels
3249 .iter()
3250 .map(|label| format!("`{label}`"))
3251 .collect::<Vec<_>>()
3252 .join(", ")
3253}
3254
3255fn format_string_label_list(labels: &[&str]) -> String {
3256 labels
3257 .iter()
3258 .map(|label| format!("`{label}`"))
3259 .collect::<Vec<_>>()
3260 .join(", ")
3261}
3262
3263fn push_unique_label<'a>(labels: &mut Vec<&'a str>, value: &'a str) {
3264 if !labels.contains(&value) {
3265 labels.push(value);
3266 }
3267}
3268
3269fn format_delta_ms(value: f64) -> String {
3270 format!("{value:+.2}")
3271}
3272
3273fn format_delta_pct(value: f64) -> String {
3274 format!("{value:+.1}%")
3275}
3276
3277fn push_optional_sample(samples: &mut Option<Vec<f64>>, value: Option<f64>) {
3278 if let Some(value) = value {
3279 samples.get_or_insert_with(Vec::new).push(value);
3280 }
3281}
3282
3283fn bytes_to_mib(value: f64) -> f64 {
3284 value / (1024.0 * 1024.0)
3285}
3286
3287fn micros_to_ms(value: f64) -> f64 {
3288 value / 1000.0
3289}
3290
3291fn hotspot_wall_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3292 Some(scenario.wall_stats.mean_ms)
3293}
3294
3295fn hotspot_wall_stddev_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3296 Some(scenario.wall_stats.stddev_ms)
3297}
3298
3299fn hotspot_wall_range_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3300 Some(scenario.wall_range_ms())
3301}
3302
3303fn hotspot_guest_import_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3304 scenario
3305 .guest_import_stats
3306 .as_ref()
3307 .map(|stats| stats.mean_ms)
3308}
3309
3310fn hotspot_startup_overhead_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3311 scenario
3312 .startup_overhead_stats
3313 .as_ref()
3314 .map(|stats| stats.mean_ms)
3315}
3316
3317fn hotspot_context_setup_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3318 Some(scenario.phase_stats.context_setup_ms.mean_ms)
3319}
3320
3321fn hotspot_startup_phase_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3322 Some(scenario.phase_stats.startup_ms.mean_ms)
3323}
3324
3325fn hotspot_guest_execution_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3326 scenario
3327 .phase_stats
3328 .guest_execution_ms
3329 .as_ref()
3330 .map(|stats| stats.mean_ms)
3331}
3332
3333fn hotspot_completion_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3334 Some(scenario.phase_stats.completion_ms.mean_ms)
3335}
3336
3337fn hotspot_startup_share_pct(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3338 scenario.mean_startup_share_pct()
3339}
3340
3341fn hotspot_rss_mean_mib(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3342 scenario
3343 .resource_usage_stats
3344 .as_ref()?
3345 .rss_bytes
3346 .as_ref()
3347 .map(|stats| bytes_to_mib(stats.mean))
3348}
3349
3350fn hotspot_heap_mean_mib(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3351 scenario
3352 .resource_usage_stats
3353 .as_ref()?
3354 .heap_used_bytes
3355 .as_ref()
3356 .map(|stats| bytes_to_mib(stats.mean))
3357}
3358
3359fn hotspot_total_cpu_mean_ms(scenario: &BenchmarkScenarioReport) -> Option<f64> {
3360 scenario
3361 .resource_usage_stats
3362 .as_ref()?
3363 .cpu_total_us
3364 .as_ref()
3365 .map(|stats| micros_to_ms(stats.mean))
3366}
3367
3368#[cfg(test)]
3369mod tests {
3370 use super::*;
3371 use std::cell::RefCell;
3372 use tempfile::tempdir;
3373
3374 fn synthetic_transport_reports() -> Vec<BenchmarkTransportRttReport> {
3375 TRANSPORT_RTT_PAYLOAD_BYTES
3376 .iter()
3377 .enumerate()
3378 .map(|(index, payload_bytes)| {
3379 let sample = index as f64 + 1.0;
3380 BenchmarkTransportRttReport {
3381 channel: TRANSPORT_RTT_CHANNEL,
3382 payload_bytes: *payload_bytes,
3383 samples_ms: vec![sample],
3384 stats: compute_stats(&[sample]),
3385 }
3386 })
3387 .collect()
3388 }
3389
3390 fn synthetic_scenario_report(
3391 definition: ScenarioDefinition,
3392 wall_sample_ms: f64,
3393 ) -> BenchmarkScenarioReport {
3394 let context_setup_ms = wall_sample_ms / 5.0;
3395 let startup_ms = wall_sample_ms / 4.0;
3396 let guest_execution_ms = definition
3397 .expect_import_metric
3398 .then_some(wall_sample_ms / 3.0);
3399 let completion_ms =
3400 wall_sample_ms - context_setup_ms - startup_ms - guest_execution_ms.unwrap_or(0.0);
3401 let startup_overhead_ms = definition
3402 .expect_import_metric
3403 .then_some(context_setup_ms + startup_ms + completion_ms);
3404 let resource_usage_samples = BenchmarkResourceUsage {
3405 rss_bytes: Some(vec![64.0 * 1024.0 * 1024.0]),
3406 heap_used_bytes: Some(vec![12.0 * 1024.0 * 1024.0]),
3407 cpu_user_us: None,
3408 cpu_system_us: None,
3409 cpu_total_us: Some(vec![wall_sample_ms * 1000.0]),
3410 };
3411
3412 BenchmarkScenarioReport {
3413 id: definition.id,
3414 workload: definition.workload,
3415 runtime: definition.runtime.label(),
3416 mode: definition.mode.label(),
3417 description: definition.description,
3418 fixture: definition.fixture,
3419 compile_cache: definition.compile_cache.label(),
3420 wall_samples_ms: vec![wall_sample_ms],
3421 wall_stats: compute_stats(&[wall_sample_ms]),
3422 guest_import_samples_ms: guest_execution_ms.map(|sample| vec![sample]),
3423 guest_import_stats: guest_execution_ms.map(|sample| compute_stats(&[sample])),
3424 startup_overhead_samples_ms: startup_overhead_ms.map(|sample| vec![sample]),
3425 startup_overhead_stats: startup_overhead_ms.map(|sample| compute_stats(&[sample])),
3426 phase_samples_ms: BenchmarkScenarioPhases {
3427 context_setup_ms: vec![context_setup_ms],
3428 startup_ms: vec![startup_ms],
3429 guest_execution_ms: guest_execution_ms.map(|sample| vec![sample]),
3430 completion_ms: vec![completion_ms],
3431 },
3432 phase_stats: BenchmarkScenarioPhases {
3433 context_setup_ms: compute_stats(&[context_setup_ms]),
3434 startup_ms: compute_stats(&[startup_ms]),
3435 guest_execution_ms: guest_execution_ms.map(|sample| compute_stats(&[sample])),
3436 completion_ms: compute_stats(&[completion_ms]),
3437 },
3438 resource_usage_stats: compute_resource_usage_stats(&resource_usage_samples),
3439 resource_usage_samples: Some(resource_usage_samples),
3440 }
3441 }
3442
3443 fn synthetic_host() -> BenchmarkHost {
3444 BenchmarkHost {
3445 node_binary: String::from("node"),
3446 node_version: String::from("v22.0.0"),
3447 os: "linux",
3448 arch: "x86_64",
3449 logical_cpus: 8,
3450 }
3451 }
3452
3453 #[test]
3454 fn javascript_benchmark_config_rejects_unbounded_iteration_counts() {
3455 assert!(matches!(
3456 validate_benchmark_config(&JavascriptBenchmarkConfig {
3457 iterations: 0,
3458 warmup_iterations: 0,
3459 }),
3460 Err(JavascriptBenchmarkError::InvalidConfig(
3461 "iterations must be greater than zero"
3462 ))
3463 ));
3464 assert!(matches!(
3465 validate_benchmark_config(&JavascriptBenchmarkConfig {
3466 iterations: MAX_BENCHMARK_ITERATIONS + 1,
3467 warmup_iterations: 0,
3468 }),
3469 Err(JavascriptBenchmarkError::InvalidConfig(
3470 "iterations must be less than or equal to 1000"
3471 ))
3472 ));
3473 assert!(matches!(
3474 validate_benchmark_config(&JavascriptBenchmarkConfig {
3475 iterations: 1,
3476 warmup_iterations: MAX_BENCHMARK_WARMUP_ITERATIONS + 1,
3477 }),
3478 Err(JavascriptBenchmarkError::InvalidConfig(
3479 "warmup iterations must be less than or equal to 1000"
3480 ))
3481 ));
3482 }
3483
3484 #[test]
3485 fn javascript_benchmark_orchestration_resumes_completed_stages_from_run_state() {
3486 let tempdir = tempdir().expect("create tempdir");
3487 let repo_root = tempdir.path().join("repo");
3488 let artifact_dir = tempdir.path().join("artifacts");
3489 fs::create_dir_all(&repo_root).expect("create repo root");
3490
3491 let config = JavascriptBenchmarkConfig {
3492 iterations: 1,
3493 warmup_iterations: 0,
3494 };
3495 let host = synthetic_host();
3496 let definitions = benchmark_scenarios();
3497 let mut state = StoredBenchmarkRunState::new(&config, &host, &repo_root);
3498 state.record_transport_rtt(&synthetic_transport_reports());
3499 state.record_scenario(&synthetic_scenario_report(definitions[0], 10.0));
3500 persist_benchmark_run_state(&benchmark_run_state_path(&artifact_dir), &state)
3501 .expect("persist initial run state");
3502
3503 let transport_calls = RefCell::new(0usize);
3504 let scenario_calls = RefCell::new(Vec::new());
3505 let (report, resumed_stage_count, _) = orchestrate_javascript_benchmark_report(
3506 &config,
3507 &repo_root,
3508 &host,
3509 &artifact_dir,
3510 || {
3511 *transport_calls.borrow_mut() += 1;
3512 Ok(synthetic_transport_reports())
3513 },
3514 |definition| {
3515 scenario_calls.borrow_mut().push(definition.id.to_owned());
3516 Ok(synthetic_scenario_report(definition, 20.0))
3517 },
3518 )
3519 .expect("resume benchmark orchestration");
3520
3521 assert_eq!(resumed_stage_count, 2);
3522 assert_eq!(*transport_calls.borrow(), 0);
3523 assert_eq!(
3524 scenario_calls.borrow().as_slice(),
3525 &definitions[1..]
3526 .iter()
3527 .map(|definition| definition.id.to_owned())
3528 .collect::<Vec<_>>()
3529 );
3530 assert_eq!(
3531 report.transport_rtt.len(),
3532 TRANSPORT_RTT_PAYLOAD_BYTES.len()
3533 );
3534 assert_eq!(report.scenarios.len(), definitions.len());
3535 assert_eq!(report.scenarios[0].id, definitions[0].id);
3536 assert_eq!(report.scenarios[1].id, definitions[1].id);
3537 }
3538
3539 #[test]
3540 fn javascript_benchmark_orchestration_persists_completed_stages_before_failure() {
3541 let tempdir = tempdir().expect("create tempdir");
3542 let repo_root = tempdir.path().join("repo");
3543 let artifact_dir = tempdir.path().join("artifacts");
3544 fs::create_dir_all(&repo_root).expect("create repo root");
3545
3546 let config = JavascriptBenchmarkConfig {
3547 iterations: 1,
3548 warmup_iterations: 0,
3549 };
3550 let host = synthetic_host();
3551 let state_path = benchmark_run_state_path(&artifact_dir);
3552 let failure = orchestrate_javascript_benchmark_report(
3553 &config,
3554 &repo_root,
3555 &host,
3556 &artifact_dir,
3557 || Ok(synthetic_transport_reports()),
3558 |definition| {
3559 if definition.id == "cold-local-import" {
3560 Err(JavascriptBenchmarkError::InvalidConfig("synthetic failure"))
3561 } else {
3562 Ok(synthetic_scenario_report(definition, 15.0))
3563 }
3564 },
3565 )
3566 .expect_err("expected synthetic orchestration failure");
3567
3568 assert!(matches!(
3569 failure,
3570 JavascriptBenchmarkError::InvalidConfig("synthetic failure")
3571 ));
3572
3573 let stored_state = serde_json::from_str::<StoredBenchmarkRunState>(
3574 &fs::read_to_string(&state_path).expect("read persisted run state"),
3575 )
3576 .expect("parse persisted run state");
3577 assert!(stored_state.transport_rtt.is_some());
3578 assert_eq!(
3579 stored_state
3580 .scenarios
3581 .iter()
3582 .map(|scenario| scenario.id.as_str())
3583 .collect::<Vec<_>>(),
3584 vec!["isolate-startup", "prewarmed-isolate-startup"]
3585 );
3586 }
3587}