1use crate::crud_flow::{CrudFlowConfig, CrudFlowDetector};
4use crate::data_driven::{DataDistribution, DataDrivenConfig, DataDrivenGenerator, DataMapping};
5use crate::dynamic_params::{DynamicParamProcessor, DynamicPlaceholder};
6use crate::error::{BenchError, Result};
7use crate::executor::K6Executor;
8use crate::invalid_data::{InvalidDataConfig, InvalidDataGenerator};
9use crate::k6_gen::{K6Config, K6ScriptGenerator};
10use crate::mock_integration::{
11 MockIntegrationConfig, MockIntegrationGenerator, MockServerDetector,
12};
13use crate::owasp_api::{OwaspApiConfig, OwaspApiGenerator, OwaspCategory, ReportFormat};
14use crate::parallel_executor::{AggregatedResults, ParallelExecutor};
15use crate::parallel_requests::{ParallelConfig, ParallelRequestGenerator};
16use crate::param_overrides::ParameterOverrides;
17use crate::reporter::TerminalReporter;
18use crate::request_gen::RequestGenerator;
19use crate::scenarios::LoadScenario;
20use crate::security_payloads::{
21 SecurityCategory, SecurityPayload, SecurityPayloads, SecurityTestConfig, SecurityTestGenerator,
22};
23use crate::spec_dependencies::{
24 topological_sort, DependencyDetector, ExtractedValues, SpecDependencyConfig,
25};
26use crate::spec_parser::SpecParser;
27use crate::target_parser::parse_targets_file;
28use crate::wafbench::WafBenchLoader;
29use mockforge_openapi::multi_spec::{
30 load_specs_from_directory, load_specs_from_files, merge_specs, ConflictStrategy,
31};
32use mockforge_openapi::spec::OpenApiSpec;
33use std::collections::{HashMap, HashSet};
34use std::path::{Path, PathBuf};
35use std::str::FromStr;
36
37pub fn parse_header_string(input: &str) -> Result<HashMap<String, String>> {
45 let mut headers = HashMap::new();
46
47 for pair in input.split(',') {
48 let parts: Vec<&str> = pair.splitn(2, ':').collect();
49 if parts.len() != 2 {
50 return Err(BenchError::Other(format!(
51 "Invalid header format: '{}'. Expected 'Key:Value'",
52 pair
53 )));
54 }
55 headers.insert(parts[0].trim().to_string(), parts[1].trim().to_string());
56 }
57
58 Ok(headers)
59}
60
61pub struct BenchCommand {
63 pub spec: Vec<PathBuf>,
65 pub spec_dir: Option<PathBuf>,
67 pub merge_conflicts: String,
69 pub spec_mode: String,
71 pub dependency_config: Option<PathBuf>,
73 pub target: String,
74 pub base_path: Option<String>,
77 pub duration: String,
78 pub vus: u32,
79 pub target_rps: Option<u32>,
85 pub no_keep_alive: bool,
90 pub scenario: String,
91 pub operations: Option<String>,
92 pub exclude_operations: Option<String>,
96 pub auth: Option<String>,
97 pub headers: Option<String>,
98 pub output: PathBuf,
99 pub generate_only: bool,
100 pub script_output: Option<PathBuf>,
101 pub threshold_percentile: String,
102 pub threshold_ms: u64,
103 pub max_error_rate: f64,
104 pub verbose: bool,
105 pub skip_tls_verify: bool,
106 pub chunked_request_bodies: bool,
111 pub targets_file: Option<PathBuf>,
113 pub max_concurrency: Option<u32>,
115 pub results_format: String,
117 pub params_file: Option<PathBuf>,
122
123 pub crud_flow: bool,
126 pub flow_config: Option<PathBuf>,
128 pub extract_fields: Option<String>,
130
131 pub parallel_create: Option<u32>,
134
135 pub data_file: Option<PathBuf>,
138 pub data_distribution: String,
140 pub data_mappings: Option<String>,
142 pub per_uri_control: bool,
144
145 pub error_rate: Option<f64>,
148 pub error_types: Option<String>,
150
151 pub security_test: bool,
154 pub security_payloads: Option<PathBuf>,
156 pub security_categories: Option<String>,
158 pub security_target_fields: Option<String>,
160
161 pub wafbench_dir: Option<String>,
164 pub wafbench_cycle_all: bool,
166
167 pub conformance: bool,
170 pub conformance_api_key: Option<String>,
172 pub conformance_basic_auth: Option<String>,
174 pub conformance_report: PathBuf,
176 pub conformance_categories: Option<String>,
178 pub conformance_report_format: String,
180 pub conformance_headers: Vec<String>,
183 pub conformance_all_operations: bool,
186 pub conformance_custom: Option<PathBuf>,
188 pub conformance_delay_ms: u64,
191 pub use_k6: bool,
193 pub conformance_custom_filter: Option<String>,
197 pub export_requests: bool,
200 pub validate_requests: bool,
203 pub conformance_self_test: bool,
210 pub conformance_self_test_capture: bool,
214
215 pub source_ips: Vec<String>,
220 pub geo_source_ips: Vec<String>,
224 pub geo_source_headers: Vec<String>,
228
229 pub report_missed_cap: Option<u32>,
236
237 pub owasp_api_top10: bool,
240 pub owasp_categories: Option<String>,
242 pub owasp_auth_header: String,
244 pub owasp_auth_token: Option<String>,
246 pub owasp_admin_paths: Option<PathBuf>,
248 pub owasp_id_fields: Option<String>,
250 pub owasp_report: Option<PathBuf>,
252 pub owasp_report_format: String,
254 pub owasp_iterations: u32,
256}
257
258fn parse_ip_list(raw: &[String], flag_name: &str) -> Vec<std::net::IpAddr> {
272 use std::net::IpAddr;
273 const MAX_CIDR_EXPANSION: usize = 256;
274 let mut out = Vec::new();
275 for entry in raw {
276 for piece in entry.split(',') {
277 let s = piece.trim();
278 if s.is_empty() {
279 continue;
280 }
281 if let Some((addr_part, prefix_part)) = s.split_once('/') {
283 let prefix: u32 = match prefix_part.parse() {
284 Ok(p) => p,
285 Err(e) => {
286 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR prefix: {e}");
287 continue;
288 }
289 };
290 let net_addr: IpAddr = match addr_part.parse() {
291 Ok(a) => a,
292 Err(e) => {
293 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR address: {e}");
294 continue;
295 }
296 };
297 expand_cidr(net_addr, prefix, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
298 continue;
299 }
300 if let Some((start_str, end_str)) = s.split_once('-') {
306 let start_s = start_str.trim();
307 let end_s = end_str.trim();
308 if start_s.contains(':') || end_s.contains(':') {
312 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{s}': IPv6 range syntax not supported (use CIDR like 2001:db8::/126 instead)");
313 continue;
314 }
315 let start: IpAddr = match start_s.parse() {
316 Ok(a) => a,
317 Err(e) => {
318 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range start: {e}");
319 continue;
320 }
321 };
322 let end: IpAddr = match end_s.parse() {
323 Ok(a) => a,
324 Err(e) => {
325 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range end: {e}");
326 continue;
327 }
328 };
329 expand_range(start, end, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
330 continue;
331 }
332 match s.parse::<IpAddr>() {
334 Ok(ip) => out.push(ip),
335 Err(e) => {
336 tracing::warn!(target: "mockforge::bench", "ignoring malformed --{flag_name} value '{s}': {e}");
337 }
338 }
339 }
340 }
341 out
342}
343
344fn expand_range(
348 start: std::net::IpAddr,
349 end: std::net::IpAddr,
350 cap: usize,
351 flag_name: &str,
352 raw: &str,
353 out: &mut Vec<std::net::IpAddr>,
354) {
355 use std::net::{IpAddr, Ipv4Addr};
356 let (start_v4, end_v4) = match (start, end) {
357 (IpAddr::V4(a), IpAddr::V4(b)) => (a, b),
358 _ => {
359 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range start/end must both be IPv4");
360 return;
361 }
362 };
363 let start_u32 = u32::from(start_v4);
364 let end_u32 = u32::from(end_v4);
365 if end_u32 < start_u32 {
366 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range end {end_v4} is before start {start_v4}");
367 return;
368 }
369 let total = (end_u32 - start_u32).saturating_add(1) as usize;
370 let take = total.min(cap);
371 if total > cap {
372 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range has {total} addresses, capping at {cap}");
373 }
374 for i in 0..take as u32 {
375 out.push(IpAddr::V4(Ipv4Addr::from(start_u32 + i)));
376 }
377}
378
379fn expand_cidr(
383 net: std::net::IpAddr,
384 prefix: u32,
385 cap: usize,
386 flag_name: &str,
387 raw: &str,
388 out: &mut Vec<std::net::IpAddr>,
389) {
390 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
391 match net {
392 IpAddr::V4(ipv4) => {
393 if prefix > 32 {
394 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv4 prefix must be <= 32");
395 return;
396 }
397 let total: u64 = 1u64 << (32 - prefix);
398 let take = total.min(cap as u64) as u32;
399 if total > cap as u64 {
400 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': CIDR has {total} addresses, capping at {cap}");
401 }
402 let mask: u32 = if prefix == 0 {
403 0
404 } else {
405 !0u32 << (32 - prefix)
406 };
407 let net_u32 = u32::from(ipv4) & mask;
408 for i in 0..take {
409 out.push(IpAddr::V4(Ipv4Addr::from(net_u32.wrapping_add(i))));
410 }
411 }
412 IpAddr::V6(ipv6) => {
413 if prefix > 128 {
414 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv6 prefix must be <= 128");
415 return;
416 }
417 let mask: u128 = if prefix == 0 {
421 0
422 } else {
423 !0u128 << (128 - prefix)
424 };
425 let net_u128 = u128::from(ipv6) & mask;
426 let remaining_bits = 128 - prefix;
427 let total_capped = if remaining_bits >= 64 {
430 cap as u128
431 } else {
432 (1u128 << remaining_bits).min(cap as u128)
433 };
434 if remaining_bits < 128 && (1u128 << remaining_bits) > cap as u128 {
435 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': IPv6 CIDR exceeds {cap} addresses, capping");
436 }
437 for i in 0..total_capped {
438 out.push(IpAddr::V6(Ipv6Addr::from(net_u128.wrapping_add(i))));
439 }
440 }
441 }
442}
443
444impl BenchCommand {
445 pub async fn load_and_merge_specs(&self) -> Result<OpenApiSpec> {
447 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
448
449 if !self.spec.is_empty() {
451 let specs = load_specs_from_files(self.spec.clone())
452 .await
453 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
454 all_specs.extend(specs);
455 }
456
457 if let Some(spec_dir) = &self.spec_dir {
459 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
460 BenchError::Other(format!("Failed to load specs from directory: {}", e))
461 })?;
462 all_specs.extend(dir_specs);
463 }
464
465 if all_specs.is_empty() {
466 return Err(BenchError::Other(
467 "No spec files provided. Use --spec or --spec-dir.".to_string(),
468 ));
469 }
470
471 if all_specs.len() == 1 {
473 return Ok(all_specs.into_iter().next().expect("checked len() == 1 above").1);
475 }
476
477 let conflict_strategy = match self.merge_conflicts.as_str() {
479 "first" => ConflictStrategy::First,
480 "last" => ConflictStrategy::Last,
481 _ => ConflictStrategy::Error,
482 };
483
484 merge_specs(all_specs, conflict_strategy)
485 .map_err(|e| BenchError::Other(format!("Failed to merge specs: {}", e)))
486 }
487
488 fn get_spec_display_name(&self) -> String {
490 if self.spec.len() == 1 {
491 self.spec[0].to_string_lossy().to_string()
492 } else if !self.spec.is_empty() {
493 format!("{} spec files", self.spec.len())
494 } else if let Some(dir) = &self.spec_dir {
495 format!("specs from {}", dir.display())
496 } else {
497 "no specs".to_string()
498 }
499 }
500
501 pub async fn execute(&self) -> Result<()> {
503 if self.conformance_self_test && self.use_k6 {
510 TerminalReporter::print_warning(
511 "--use-k6 has no effect with --conformance-self-test: the self-test driver runs and returns before k6 is invoked. Drop one or the other depending on whether you want the spec-driven self-test or a k6 bench run.",
512 );
513 }
514
515 if let Some(targets_file) = &self.targets_file {
517 if self.conformance {
518 return self.execute_multi_target_conformance(targets_file).await;
519 }
520 return self.execute_multi_target(targets_file).await;
521 }
522
523 if self.spec_mode == "sequential" && (self.spec.len() > 1 || self.spec_dir.is_some()) {
525 return self.execute_sequential_specs().await;
526 }
527
528 TerminalReporter::print_header(
531 &self.get_spec_display_name(),
532 &self.target,
533 0, &self.scenario,
535 Self::parse_duration(&self.duration)?,
536 );
537
538 if !K6Executor::is_k6_installed() {
540 TerminalReporter::print_error("k6 is not installed");
541 TerminalReporter::print_warning(
542 "Install k6 from: https://k6.io/docs/get-started/installation/",
543 );
544 return Err(BenchError::K6NotFound);
545 }
546
547 if self.conformance {
549 return self.execute_conformance_test().await;
550 }
551
552 TerminalReporter::print_progress("Loading OpenAPI specification(s)...");
554 let merged_spec = self.load_and_merge_specs().await?;
555 let parser = SpecParser::from_spec(merged_spec);
556 if self.spec.len() > 1 || self.spec_dir.is_some() {
557 TerminalReporter::print_success(&format!(
558 "Loaded and merged {} specification(s)",
559 self.spec.len() + self.spec_dir.as_ref().map(|_| 1).unwrap_or(0)
560 ));
561 } else {
562 TerminalReporter::print_success("Specification loaded");
563 }
564
565 let mock_config = self.build_mock_config().await;
567 if mock_config.is_mock_server {
568 TerminalReporter::print_progress("Mock server integration enabled");
569 }
570
571 if self.crud_flow {
573 return self.execute_crud_flow(&parser).await;
574 }
575
576 if self.owasp_api_top10 {
578 return self.execute_owasp_test(&parser).await;
579 }
580
581 TerminalReporter::print_progress("Extracting API operations...");
583 let mut operations = if let Some(filter) = &self.operations {
584 parser.filter_operations(filter)?
585 } else {
586 parser.get_operations()
587 };
588
589 if let Some(exclude) = &self.exclude_operations {
591 let before_count = operations.len();
592 operations = parser.exclude_operations(operations, exclude)?;
593 let excluded_count = before_count - operations.len();
594 if excluded_count > 0 {
595 TerminalReporter::print_progress(&format!(
596 "Excluded {} operations matching '{}'",
597 excluded_count, exclude
598 ));
599 }
600 }
601
602 if operations.is_empty() {
603 return Err(BenchError::Other("No operations found in spec".to_string()));
604 }
605
606 TerminalReporter::print_success(&format!("Found {} operations", operations.len()));
607
608 let param_overrides = if let Some(params_file) = &self.params_file {
610 TerminalReporter::print_progress("Loading parameter overrides...");
611 let overrides = ParameterOverrides::from_file(params_file)?;
612 TerminalReporter::print_success(&format!(
613 "Loaded parameter overrides ({} operation-specific, {} defaults)",
614 overrides.operations.len(),
615 if overrides.defaults.is_empty() { 0 } else { 1 }
616 ));
617 Some(overrides)
618 } else {
619 None
620 };
621
622 TerminalReporter::print_progress("Generating request templates...");
624 let templates: Vec<_> = operations
625 .iter()
626 .map(|op| {
627 let op_overrides = param_overrides.as_ref().map(|po| {
628 po.get_for_operation(op.operation_id.as_deref(), &op.method, &op.path)
629 });
630 RequestGenerator::generate_template_with_overrides(op, op_overrides.as_ref())
631 })
632 .collect::<Result<Vec<_>>>()?;
633 TerminalReporter::print_success("Request templates generated");
634
635 let custom_headers = self.parse_headers()?;
637
638 let base_path = self.resolve_base_path(&parser);
640 if let Some(ref bp) = base_path {
641 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
642 }
643
644 TerminalReporter::print_progress("Generating k6 load test script...");
646 let scenario =
647 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
648
649 let security_testing_enabled = self.security_test || self.wafbench_dir.is_some();
650
651 let num_ops = operations.len() as u32;
669 if let Some(rps) = self.target_rps {
670 let probe =
671 crate::preflight::probe_target_latency(&self.target, 3, self.skip_tls_verify).await;
672
673 let (required_vus, basis) = match probe {
674 Some(p) => (
675 p.required_vus(rps, num_ops),
676 format!("avg {:.1}ms (measured)", p.avg_latency.as_secs_f64() * 1000.0),
677 ),
678 None => {
679 let fallback = (rps as u64)
681 .saturating_mul(num_ops.max(1) as u64)
682 .div_ceil(10)
683 .min(u32::MAX as u64) as u32;
684 (fallback, "~100ms (default — probe failed)".to_string())
685 }
686 };
687
688 if self.vus < required_vus {
689 const VU_RECOMMENDATION_CAP: u32 = 1000;
695 let recommendation = required_vus.max(self.vus + 1);
696 if recommendation > VU_RECOMMENDATION_CAP {
697 TerminalReporter::print_warning(&format!(
698 "Workload is very large: --rps {} × {} ops/iteration × {} \
699 baseline ⇒ ~{} VUs needed end-to-end, far beyond what's \
700 practical to drive. Two ways to fix:\n 1. Reduce \
701 operations per iteration with `--operations 'pattern,…'` \
702 (or `--exclude-operations`) to focus the bench on a \
703 representative subset.\n 2. Drop `--rps` and use \
704 `--vus {}` alone — closed-model load runs as fast as \
705 the VU pool allows, bounded by latency, with no per-\
706 iteration deadline. Expect 1-iteration coverage of ~{} \
707 operations in {}s.",
708 rps,
709 num_ops,
710 basis,
711 recommendation,
712 self.vus.max(5),
713 num_ops,
714 Self::parse_duration(&self.duration).unwrap_or(0),
715 ));
716 } else {
717 TerminalReporter::print_warning(&format!(
718 "--vus {} may be insufficient for --rps {} × {} ops/iteration \
719 (baseline latency {}). k6's constant-arrival-rate counts ITERATIONS \
720 and each runs every operation in the spec — required ≈ rps × ops × \
721 latency_secs VUs. Bump --vus to ~{} if you see \"Insufficient VUs\" \
722 warnings.",
723 self.vus, rps, num_ops, basis, recommendation,
724 ));
725 }
726 } else if probe.is_some() {
727 TerminalReporter::print_progress(&format!(
728 "Pre-flight probe: target latency {}, {} ops/iteration — --vus {} \
729 is sufficient for --rps {}",
730 basis, num_ops, self.vus, rps,
731 ));
732 }
733 }
734
735 let k6_config = K6Config {
736 target_url: self.target.clone(),
737 base_path,
738 scenario,
739 duration_secs: Self::parse_duration(&self.duration)?,
740 max_vus: self.vus,
741 threshold_percentile: self.threshold_percentile.clone(),
742 threshold_ms: self.threshold_ms,
743 max_error_rate: self.max_error_rate,
744 auth_header: self.auth.clone(),
745 custom_headers,
746 skip_tls_verify: self.skip_tls_verify,
747 security_testing_enabled,
748 chunked_request_bodies: self.chunked_request_bodies,
749 target_rps: self.target_rps,
750 no_keep_alive: self.no_keep_alive,
751 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
757 .into_iter()
758 .map(|ip| ip.to_string())
759 .collect(),
760 geo_source_headers: if self.geo_source_headers.is_empty()
761 && !self.geo_source_ips.is_empty()
762 {
763 crate::conformance::self_test::default_geo_source_headers()
764 } else {
765 self.geo_source_headers.clone()
766 },
767 };
768
769 let generator = K6ScriptGenerator::new(k6_config, templates);
770 let mut script = generator.generate()?;
771 TerminalReporter::print_success("k6 script generated");
772
773 let has_advanced_features = self.data_file.is_some()
775 || self.error_rate.is_some()
776 || self.security_test
777 || self.parallel_create.is_some()
778 || self.wafbench_dir.is_some();
779
780 if has_advanced_features {
782 script = self.generate_enhanced_script(&script)?;
783 }
784
785 if mock_config.is_mock_server {
787 let setup_code = MockIntegrationGenerator::generate_setup(&mock_config);
788 let teardown_code = MockIntegrationGenerator::generate_teardown(&mock_config);
789 let helper_code = MockIntegrationGenerator::generate_vu_id_helper();
790
791 if let Some(import_end) = script.find("export const options") {
793 script.insert_str(
794 import_end,
795 &format!(
796 "\n// === Mock Server Integration ===\n{}\n{}\n{}\n",
797 helper_code, setup_code, teardown_code
798 ),
799 );
800 }
801 }
802
803 TerminalReporter::print_progress("Validating k6 script...");
805 let validation_errors = K6ScriptGenerator::validate_script(&script);
806 if !validation_errors.is_empty() {
807 TerminalReporter::print_error("Script validation failed");
808 for error in &validation_errors {
809 eprintln!(" {}", error);
810 }
811 return Err(BenchError::Other(format!(
812 "Generated k6 script has {} validation error(s). Please check the output above.",
813 validation_errors.len()
814 )));
815 }
816 TerminalReporter::print_success("Script validation passed");
817
818 let script_path = if let Some(output) = &self.script_output {
820 output.clone()
821 } else {
822 self.output.join("k6-script.js")
823 };
824
825 if let Some(parent) = script_path.parent() {
826 std::fs::create_dir_all(parent)?;
827 }
828 std::fs::write(&script_path, &script)?;
829 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
830
831 if self.generate_only {
833 println!("\nScript generated successfully. Run it with:");
834 println!(" k6 run {}", script_path.display());
835 return Ok(());
836 }
837
838 TerminalReporter::print_progress("Executing load test...");
840 let executor = K6Executor::new()?.with_local_ips(self.source_ips.join(","));
841
842 std::fs::create_dir_all(&self.output)?;
843
844 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
845
846 let duration_secs = Self::parse_duration(&self.duration)?;
848 TerminalReporter::print_summary_full(
849 &results,
850 duration_secs,
851 self.no_keep_alive,
852 Some(num_ops),
853 );
854
855 println!("\nResults saved to: {}", self.output.display());
856
857 Ok(())
858 }
859
860 async fn execute_multi_target(&self, targets_file: &Path) -> Result<()> {
862 TerminalReporter::print_progress("Parsing targets file...");
863 let targets = parse_targets_file(targets_file)?;
864 let num_targets = targets.len();
865 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
866
867 if targets.is_empty() {
868 return Err(BenchError::Other("No targets found in file".to_string()));
869 }
870
871 let max_concurrency = self.max_concurrency.unwrap_or(10) as usize;
873 let max_concurrency = max_concurrency.min(num_targets); TerminalReporter::print_header(
877 &self.get_spec_display_name(),
878 &format!("{} targets", num_targets),
879 0,
880 &self.scenario,
881 Self::parse_duration(&self.duration)?,
882 );
883
884 let executor = ParallelExecutor::new(
886 BenchCommand {
887 spec: self.spec.clone(),
889 spec_dir: self.spec_dir.clone(),
890 merge_conflicts: self.merge_conflicts.clone(),
891 spec_mode: self.spec_mode.clone(),
892 dependency_config: self.dependency_config.clone(),
893 target: self.target.clone(), base_path: self.base_path.clone(),
895 duration: self.duration.clone(),
896 vus: self.vus,
897 target_rps: self.target_rps,
898 no_keep_alive: self.no_keep_alive,
899 scenario: self.scenario.clone(),
900 operations: self.operations.clone(),
901 exclude_operations: self.exclude_operations.clone(),
902 auth: self.auth.clone(),
903 headers: self.headers.clone(),
904 output: self.output.clone(),
905 generate_only: self.generate_only,
906 script_output: self.script_output.clone(),
907 threshold_percentile: self.threshold_percentile.clone(),
908 threshold_ms: self.threshold_ms,
909 max_error_rate: self.max_error_rate,
910 verbose: self.verbose,
911 skip_tls_verify: self.skip_tls_verify,
912 chunked_request_bodies: self.chunked_request_bodies,
913 targets_file: None,
914 max_concurrency: None,
915 results_format: self.results_format.clone(),
916 params_file: self.params_file.clone(),
917 crud_flow: self.crud_flow,
918 flow_config: self.flow_config.clone(),
919 extract_fields: self.extract_fields.clone(),
920 parallel_create: self.parallel_create,
921 data_file: self.data_file.clone(),
922 data_distribution: self.data_distribution.clone(),
923 data_mappings: self.data_mappings.clone(),
924 per_uri_control: self.per_uri_control,
925 error_rate: self.error_rate,
926 error_types: self.error_types.clone(),
927 security_test: self.security_test,
928 security_payloads: self.security_payloads.clone(),
929 security_categories: self.security_categories.clone(),
930 security_target_fields: self.security_target_fields.clone(),
931 wafbench_dir: self.wafbench_dir.clone(),
932 wafbench_cycle_all: self.wafbench_cycle_all,
933 owasp_api_top10: self.owasp_api_top10,
934 owasp_categories: self.owasp_categories.clone(),
935 owasp_auth_header: self.owasp_auth_header.clone(),
936 owasp_auth_token: self.owasp_auth_token.clone(),
937 owasp_admin_paths: self.owasp_admin_paths.clone(),
938 owasp_id_fields: self.owasp_id_fields.clone(),
939 owasp_report: self.owasp_report.clone(),
940 owasp_report_format: self.owasp_report_format.clone(),
941 owasp_iterations: self.owasp_iterations,
942 conformance: false,
943 conformance_api_key: None,
944 conformance_basic_auth: None,
945 conformance_report: PathBuf::from("conformance-report.json"),
946 conformance_categories: None,
947 conformance_report_format: "json".to_string(),
948 conformance_headers: vec![],
949 conformance_all_operations: false,
950 conformance_custom: None,
951 conformance_delay_ms: 0,
952 use_k6: false,
953 conformance_custom_filter: None,
954 export_requests: false,
955 validate_requests: false,
956 conformance_self_test: false,
957 conformance_self_test_capture: false,
958 source_ips: Vec::new(),
959 geo_source_ips: Vec::new(),
960 geo_source_headers: Vec::new(),
961 report_missed_cap: None,
962 },
963 targets,
964 max_concurrency,
965 );
966
967 let start_time = std::time::Instant::now();
969 let aggregated_results = executor.execute_all().await?;
970 let elapsed = start_time.elapsed();
971
972 self.report_multi_target_results(&aggregated_results, elapsed)?;
974
975 Ok(())
976 }
977
978 fn report_multi_target_results(
980 &self,
981 results: &AggregatedResults,
982 elapsed: std::time::Duration,
983 ) -> Result<()> {
984 TerminalReporter::print_multi_target_summary(results);
986
987 let total_secs = elapsed.as_secs();
989 let hours = total_secs / 3600;
990 let minutes = (total_secs % 3600) / 60;
991 let seconds = total_secs % 60;
992 if hours > 0 {
993 println!("\n Total Elapsed Time: {}h {}m {}s", hours, minutes, seconds);
994 } else if minutes > 0 {
995 println!("\n Total Elapsed Time: {}m {}s", minutes, seconds);
996 } else {
997 println!("\n Total Elapsed Time: {}s", seconds);
998 }
999
1000 if self.results_format == "aggregated" || self.results_format == "both" {
1002 let summary_path = self.output.join("aggregated_summary.json");
1003 let summary_json = serde_json::json!({
1004 "total_elapsed_seconds": elapsed.as_secs(),
1005 "total_targets": results.total_targets,
1006 "successful_targets": results.successful_targets,
1007 "failed_targets": results.failed_targets,
1008 "aggregated_metrics": {
1009 "total_requests": results.aggregated_metrics.total_requests,
1010 "total_failed_requests": results.aggregated_metrics.total_failed_requests,
1011 "avg_duration_ms": results.aggregated_metrics.avg_duration_ms,
1012 "p95_duration_ms": results.aggregated_metrics.p95_duration_ms,
1013 "p99_duration_ms": results.aggregated_metrics.p99_duration_ms,
1014 "error_rate": results.aggregated_metrics.error_rate,
1015 "total_rps": results.aggregated_metrics.total_rps,
1016 "avg_rps": results.aggregated_metrics.avg_rps,
1017 "total_vus_max": results.aggregated_metrics.total_vus_max,
1018 },
1019 "target_results": results.target_results.iter().map(|r| {
1020 serde_json::json!({
1021 "target_url": r.target_url,
1022 "target_index": r.target_index,
1023 "success": r.success,
1024 "error": r.error,
1025 "total_requests": r.results.total_requests,
1026 "failed_requests": r.results.failed_requests,
1027 "avg_duration_ms": r.results.avg_duration_ms,
1028 "min_duration_ms": r.results.min_duration_ms,
1029 "med_duration_ms": r.results.med_duration_ms,
1030 "p90_duration_ms": r.results.p90_duration_ms,
1031 "p95_duration_ms": r.results.p95_duration_ms,
1032 "p99_duration_ms": r.results.p99_duration_ms,
1033 "max_duration_ms": r.results.max_duration_ms,
1034 "rps": r.results.rps,
1035 "vus_max": r.results.vus_max,
1036 "output_dir": r.output_dir.to_string_lossy(),
1037 })
1038 }).collect::<Vec<_>>(),
1039 });
1040
1041 std::fs::write(&summary_path, serde_json::to_string_pretty(&summary_json)?)?;
1042 TerminalReporter::print_success(&format!(
1043 "Aggregated summary saved to: {}",
1044 summary_path.display()
1045 ));
1046 }
1047
1048 let csv_path = self.output.join("all_targets.csv");
1050 let mut csv = String::from(
1051 "target_url,success,requests,failed,rps,vus,min_ms,avg_ms,med_ms,p90_ms,p95_ms,p99_ms,max_ms,error\n",
1052 );
1053 for r in &results.target_results {
1054 csv.push_str(&format!(
1055 "{},{},{},{},{:.1},{},{:.1},{:.1},{:.1},{:.1},{:.1},{:.1},{:.1},{}\n",
1056 r.target_url,
1057 r.success,
1058 r.results.total_requests,
1059 r.results.failed_requests,
1060 r.results.rps,
1061 r.results.vus_max,
1062 r.results.min_duration_ms,
1063 r.results.avg_duration_ms,
1064 r.results.med_duration_ms,
1065 r.results.p90_duration_ms,
1066 r.results.p95_duration_ms,
1067 r.results.p99_duration_ms,
1068 r.results.max_duration_ms,
1069 r.error.as_deref().unwrap_or(""),
1070 ));
1071 }
1072 let _ = std::fs::write(&csv_path, &csv);
1073
1074 println!("\nResults saved to: {}", self.output.display());
1075 println!(" - Per-target results: {}", self.output.join("target_*").display());
1076 println!(" - All targets CSV: {}", csv_path.display());
1077 if self.results_format == "aggregated" || self.results_format == "both" {
1078 println!(
1079 " - Aggregated summary: {}",
1080 self.output.join("aggregated_summary.json").display()
1081 );
1082 }
1083
1084 Ok(())
1085 }
1086
1087 pub fn parse_duration(duration: &str) -> Result<u64> {
1089 let duration = duration.trim();
1090
1091 if let Some(secs) = duration.strip_suffix('s') {
1092 secs.parse::<u64>()
1093 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1094 } else if let Some(mins) = duration.strip_suffix('m') {
1095 mins.parse::<u64>()
1096 .map(|m| m * 60)
1097 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1098 } else if let Some(hours) = duration.strip_suffix('h') {
1099 hours
1100 .parse::<u64>()
1101 .map(|h| h * 3600)
1102 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1103 } else {
1104 duration
1106 .parse::<u64>()
1107 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1108 }
1109 }
1110
1111 pub fn parse_headers(&self) -> Result<HashMap<String, String>> {
1113 match &self.headers {
1114 Some(s) => parse_header_string(s),
1115 None => Ok(HashMap::new()),
1116 }
1117 }
1118
1119 fn parse_extracted_values(output_dir: &Path) -> Result<ExtractedValues> {
1120 let extracted_path = output_dir.join("extracted_values.json");
1121 if !extracted_path.exists() {
1122 return Ok(ExtractedValues::new());
1123 }
1124
1125 let content = std::fs::read_to_string(&extracted_path)
1126 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1127 let parsed: serde_json::Value = serde_json::from_str(&content)
1128 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1129
1130 let mut extracted = ExtractedValues::new();
1131 if let Some(values) = parsed.as_object() {
1132 for (key, value) in values {
1133 extracted.set(key.clone(), value.clone());
1134 }
1135 }
1136
1137 Ok(extracted)
1138 }
1139
1140 fn resolve_base_path(&self, parser: &SpecParser) -> Option<String> {
1149 if let Some(cli_base_path) = &self.base_path {
1151 if cli_base_path.is_empty() {
1152 return None;
1154 }
1155 return Some(cli_base_path.clone());
1156 }
1157
1158 parser.get_base_path()
1160 }
1161
1162 async fn build_mock_config(&self) -> MockIntegrationConfig {
1164 if MockServerDetector::looks_like_mock_server(&self.target) {
1166 if let Ok(info) = MockServerDetector::detect(&self.target).await {
1168 if info.is_mockforge {
1169 TerminalReporter::print_success(&format!(
1170 "Detected MockForge server (version: {})",
1171 info.version.as_deref().unwrap_or("unknown")
1172 ));
1173 return MockIntegrationConfig::mock_server();
1174 }
1175 }
1176 }
1177 MockIntegrationConfig::real_api()
1178 }
1179
1180 fn build_crud_flow_config(&self) -> Option<CrudFlowConfig> {
1182 if !self.crud_flow {
1183 return None;
1184 }
1185
1186 if let Some(config_path) = &self.flow_config {
1188 match CrudFlowConfig::from_file(config_path) {
1189 Ok(config) => return Some(config),
1190 Err(e) => {
1191 TerminalReporter::print_warning(&format!(
1192 "Failed to load flow config: {}. Using auto-detection.",
1193 e
1194 ));
1195 }
1196 }
1197 }
1198
1199 let extract_fields = self
1201 .extract_fields
1202 .as_ref()
1203 .map(|f| f.split(',').map(|s| s.trim().to_string()).collect())
1204 .unwrap_or_else(|| vec!["id".to_string(), "uuid".to_string()]);
1205
1206 Some(CrudFlowConfig {
1207 flows: Vec::new(), default_extract_fields: extract_fields,
1209 })
1210 }
1211
1212 fn build_data_driven_config(&self) -> Option<DataDrivenConfig> {
1214 let data_file = self.data_file.as_ref()?;
1215
1216 let distribution = DataDistribution::from_str(&self.data_distribution)
1217 .unwrap_or(DataDistribution::UniquePerVu);
1218
1219 let mappings = self
1220 .data_mappings
1221 .as_ref()
1222 .map(|m| DataMapping::parse_mappings(m).unwrap_or_default())
1223 .unwrap_or_default();
1224
1225 Some(DataDrivenConfig {
1226 file_path: data_file.to_string_lossy().to_string(),
1227 distribution,
1228 mappings,
1229 csv_has_header: true,
1230 per_uri_control: self.per_uri_control,
1231 per_uri_columns: crate::data_driven::PerUriColumns::default(),
1232 })
1233 }
1234
1235 fn build_invalid_data_config(&self) -> Option<InvalidDataConfig> {
1237 let error_rate = self.error_rate?;
1238
1239 let error_types = self
1240 .error_types
1241 .as_ref()
1242 .map(|types| InvalidDataConfig::parse_error_types(types).unwrap_or_default())
1243 .unwrap_or_default();
1244
1245 Some(InvalidDataConfig {
1246 error_rate,
1247 error_types,
1248 target_fields: Vec::new(),
1249 })
1250 }
1251
1252 fn build_security_config(&self) -> Option<SecurityTestConfig> {
1254 if !self.security_test {
1255 return None;
1256 }
1257
1258 let categories = self
1259 .security_categories
1260 .as_ref()
1261 .map(|cats| SecurityTestConfig::parse_categories(cats).unwrap_or_default())
1262 .unwrap_or_else(|| {
1263 let mut default = HashSet::new();
1264 default.insert(SecurityCategory::SqlInjection);
1265 default.insert(SecurityCategory::Xss);
1266 default
1267 });
1268
1269 let target_fields = self
1270 .security_target_fields
1271 .as_ref()
1272 .map(|fields| fields.split(',').map(|f| f.trim().to_string()).collect())
1273 .unwrap_or_default();
1274
1275 let custom_payloads_file =
1276 self.security_payloads.as_ref().map(|p| p.to_string_lossy().to_string());
1277
1278 Some(SecurityTestConfig {
1279 enabled: true,
1280 categories,
1281 target_fields,
1282 custom_payloads_file,
1283 include_high_risk: false,
1284 })
1285 }
1286
1287 fn build_parallel_config(&self) -> Option<ParallelConfig> {
1289 let count = self.parallel_create?;
1290
1291 Some(ParallelConfig::new(count))
1292 }
1293
1294 fn load_wafbench_payloads(&self) -> Vec<SecurityPayload> {
1296 let Some(ref wafbench_dir) = self.wafbench_dir else {
1297 return Vec::new();
1298 };
1299
1300 let mut loader = WafBenchLoader::new();
1301
1302 if let Err(e) = loader.load_from_pattern(wafbench_dir) {
1303 TerminalReporter::print_warning(&format!("Failed to load WAFBench tests: {}", e));
1304 return Vec::new();
1305 }
1306
1307 let stats = loader.stats();
1308
1309 if stats.files_processed == 0 {
1310 TerminalReporter::print_warning(&format!(
1311 "No WAFBench YAML files found matching '{}'",
1312 wafbench_dir
1313 ));
1314 if !stats.parse_errors.is_empty() {
1316 TerminalReporter::print_warning("Some files were found but failed to parse:");
1317 for error in &stats.parse_errors {
1318 TerminalReporter::print_warning(&format!(" - {}", error));
1319 }
1320 }
1321 return Vec::new();
1322 }
1323
1324 TerminalReporter::print_progress(&format!(
1325 "Loaded {} WAFBench files, {} test cases, {} payloads",
1326 stats.files_processed, stats.test_cases_loaded, stats.payloads_extracted
1327 ));
1328
1329 for (category, count) in &stats.by_category {
1331 TerminalReporter::print_progress(&format!(" - {}: {} tests", category, count));
1332 }
1333
1334 for error in &stats.parse_errors {
1336 TerminalReporter::print_warning(&format!(" Parse error: {}", error));
1337 }
1338
1339 loader.to_security_payloads()
1340 }
1341
1342 pub(crate) fn generate_enhanced_script(&self, base_script: &str) -> Result<String> {
1344 let mut enhanced_script = base_script.to_string();
1345 let mut additional_code = String::new();
1346
1347 if let Some(config) = self.build_data_driven_config() {
1349 TerminalReporter::print_progress("Adding data-driven testing support...");
1350 additional_code.push_str(&DataDrivenGenerator::generate_setup(&config));
1351 additional_code.push('\n');
1352 TerminalReporter::print_success("Data-driven testing enabled");
1353 }
1354
1355 if let Some(config) = self.build_invalid_data_config() {
1357 TerminalReporter::print_progress("Adding invalid data testing support...");
1358 additional_code.push_str(&InvalidDataGenerator::generate_invalidation_logic());
1359 additional_code.push('\n');
1360 additional_code
1361 .push_str(&InvalidDataGenerator::generate_should_invalidate(config.error_rate));
1362 additional_code.push('\n');
1363 additional_code
1364 .push_str(&InvalidDataGenerator::generate_type_selection(&config.error_types));
1365 additional_code.push('\n');
1366 TerminalReporter::print_success(&format!(
1367 "Invalid data testing enabled ({}% error rate)",
1368 (self.error_rate.unwrap_or(0.0) * 100.0) as u32
1369 ));
1370 }
1371
1372 let security_config = self.build_security_config();
1374 let wafbench_payloads = self.load_wafbench_payloads();
1375 let security_requested = security_config.is_some() || self.wafbench_dir.is_some();
1376
1377 if security_config.is_some() || !wafbench_payloads.is_empty() {
1378 TerminalReporter::print_progress("Adding security testing support...");
1379
1380 let mut payload_list: Vec<SecurityPayload> = Vec::new();
1382
1383 if let Some(ref config) = security_config {
1384 payload_list.extend(SecurityPayloads::get_payloads(config));
1385 }
1386
1387 if !wafbench_payloads.is_empty() {
1389 TerminalReporter::print_progress(&format!(
1390 "Loading {} WAFBench attack patterns...",
1391 wafbench_payloads.len()
1392 ));
1393 payload_list.extend(wafbench_payloads);
1394 }
1395
1396 let target_fields =
1397 security_config.as_ref().map(|c| c.target_fields.clone()).unwrap_or_default();
1398
1399 additional_code.push_str(&SecurityTestGenerator::generate_payload_selection(
1400 &payload_list,
1401 self.wafbench_cycle_all,
1402 ));
1403 additional_code.push('\n');
1404 additional_code
1405 .push_str(&SecurityTestGenerator::generate_apply_payload(&target_fields));
1406 additional_code.push('\n');
1407 additional_code.push_str(&SecurityTestGenerator::generate_security_checks());
1408 additional_code.push('\n');
1409
1410 let mode = if self.wafbench_cycle_all {
1411 "cycle-all"
1412 } else {
1413 "random"
1414 };
1415 TerminalReporter::print_success(&format!(
1416 "Security testing enabled ({} payloads, {} mode)",
1417 payload_list.len(),
1418 mode
1419 ));
1420 } else if security_requested {
1421 TerminalReporter::print_warning(
1425 "Security testing was requested but no payloads were loaded. \
1426 Ensure --wafbench-dir points to valid CRS YAML files or add --security-test.",
1427 );
1428 additional_code
1429 .push_str(&SecurityTestGenerator::generate_payload_selection(&[], false));
1430 additional_code.push('\n');
1431 additional_code.push_str(&SecurityTestGenerator::generate_apply_payload(&[]));
1432 additional_code.push('\n');
1433 }
1434
1435 if let Some(config) = self.build_parallel_config() {
1437 TerminalReporter::print_progress("Adding parallel execution support...");
1438 additional_code.push_str(&ParallelRequestGenerator::generate_batch_helper(&config));
1439 additional_code.push('\n');
1440 TerminalReporter::print_success(&format!(
1441 "Parallel execution enabled (count: {})",
1442 config.count
1443 ));
1444 }
1445
1446 if !additional_code.is_empty() {
1448 if let Some(import_end) = enhanced_script.find("export const options") {
1450 enhanced_script.insert_str(
1451 import_end,
1452 &format!("\n// === Advanced Testing Features ===\n{}\n", additional_code),
1453 );
1454 }
1455 }
1456
1457 Ok(enhanced_script)
1458 }
1459
1460 async fn execute_sequential_specs(&self) -> Result<()> {
1462 TerminalReporter::print_progress("Sequential spec mode: Loading specs individually...");
1463
1464 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
1466
1467 if !self.spec.is_empty() {
1468 let specs = load_specs_from_files(self.spec.clone())
1469 .await
1470 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
1471 all_specs.extend(specs);
1472 }
1473
1474 if let Some(spec_dir) = &self.spec_dir {
1475 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
1476 BenchError::Other(format!("Failed to load specs from directory: {}", e))
1477 })?;
1478 all_specs.extend(dir_specs);
1479 }
1480
1481 if all_specs.is_empty() {
1482 return Err(BenchError::Other(
1483 "No spec files found for sequential execution".to_string(),
1484 ));
1485 }
1486
1487 TerminalReporter::print_success(&format!("Loaded {} spec(s)", all_specs.len()));
1488
1489 let execution_order = if let Some(config_path) = &self.dependency_config {
1491 TerminalReporter::print_progress("Loading dependency configuration...");
1492 let config = SpecDependencyConfig::from_file(config_path)?;
1493
1494 if !config.disable_auto_detect && config.execution_order.is_empty() {
1495 self.detect_and_sort_specs(&all_specs)?
1497 } else {
1498 config.execution_order.iter().flat_map(|g| g.specs.clone()).collect()
1500 }
1501 } else {
1502 self.detect_and_sort_specs(&all_specs)?
1504 };
1505
1506 TerminalReporter::print_success(&format!(
1507 "Execution order: {}",
1508 execution_order
1509 .iter()
1510 .map(|p| p.file_name().unwrap_or_default().to_string_lossy().to_string())
1511 .collect::<Vec<_>>()
1512 .join(" → ")
1513 ));
1514
1515 let mut extracted_values = ExtractedValues::new();
1517 let total_specs = execution_order.len();
1518
1519 for (index, spec_path) in execution_order.iter().enumerate() {
1520 let spec_name = spec_path.file_name().unwrap_or_default().to_string_lossy().to_string();
1521
1522 TerminalReporter::print_progress(&format!(
1523 "[{}/{}] Executing spec: {}",
1524 index + 1,
1525 total_specs,
1526 spec_name
1527 ));
1528
1529 let spec = all_specs
1531 .iter()
1532 .find(|(p, _)| {
1533 p == spec_path
1534 || p.file_name() == spec_path.file_name()
1535 || p.file_name() == Some(spec_path.as_os_str())
1536 })
1537 .map(|(_, s)| s.clone())
1538 .ok_or_else(|| {
1539 BenchError::Other(format!("Spec not found: {}", spec_path.display()))
1540 })?;
1541
1542 let new_values = self.execute_single_spec(&spec, &spec_name, &extracted_values).await?;
1544
1545 extracted_values.merge(&new_values);
1547
1548 TerminalReporter::print_success(&format!(
1549 "[{}/{}] Completed: {} (extracted {} values)",
1550 index + 1,
1551 total_specs,
1552 spec_name,
1553 new_values.values.len()
1554 ));
1555 }
1556
1557 TerminalReporter::print_success(&format!(
1558 "Sequential execution complete: {} specs executed",
1559 total_specs
1560 ));
1561
1562 Ok(())
1563 }
1564
1565 fn detect_and_sort_specs(&self, specs: &[(PathBuf, OpenApiSpec)]) -> Result<Vec<PathBuf>> {
1567 TerminalReporter::print_progress("Auto-detecting spec dependencies...");
1568
1569 let mut detector = DependencyDetector::new();
1570 let dependencies = detector.detect_dependencies(specs);
1571
1572 if dependencies.is_empty() {
1573 TerminalReporter::print_progress("No dependencies detected, using file order");
1574 return Ok(specs.iter().map(|(p, _)| p.clone()).collect());
1575 }
1576
1577 TerminalReporter::print_progress(&format!(
1578 "Detected {} cross-spec dependencies",
1579 dependencies.len()
1580 ));
1581
1582 for dep in &dependencies {
1583 TerminalReporter::print_progress(&format!(
1584 " {} → {} (via field '{}')",
1585 dep.dependency_spec.file_name().unwrap_or_default().to_string_lossy(),
1586 dep.dependent_spec.file_name().unwrap_or_default().to_string_lossy(),
1587 dep.field_name
1588 ));
1589 }
1590
1591 topological_sort(specs, &dependencies)
1592 }
1593
1594 async fn execute_single_spec(
1596 &self,
1597 spec: &OpenApiSpec,
1598 spec_name: &str,
1599 _external_values: &ExtractedValues,
1600 ) -> Result<ExtractedValues> {
1601 let parser = SpecParser::from_spec(spec.clone());
1602
1603 if self.crud_flow {
1605 self.execute_crud_flow_with_extraction(&parser, spec_name).await
1607 } else {
1608 self.execute_standard_spec(&parser, spec_name).await?;
1610 Ok(ExtractedValues::new())
1611 }
1612 }
1613
1614 async fn execute_crud_flow_with_extraction(
1616 &self,
1617 parser: &SpecParser,
1618 spec_name: &str,
1619 ) -> Result<ExtractedValues> {
1620 let operations = parser.get_operations();
1621 let flows = CrudFlowDetector::detect_flows(&operations);
1622
1623 if flows.is_empty() {
1624 TerminalReporter::print_warning(&format!("No CRUD flows detected in {}", spec_name));
1625 return Ok(ExtractedValues::new());
1626 }
1627
1628 TerminalReporter::print_progress(&format!(
1629 " {} CRUD flow(s) in {}",
1630 flows.len(),
1631 spec_name
1632 ));
1633
1634 let mut handlebars = handlebars::Handlebars::new();
1636 handlebars.register_helper(
1638 "json",
1639 Box::new(
1640 |h: &handlebars::Helper,
1641 _: &handlebars::Handlebars,
1642 _: &handlebars::Context,
1643 _: &mut handlebars::RenderContext,
1644 out: &mut dyn handlebars::Output|
1645 -> handlebars::HelperResult {
1646 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
1647 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
1648 Ok(())
1649 },
1650 ),
1651 );
1652 let template = include_str!("templates/k6_crud_flow.hbs");
1653 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
1654
1655 let custom_headers = self.parse_headers()?;
1656 let config = self.build_crud_flow_config().unwrap_or_default();
1657
1658 let param_overrides = if let Some(params_file) = &self.params_file {
1660 let overrides = ParameterOverrides::from_file(params_file)?;
1661 Some(overrides)
1662 } else {
1663 None
1664 };
1665
1666 let duration_secs = Self::parse_duration(&self.duration)?;
1668 let scenario =
1669 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
1670 let stages = scenario.generate_stages(duration_secs, self.vus);
1671
1672 let api_base_path = self.resolve_base_path(parser);
1674
1675 let mut all_headers = custom_headers.clone();
1677 if let Some(auth) = &self.auth {
1678 all_headers.insert("Authorization".to_string(), auth.clone());
1679 }
1680 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
1681
1682 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
1684
1685 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
1686 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
1690 serde_json::json!({
1691 "name": sanitized_name.clone(),
1692 "display_name": f.name,
1693 "base_path": f.base_path,
1694 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
1695 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
1697 let method_raw = if !parts.is_empty() {
1698 parts[0].to_uppercase()
1699 } else {
1700 "GET".to_string()
1701 };
1702 let method = if !parts.is_empty() {
1703 let m = parts[0].to_lowercase();
1704 if m == "delete" { "del".to_string() } else { m }
1706 } else {
1707 "get".to_string()
1708 };
1709 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
1710 let path = if let Some(ref bp) = api_base_path {
1712 format!("{}{}", bp, raw_path)
1713 } else {
1714 raw_path.to_string()
1715 };
1716 let is_get_or_head = method == "get" || method == "head";
1717 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
1719
1720 let body_value = if has_body {
1722 param_overrides.as_ref()
1723 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
1724 .and_then(|oo| oo.body)
1725 .unwrap_or_else(|| serde_json::json!({}))
1726 } else {
1727 serde_json::json!({})
1728 };
1729
1730 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
1732
1733 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
1735 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
1736
1737 serde_json::json!({
1738 "operation": s.operation,
1739 "method": method,
1740 "path": path,
1741 "extract": s.extract,
1742 "use_values": s.use_values,
1743 "use_body": s.use_body,
1744 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
1745 "inject_attacks": s.inject_attacks,
1746 "attack_types": s.attack_types,
1747 "description": s.description,
1748 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
1749 "is_get_or_head": is_get_or_head,
1750 "has_body": has_body,
1751 "body": processed_body.value,
1752 "body_is_dynamic": body_is_dynamic,
1753 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
1754 })
1755 }).collect::<Vec<_>>(),
1756 })
1757 }).collect();
1758
1759 for flow_data in &flows_data {
1761 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
1762 for step in steps {
1763 if let Some(placeholders_arr) =
1764 step.get("_placeholders").and_then(|p| p.as_array())
1765 {
1766 for p_str in placeholders_arr {
1767 if let Some(p_name) = p_str.as_str() {
1768 match p_name {
1769 "VU" => {
1770 all_placeholders.insert(DynamicPlaceholder::VU);
1771 }
1772 "Iteration" => {
1773 all_placeholders.insert(DynamicPlaceholder::Iteration);
1774 }
1775 "Timestamp" => {
1776 all_placeholders.insert(DynamicPlaceholder::Timestamp);
1777 }
1778 "UUID" => {
1779 all_placeholders.insert(DynamicPlaceholder::UUID);
1780 }
1781 "Random" => {
1782 all_placeholders.insert(DynamicPlaceholder::Random);
1783 }
1784 "Counter" => {
1785 all_placeholders.insert(DynamicPlaceholder::Counter);
1786 }
1787 "Date" => {
1788 all_placeholders.insert(DynamicPlaceholder::Date);
1789 }
1790 "VuIter" => {
1791 all_placeholders.insert(DynamicPlaceholder::VuIter);
1792 }
1793 _ => {}
1794 }
1795 }
1796 }
1797 }
1798 }
1799 }
1800 }
1801
1802 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
1804 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
1805
1806 let security_testing_enabled = self.wafbench_dir.is_some() || self.security_test;
1808
1809 let data = serde_json::json!({
1810 "base_url": self.target,
1811 "flows": flows_data,
1812 "extract_fields": config.default_extract_fields,
1813 "duration_secs": duration_secs,
1814 "max_vus": self.vus,
1815 "auth_header": self.auth,
1816 "custom_headers": custom_headers,
1817 "skip_tls_verify": self.skip_tls_verify,
1818 "stages": stages.iter().map(|s| serde_json::json!({
1820 "duration": s.duration,
1821 "target": s.target,
1822 })).collect::<Vec<_>>(),
1823 "threshold_percentile": self.threshold_percentile,
1824 "threshold_ms": self.threshold_ms,
1825 "max_error_rate": self.max_error_rate,
1826 "headers": headers_json,
1827 "dynamic_imports": required_imports,
1828 "dynamic_globals": required_globals,
1829 "extracted_values_output_path": output_dir.join("extracted_values.json").to_string_lossy(),
1830 "security_testing_enabled": security_testing_enabled,
1832 "has_custom_headers": !custom_headers.is_empty(),
1833 });
1834
1835 let mut script = handlebars
1836 .render_template(template, &data)
1837 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
1838
1839 if security_testing_enabled {
1841 script = self.generate_enhanced_script(&script)?;
1842 }
1843
1844 let script_path =
1846 self.output.join(format!("k6-{}-crud-flow.js", spec_name.replace('.', "_")));
1847
1848 std::fs::create_dir_all(self.output.clone())?;
1849 std::fs::write(&script_path, &script)?;
1850
1851 if !self.generate_only {
1852 let executor = K6Executor::new()?.with_local_ips(self.source_ips.join(","));
1853 std::fs::create_dir_all(&output_dir)?;
1854
1855 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
1856
1857 let extracted = Self::parse_extracted_values(&output_dir)?;
1858 TerminalReporter::print_progress(&format!(
1859 " Extracted {} value(s) from {}",
1860 extracted.values.len(),
1861 spec_name
1862 ));
1863 return Ok(extracted);
1864 }
1865
1866 Ok(ExtractedValues::new())
1867 }
1868
1869 async fn execute_standard_spec(&self, parser: &SpecParser, spec_name: &str) -> Result<()> {
1871 let mut operations = if let Some(filter) = &self.operations {
1872 parser.filter_operations(filter)?
1873 } else {
1874 parser.get_operations()
1875 };
1876
1877 if let Some(exclude) = &self.exclude_operations {
1878 operations = parser.exclude_operations(operations, exclude)?;
1879 }
1880
1881 if operations.is_empty() {
1882 TerminalReporter::print_warning(&format!("No operations found in {}", spec_name));
1883 return Ok(());
1884 }
1885
1886 TerminalReporter::print_progress(&format!(
1887 " {} operations in {}",
1888 operations.len(),
1889 spec_name
1890 ));
1891
1892 let templates: Vec<_> = operations
1894 .iter()
1895 .map(RequestGenerator::generate_template)
1896 .collect::<Result<Vec<_>>>()?;
1897
1898 let custom_headers = self.parse_headers()?;
1900
1901 let base_path = self.resolve_base_path(parser);
1903
1904 let scenario =
1906 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
1907
1908 let security_testing_enabled = self.security_test || self.wafbench_dir.is_some();
1909
1910 let k6_config = K6Config {
1911 target_url: self.target.clone(),
1912 base_path,
1913 scenario,
1914 duration_secs: Self::parse_duration(&self.duration)?,
1915 max_vus: self.vus,
1916 threshold_percentile: self.threshold_percentile.clone(),
1917 threshold_ms: self.threshold_ms,
1918 max_error_rate: self.max_error_rate,
1919 auth_header: self.auth.clone(),
1920 custom_headers,
1921 skip_tls_verify: self.skip_tls_verify,
1922 security_testing_enabled,
1923 chunked_request_bodies: self.chunked_request_bodies,
1924 target_rps: self.target_rps,
1925 no_keep_alive: self.no_keep_alive,
1926 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
1928 .into_iter()
1929 .map(|ip| ip.to_string())
1930 .collect(),
1931 geo_source_headers: if self.geo_source_headers.is_empty()
1932 && !self.geo_source_ips.is_empty()
1933 {
1934 crate::conformance::self_test::default_geo_source_headers()
1935 } else {
1936 self.geo_source_headers.clone()
1937 },
1938 };
1939
1940 let generator = K6ScriptGenerator::new(k6_config, templates);
1941 let mut script = generator.generate()?;
1942
1943 let has_advanced_features = self.data_file.is_some()
1945 || self.error_rate.is_some()
1946 || self.security_test
1947 || self.parallel_create.is_some()
1948 || self.wafbench_dir.is_some();
1949
1950 if has_advanced_features {
1951 script = self.generate_enhanced_script(&script)?;
1952 }
1953
1954 let script_path = self.output.join(format!("k6-{}.js", spec_name.replace('.', "_")));
1956
1957 std::fs::create_dir_all(self.output.clone())?;
1958 std::fs::write(&script_path, &script)?;
1959
1960 if !self.generate_only {
1961 let executor = K6Executor::new()?.with_local_ips(self.source_ips.join(","));
1962 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
1963 std::fs::create_dir_all(&output_dir)?;
1964
1965 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
1966 }
1967
1968 Ok(())
1969 }
1970
1971 async fn execute_crud_flow(&self, parser: &SpecParser) -> Result<()> {
1973 let config = self.build_crud_flow_config().unwrap_or_default();
1975
1976 let flows = if !config.flows.is_empty() {
1978 TerminalReporter::print_progress("Using custom flow configuration...");
1979 config.flows.clone()
1980 } else {
1981 TerminalReporter::print_progress("Detecting CRUD operations...");
1982 let operations = parser.get_operations();
1983 CrudFlowDetector::detect_flows(&operations)
1984 };
1985
1986 if flows.is_empty() {
1987 return Err(BenchError::Other(
1988 "No CRUD flows detected in spec. Ensure spec has POST/GET/PUT/DELETE operations on related paths.".to_string(),
1989 ));
1990 }
1991
1992 if config.flows.is_empty() {
1993 TerminalReporter::print_success(&format!("Detected {} CRUD flow(s)", flows.len()));
1994 } else {
1995 TerminalReporter::print_success(&format!("Loaded {} custom flow(s)", flows.len()));
1996 }
1997
1998 for flow in &flows {
1999 TerminalReporter::print_progress(&format!(
2000 " - {}: {} steps",
2001 flow.name,
2002 flow.steps.len()
2003 ));
2004 }
2005
2006 let mut handlebars = handlebars::Handlebars::new();
2008 handlebars.register_helper(
2010 "json",
2011 Box::new(
2012 |h: &handlebars::Helper,
2013 _: &handlebars::Handlebars,
2014 _: &handlebars::Context,
2015 _: &mut handlebars::RenderContext,
2016 out: &mut dyn handlebars::Output|
2017 -> handlebars::HelperResult {
2018 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2019 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2020 Ok(())
2021 },
2022 ),
2023 );
2024 let template = include_str!("templates/k6_crud_flow.hbs");
2025
2026 let custom_headers = self.parse_headers()?;
2027
2028 let param_overrides = if let Some(params_file) = &self.params_file {
2030 TerminalReporter::print_progress("Loading parameter overrides...");
2031 let overrides = ParameterOverrides::from_file(params_file)?;
2032 TerminalReporter::print_success(&format!(
2033 "Loaded parameter overrides ({} operation-specific, {} defaults)",
2034 overrides.operations.len(),
2035 if overrides.defaults.is_empty() { 0 } else { 1 }
2036 ));
2037 Some(overrides)
2038 } else {
2039 None
2040 };
2041
2042 let duration_secs = Self::parse_duration(&self.duration)?;
2044 let scenario =
2045 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2046 let stages = scenario.generate_stages(duration_secs, self.vus);
2047
2048 let api_base_path = self.resolve_base_path(parser);
2050 if let Some(ref bp) = api_base_path {
2051 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
2052 }
2053
2054 let mut all_headers = custom_headers.clone();
2056 if let Some(auth) = &self.auth {
2057 all_headers.insert("Authorization".to_string(), auth.clone());
2058 }
2059 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2060
2061 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2063
2064 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2065 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2070 serde_json::json!({
2071 "name": sanitized_name.clone(), "display_name": f.name, "base_path": f.base_path,
2074 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2075 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2077 let method_raw = if !parts.is_empty() {
2078 parts[0].to_uppercase()
2079 } else {
2080 "GET".to_string()
2081 };
2082 let method = if !parts.is_empty() {
2083 let m = parts[0].to_lowercase();
2084 if m == "delete" { "del".to_string() } else { m }
2086 } else {
2087 "get".to_string()
2088 };
2089 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2090 let path = if let Some(ref bp) = api_base_path {
2092 format!("{}{}", bp, raw_path)
2093 } else {
2094 raw_path.to_string()
2095 };
2096 let is_get_or_head = method == "get" || method == "head";
2097 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2099
2100 let body_value = if has_body {
2102 param_overrides.as_ref()
2103 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2104 .and_then(|oo| oo.body)
2105 .unwrap_or_else(|| serde_json::json!({}))
2106 } else {
2107 serde_json::json!({})
2108 };
2109
2110 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2112 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2117 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2118
2119 serde_json::json!({
2120 "operation": s.operation,
2121 "method": method,
2122 "path": path,
2123 "extract": s.extract,
2124 "use_values": s.use_values,
2125 "use_body": s.use_body,
2126 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2127 "inject_attacks": s.inject_attacks,
2128 "attack_types": s.attack_types,
2129 "description": s.description,
2130 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2131 "is_get_or_head": is_get_or_head,
2132 "has_body": has_body,
2133 "body": processed_body.value,
2134 "body_is_dynamic": body_is_dynamic,
2135 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2136 })
2137 }).collect::<Vec<_>>(),
2138 })
2139 }).collect();
2140
2141 for flow_data in &flows_data {
2143 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2144 for step in steps {
2145 if let Some(placeholders_arr) =
2146 step.get("_placeholders").and_then(|p| p.as_array())
2147 {
2148 for p_str in placeholders_arr {
2149 if let Some(p_name) = p_str.as_str() {
2150 match p_name {
2152 "VU" => {
2153 all_placeholders.insert(DynamicPlaceholder::VU);
2154 }
2155 "Iteration" => {
2156 all_placeholders.insert(DynamicPlaceholder::Iteration);
2157 }
2158 "Timestamp" => {
2159 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2160 }
2161 "UUID" => {
2162 all_placeholders.insert(DynamicPlaceholder::UUID);
2163 }
2164 "Random" => {
2165 all_placeholders.insert(DynamicPlaceholder::Random);
2166 }
2167 "Counter" => {
2168 all_placeholders.insert(DynamicPlaceholder::Counter);
2169 }
2170 "Date" => {
2171 all_placeholders.insert(DynamicPlaceholder::Date);
2172 }
2173 "VuIter" => {
2174 all_placeholders.insert(DynamicPlaceholder::VuIter);
2175 }
2176 _ => {}
2177 }
2178 }
2179 }
2180 }
2181 }
2182 }
2183 }
2184
2185 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2187 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2188
2189 let invalid_data_config = self.build_invalid_data_config();
2191 let error_injection_enabled = invalid_data_config.is_some();
2192 let error_rate = self.error_rate.unwrap_or(0.0);
2193 let error_types: Vec<String> = invalid_data_config
2194 .as_ref()
2195 .map(|c| c.error_types.iter().map(|t| format!("{:?}", t)).collect())
2196 .unwrap_or_default();
2197
2198 if error_injection_enabled {
2199 TerminalReporter::print_progress(&format!(
2200 "Error injection enabled ({}% rate)",
2201 (error_rate * 100.0) as u32
2202 ));
2203 }
2204
2205 let security_testing_enabled = self.wafbench_dir.is_some() || self.security_test;
2207
2208 let data = serde_json::json!({
2209 "base_url": self.target,
2210 "flows": flows_data,
2211 "extract_fields": config.default_extract_fields,
2212 "duration_secs": duration_secs,
2213 "max_vus": self.vus,
2214 "auth_header": self.auth,
2215 "custom_headers": custom_headers,
2216 "skip_tls_verify": self.skip_tls_verify,
2217 "stages": stages.iter().map(|s| serde_json::json!({
2219 "duration": s.duration,
2220 "target": s.target,
2221 })).collect::<Vec<_>>(),
2222 "threshold_percentile": self.threshold_percentile,
2223 "threshold_ms": self.threshold_ms,
2224 "max_error_rate": self.max_error_rate,
2225 "headers": headers_json,
2226 "dynamic_imports": required_imports,
2227 "dynamic_globals": required_globals,
2228 "extracted_values_output_path": self
2229 .output
2230 .join("crud_flow_extracted_values.json")
2231 .to_string_lossy(),
2232 "error_injection_enabled": error_injection_enabled,
2234 "error_rate": error_rate,
2235 "error_types": error_types,
2236 "security_testing_enabled": security_testing_enabled,
2238 "has_custom_headers": !custom_headers.is_empty(),
2239 });
2240
2241 let mut script = handlebars
2242 .render_template(template, &data)
2243 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2244
2245 if security_testing_enabled {
2247 script = self.generate_enhanced_script(&script)?;
2248 }
2249
2250 TerminalReporter::print_progress("Validating CRUD flow script...");
2252 let validation_errors = K6ScriptGenerator::validate_script(&script);
2253 if !validation_errors.is_empty() {
2254 TerminalReporter::print_error("CRUD flow script validation failed");
2255 for error in &validation_errors {
2256 eprintln!(" {}", error);
2257 }
2258 return Err(BenchError::Other(format!(
2259 "CRUD flow script validation failed with {} error(s)",
2260 validation_errors.len()
2261 )));
2262 }
2263
2264 TerminalReporter::print_success("CRUD flow script generated");
2265
2266 let script_path = if let Some(output) = &self.script_output {
2268 output.clone()
2269 } else {
2270 self.output.join("k6-crud-flow-script.js")
2271 };
2272
2273 if let Some(parent) = script_path.parent() {
2274 std::fs::create_dir_all(parent)?;
2275 }
2276 std::fs::write(&script_path, &script)?;
2277 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
2278
2279 if self.generate_only {
2280 println!("\nScript generated successfully. Run it with:");
2281 println!(" k6 run {}", script_path.display());
2282 return Ok(());
2283 }
2284
2285 TerminalReporter::print_progress("Executing CRUD flow test...");
2287 let executor = K6Executor::new()?.with_local_ips(self.source_ips.join(","));
2288 std::fs::create_dir_all(&self.output)?;
2289
2290 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
2291
2292 let duration_secs = Self::parse_duration(&self.duration)?;
2293 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
2294
2295 Ok(())
2296 }
2297
2298 async fn execute_conformance_test(&self) -> Result<()> {
2300 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
2301 use crate::conformance::report::ConformanceReport;
2302 use crate::conformance::spec::ConformanceFeature;
2303
2304 TerminalReporter::print_progress("OpenAPI 3.0.0 Conformance Testing Mode");
2305
2306 TerminalReporter::print_progress(
2309 "Conformance mode runs 1 VU, 1 iteration per endpoint (--vus and -d are ignored)",
2310 );
2311
2312 let categories = self.conformance_categories.as_ref().map(|cats_str| {
2314 cats_str
2315 .split(',')
2316 .filter_map(|s| {
2317 let trimmed = s.trim();
2318 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
2319 Some(canonical.to_string())
2320 } else {
2321 TerminalReporter::print_warning(&format!(
2322 "Unknown conformance category: '{}'. Valid categories: {}",
2323 trimmed,
2324 ConformanceFeature::cli_category_names()
2325 .iter()
2326 .map(|(cli, _)| *cli)
2327 .collect::<Vec<_>>()
2328 .join(", ")
2329 ));
2330 None
2331 }
2332 })
2333 .collect::<Vec<String>>()
2334 });
2335
2336 let custom_headers: Vec<(String, String)> = self
2338 .conformance_headers
2339 .iter()
2340 .filter_map(|h| {
2341 let (name, value) = h.split_once(':')?;
2342 Some((name.trim().to_string(), value.trim().to_string()))
2343 })
2344 .collect();
2345
2346 if !custom_headers.is_empty() {
2347 TerminalReporter::print_progress(&format!(
2348 "Using {} custom header(s) for authentication",
2349 custom_headers.len()
2350 ));
2351 }
2352
2353 if self.conformance_delay_ms > 0 {
2354 TerminalReporter::print_progress(&format!(
2355 "Using {}ms delay between conformance requests",
2356 self.conformance_delay_ms
2357 ));
2358 }
2359
2360 std::fs::create_dir_all(&self.output)?;
2362
2363 let config = ConformanceConfig {
2364 target_url: self.target.clone(),
2365 api_key: self.conformance_api_key.clone(),
2366 basic_auth: self.conformance_basic_auth.clone(),
2367 skip_tls_verify: self.skip_tls_verify,
2368 categories,
2369 base_path: self.base_path.clone(),
2370 custom_headers,
2371 output_dir: Some(self.output.clone()),
2372 all_operations: self.conformance_all_operations,
2373 custom_checks_file: self.conformance_custom.clone(),
2374 request_delay_ms: self.conformance_delay_ms,
2375 custom_filter: self.conformance_custom_filter.clone(),
2376 export_requests: self.export_requests,
2377 validate_requests: self.validate_requests,
2378 };
2379
2380 let mut resolved_base_path: Option<String> = None;
2388 let annotated_ops = if !self.spec.is_empty() {
2389 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
2390 let parser = SpecParser::from_file(&self.spec[0]).await?;
2391 resolved_base_path = self.resolve_base_path(&parser);
2392
2393 let mut operations = if let Some(filter) = &self.operations {
2398 parser.filter_operations(filter)?
2399 } else {
2400 parser.get_operations()
2401 };
2402 if let Some(exclude) = &self.exclude_operations {
2403 let before_count = operations.len();
2404 operations = parser.exclude_operations(operations, exclude)?;
2405 let excluded_count = before_count - operations.len();
2406 if excluded_count > 0 {
2407 TerminalReporter::print_progress(&format!(
2408 "Excluded {} operations matching '{}'",
2409 excluded_count, exclude
2410 ));
2411 }
2412 }
2413
2414 let annotated =
2415 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
2416 &operations,
2417 parser.spec(),
2418 );
2419 TerminalReporter::print_success(&format!(
2420 "Analyzed {} operations, found {} feature annotations",
2421 operations.len(),
2422 annotated.iter().map(|a| a.features.len()).sum::<usize>()
2423 ));
2424 Some(annotated)
2425 } else {
2426 None
2427 };
2428
2429 if self.conformance_self_test {
2436 let Some(ops) = annotated_ops else {
2437 TerminalReporter::print_error(
2438 "--conformance-self-test requires --spec; no operations to test",
2439 );
2440 return Ok(());
2441 };
2442 let cfg = crate::conformance::self_test::SelfTestConfig {
2443 target_url: self.target.clone(),
2444 skip_tls_verify: self.skip_tls_verify,
2445 timeout: std::time::Duration::from_secs(30),
2446 extra_headers: self
2450 .conformance_headers
2451 .iter()
2452 .filter_map(|h| {
2453 let (n, v) = h.split_once(':')?;
2454 Some((n.trim().to_string(), v.trim().to_string()))
2455 })
2456 .collect(),
2457 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
2458 base_path: resolved_base_path.clone(),
2462 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
2466 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
2467 geo_source_headers: if self.geo_source_headers.is_empty() {
2468 crate::conformance::self_test::default_geo_source_headers()
2469 } else {
2470 self.geo_source_headers.clone()
2471 },
2472 capture: if self.conformance_self_test_capture {
2476 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
2477 } else {
2478 None
2479 },
2480 };
2481 let capture_sink = cfg.capture.clone();
2482 TerminalReporter::print_progress(&format!(
2483 "Self-test mode: driving {} operations with positive + per-category negative cases",
2484 ops.len()
2485 ));
2486 let report = crate::conformance::self_test::run_self_test(&ops, &cfg)
2487 .await
2488 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
2489 if let Some(sink) = capture_sink {
2496 if let Ok(guard) = sink.lock() {
2497 let jsonl_path = self.output.join("conformance-self-test-requests.jsonl");
2498 let mut lines = String::with_capacity(guard.len() * 256);
2499 for entry in guard.iter() {
2500 if let Ok(line) = serde_json::to_string(entry) {
2501 lines.push_str(&line);
2502 lines.push('\n');
2503 }
2504 }
2505 let _ = std::fs::write(&jsonl_path, lines);
2506 let html_path = self.output.join("conformance-self-test-requests.html");
2507 let html =
2508 crate::conformance::capture_html::render_capture_html(guard.as_slice());
2509 let _ = std::fs::write(&html_path, html);
2510 TerminalReporter::print_progress(&format!(
2511 "Self-test request/response capture written to {} ({} entries) + {}",
2512 jsonl_path.display(),
2513 guard.len(),
2514 html_path.display(),
2515 ));
2516 }
2517 }
2518 TerminalReporter::print_progress(&report.render_summary());
2519 let json_path = self.output.join("conformance-self-test.json");
2523 if let Ok(json) = serde_json::to_string_pretty(&report) {
2524 let _ = std::fs::write(&json_path, json);
2525 TerminalReporter::print_progress(&format!(
2526 "Self-test report written to {}",
2527 json_path.display()
2528 ));
2529 }
2530 if let Some(status) = report.detect_target_misconfiguration() {
2539 let hint = match status {
2540 404 => " Likely cause: spec paths don't match deployed routes — check --base-path and the spec's `servers` block.",
2541 401 | 403 => " Likely cause: authentication header is missing or invalid — check --conformance-header.",
2542 _ => "",
2543 };
2544 TerminalReporter::print_warning(&format!(
2545 "Self-test misconfiguration: every positive case returned {status}.{hint} Negative results below are meaningless under this condition."
2546 ));
2547 } else if !report.all_passed() {
2548 TerminalReporter::print_warning(
2549 "Self-test detected gaps — server let through at least one request that should have been a 4xx",
2550 );
2551 } else {
2552 TerminalReporter::print_success(
2553 "Self-test passed — all positive cases accepted and all negative cases rejected",
2554 );
2555 }
2556 let html_path = self.output.join("conformance-report.html");
2563 let audit_path = self.output.join("conformance-spec-audit.json");
2564 let audit_value = std::fs::read_to_string(&audit_path)
2565 .ok()
2566 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok());
2567 let render_opts = crate::conformance::report_html::RenderOptions {
2572 missed_cap: match self.report_missed_cap {
2573 Some(0) => None,
2574 Some(n) => Some(n as usize),
2575 None => Some(200),
2576 },
2577 };
2578 let html = crate::conformance::report_html::render_html_with_options(
2579 &report,
2580 audit_value.as_ref(),
2581 &render_opts,
2582 );
2583 if std::fs::write(&html_path, html).is_ok() {
2584 TerminalReporter::print_progress(&format!(
2585 "HTML report written to {}",
2586 html_path.display()
2587 ));
2588 }
2589 return Ok(());
2590 }
2591
2592 if self.validate_requests && !self.spec.is_empty() {
2594 TerminalReporter::print_progress("Validating requests against OpenAPI spec...");
2595 let violation_count = crate::conformance::request_validator::run_request_validation(
2596 &self.spec,
2597 self.conformance_custom.as_deref(),
2598 self.base_path.as_deref(),
2599 &self.output,
2600 )
2601 .await?;
2602 if violation_count > 0 {
2603 TerminalReporter::print_warning(&format!(
2604 "{} request validation violation(s) found — see conformance-request-violations.json",
2605 violation_count
2606 ));
2607 } else {
2608 TerminalReporter::print_success("All requests conform to the OpenAPI spec");
2609 }
2610 }
2611
2612 if self.generate_only || self.use_k6 {
2614 let script = if let Some(annotated) = &annotated_ops {
2615 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
2616 config,
2617 annotated.clone(),
2618 );
2619 let op_count = gen.operation_count();
2620 let (script, check_count) = gen.generate()?;
2621 TerminalReporter::print_success(&format!(
2622 "Conformance: {} operations analyzed, {} unique checks generated",
2623 op_count, check_count
2624 ));
2625 script
2626 } else {
2627 let generator = ConformanceGenerator::new(config);
2628 generator.generate()?
2629 };
2630
2631 let script_path = self.output.join("k6-conformance.js");
2632 std::fs::write(&script_path, &script).map_err(|e| {
2633 BenchError::Other(format!("Failed to write conformance script: {}", e))
2634 })?;
2635 TerminalReporter::print_success(&format!(
2636 "Conformance script generated: {}",
2637 script_path.display()
2638 ));
2639
2640 if self.generate_only {
2641 println!("\nScript generated. Run with:");
2642 println!(" k6 run {}", script_path.display());
2643 return Ok(());
2644 }
2645
2646 if !K6Executor::is_k6_installed() {
2648 TerminalReporter::print_error("k6 is not installed");
2649 TerminalReporter::print_warning(
2650 "Install k6 from: https://k6.io/docs/get-started/installation/",
2651 );
2652 return Err(BenchError::K6NotFound);
2653 }
2654
2655 TerminalReporter::print_progress("Running conformance tests via k6...");
2656 let executor = K6Executor::new()?.with_local_ips(self.source_ips.join(","));
2657 executor.execute(&script_path, Some(&self.output), self.verbose).await?;
2658
2659 let report_path = self.output.join("conformance-report.json");
2660 if report_path.exists() {
2661 let report = ConformanceReport::from_file(&report_path)?;
2662 report.print_report_with_options(self.conformance_all_operations);
2663 self.save_conformance_report(&report, &report_path)?;
2664 } else {
2665 TerminalReporter::print_warning(
2666 "Conformance report not generated (k6 handleSummary may not have run)",
2667 );
2668 }
2669
2670 return Ok(());
2671 }
2672
2673 TerminalReporter::print_progress("Running conformance tests (native executor)...");
2675
2676 let mut executor = crate::conformance::executor::NativeConformanceExecutor::new(config)?;
2677
2678 executor = if let Some(annotated) = &annotated_ops {
2679 executor.with_spec_driven_checks(annotated)
2680 } else {
2681 executor.with_reference_checks()
2682 };
2683 executor = executor.with_custom_checks()?;
2684
2685 TerminalReporter::print_success(&format!(
2686 "Executing {} conformance checks...",
2687 executor.check_count()
2688 ));
2689
2690 let report = executor.execute().await?;
2691 report.print_report_with_options(self.conformance_all_operations);
2692
2693 let failure_details = report.failure_details();
2695 if !failure_details.is_empty() {
2696 let details_path = self.output.join("conformance-failure-details.json");
2697 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
2698 let _ = std::fs::write(&details_path, json);
2699 TerminalReporter::print_success(&format!(
2700 "Failure details saved to: {}",
2701 details_path.display()
2702 ));
2703 }
2704 }
2705
2706 let report_path = self.output.join("conformance-report.json");
2708 let report_json = serde_json::to_string_pretty(&report.to_json())
2709 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
2710 std::fs::write(&report_path, &report_json)
2711 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
2712 TerminalReporter::print_success(&format!("Report saved to: {}", report_path.display()));
2713
2714 self.save_conformance_report(&report, &report_path)?;
2715
2716 Ok(())
2717 }
2718
2719 fn save_conformance_report(
2721 &self,
2722 report: &crate::conformance::report::ConformanceReport,
2723 report_path: &Path,
2724 ) -> Result<()> {
2725 if self.conformance_report_format == "sarif" {
2726 use crate::conformance::sarif::ConformanceSarifReport;
2727 ConformanceSarifReport::write(report, &self.target, &self.conformance_report)?;
2728 TerminalReporter::print_success(&format!(
2729 "SARIF report saved to: {}",
2730 self.conformance_report.display()
2731 ));
2732 } else if self.conformance_report != *report_path {
2733 std::fs::copy(report_path, &self.conformance_report)?;
2734 TerminalReporter::print_success(&format!(
2735 "Report saved to: {}",
2736 self.conformance_report.display()
2737 ));
2738 }
2739 Ok(())
2740 }
2741
2742 async fn execute_multi_target_conformance(&self, targets_file: &Path) -> Result<()> {
2748 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
2749 use crate::conformance::report::ConformanceReport;
2750 use crate::conformance::spec::ConformanceFeature;
2751
2752 TerminalReporter::print_progress("Multi-target OpenAPI 3.0.0 Conformance Testing Mode");
2753
2754 TerminalReporter::print_progress("Parsing targets file...");
2756 let targets = parse_targets_file(targets_file)?;
2757 let num_targets = targets.len();
2758 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
2759
2760 if targets.is_empty() {
2761 return Err(BenchError::Other("No targets found in file".to_string()));
2762 }
2763
2764 TerminalReporter::print_progress(
2765 "Conformance mode runs 1 VU, 1 iteration per endpoint (--vus and -d are ignored)",
2766 );
2767
2768 let categories = self.conformance_categories.as_ref().map(|cats_str| {
2770 cats_str
2771 .split(',')
2772 .filter_map(|s| {
2773 let trimmed = s.trim();
2774 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
2775 Some(canonical.to_string())
2776 } else {
2777 TerminalReporter::print_warning(&format!(
2778 "Unknown conformance category: '{}'. Valid categories: {}",
2779 trimmed,
2780 ConformanceFeature::cli_category_names()
2781 .iter()
2782 .map(|(cli, _)| *cli)
2783 .collect::<Vec<_>>()
2784 .join(", ")
2785 ));
2786 None
2787 }
2788 })
2789 .collect::<Vec<String>>()
2790 });
2791
2792 let base_custom_headers: Vec<(String, String)> = self
2794 .conformance_headers
2795 .iter()
2796 .filter_map(|h| {
2797 let (name, value) = h.split_once(':')?;
2798 Some((name.trim().to_string(), value.trim().to_string()))
2799 })
2800 .collect();
2801
2802 if !base_custom_headers.is_empty() {
2803 TerminalReporter::print_progress(&format!(
2804 "Using {} base custom header(s) for authentication",
2805 base_custom_headers.len()
2806 ));
2807 }
2808
2809 let annotated_ops = if !self.spec.is_empty() {
2811 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
2812 let parser = SpecParser::from_file(&self.spec[0]).await?;
2813 let operations = parser.get_operations();
2814 let annotated =
2815 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
2816 &operations,
2817 parser.spec(),
2818 );
2819 TerminalReporter::print_success(&format!(
2820 "Analyzed {} operations, found {} feature annotations",
2821 operations.len(),
2822 annotated.iter().map(|a| a.features.len()).sum::<usize>()
2823 ));
2824 Some(annotated)
2825 } else {
2826 None
2827 };
2828
2829 std::fs::create_dir_all(&self.output)?;
2831
2832 struct TargetResult {
2834 url: String,
2835 passed: usize,
2836 failed: usize,
2837 elapsed: std::time::Duration,
2838 report_json: serde_json::Value,
2839 owasp_coverage: Vec<crate::conformance::report::OwaspCoverageEntry>,
2840 }
2841
2842 let mut target_results: Vec<TargetResult> = Vec::with_capacity(num_targets);
2843 let total_start = std::time::Instant::now();
2844
2845 for (idx, target) in targets.iter().enumerate() {
2846 tracing::info!(
2847 "Running conformance tests against target {}/{}: {}",
2848 idx + 1,
2849 num_targets,
2850 target.url
2851 );
2852 TerminalReporter::print_progress(&format!(
2853 "\n--- Target {}/{}: {} ---",
2854 idx + 1,
2855 num_targets,
2856 target.url
2857 ));
2858
2859 let mut merged_headers = base_custom_headers.clone();
2861 if let Some(ref target_headers) = target.headers {
2862 for (name, value) in target_headers {
2863 if let Some(existing) = merged_headers.iter_mut().find(|(n, _)| n == name) {
2865 existing.1 = value.clone();
2866 } else {
2867 merged_headers.push((name.clone(), value.clone()));
2868 }
2869 }
2870 }
2871 if let Some(ref auth) = target.auth {
2873 if let Some(existing) =
2874 merged_headers.iter_mut().find(|(n, _)| n.eq_ignore_ascii_case("Authorization"))
2875 {
2876 existing.1 = auth.clone();
2877 } else {
2878 merged_headers.push(("Authorization".to_string(), auth.clone()));
2879 }
2880 }
2881
2882 let target_dir = self.output.join(format!("target_{}", idx));
2888 std::fs::create_dir_all(&target_dir)?;
2889
2890 let config = ConformanceConfig {
2891 target_url: target.url.clone(),
2892 api_key: self.conformance_api_key.clone(),
2893 basic_auth: self.conformance_basic_auth.clone(),
2894 skip_tls_verify: self.skip_tls_verify,
2895 categories: categories.clone(),
2896 base_path: self.base_path.clone(),
2897 custom_headers: merged_headers,
2898 output_dir: Some(target_dir.clone()),
2899 all_operations: self.conformance_all_operations,
2900 custom_checks_file: self.conformance_custom.clone(),
2901 request_delay_ms: self.conformance_delay_ms,
2902 custom_filter: self.conformance_custom_filter.clone(),
2903 export_requests: self.export_requests,
2904 validate_requests: self.validate_requests,
2905 };
2906
2907 let target_start = std::time::Instant::now();
2908 let report = if self.use_k6 {
2909 if !K6Executor::is_k6_installed() {
2910 TerminalReporter::print_error("k6 is not installed");
2911 TerminalReporter::print_warning(
2912 "Install k6 from: https://k6.io/docs/get-started/installation/",
2913 );
2914 return Err(BenchError::K6NotFound);
2915 }
2916
2917 let script = if let Some(ref annotated) = annotated_ops {
2918 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
2919 config.clone(),
2920 annotated.clone(),
2921 );
2922 let (script, _check_count) = gen.generate()?;
2923 script
2924 } else {
2925 let generator = ConformanceGenerator::new(config.clone());
2926 generator.generate()?
2927 };
2928
2929 let script_path = target_dir.join("k6-conformance.js");
2930 std::fs::write(&script_path, &script).map_err(|e| {
2931 BenchError::Other(format!("Failed to write conformance script: {}", e))
2932 })?;
2933 TerminalReporter::print_success(&format!(
2934 "Conformance script generated: {}",
2935 script_path.display()
2936 ));
2937
2938 TerminalReporter::print_progress(&format!(
2939 "Running conformance tests via k6 against {}...",
2940 target.url
2941 ));
2942 let k6 = K6Executor::new()?.with_local_ips(self.source_ips.join(","));
2943 let api_port = 6565u16.saturating_add(idx as u16);
2945 k6.execute_with_port(&script_path, Some(&target_dir), self.verbose, Some(api_port))
2946 .await?;
2947
2948 let report_path = target_dir.join("conformance-report.json");
2949 if report_path.exists() {
2950 ConformanceReport::from_file(&report_path)?
2951 } else {
2952 TerminalReporter::print_warning(&format!(
2953 "Conformance report not generated for target {} (k6 handleSummary may not have run)",
2954 target.url
2955 ));
2956 continue;
2957 }
2958 } else {
2959 let mut executor =
2960 crate::conformance::executor::NativeConformanceExecutor::new(config)?;
2961
2962 executor = if let Some(ref annotated) = annotated_ops {
2963 executor.with_spec_driven_checks(annotated)
2964 } else {
2965 executor.with_reference_checks()
2966 };
2967 executor = executor.with_custom_checks()?;
2968
2969 TerminalReporter::print_success(&format!(
2970 "Executing {} conformance checks against {}...",
2971 executor.check_count(),
2972 target.url
2973 ));
2974
2975 executor.execute().await?
2976 };
2977 let target_elapsed = target_start.elapsed();
2978
2979 let report_json = report.to_json();
2980
2981 let passed = report_json["summary"]["passed"].as_u64().unwrap_or(0) as usize;
2983 let failed = report_json["summary"]["failed"].as_u64().unwrap_or(0) as usize;
2984 let total_checks = passed + failed;
2985 let rate = if total_checks == 0 {
2986 0.0
2987 } else {
2988 (passed as f64 / total_checks as f64) * 100.0
2989 };
2990
2991 TerminalReporter::print_success(&format!(
2992 "Target {}: {}/{} passed ({:.1}%) in {:.1}s",
2993 target.url,
2994 passed,
2995 total_checks,
2996 rate,
2997 target_elapsed.as_secs_f64()
2998 ));
2999
3000 let target_report_path = target_dir.join("conformance-report.json");
3002 let report_str = serde_json::to_string_pretty(&report_json)
3003 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
3004 std::fs::write(&target_report_path, &report_str)
3005 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
3006
3007 let failure_details = report.failure_details();
3009 if !failure_details.is_empty() {
3010 let details_path = target_dir.join("conformance-failure-details.json");
3011 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
3012 let _ = std::fs::write(&details_path, json);
3013 }
3014 }
3015
3016 let owasp_coverage = report.owasp_coverage_data();
3018
3019 target_results.push(TargetResult {
3020 url: target.url.clone(),
3021 passed,
3022 failed,
3023 elapsed: target_elapsed,
3024 report_json,
3025 owasp_coverage,
3026 });
3027 }
3028
3029 let total_elapsed = total_start.elapsed();
3030
3031 println!("\n{}", "=".repeat(80));
3033 println!(" Multi-Target Conformance Summary");
3034 println!("{}", "=".repeat(80));
3035 println!(
3036 " {:<40} {:>8} {:>8} {:>8} {:>8}",
3037 "Target URL", "Passed", "Failed", "Rate", "Time"
3038 );
3039 println!(" {}", "-".repeat(76));
3040
3041 let mut total_passed = 0usize;
3042 let mut total_failed = 0usize;
3043
3044 for result in &target_results {
3045 let total_checks = result.passed + result.failed;
3046 let rate = if total_checks == 0 {
3047 0.0
3048 } else {
3049 (result.passed as f64 / total_checks as f64) * 100.0
3050 };
3051
3052 let display_url = if result.url.len() > 38 {
3054 format!("{}...", &result.url[..35])
3055 } else {
3056 result.url.clone()
3057 };
3058
3059 println!(
3060 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
3061 display_url,
3062 result.passed,
3063 result.failed,
3064 rate,
3065 result.elapsed.as_secs_f64()
3066 );
3067
3068 total_passed += result.passed;
3069 total_failed += result.failed;
3070 }
3071
3072 let grand_total = total_passed + total_failed;
3073 let overall_rate = if grand_total == 0 {
3074 0.0
3075 } else {
3076 (total_passed as f64 / grand_total as f64) * 100.0
3077 };
3078
3079 println!(" {}", "-".repeat(76));
3080 println!(
3081 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
3082 format!("TOTAL ({} targets)", num_targets),
3083 total_passed,
3084 total_failed,
3085 overall_rate,
3086 total_elapsed.as_secs_f64()
3087 );
3088 println!("{}", "=".repeat(80));
3089
3090 for result in &target_results {
3092 println!("\n OWASP API Security Top 10 Coverage for {}:", result.url);
3093 for entry in &result.owasp_coverage {
3094 let status = if !entry.tested {
3095 "-"
3096 } else if entry.all_passed {
3097 "pass"
3098 } else {
3099 "FAIL"
3100 };
3101 let via = if entry.via_categories.is_empty() {
3102 String::new()
3103 } else {
3104 format!(" (via {})", entry.via_categories.join(", "))
3105 };
3106 println!(" {:<12} {:<40} {}{}", entry.id, entry.name, status, via);
3107 }
3108 }
3109
3110 let per_target_summaries: Vec<serde_json::Value> = target_results
3112 .iter()
3113 .enumerate()
3114 .map(|(idx, r)| {
3115 let total_checks = r.passed + r.failed;
3116 let rate = if total_checks == 0 {
3117 0.0
3118 } else {
3119 (r.passed as f64 / total_checks as f64) * 100.0
3120 };
3121 let owasp_json: Vec<serde_json::Value> = r
3122 .owasp_coverage
3123 .iter()
3124 .map(|e| {
3125 serde_json::json!({
3126 "id": e.id,
3127 "name": e.name,
3128 "tested": e.tested,
3129 "all_passed": e.all_passed,
3130 "via_categories": e.via_categories,
3131 })
3132 })
3133 .collect();
3134 serde_json::json!({
3135 "target_url": r.url,
3136 "target_index": idx,
3137 "checks_passed": r.passed,
3138 "checks_failed": r.failed,
3139 "total_checks": total_checks,
3140 "pass_rate": rate,
3141 "elapsed_seconds": r.elapsed.as_secs_f64(),
3142 "report": r.report_json,
3143 "owasp_coverage": owasp_json,
3144 })
3145 })
3146 .collect();
3147
3148 let combined_summary = serde_json::json!({
3149 "total_targets": num_targets,
3150 "total_checks_passed": total_passed,
3151 "total_checks_failed": total_failed,
3152 "overall_pass_rate": overall_rate,
3153 "total_elapsed_seconds": total_elapsed.as_secs_f64(),
3154 "targets": per_target_summaries,
3155 });
3156
3157 let summary_path = self.output.join("multi-target-conformance-summary.json");
3158 let summary_str = serde_json::to_string_pretty(&combined_summary)
3159 .map_err(|e| BenchError::Other(format!("Failed to serialize summary: {}", e)))?;
3160 std::fs::write(&summary_path, &summary_str)
3161 .map_err(|e| BenchError::Other(format!("Failed to write summary: {}", e)))?;
3162 TerminalReporter::print_success(&format!(
3163 "Combined summary saved to: {}",
3164 summary_path.display()
3165 ));
3166
3167 Ok(())
3168 }
3169
3170 async fn execute_owasp_test(&self, parser: &SpecParser) -> Result<()> {
3172 TerminalReporter::print_progress("OWASP API Security Top 10 Testing Mode");
3173
3174 let custom_headers = self.parse_headers()?;
3176
3177 let mut config = OwaspApiConfig::new()
3179 .with_auth_header(&self.owasp_auth_header)
3180 .with_verbose(self.verbose)
3181 .with_insecure(self.skip_tls_verify)
3182 .with_concurrency(self.vus as usize)
3183 .with_iterations(self.owasp_iterations as usize)
3184 .with_base_path(self.base_path.clone())
3185 .with_custom_headers(custom_headers);
3186
3187 if let Some(ref token) = self.owasp_auth_token {
3189 config = config.with_valid_auth_token(token);
3190 }
3191
3192 if let Some(ref cats_str) = self.owasp_categories {
3194 let categories: Vec<OwaspCategory> = cats_str
3195 .split(',')
3196 .filter_map(|s| {
3197 let trimmed = s.trim();
3198 match trimmed.parse::<OwaspCategory>() {
3199 Ok(cat) => Some(cat),
3200 Err(e) => {
3201 TerminalReporter::print_warning(&e);
3202 None
3203 }
3204 }
3205 })
3206 .collect();
3207
3208 if !categories.is_empty() {
3209 config = config.with_categories(categories);
3210 }
3211 }
3212
3213 if let Some(ref admin_paths_file) = self.owasp_admin_paths {
3215 config.admin_paths_file = Some(admin_paths_file.clone());
3216 if let Err(e) = config.load_admin_paths() {
3217 TerminalReporter::print_warning(&format!("Failed to load admin paths file: {}", e));
3218 }
3219 }
3220
3221 if let Some(ref id_fields_str) = self.owasp_id_fields {
3223 let id_fields: Vec<String> = id_fields_str
3224 .split(',')
3225 .map(|s| s.trim().to_string())
3226 .filter(|s| !s.is_empty())
3227 .collect();
3228 if !id_fields.is_empty() {
3229 config = config.with_id_fields(id_fields);
3230 }
3231 }
3232
3233 if let Some(ref report_path) = self.owasp_report {
3235 config = config.with_report_path(report_path);
3236 }
3237 if let Ok(format) = self.owasp_report_format.parse::<ReportFormat>() {
3238 config = config.with_report_format(format);
3239 }
3240
3241 let categories = config.categories_to_test();
3243 TerminalReporter::print_success(&format!(
3244 "Testing {} OWASP categories: {}",
3245 categories.len(),
3246 categories.iter().map(|c| c.cli_name()).collect::<Vec<_>>().join(", ")
3247 ));
3248
3249 if config.valid_auth_token.is_some() {
3250 TerminalReporter::print_progress("Using provided auth token for baseline requests");
3251 }
3252
3253 TerminalReporter::print_progress("Generating OWASP security test script...");
3255 let generator = OwaspApiGenerator::new(config, self.target.clone(), parser);
3256
3257 let script = generator.generate()?;
3259 TerminalReporter::print_success("OWASP security test script generated");
3260
3261 let script_path = if let Some(output) = &self.script_output {
3263 output.clone()
3264 } else {
3265 self.output.join("k6-owasp-security-test.js")
3266 };
3267
3268 if let Some(parent) = script_path.parent() {
3269 std::fs::create_dir_all(parent)?;
3270 }
3271 std::fs::write(&script_path, &script)?;
3272 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
3273
3274 if self.generate_only {
3276 println!("\nOWASP security test script generated. Run it with:");
3277 println!(" k6 run {}", script_path.display());
3278 return Ok(());
3279 }
3280
3281 TerminalReporter::print_progress("Executing OWASP security tests...");
3283 let executor = K6Executor::new()?.with_local_ips(self.source_ips.join(","));
3284 std::fs::create_dir_all(&self.output)?;
3285
3286 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
3287
3288 let duration_secs = Self::parse_duration(&self.duration)?;
3289 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
3290
3291 println!("\nOWASP security test results saved to: {}", self.output.display());
3292
3293 Ok(())
3294 }
3295}
3296
3297#[cfg(test)]
3298mod tests {
3299 use super::*;
3300 use tempfile::tempdir;
3301
3302 #[test]
3303 fn test_parse_duration() {
3304 assert_eq!(BenchCommand::parse_duration("30s").unwrap(), 30);
3305 assert_eq!(BenchCommand::parse_duration("5m").unwrap(), 300);
3306 assert_eq!(BenchCommand::parse_duration("1h").unwrap(), 3600);
3307 assert_eq!(BenchCommand::parse_duration("60").unwrap(), 60);
3308 }
3309
3310 #[test]
3314 fn parse_ip_list_ipv4_range_inclusive() {
3315 let v = parse_ip_list(&["10.0.0.5-10.0.0.27".into()], "source-ip");
3316 assert_eq!(v.len(), 23);
3317 assert_eq!(v.first().unwrap().to_string(), "10.0.0.5");
3318 assert_eq!(v.last().unwrap().to_string(), "10.0.0.27");
3319 }
3320
3321 #[test]
3324 fn parse_ip_list_range_rejects_backwards() {
3325 let v = parse_ip_list(&["10.0.0.10-10.0.0.5".into()], "source-ip");
3326 assert!(v.is_empty(), "backwards range should produce no IPs; got {v:?}");
3327 }
3328
3329 #[test]
3333 fn parse_ip_list_rejects_ipv6_range_syntax() {
3334 let v = parse_ip_list(&["2001:db8::1-2001:db8::5".into()], "geo-source-ip");
3335 assert!(v.is_empty(), "IPv6 range should be rejected; got {v:?}");
3336 }
3337
3338 #[test]
3340 fn parse_ip_list_range_capped_at_256() {
3341 let v = parse_ip_list(&["10.0.0.0-10.0.5.0".into()], "source-ip");
3342 assert_eq!(v.len(), 256);
3343 assert_eq!(v.first().unwrap().to_string(), "10.0.0.0");
3344 }
3345
3346 #[test]
3349 fn parse_ip_list_plain_and_comma() {
3350 let v = parse_ip_list(&["10.0.0.5".into(), "10.0.0.6,10.0.0.7".into()], "source-ip");
3351 assert_eq!(v.len(), 3);
3352 assert_eq!(v[0].to_string(), "10.0.0.5");
3353 assert_eq!(v[2].to_string(), "10.0.0.7");
3354 }
3355
3356 #[test]
3359 fn parse_ip_list_ipv4_cidr_29_expands_to_8() {
3360 let v = parse_ip_list(&["10.0.0.0/29".into()], "source-ip");
3361 assert_eq!(v.len(), 8);
3362 assert_eq!(v[0].to_string(), "10.0.0.0");
3363 assert_eq!(v[7].to_string(), "10.0.0.7");
3364 }
3365
3366 #[test]
3369 fn parse_ip_list_ipv4_cidr_8_capped_at_256() {
3370 let v = parse_ip_list(&["10.0.0.0/8".into()], "source-ip");
3371 assert_eq!(v.len(), 256);
3372 assert_eq!(v[0].to_string(), "10.0.0.0");
3373 assert_eq!(v[255].to_string(), "10.0.0.255");
3374 }
3375
3376 #[test]
3378 fn parse_ip_list_ipv6_cidr_126_expands_to_4() {
3379 let v = parse_ip_list(&["2001:db8::/126".into()], "geo-source-ip");
3380 assert_eq!(v.len(), 4);
3381 assert!(v[0].is_ipv6());
3382 assert_eq!(v[0].to_string(), "2001:db8::");
3383 assert_eq!(v[3].to_string(), "2001:db8::3");
3384 }
3385
3386 #[test]
3388 fn parse_ip_list_mixed_v4_v6_cidr() {
3389 let v = parse_ip_list(&["10.0.0.0/30,2001:db8::1,203.0.113.42".into()], "geo-source-ip");
3390 assert_eq!(v.len(), 6); assert!(v.iter().any(|ip| ip.to_string() == "2001:db8::1"));
3392 assert!(v.iter().any(|ip| ip.to_string() == "203.0.113.42"));
3393 }
3394
3395 #[test]
3398 fn parse_ip_list_skips_malformed() {
3399 let v = parse_ip_list(
3400 &[
3401 "10.0.0.5".into(),
3402 "not-an-ip".into(),
3403 "10.0.0.6".into(),
3404 "/24".into(),
3405 "1.2.3.4/200".into(),
3406 ],
3407 "source-ip",
3408 );
3409 assert_eq!(v.len(), 2);
3410 assert_eq!(v[0].to_string(), "10.0.0.5");
3411 assert_eq!(v[1].to_string(), "10.0.0.6");
3412 }
3413
3414 #[test]
3415 fn test_parse_duration_invalid() {
3416 assert!(BenchCommand::parse_duration("invalid").is_err());
3417 assert!(BenchCommand::parse_duration("30x").is_err());
3418 }
3419
3420 #[test]
3421 fn test_parse_headers() {
3422 let cmd = BenchCommand {
3423 spec: vec![PathBuf::from("test.yaml")],
3424 spec_dir: None,
3425 merge_conflicts: "error".to_string(),
3426 spec_mode: "merge".to_string(),
3427 dependency_config: None,
3428 target: "http://localhost".to_string(),
3429 base_path: None,
3430 duration: "1m".to_string(),
3431 vus: 10,
3432 scenario: "ramp-up".to_string(),
3433 operations: None,
3434 exclude_operations: None,
3435 auth: None,
3436 headers: Some("X-API-Key:test123,X-Client-ID:client456".to_string()),
3437 output: PathBuf::from("output"),
3438 generate_only: false,
3439 script_output: None,
3440 threshold_percentile: "p(95)".to_string(),
3441 threshold_ms: 500,
3442 max_error_rate: 0.05,
3443 verbose: false,
3444 skip_tls_verify: false,
3445 chunked_request_bodies: false,
3446 target_rps: None,
3447 no_keep_alive: false,
3448 targets_file: None,
3449 max_concurrency: None,
3450 results_format: "both".to_string(),
3451 params_file: None,
3452 crud_flow: false,
3453 flow_config: None,
3454 extract_fields: None,
3455 parallel_create: None,
3456 data_file: None,
3457 data_distribution: "unique-per-vu".to_string(),
3458 data_mappings: None,
3459 per_uri_control: false,
3460 error_rate: None,
3461 error_types: None,
3462 security_test: false,
3463 security_payloads: None,
3464 security_categories: None,
3465 security_target_fields: None,
3466 wafbench_dir: None,
3467 wafbench_cycle_all: false,
3468 owasp_api_top10: false,
3469 owasp_categories: None,
3470 owasp_auth_header: "Authorization".to_string(),
3471 owasp_auth_token: None,
3472 owasp_admin_paths: None,
3473 owasp_id_fields: None,
3474 owasp_report: None,
3475 owasp_report_format: "json".to_string(),
3476 owasp_iterations: 1,
3477 conformance: false,
3478 conformance_api_key: None,
3479 conformance_basic_auth: None,
3480 conformance_report: PathBuf::from("conformance-report.json"),
3481 conformance_categories: None,
3482 conformance_report_format: "json".to_string(),
3483 conformance_headers: vec![],
3484 conformance_all_operations: false,
3485 conformance_custom: None,
3486 conformance_delay_ms: 0,
3487 use_k6: false,
3488 conformance_custom_filter: None,
3489 export_requests: false,
3490 validate_requests: false,
3491 conformance_self_test: false,
3492 conformance_self_test_capture: false,
3493 source_ips: Vec::new(),
3494 geo_source_ips: Vec::new(),
3495 geo_source_headers: Vec::new(),
3496 report_missed_cap: None,
3497 };
3498
3499 let headers = cmd.parse_headers().unwrap();
3500 assert_eq!(headers.get("X-API-Key"), Some(&"test123".to_string()));
3501 assert_eq!(headers.get("X-Client-ID"), Some(&"client456".to_string()));
3502 }
3503
3504 #[test]
3505 fn test_get_spec_display_name() {
3506 let cmd = BenchCommand {
3507 spec: vec![PathBuf::from("test.yaml")],
3508 spec_dir: None,
3509 merge_conflicts: "error".to_string(),
3510 spec_mode: "merge".to_string(),
3511 dependency_config: None,
3512 target: "http://localhost".to_string(),
3513 base_path: None,
3514 duration: "1m".to_string(),
3515 vus: 10,
3516 scenario: "ramp-up".to_string(),
3517 operations: None,
3518 exclude_operations: None,
3519 auth: None,
3520 headers: None,
3521 output: PathBuf::from("output"),
3522 generate_only: false,
3523 script_output: None,
3524 threshold_percentile: "p(95)".to_string(),
3525 threshold_ms: 500,
3526 max_error_rate: 0.05,
3527 verbose: false,
3528 skip_tls_verify: false,
3529 chunked_request_bodies: false,
3530 target_rps: None,
3531 no_keep_alive: false,
3532 targets_file: None,
3533 max_concurrency: None,
3534 results_format: "both".to_string(),
3535 params_file: None,
3536 crud_flow: false,
3537 flow_config: None,
3538 extract_fields: None,
3539 parallel_create: None,
3540 data_file: None,
3541 data_distribution: "unique-per-vu".to_string(),
3542 data_mappings: None,
3543 per_uri_control: false,
3544 error_rate: None,
3545 error_types: None,
3546 security_test: false,
3547 security_payloads: None,
3548 security_categories: None,
3549 security_target_fields: None,
3550 wafbench_dir: None,
3551 wafbench_cycle_all: false,
3552 owasp_api_top10: false,
3553 owasp_categories: None,
3554 owasp_auth_header: "Authorization".to_string(),
3555 owasp_auth_token: None,
3556 owasp_admin_paths: None,
3557 owasp_id_fields: None,
3558 owasp_report: None,
3559 owasp_report_format: "json".to_string(),
3560 owasp_iterations: 1,
3561 conformance: false,
3562 conformance_api_key: None,
3563 conformance_basic_auth: None,
3564 conformance_report: PathBuf::from("conformance-report.json"),
3565 conformance_categories: None,
3566 conformance_report_format: "json".to_string(),
3567 conformance_headers: vec![],
3568 conformance_all_operations: false,
3569 conformance_custom: None,
3570 conformance_delay_ms: 0,
3571 use_k6: false,
3572 conformance_custom_filter: None,
3573 export_requests: false,
3574 validate_requests: false,
3575 conformance_self_test: false,
3576 conformance_self_test_capture: false,
3577 source_ips: Vec::new(),
3578 geo_source_ips: Vec::new(),
3579 geo_source_headers: Vec::new(),
3580 report_missed_cap: None,
3581 };
3582
3583 assert_eq!(cmd.get_spec_display_name(), "test.yaml");
3584
3585 let cmd_multi = BenchCommand {
3587 spec: vec![PathBuf::from("a.yaml"), PathBuf::from("b.yaml")],
3588 spec_dir: None,
3589 merge_conflicts: "error".to_string(),
3590 spec_mode: "merge".to_string(),
3591 dependency_config: None,
3592 target: "http://localhost".to_string(),
3593 base_path: None,
3594 duration: "1m".to_string(),
3595 vus: 10,
3596 scenario: "ramp-up".to_string(),
3597 operations: None,
3598 exclude_operations: None,
3599 auth: None,
3600 headers: None,
3601 output: PathBuf::from("output"),
3602 generate_only: false,
3603 script_output: None,
3604 threshold_percentile: "p(95)".to_string(),
3605 threshold_ms: 500,
3606 max_error_rate: 0.05,
3607 verbose: false,
3608 skip_tls_verify: false,
3609 chunked_request_bodies: false,
3610 target_rps: None,
3611 no_keep_alive: false,
3612 targets_file: None,
3613 max_concurrency: None,
3614 results_format: "both".to_string(),
3615 params_file: None,
3616 crud_flow: false,
3617 flow_config: None,
3618 extract_fields: None,
3619 parallel_create: None,
3620 data_file: None,
3621 data_distribution: "unique-per-vu".to_string(),
3622 data_mappings: None,
3623 per_uri_control: false,
3624 error_rate: None,
3625 error_types: None,
3626 security_test: false,
3627 security_payloads: None,
3628 security_categories: None,
3629 security_target_fields: None,
3630 wafbench_dir: None,
3631 wafbench_cycle_all: false,
3632 owasp_api_top10: false,
3633 owasp_categories: None,
3634 owasp_auth_header: "Authorization".to_string(),
3635 owasp_auth_token: None,
3636 owasp_admin_paths: None,
3637 owasp_id_fields: None,
3638 owasp_report: None,
3639 owasp_report_format: "json".to_string(),
3640 owasp_iterations: 1,
3641 conformance: false,
3642 conformance_api_key: None,
3643 conformance_basic_auth: None,
3644 conformance_report: PathBuf::from("conformance-report.json"),
3645 conformance_categories: None,
3646 conformance_report_format: "json".to_string(),
3647 conformance_headers: vec![],
3648 conformance_all_operations: false,
3649 conformance_custom: None,
3650 conformance_delay_ms: 0,
3651 use_k6: false,
3652 conformance_custom_filter: None,
3653 export_requests: false,
3654 validate_requests: false,
3655 conformance_self_test: false,
3656 conformance_self_test_capture: false,
3657 source_ips: Vec::new(),
3658 geo_source_ips: Vec::new(),
3659 geo_source_headers: Vec::new(),
3660 report_missed_cap: None,
3661 };
3662
3663 assert_eq!(cmd_multi.get_spec_display_name(), "2 spec files");
3664 }
3665
3666 #[test]
3667 fn test_parse_extracted_values_from_output_dir() {
3668 let dir = tempdir().unwrap();
3669 let path = dir.path().join("extracted_values.json");
3670 std::fs::write(
3671 &path,
3672 r#"{
3673 "pool_id": "abc123",
3674 "count": 0,
3675 "enabled": false,
3676 "metadata": { "owner": "team-a" }
3677}"#,
3678 )
3679 .unwrap();
3680
3681 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
3682 assert_eq!(extracted.get("pool_id"), Some(&serde_json::json!("abc123")));
3683 assert_eq!(extracted.get("count"), Some(&serde_json::json!(0)));
3684 assert_eq!(extracted.get("enabled"), Some(&serde_json::json!(false)));
3685 assert_eq!(extracted.get("metadata"), Some(&serde_json::json!({"owner": "team-a"})));
3686 }
3687
3688 #[test]
3689 fn test_parse_extracted_values_missing_file() {
3690 let dir = tempdir().unwrap();
3691 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
3692 assert!(extracted.values.is_empty());
3693 }
3694}