1use crate::crud_flow::{CrudFlowConfig, CrudFlowDetector};
4use crate::data_driven::{DataDistribution, DataDrivenConfig, DataDrivenGenerator, DataMapping};
5use crate::dynamic_params::{DynamicParamProcessor, DynamicPlaceholder};
6use crate::error::{BenchError, Result};
7use crate::executor::K6Executor;
8use crate::invalid_data::{InvalidDataConfig, InvalidDataGenerator};
9use crate::k6_gen::{K6Config, K6ScriptGenerator};
10use crate::mock_integration::{
11 MockIntegrationConfig, MockIntegrationGenerator, MockServerDetector,
12};
13use crate::owasp_api::{OwaspApiConfig, OwaspApiGenerator, OwaspCategory, ReportFormat};
14use crate::parallel_executor::{AggregatedResults, ParallelExecutor};
15use crate::parallel_requests::{ParallelConfig, ParallelRequestGenerator};
16use crate::param_overrides::ParameterOverrides;
17use crate::reporter::TerminalReporter;
18use crate::request_gen::RequestGenerator;
19use crate::scenarios::LoadScenario;
20use crate::security_payloads::{
21 SecurityCategory, SecurityPayload, SecurityPayloads, SecurityTestConfig, SecurityTestGenerator,
22};
23use crate::spec_dependencies::{
24 topological_sort, DependencyDetector, ExtractedValues, SpecDependencyConfig,
25};
26use crate::spec_parser::SpecParser;
27use crate::target_parser::parse_targets_file;
28use crate::wafbench::WafBenchLoader;
29use mockforge_openapi::multi_spec::{
30 load_specs_from_directory, load_specs_from_files, merge_specs, ConflictStrategy,
31};
32use mockforge_openapi::spec::OpenApiSpec;
33use std::collections::{HashMap, HashSet};
34use std::path::{Path, PathBuf};
35use std::str::FromStr;
36
37pub fn parse_header_string(inputs: &[String]) -> Result<HashMap<String, String>> {
45 let mut headers = HashMap::new();
46
47 for pair in inputs {
48 let pair = pair.trim();
49 if pair.is_empty() {
50 continue;
51 }
52 let parts: Vec<&str> = pair.splitn(2, ':').collect();
53 if parts.len() != 2 {
54 return Err(BenchError::Other(format!(
55 "Invalid header format: '{}'. Expected 'Key:Value'",
56 pair
57 )));
58 }
59 headers.insert(parts[0].trim().to_string(), parts[1].trim().to_string());
60 }
61
62 Ok(headers)
63}
64
65const CONFORMANCE_REPLACES_LOAD_ADVISORY: &str =
85 "Conformance mode REPLACES the load run: 1 VU, 1 iteration per endpoint. \
86 --vus, --rps and -d are ignored. Run bench a second time without \
87 --conformance if you also want a load test.";
88
89pub struct BenchCommand {
91 pub spec: Vec<PathBuf>,
93 pub spec_dir: Option<PathBuf>,
95 pub merge_conflicts: String,
97 pub spec_mode: String,
99 pub dependency_config: Option<PathBuf>,
101 pub target: String,
102 pub base_path: Option<String>,
105 pub duration: String,
106 pub vus: u32,
107 pub target_rps: Option<u32>,
113 pub no_keep_alive: bool,
118 pub scenario: String,
119 pub operations: Option<String>,
120 pub exclude_operations: Option<String>,
124 pub auth: Option<String>,
125 pub headers: Vec<String>,
128 pub output: PathBuf,
129 pub generate_only: bool,
130 pub script_output: Option<PathBuf>,
131 pub threshold_percentile: String,
132 pub threshold_ms: u64,
133 pub max_error_rate: f64,
134 pub abort_on_error: bool,
139 pub abort_on_error_rate: f64,
143 pub per_op_metrics: Option<bool>,
147 pub verbose: bool,
148 pub skip_tls_verify: bool,
149 pub chunked_request_bodies: bool,
154 pub targets_file: Option<PathBuf>,
156 pub max_concurrency: Option<u32>,
158 pub results_format: String,
160 pub params_file: Option<PathBuf>,
165
166 pub crud_flow: bool,
169 pub flow_config: Option<PathBuf>,
171 pub extract_fields: Option<String>,
173
174 pub parallel_create: Option<u32>,
177
178 pub data_file: Option<PathBuf>,
181 pub data_distribution: String,
183 pub data_mappings: Option<String>,
185 pub per_uri_control: bool,
187
188 pub error_rate: Option<f64>,
191 pub error_types: Option<String>,
193
194 pub security_test: bool,
197 pub security_payloads: Option<PathBuf>,
199 pub security_categories: Option<String>,
201 pub security_target_fields: Option<String>,
203
204 pub wafbench_dir: Option<String>,
207 pub wafbench_cycle_all: bool,
209 pub wafbench_verbatim: bool,
212
213 pub conformance: bool,
216 pub conformance_api_key: Option<String>,
218 pub conformance_basic_auth: Option<String>,
220 pub conformance_report: PathBuf,
222 pub conformance_categories: Option<String>,
224 pub conformance_report_format: String,
226 pub conformance_headers: Vec<String>,
229 pub conformance_all_operations: bool,
232 pub conformance_custom: Option<PathBuf>,
234 pub conformance_delay_ms: u64,
237 pub use_k6: bool,
239 pub conformance_custom_filter: Option<String>,
243 pub export_requests: bool,
246 pub validate_requests: bool,
249 pub conformance_self_test: bool,
256 pub conformance_self_test_capture: bool,
260 pub validate_response_schemas: bool,
266 pub conformance_self_test_iterations: u32,
271 pub conformance_self_test_duration: Option<String>,
276
277 pub source_ips: Vec<String>,
282 pub geo_source_ips: Vec<String>,
286 pub geo_source_headers: Vec<String>,
290
291 pub report_missed_cap: Option<u32>,
298
299 pub discard_response_bodies: bool,
306
307 pub dns_policy: Option<String>,
313
314 pub owasp_api_top10: bool,
317 pub owasp_categories: Option<String>,
319 pub owasp_auth_header: String,
321 pub owasp_auth_token: Option<String>,
323 pub owasp_admin_paths: Option<PathBuf>,
325 pub owasp_id_fields: Option<String>,
327 pub owasp_report: Option<PathBuf>,
329 pub owasp_report_format: String,
331 pub owasp_iterations: u32,
333}
334
335fn parse_ip_list(raw: &[String], flag_name: &str) -> Vec<std::net::IpAddr> {
349 use std::net::IpAddr;
350 const MAX_CIDR_EXPANSION: usize = 256;
351 let mut out = Vec::new();
352 for entry in raw {
353 for piece in entry.split(',') {
354 let s = piece.trim();
355 if s.is_empty() {
356 continue;
357 }
358 if let Some((addr_part, prefix_part)) = s.split_once('/') {
360 let prefix: u32 = match prefix_part.parse() {
361 Ok(p) => p,
362 Err(e) => {
363 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR prefix: {e}");
364 continue;
365 }
366 };
367 let net_addr: IpAddr = match addr_part.parse() {
368 Ok(a) => a,
369 Err(e) => {
370 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR address: {e}");
371 continue;
372 }
373 };
374 expand_cidr(net_addr, prefix, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
375 continue;
376 }
377 if let Some((start_str, end_str)) = s.split_once('-') {
383 let start_s = start_str.trim();
384 let end_s = end_str.trim();
385 if start_s.contains(':') || end_s.contains(':') {
389 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{s}': IPv6 range syntax not supported (use CIDR like 2001:db8::/126 instead)");
390 continue;
391 }
392 let start: IpAddr = match start_s.parse() {
393 Ok(a) => a,
394 Err(e) => {
395 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range start: {e}");
396 continue;
397 }
398 };
399 let end: IpAddr = match end_s.parse() {
400 Ok(a) => a,
401 Err(e) => {
402 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range end: {e}");
403 continue;
404 }
405 };
406 expand_range(start, end, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
407 continue;
408 }
409 match s.parse::<IpAddr>() {
411 Ok(ip) => out.push(ip),
412 Err(e) => {
413 tracing::warn!(target: "mockforge::bench", "ignoring malformed --{flag_name} value '{s}': {e}");
414 }
415 }
416 }
417 }
418 out
419}
420
421fn expand_range(
425 start: std::net::IpAddr,
426 end: std::net::IpAddr,
427 cap: usize,
428 flag_name: &str,
429 raw: &str,
430 out: &mut Vec<std::net::IpAddr>,
431) {
432 use std::net::{IpAddr, Ipv4Addr};
433 let (start_v4, end_v4) = match (start, end) {
434 (IpAddr::V4(a), IpAddr::V4(b)) => (a, b),
435 _ => {
436 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range start/end must both be IPv4");
437 return;
438 }
439 };
440 let start_u32 = u32::from(start_v4);
441 let end_u32 = u32::from(end_v4);
442 if end_u32 < start_u32 {
443 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range end {end_v4} is before start {start_v4}");
444 return;
445 }
446 let total = (end_u32 - start_u32).saturating_add(1) as usize;
447 let take = total.min(cap);
448 if total > cap {
449 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range has {total} addresses, capping at {cap}");
450 }
451 for i in 0..take as u32 {
452 out.push(IpAddr::V4(Ipv4Addr::from(start_u32 + i)));
453 }
454}
455
456fn expand_cidr(
460 net: std::net::IpAddr,
461 prefix: u32,
462 cap: usize,
463 flag_name: &str,
464 raw: &str,
465 out: &mut Vec<std::net::IpAddr>,
466) {
467 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
468 match net {
469 IpAddr::V4(ipv4) => {
470 if prefix > 32 {
471 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv4 prefix must be <= 32");
472 return;
473 }
474 let total: u64 = 1u64 << (32 - prefix);
475 let take = total.min(cap as u64) as u32;
476 if total > cap as u64 {
477 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': CIDR has {total} addresses, capping at {cap}");
478 }
479 let mask: u32 = if prefix == 0 {
480 0
481 } else {
482 !0u32 << (32 - prefix)
483 };
484 let net_u32 = u32::from(ipv4) & mask;
485 for i in 0..take {
486 out.push(IpAddr::V4(Ipv4Addr::from(net_u32.wrapping_add(i))));
487 }
488 }
489 IpAddr::V6(ipv6) => {
490 if prefix > 128 {
491 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv6 prefix must be <= 128");
492 return;
493 }
494 let mask: u128 = if prefix == 0 {
498 0
499 } else {
500 !0u128 << (128 - prefix)
501 };
502 let net_u128 = u128::from(ipv6) & mask;
503 let remaining_bits = 128 - prefix;
504 let total_capped = if remaining_bits >= 64 {
507 cap as u128
508 } else {
509 (1u128 << remaining_bits).min(cap as u128)
510 };
511 if remaining_bits < 128 && (1u128 << remaining_bits) > cap as u128 {
512 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': IPv6 CIDR exceeds {cap} addresses, capping");
513 }
514 for i in 0..total_capped {
515 out.push(IpAddr::V6(Ipv6Addr::from(net_u128.wrapping_add(i))));
516 }
517 }
518 }
519}
520
521impl BenchCommand {
522 pub fn security_testing_enabled(&self) -> bool {
538 if self.wafbench_verbatim {
539 return false;
540 }
541 self.security_test || self.wafbench_dir.is_some()
542 }
543
544 pub async fn load_and_merge_specs(&self) -> Result<OpenApiSpec> {
546 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
547
548 if !self.spec.is_empty() {
550 let specs = load_specs_from_files(self.spec.clone())
551 .await
552 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
553 all_specs.extend(specs);
554 }
555
556 if let Some(spec_dir) = &self.spec_dir {
558 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
559 BenchError::Other(format!("Failed to load specs from directory: {}", e))
560 })?;
561 all_specs.extend(dir_specs);
562 }
563
564 if all_specs.is_empty() {
565 return Err(BenchError::Other(
566 "No spec files provided. Use --spec or --spec-dir.".to_string(),
567 ));
568 }
569
570 if all_specs.len() == 1 {
572 return Ok(all_specs.into_iter().next().expect("checked len() == 1 above").1);
574 }
575
576 let conflict_strategy = match self.merge_conflicts.as_str() {
578 "first" => ConflictStrategy::First,
579 "last" => ConflictStrategy::Last,
580 _ => ConflictStrategy::Error,
581 };
582
583 merge_specs(all_specs, conflict_strategy)
584 .map_err(|e| BenchError::Other(format!("Failed to merge specs: {}", e)))
585 }
586
587 fn get_spec_display_name(&self) -> String {
589 if self.spec.len() == 1 {
590 self.spec[0].to_string_lossy().to_string()
591 } else if !self.spec.is_empty() {
592 format!("{} spec files", self.spec.len())
593 } else if let Some(dir) = &self.spec_dir {
594 format!("specs from {}", dir.display())
595 } else {
596 "no specs".to_string()
597 }
598 }
599
600 fn advise_capacity(&self) {
607 let target_count = self
608 .targets_file
609 .as_ref()
610 .and_then(|p| std::fs::read_to_string(p).ok())
611 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
612 .and_then(|v| v.as_array().map(|a| a.len()))
613 .unwrap_or(1);
614 let vus = self.vus.max(1);
615 let rps_total = self.target_rps.unwrap_or(0) as usize * target_count.max(1);
616 let load_product = target_count * vus as usize;
620 if load_product >= 150 {
621 let est_ram_gb =
622 (vus as usize * 50) / 1024 + (target_count * 10 * 2) / 1024 + target_count / 2;
623 let est_cores = ((vus as usize) / 50).max(2);
624 TerminalReporter::print_warning(&format!(
625 "Capacity advisory: targets={target_count}, VUs={vus}, RPS-total≈{rps_total}. \
626 Single-client estimate: ~{est_cores} CPU cores, ~{est_ram_gb} GB RAM. \
627 If your machine is below that, expect OOM hangs partway through the run. \
628 See https://docs.mockforge.dev/reference/bench-capacity-sizing.html \
629 for the sizing table and sharding guide."
630 ));
631 }
632 }
633
634 pub async fn execute(&self) -> Result<()> {
636 if self.conformance_self_test && self.use_k6 {
643 TerminalReporter::print_warning(
644 "--use-k6 has no effect with --conformance-self-test: the self-test driver runs and returns before k6 is invoked. Drop one or the other depending on whether you want the spec-driven self-test or a k6 bench run.",
645 );
646 }
647
648 self.advise_capacity();
654
655 if let Some(targets_file) = &self.targets_file {
657 if self.conformance && self.conformance_self_test {
666 return self.execute_multi_target_self_test(targets_file).await;
667 }
668 if self.conformance {
669 return self.execute_multi_target_conformance(targets_file).await;
670 }
671 return self.execute_multi_target(targets_file).await;
672 }
673
674 if self.spec_mode == "sequential" && (self.spec.len() > 1 || self.spec_dir.is_some()) {
676 return self.execute_sequential_specs().await;
677 }
678
679 TerminalReporter::print_header(
682 &self.get_spec_display_name(),
683 &self.target,
684 0, &self.scenario,
686 Self::parse_duration(&self.duration)?,
687 );
688
689 if !K6Executor::is_k6_installed() {
691 TerminalReporter::print_error("k6 is not installed");
692 TerminalReporter::print_warning(
693 "Install k6 from: https://k6.io/docs/get-started/installation/",
694 );
695 return Err(BenchError::K6NotFound);
696 }
697 K6Executor::warn_if_pre_v1().await;
698
699 if self.conformance {
701 return self.execute_conformance_test().await;
702 }
703
704 let spec_supplied = !self.spec.is_empty() || self.spec_dir.is_some();
711 let merged_spec = if self.wafbench_verbatim && !spec_supplied {
712 tracing::info!(
713 target: "mockforge::bench",
714 "--wafbench-verbatim without --spec: sending only the traffic file's requests"
715 );
716 OpenApiSpec {
717 spec: Default::default(),
718 file_path: None,
719 raw_document: None,
720 }
721 } else {
722 TerminalReporter::print_progress("Loading OpenAPI specification(s)...");
723 self.load_and_merge_specs().await?
724 };
725 let parser = SpecParser::from_spec(merged_spec);
726 if self.spec.len() > 1 || self.spec_dir.is_some() {
727 TerminalReporter::print_success(&format!(
728 "Loaded and merged {} specification(s)",
729 self.spec.len() + self.spec_dir.as_ref().map(|_| 1).unwrap_or(0)
730 ));
731 } else {
732 TerminalReporter::print_success("Specification loaded");
733 }
734
735 let mock_config = self.build_mock_config().await;
737 if mock_config.is_mock_server {
738 TerminalReporter::print_progress("Mock server integration enabled");
739 }
740
741 if self.crud_flow {
743 return self.execute_crud_flow(&parser).await;
744 }
745
746 if self.owasp_api_top10 {
748 return self.execute_owasp_test(&parser).await;
749 }
750
751 TerminalReporter::print_progress("Extracting API operations...");
753 let mut operations = if let Some(filter) = &self.operations {
754 parser.filter_operations(filter)?
755 } else {
756 parser.get_operations()
757 };
758
759 if let Some(exclude) = &self.exclude_operations {
761 let before_count = operations.len();
762 operations = parser.exclude_operations(operations, exclude)?;
763 let excluded_count = before_count - operations.len();
764 if excluded_count > 0 {
765 TerminalReporter::print_progress(&format!(
766 "Excluded {} operations matching '{}'",
767 excluded_count, exclude
768 ));
769 }
770 }
771
772 if operations.is_empty() && !self.wafbench_verbatim {
778 return Err(BenchError::Other("No operations found in spec".to_string()));
779 }
780
781 TerminalReporter::print_success(&format!("Found {} operations", operations.len()));
782
783 let param_overrides = if let Some(params_file) = &self.params_file {
785 TerminalReporter::print_progress("Loading parameter overrides...");
786 let overrides = ParameterOverrides::from_file(params_file)?;
787 TerminalReporter::print_success(&format!(
788 "Loaded parameter overrides ({} operation-specific, {} defaults)",
789 overrides.operations.len(),
790 if overrides.defaults.is_empty() { 0 } else { 1 }
791 ));
792 Some(overrides)
793 } else {
794 None
795 };
796
797 TerminalReporter::print_progress("Generating request templates...");
799 let templates: Vec<_> = operations
800 .iter()
801 .map(|op| {
802 let op_overrides = param_overrides.as_ref().map(|po| {
803 po.get_for_operation(op.operation_id.as_deref(), &op.method, &op.path)
804 });
805 RequestGenerator::generate_template_with_overrides(op, op_overrides.as_ref())
806 })
807 .collect::<Result<Vec<_>>>()?;
808 TerminalReporter::print_success("Request templates generated");
809
810 let templates = if self.wafbench_verbatim {
816 let verbatim = self.load_verbatim_templates()?;
817 if verbatim.is_empty() {
818 return Err(BenchError::Other(
819 "--wafbench-verbatim was set but no traffic cases were loaded. Check \
820 --wafbench-dir points at a file, directory or glob containing cases with \
821 a `request.uri`."
822 .to_string(),
823 ));
824 }
825 TerminalReporter::print_success(&format!(
826 "Verbatim mode: {} request(s) will be sent exactly as written (spec endpoints not used)",
827 verbatim.len()
828 ));
829 verbatim
830 } else {
831 templates
832 };
833
834 let custom_headers = self.parse_headers()?;
836
837 let force_http1 = crate::request_gen::should_force_k6_http1(
840 self.wafbench_verbatim,
841 &templates,
842 &custom_headers,
843 );
844
845 let base_path = self.resolve_base_path(&parser);
847 if let Some(ref bp) = base_path {
848 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
849 }
850
851 TerminalReporter::print_progress("Generating k6 load test script...");
853 let scenario =
854 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
855
856 let security_testing_enabled = self.security_testing_enabled();
857
858 let num_ops = operations.len() as u32;
876 if let Some(rps) = self.target_rps {
877 let probe =
878 crate::preflight::probe_target_latency(&self.target, 3, self.skip_tls_verify).await;
879
880 let (required_vus, basis) = match probe {
881 Some(p) => (
882 p.required_vus(rps, num_ops),
883 format!("avg {:.1}ms (measured)", p.avg_latency.as_secs_f64() * 1000.0),
884 ),
885 None => {
886 let fallback = (rps as u64)
888 .saturating_mul(num_ops.max(1) as u64)
889 .div_ceil(10)
890 .min(u32::MAX as u64) as u32;
891 (fallback, "~100ms (default — probe failed)".to_string())
892 }
893 };
894
895 if self.vus < required_vus {
896 const VU_RECOMMENDATION_CAP: u32 = 1000;
902 let recommendation = required_vus.max(self.vus + 1);
903 if recommendation > VU_RECOMMENDATION_CAP {
904 TerminalReporter::print_warning(&format!(
905 "Workload is very large: --rps {} × {} ops/iteration × {} \
906 baseline ⇒ ~{} VUs needed end-to-end, far beyond what's \
907 practical to drive. Two ways to fix:\n 1. Reduce \
908 operations per iteration with `--operations 'pattern,…'` \
909 (or `--exclude-operations`) to focus the bench on a \
910 representative subset.\n 2. Drop `--rps` and use \
911 `--vus {}` alone — closed-model load runs as fast as \
912 the VU pool allows, bounded by latency, with no per-\
913 iteration deadline. Expect 1-iteration coverage of ~{} \
914 operations in {}s.",
915 rps,
916 num_ops,
917 basis,
918 recommendation,
919 self.vus.max(5),
920 num_ops,
921 Self::parse_duration(&self.duration).unwrap_or(0),
922 ));
923 } else {
924 TerminalReporter::print_warning(&format!(
925 "--vus {} may be insufficient for --rps {} × {} ops/iteration \
926 (baseline latency {}). k6's constant-arrival-rate counts ITERATIONS \
927 and each runs every operation in the spec — required ≈ rps × ops × \
928 latency_secs VUs. Bump --vus to ~{} if you see \"Insufficient VUs\" \
929 warnings.",
930 self.vus, rps, num_ops, basis, recommendation,
931 ));
932 }
933 } else if probe.is_some() {
934 TerminalReporter::print_progress(&format!(
935 "Pre-flight probe: target latency {}, {} ops/iteration — --vus {} \
936 is sufficient for --rps {}",
937 basis, num_ops, self.vus, rps,
938 ));
939 }
940 }
941
942 let k6_config = K6Config {
943 target_url: self.target.clone(),
944 base_path,
945 scenario,
946 duration_secs: Self::parse_duration(&self.duration)?,
947 max_vus: self.vus,
948 threshold_percentile: self.threshold_percentile.clone(),
949 threshold_ms: self.threshold_ms,
950 max_error_rate: self.max_error_rate,
951 auth_header: self.auth.clone(),
952 custom_headers,
953 skip_tls_verify: self.skip_tls_verify,
954 security_testing_enabled,
955 chunked_request_bodies: self.chunked_request_bodies,
956 target_rps: self.target_rps,
957 no_keep_alive: self.no_keep_alive,
958 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
964 .into_iter()
965 .map(|ip| ip.to_string())
966 .collect(),
967 geo_source_headers: if self.geo_source_headers.is_empty()
968 && !self.geo_source_ips.is_empty()
969 {
970 crate::conformance::self_test::default_geo_source_headers()
971 } else {
972 self.geo_source_headers.clone()
973 },
974 };
975
976 let duration_secs = Self::parse_duration(&self.duration)?;
979 let (per_op_metrics, per_op_warn) = crate::k6_gen::resolve_per_op_metrics(
980 self.per_op_metrics,
981 templates.len(),
982 duration_secs,
983 );
984 if let Some(msg) = per_op_warn {
985 TerminalReporter::print_warning(&msg);
986 }
987
988 let generator = K6ScriptGenerator::new(k6_config, templates)
989 .with_abort_valve(self.abort_on_error, self.abort_on_error_rate)
990 .with_force_http1(force_http1)
991 .with_per_op_metrics(per_op_metrics);
992 let mut script = generator.generate()?;
993 TerminalReporter::print_success("k6 script generated");
994
995 let has_advanced_features = self.data_file.is_some()
997 || self.error_rate.is_some()
998 || self.security_test
999 || self.parallel_create.is_some()
1000 || self.wafbench_dir.is_some();
1001
1002 if has_advanced_features {
1004 script = self.generate_enhanced_script(&script)?;
1005 }
1006
1007 if mock_config.is_mock_server {
1009 let setup_code = MockIntegrationGenerator::generate_setup(&mock_config);
1010 let teardown_code = MockIntegrationGenerator::generate_teardown(&mock_config);
1011 let helper_code = MockIntegrationGenerator::generate_vu_id_helper();
1012
1013 if let Some(import_end) = script.find("export const options") {
1015 script.insert_str(
1016 import_end,
1017 &format!(
1018 "\n// === Mock Server Integration ===\n{}\n{}\n{}\n",
1019 helper_code, setup_code, teardown_code
1020 ),
1021 );
1022 }
1023 }
1024
1025 TerminalReporter::print_progress("Validating k6 script...");
1027 let validation_errors = K6ScriptGenerator::validate_script(&script);
1028 if !validation_errors.is_empty() {
1029 TerminalReporter::print_error("Script validation failed");
1030 for error in &validation_errors {
1031 eprintln!(" {}", error);
1032 }
1033 return Err(BenchError::Other(format!(
1034 "Generated k6 script has {} validation error(s). Please check the output above.",
1035 validation_errors.len()
1036 )));
1037 }
1038 TerminalReporter::print_success("Script validation passed");
1039
1040 let script_path = if let Some(output) = &self.script_output {
1042 output.clone()
1043 } else {
1044 self.output.join("k6-script.js")
1045 };
1046
1047 if let Some(parent) = script_path.parent() {
1048 std::fs::create_dir_all(parent)?;
1049 }
1050 std::fs::write(&script_path, &script)?;
1051 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
1052
1053 if self.generate_only {
1055 Self::print_k6_run_hint(&script_path, force_http1);
1056 return Ok(());
1057 }
1058
1059 TerminalReporter::print_progress("Executing load test...");
1061 if force_http1 {
1062 TerminalReporter::print_progress(
1063 "Forcing HTTP/1.1 (GODEBUG=http2client=0): Connection headers are hop-by-hop and HTTP/2 rejects them. The header stays on the wire.",
1064 );
1065 }
1066 let executor = K6Executor::new()?
1070 .with_local_ips(self.source_ips.join(","))
1071 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
1072 .with_discard_response_bodies(self.discard_response_bodies)
1073 .with_force_http1(force_http1);
1074
1075 std::fs::create_dir_all(&self.output)?;
1076
1077 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
1078
1079 let duration_secs = Self::parse_duration(&self.duration)?;
1081 TerminalReporter::print_summary_full(
1082 &results,
1083 duration_secs,
1084 self.no_keep_alive,
1085 Some(num_ops),
1086 );
1087
1088 self.reprint_traffic_file_breakdown();
1089 println!("\nResults saved to: {}", self.output.display());
1090
1091 Ok(())
1092 }
1093
1094 async fn execute_multi_target(&self, targets_file: &Path) -> Result<()> {
1096 TerminalReporter::print_progress("Parsing targets file...");
1097 let targets = parse_targets_file(targets_file)?;
1098 let num_targets = targets.len();
1099 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
1100
1101 if targets.is_empty() {
1102 return Err(BenchError::Other("No targets found in file".to_string()));
1103 }
1104
1105 let max_concurrency = self.max_concurrency.map(|n| n as usize);
1109
1110 TerminalReporter::print_header(
1112 &self.get_spec_display_name(),
1113 &format!("{} targets", num_targets),
1114 0,
1115 &self.scenario,
1116 Self::parse_duration(&self.duration)?,
1117 );
1118
1119 let executor = ParallelExecutor::new(
1121 BenchCommand {
1122 spec: self.spec.clone(),
1124 spec_dir: self.spec_dir.clone(),
1125 merge_conflicts: self.merge_conflicts.clone(),
1126 spec_mode: self.spec_mode.clone(),
1127 dependency_config: self.dependency_config.clone(),
1128 target: self.target.clone(), base_path: self.base_path.clone(),
1130 duration: self.duration.clone(),
1131 vus: self.vus,
1132 target_rps: self.target_rps,
1133 no_keep_alive: self.no_keep_alive,
1134 scenario: self.scenario.clone(),
1135 operations: self.operations.clone(),
1136 exclude_operations: self.exclude_operations.clone(),
1137 auth: self.auth.clone(),
1138 headers: self.headers.clone(),
1139 output: self.output.clone(),
1140 generate_only: self.generate_only,
1141 script_output: self.script_output.clone(),
1142 threshold_percentile: self.threshold_percentile.clone(),
1143 threshold_ms: self.threshold_ms,
1144 max_error_rate: self.max_error_rate,
1145 abort_on_error: self.abort_on_error,
1146 abort_on_error_rate: self.abort_on_error_rate,
1147 per_op_metrics: self.per_op_metrics,
1148 verbose: self.verbose,
1149 skip_tls_verify: self.skip_tls_verify,
1150 chunked_request_bodies: self.chunked_request_bodies,
1151 targets_file: None,
1152 max_concurrency: None,
1153 results_format: self.results_format.clone(),
1154 params_file: self.params_file.clone(),
1155 crud_flow: self.crud_flow,
1156 flow_config: self.flow_config.clone(),
1157 extract_fields: self.extract_fields.clone(),
1158 parallel_create: self.parallel_create,
1159 data_file: self.data_file.clone(),
1160 data_distribution: self.data_distribution.clone(),
1161 data_mappings: self.data_mappings.clone(),
1162 per_uri_control: self.per_uri_control,
1163 error_rate: self.error_rate,
1164 error_types: self.error_types.clone(),
1165 security_test: self.security_test,
1166 security_payloads: self.security_payloads.clone(),
1167 security_categories: self.security_categories.clone(),
1168 security_target_fields: self.security_target_fields.clone(),
1169 wafbench_dir: self.wafbench_dir.clone(),
1170 wafbench_cycle_all: self.wafbench_cycle_all,
1171 wafbench_verbatim: self.wafbench_verbatim,
1172 owasp_api_top10: self.owasp_api_top10,
1173 owasp_categories: self.owasp_categories.clone(),
1174 owasp_auth_header: self.owasp_auth_header.clone(),
1175 owasp_auth_token: self.owasp_auth_token.clone(),
1176 owasp_admin_paths: self.owasp_admin_paths.clone(),
1177 owasp_id_fields: self.owasp_id_fields.clone(),
1178 owasp_report: self.owasp_report.clone(),
1179 owasp_report_format: self.owasp_report_format.clone(),
1180 owasp_iterations: self.owasp_iterations,
1181 conformance: false,
1182 conformance_api_key: self.conformance_api_key.clone(),
1198 conformance_basic_auth: self.conformance_basic_auth.clone(),
1199 conformance_report: PathBuf::from("conformance-report.json"),
1200 conformance_categories: None,
1201 conformance_report_format: "json".to_string(),
1202 conformance_headers: self.conformance_headers.clone(),
1206 conformance_all_operations: false,
1207 conformance_custom: None,
1208 conformance_delay_ms: 0,
1209 use_k6: false,
1210 conformance_custom_filter: None,
1211 export_requests: false,
1212 validate_requests: false,
1213 conformance_self_test: false,
1214 conformance_self_test_capture: false,
1215 conformance_self_test_iterations: 1,
1216 conformance_self_test_duration: None,
1217 validate_response_schemas: false,
1218 source_ips: self.source_ips.clone(),
1223 geo_source_ips: self.geo_source_ips.clone(),
1224 geo_source_headers: self.geo_source_headers.clone(),
1225 report_missed_cap: None,
1226 discard_response_bodies: self.discard_response_bodies,
1230 dns_policy: self.dns_policy.clone(),
1233 },
1234 targets,
1235 max_concurrency,
1236 );
1237
1238 let start_time = std::time::Instant::now();
1240 let aggregated_results = executor.execute_all().await?;
1241 let elapsed = start_time.elapsed();
1242
1243 self.report_multi_target_results(&aggregated_results, elapsed)?;
1245
1246 Ok(())
1247 }
1248
1249 fn report_multi_target_results(
1251 &self,
1252 results: &AggregatedResults,
1253 elapsed: std::time::Duration,
1254 ) -> Result<()> {
1255 TerminalReporter::print_multi_target_summary(results);
1257
1258 let total_secs = elapsed.as_secs();
1260 let hours = total_secs / 3600;
1261 let minutes = (total_secs % 3600) / 60;
1262 let seconds = total_secs % 60;
1263 if hours > 0 {
1264 println!("\n Total Elapsed Time: {}h {}m {}s", hours, minutes, seconds);
1265 } else if minutes > 0 {
1266 println!("\n Total Elapsed Time: {}m {}s", minutes, seconds);
1267 } else {
1268 println!("\n Total Elapsed Time: {}s", seconds);
1269 }
1270
1271 if self.results_format == "aggregated" || self.results_format == "both" {
1273 let summary_path = self.output.join("aggregated_summary.json");
1274 let summary_json = serde_json::json!({
1275 "total_elapsed_seconds": elapsed.as_secs(),
1276 "total_targets": results.total_targets,
1277 "successful_targets": results.successful_targets,
1278 "failed_targets": results.failed_targets,
1279 "aggregated_metrics": {
1280 "total_requests": results.aggregated_metrics.total_requests,
1281 "total_failed_requests": results.aggregated_metrics.total_failed_requests,
1282 "avg_duration_ms": results.aggregated_metrics.avg_duration_ms,
1283 "p95_duration_ms": results.aggregated_metrics.p95_duration_ms,
1284 "p99_duration_ms": results.aggregated_metrics.p99_duration_ms,
1285 "error_rate": results.aggregated_metrics.error_rate,
1286 "total_rps": results.aggregated_metrics.total_rps,
1287 "avg_rps": results.aggregated_metrics.avg_rps,
1288 "total_vus_max": results.aggregated_metrics.total_vus_max,
1289 },
1290 "target_results": results.target_results.iter().map(|r| {
1291 serde_json::json!({
1292 "target_url": r.target_url,
1293 "target_index": r.target_index,
1294 "success": r.success,
1295 "error": r.error,
1296 "total_requests": r.results.total_requests,
1297 "failed_requests": r.results.failed_requests,
1298 "avg_duration_ms": r.results.avg_duration_ms,
1299 "min_duration_ms": r.results.min_duration_ms,
1300 "med_duration_ms": r.results.med_duration_ms,
1301 "p90_duration_ms": r.results.p90_duration_ms,
1302 "p95_duration_ms": r.results.p95_duration_ms,
1303 "p99_duration_ms": r.results.p99_duration_ms,
1304 "max_duration_ms": r.results.max_duration_ms,
1305 "rps": r.results.rps,
1306 "vus_max": r.results.vus_max,
1307 "output_dir": r.output_dir.to_string_lossy(),
1308 })
1309 }).collect::<Vec<_>>(),
1310 });
1311
1312 std::fs::write(&summary_path, serde_json::to_string_pretty(&summary_json)?)?;
1313 TerminalReporter::print_success(&format!(
1314 "Aggregated summary saved to: {}",
1315 summary_path.display()
1316 ));
1317 }
1318
1319 let csv_path = self.output.join("all_targets.csv");
1321 let mut csv = String::from(
1322 "target_url,success,requests,failed,rps,vus,min_ms,avg_ms,med_ms,p90_ms,p95_ms,p99_ms,max_ms,error\n",
1323 );
1324 for r in &results.target_results {
1325 csv.push_str(&format!(
1326 "{},{},{},{},{:.1},{},{:.1},{:.1},{:.1},{:.1},{:.1},{:.1},{:.1},{}\n",
1327 r.target_url,
1328 r.success,
1329 r.results.total_requests,
1330 r.results.failed_requests,
1331 r.results.rps,
1332 r.results.vus_max,
1333 r.results.min_duration_ms,
1334 r.results.avg_duration_ms,
1335 r.results.med_duration_ms,
1336 r.results.p90_duration_ms,
1337 r.results.p95_duration_ms,
1338 r.results.p99_duration_ms,
1339 r.results.max_duration_ms,
1340 r.error.as_deref().unwrap_or(""),
1341 ));
1342 }
1343 let _ = std::fs::write(&csv_path, &csv);
1344
1345 self.reprint_traffic_file_breakdown();
1346 println!("\nResults saved to: {}", self.output.display());
1347 println!(" - Per-target results: {}", self.output.join("target_*").display());
1348 println!(" - All targets CSV: {}", csv_path.display());
1349 if self.results_format == "aggregated" || self.results_format == "both" {
1350 println!(
1351 " - Aggregated summary: {}",
1352 self.output.join("aggregated_summary.json").display()
1353 );
1354 }
1355
1356 Ok(())
1357 }
1358
1359 pub fn parse_duration(duration: &str) -> Result<u64> {
1361 let duration = duration.trim();
1362
1363 if let Some(secs) = duration.strip_suffix('s') {
1364 secs.parse::<u64>()
1365 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1366 } else if let Some(mins) = duration.strip_suffix('m') {
1367 mins.parse::<u64>()
1368 .map(|m| m * 60)
1369 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1370 } else if let Some(hours) = duration.strip_suffix('h') {
1371 hours
1372 .parse::<u64>()
1373 .map(|h| h * 3600)
1374 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1375 } else {
1376 duration
1378 .parse::<u64>()
1379 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1380 }
1381 }
1382
1383 fn print_k6_run_hint(script_path: &Path, force_http1: bool) {
1387 println!("\nScript generated successfully. Run it with:");
1388 if force_http1 {
1389 println!(" GODEBUG=http2client=0 k6 run {}", script_path.display());
1390 println!(
1391 " (HTTP/1.1: a Connection header is on the wire; HTTP/2 rejects it. mockforge bench sets this automatically when it invokes k6.)"
1392 );
1393 } else {
1394 println!(" k6 run {}", script_path.display());
1395 }
1396 }
1397
1398 pub(crate) fn load_verbatim_templates(
1405 &self,
1406 ) -> Result<Vec<crate::request_gen::RequestTemplate>> {
1407 let Some(pattern) = self.wafbench_dir.as_ref() else {
1408 return Err(BenchError::Other(
1409 "--wafbench-verbatim requires --wafbench-dir pointing at your traffic file(s)"
1410 .to_string(),
1411 ));
1412 };
1413
1414 let mut loader = WafBenchLoader::new();
1415 loader.load_from_pattern(pattern)?;
1416 self.emit_traffic_file_breakdown(loader.stats(), "what to expect in proxy logs");
1417
1418 Ok(crate::wafbench::traffic_cases_to_templates(loader.test_cases()))
1419 }
1420
1421 pub fn parse_headers(&self) -> Result<HashMap<String, String>> {
1423 let mut headers = parse_header_string(&self.headers)?;
1424
1425 let already_has = |hs: &HashMap<String, String>, name: &str| -> bool {
1436 hs.keys().any(|k| k.eq_ignore_ascii_case(name))
1437 };
1438
1439 if !already_has(&headers, "Authorization") {
1440 if let Some(b) = self.conformance_basic_auth.as_ref().filter(|s| !s.is_empty()) {
1441 use base64::Engine as _;
1442 let encoded = base64::engine::general_purpose::STANDARD.encode(b.as_bytes());
1443 headers.insert("Authorization".to_string(), format!("Basic {}", encoded));
1444 }
1445 }
1446
1447 for line in &self.conformance_headers {
1453 let Some((name, value)) = line.split_once(':') else {
1454 continue;
1455 };
1456 let name = name.trim();
1457 let value = value.trim();
1458 if name.is_empty() || already_has(&headers, name) {
1459 continue;
1460 }
1461 headers.insert(name.to_string(), value.to_string());
1462 }
1463
1464 if !self.conformance && self.conformance_api_key.is_some() {
1470 TerminalReporter::print_warning(
1471 "--conformance-api-key only fires under --conformance. For plain bench use --header 'X-API-Key: ...'.",
1472 );
1473 }
1474
1475 Ok(headers)
1476 }
1477
1478 fn parse_extracted_values(output_dir: &Path) -> Result<ExtractedValues> {
1479 let extracted_path = output_dir.join("extracted_values.json");
1480 if !extracted_path.exists() {
1481 return Ok(ExtractedValues::new());
1482 }
1483
1484 let content = std::fs::read_to_string(&extracted_path)
1485 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1486 let parsed: serde_json::Value = serde_json::from_str(&content)
1487 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1488
1489 let mut extracted = ExtractedValues::new();
1490 if let Some(values) = parsed.as_object() {
1491 for (key, value) in values {
1492 extracted.set(key.clone(), value.clone());
1493 }
1494 }
1495
1496 Ok(extracted)
1497 }
1498
1499 fn resolve_base_path(&self, parser: &SpecParser) -> Option<String> {
1508 if let Some(cli_base_path) = &self.base_path {
1510 if cli_base_path.is_empty() {
1511 return None;
1513 }
1514 return Some(cli_base_path.clone());
1515 }
1516
1517 parser.get_base_path()
1519 }
1520
1521 async fn build_mock_config(&self) -> MockIntegrationConfig {
1523 if MockServerDetector::looks_like_mock_server(&self.target) {
1525 if let Ok(info) = MockServerDetector::detect(&self.target).await {
1527 if info.is_mockforge {
1528 TerminalReporter::print_success(&format!(
1529 "Detected MockForge server (version: {})",
1530 info.version.as_deref().unwrap_or("unknown")
1531 ));
1532 return MockIntegrationConfig::mock_server();
1533 }
1534 }
1535 }
1536 MockIntegrationConfig::real_api()
1537 }
1538
1539 fn build_crud_flow_config(&self) -> Option<CrudFlowConfig> {
1541 if !self.crud_flow {
1542 return None;
1543 }
1544
1545 if let Some(config_path) = &self.flow_config {
1547 match CrudFlowConfig::from_file(config_path) {
1548 Ok(config) => return Some(config),
1549 Err(e) => {
1550 TerminalReporter::print_warning(&format!(
1551 "Failed to load flow config: {}. Using auto-detection.",
1552 e
1553 ));
1554 }
1555 }
1556 }
1557
1558 let extract_fields = self
1560 .extract_fields
1561 .as_ref()
1562 .map(|f| f.split(',').map(|s| s.trim().to_string()).collect())
1563 .unwrap_or_else(|| vec!["id".to_string(), "uuid".to_string()]);
1564
1565 Some(CrudFlowConfig {
1566 flows: Vec::new(), default_extract_fields: extract_fields,
1568 })
1569 }
1570
1571 fn build_data_driven_config(&self) -> Option<DataDrivenConfig> {
1573 let data_file = self.data_file.as_ref()?;
1574
1575 let distribution = DataDistribution::from_str(&self.data_distribution)
1576 .unwrap_or(DataDistribution::UniquePerVu);
1577
1578 let mappings = self
1579 .data_mappings
1580 .as_ref()
1581 .map(|m| DataMapping::parse_mappings(m).unwrap_or_default())
1582 .unwrap_or_default();
1583
1584 Some(DataDrivenConfig {
1585 file_path: data_file.to_string_lossy().to_string(),
1586 distribution,
1587 mappings,
1588 csv_has_header: true,
1589 per_uri_control: self.per_uri_control,
1590 per_uri_columns: crate::data_driven::PerUriColumns::default(),
1591 })
1592 }
1593
1594 fn build_invalid_data_config(&self) -> Option<InvalidDataConfig> {
1596 let error_rate = self.error_rate?;
1597
1598 let error_types = self
1599 .error_types
1600 .as_ref()
1601 .map(|types| InvalidDataConfig::parse_error_types(types).unwrap_or_default())
1602 .unwrap_or_default();
1603
1604 Some(InvalidDataConfig {
1605 error_rate,
1606 error_types,
1607 target_fields: Vec::new(),
1608 })
1609 }
1610
1611 fn build_security_config(&self) -> Option<SecurityTestConfig> {
1613 if !self.security_test {
1614 return None;
1615 }
1616
1617 let categories = self
1618 .security_categories
1619 .as_ref()
1620 .map(|cats| SecurityTestConfig::parse_categories(cats).unwrap_or_default())
1621 .unwrap_or_else(|| {
1622 let mut default = HashSet::new();
1623 default.insert(SecurityCategory::SqlInjection);
1624 default.insert(SecurityCategory::Xss);
1625 default
1626 });
1627
1628 let target_fields = self
1629 .security_target_fields
1630 .as_ref()
1631 .map(|fields| fields.split(',').map(|f| f.trim().to_string()).collect())
1632 .unwrap_or_default();
1633
1634 let custom_payloads_file =
1635 self.security_payloads.as_ref().map(|p| p.to_string_lossy().to_string());
1636
1637 Some(SecurityTestConfig {
1638 enabled: true,
1639 categories,
1640 target_fields,
1641 custom_payloads_file,
1642 include_high_risk: false,
1643 })
1644 }
1645
1646 fn build_parallel_config(&self) -> Option<ParallelConfig> {
1648 let count = self.parallel_create?;
1649
1650 Some(ParallelConfig::new(count))
1651 }
1652
1653 fn format_unique_total(unique: usize, rps: Option<u32>) -> String {
1656 match rps {
1657 Some(r) if r > 0 => {
1658 let projected = unique.saturating_mul(r as usize);
1659 format!(
1660 "unique_cases={unique} projected_per_second={projected} ({unique} * {r} RPS)"
1661 )
1662 }
1663 _ => format!("unique_cases={unique}"),
1664 }
1665 }
1666
1667 fn traffic_bucket_json(
1668 unique: usize,
1669 rps: Option<u32>,
1670 duration_secs: Option<u64>,
1671 ) -> serde_json::Value {
1672 let per_second = rps.filter(|&r| r > 0).map(|r| (unique as u64).saturating_mul(r as u64));
1679 let projected_over_run = match (rps.filter(|&r| r > 0), duration_secs) {
1680 (Some(r), Some(d)) => Some((unique as u64).saturating_mul(r as u64).saturating_mul(d)),
1681 _ => None,
1682 };
1683 serde_json::json!({
1684 "unique_cases": unique,
1685 "projected_per_second": per_second,
1686 "projected_over_run": projected_over_run,
1687 })
1688 }
1689
1690 fn emit_traffic_file_breakdown(&self, stats: &crate::wafbench::WafBenchStats, phase: &str) {
1693 if stats.per_file.is_empty() {
1694 return;
1695 }
1696 let rps = self.target_rps.filter(|&r| r > 0);
1697 TerminalReporter::print_success(&format!("Traffic file breakdown ({phase}):"));
1698 for file in &stats.per_file {
1699 let other = if file.other > 0 {
1700 format!(" other={}", file.other)
1701 } else {
1702 String::new()
1703 };
1704 TerminalReporter::print_progress(&format!(
1705 " {}: sent {} attack(expected 403) {} normal(expected 200) {} omitted={}{other}",
1706 file.file,
1707 Self::format_unique_total(file.sent, rps),
1708 Self::format_unique_total(file.attack, rps),
1709 Self::format_unique_total(file.normal, rps),
1710 file.omitted
1711 ));
1712 }
1713 self.write_traffic_breakdown_json(stats);
1714 }
1715
1716 fn write_traffic_breakdown_json(&self, stats: &crate::wafbench::WafBenchStats) {
1718 if stats.per_file.is_empty() {
1719 return;
1720 }
1721 let rps = self.target_rps.filter(|&r| r > 0);
1722 let duration_secs = Self::parse_duration(&self.duration).ok();
1723 let files: Vec<serde_json::Value> = stats
1724 .per_file
1725 .iter()
1726 .map(|file| {
1727 serde_json::json!({
1728 "file": file.file,
1729 "sent": Self::traffic_bucket_json(file.sent, rps, duration_secs),
1730 "attack": Self::traffic_bucket_json(file.attack, rps, duration_secs),
1731 "normal": Self::traffic_bucket_json(file.normal, rps, duration_secs),
1732 "omitted": file.omitted,
1733 "other": file.other,
1734 })
1735 })
1736 .collect();
1737 let payload = serde_json::json!({
1738 "rps": rps,
1739 "duration_secs": duration_secs,
1740 "note": "Plan, not k6 counters. unique_cases is the YAML case count (not traffic on the wire). projected_per_second is unique_cases * rps. projected_over_run is unique_cases * rps * duration_secs, assuming each k6 iteration sends every unique case. projected_* are null when --rps is unset.",
1741 "files": files,
1742 });
1743 if let Some(parent) = self.output.parent() {
1744 let _ = std::fs::create_dir_all(parent);
1745 }
1746 let _ = std::fs::create_dir_all(&self.output);
1747 let path = self.output.join("traffic-breakdown.json");
1748 if let Ok(body) = serde_json::to_string_pretty(&payload) {
1749 if std::fs::write(&path, body).is_ok() {
1750 TerminalReporter::print_progress(&format!(
1751 "Traffic breakdown written to: {}",
1752 path.display()
1753 ));
1754 }
1755 }
1756 }
1757
1758 fn reprint_traffic_file_breakdown(&self) {
1761 let path = self.output.join("traffic-breakdown.json");
1762 let Ok(raw) = std::fs::read_to_string(&path) else {
1763 return;
1764 };
1765 let Ok(v) = serde_json::from_str::<serde_json::Value>(&raw) else {
1766 return;
1767 };
1768 let Some(files) = v.get("files").and_then(|f| f.as_array()) else {
1769 return;
1770 };
1771 if files.is_empty() {
1772 return;
1773 }
1774 TerminalReporter::print_success("Traffic file breakdown (end of run):");
1775 for file in files {
1776 let name = file.get("file").and_then(|x| x.as_str()).unwrap_or("?");
1777 let bucket = |key: &str| -> String {
1778 let unique = file
1783 .get(key)
1784 .and_then(|b| b.get("unique_cases").or_else(|| b.get("unique")))
1785 .and_then(|u| u.as_u64())
1786 .unwrap_or(0) as usize;
1787 Self::format_unique_total(unique, self.target_rps.filter(|&r| r > 0))
1788 };
1789 let omitted = file.get("omitted").and_then(|o| o.as_u64()).unwrap_or(0);
1790 let other = file.get("other").and_then(|o| o.as_u64()).unwrap_or(0);
1791 let other = if other > 0 {
1792 format!(" other={other}")
1793 } else {
1794 String::new()
1795 };
1796 TerminalReporter::print_progress(&format!(
1797 " {name}: sent {} attack(expected 403) {} normal(expected 200) {} omitted={omitted}{other}",
1798 bucket("sent"),
1799 bucket("attack"),
1800 bucket("normal"),
1801 ));
1802 }
1803 TerminalReporter::print_progress(&format!(" (also in {})", path.display()));
1804 }
1805
1806 fn load_wafbench_payloads(&self) -> Result<Vec<SecurityPayload>> {
1813 let Some(ref wafbench_dir) = self.wafbench_dir else {
1814 return Ok(Vec::new());
1815 };
1816
1817 let mut loader = WafBenchLoader::new();
1818 loader.load_from_pattern(wafbench_dir)?;
1819
1820 let stats = loader.stats();
1821
1822 if stats.files_processed == 0 {
1823 let mut msg = format!(
1824 "No WAFBench YAML files found matching '{wafbench_dir}'. \
1825 --wafbench-dir is a file, a directory or a glob. A missing \
1826 file is an error, not an empty payload pool."
1827 );
1828 if !stats.parse_errors.is_empty() {
1829 msg.push_str(" Parse errors:");
1830 for error in &stats.parse_errors {
1831 msg.push_str(&format!("\n - {error}"));
1832 }
1833 }
1834 return Err(BenchError::Other(msg));
1835 }
1836
1837 TerminalReporter::print_progress(&format!(
1838 "Loaded {} WAFBench files, {} test cases, {} payloads",
1839 stats.files_processed, stats.test_cases_loaded, stats.payloads_extracted
1840 ));
1841 self.emit_traffic_file_breakdown(stats, "what to expect in proxy logs");
1842
1843 for (category, count) in &stats.by_category {
1845 TerminalReporter::print_progress(&format!(" - {}: {} tests", category, count));
1846 }
1847
1848 for error in &stats.parse_errors {
1850 TerminalReporter::print_warning(&format!(" Parse error: {}", error));
1851 }
1852
1853 Ok(loader.to_security_payloads())
1854 }
1855
1856 pub(crate) fn generate_enhanced_script(&self, base_script: &str) -> Result<String> {
1858 let mut enhanced_script = base_script.to_string();
1859 let mut additional_code = String::new();
1860
1861 if let Some(config) = self.build_data_driven_config() {
1863 TerminalReporter::print_progress("Adding data-driven testing support...");
1864 additional_code.push_str(&DataDrivenGenerator::generate_setup(&config));
1865 additional_code.push('\n');
1866 TerminalReporter::print_success("Data-driven testing enabled");
1867 }
1868
1869 if let Some(config) = self.build_invalid_data_config() {
1871 TerminalReporter::print_progress("Adding invalid data testing support...");
1872 additional_code.push_str(&InvalidDataGenerator::generate_invalidation_logic());
1873 additional_code.push('\n');
1874 additional_code
1875 .push_str(&InvalidDataGenerator::generate_should_invalidate(config.error_rate));
1876 additional_code.push('\n');
1877 additional_code
1878 .push_str(&InvalidDataGenerator::generate_type_selection(&config.error_types));
1879 additional_code.push('\n');
1880 TerminalReporter::print_success(&format!(
1881 "Invalid data testing enabled ({}% error rate)",
1882 (self.error_rate.unwrap_or(0.0) * 100.0) as u32
1883 ));
1884 }
1885
1886 let verbatim = self.wafbench_verbatim;
1893 if verbatim && self.security_test {
1894 TerminalReporter::print_warning(
1895 "--security-test is ignored under --wafbench-verbatim: verbatim mode sends your \
1896 traffic cases exactly as written and will not append attack payloads to them. \
1897 Drop --wafbench-verbatim if you want payload injection.",
1898 );
1899 }
1900 let security_config = if verbatim {
1901 None
1902 } else {
1903 self.build_security_config()
1904 };
1905 let wafbench_payloads = if verbatim {
1906 Vec::new()
1907 } else {
1908 self.load_wafbench_payloads()?
1909 };
1910 let security_requested =
1911 !verbatim && (security_config.is_some() || self.wafbench_dir.is_some());
1912
1913 if security_config.is_some() || !wafbench_payloads.is_empty() {
1914 TerminalReporter::print_progress("Adding security testing support...");
1915
1916 let mut payload_list: Vec<SecurityPayload> = Vec::new();
1918
1919 if let Some(ref config) = security_config {
1920 payload_list.extend(SecurityPayloads::get_payloads(config));
1921 }
1922
1923 if !wafbench_payloads.is_empty() {
1925 TerminalReporter::print_progress(&format!(
1926 "Loading {} WAFBench attack patterns...",
1927 wafbench_payloads.len()
1928 ));
1929 payload_list.extend(wafbench_payloads);
1930 }
1931
1932 let target_fields =
1933 security_config.as_ref().map(|c| c.target_fields.clone()).unwrap_or_default();
1934
1935 additional_code.push_str(&SecurityTestGenerator::generate_payload_selection(
1936 &payload_list,
1937 self.wafbench_cycle_all,
1938 ));
1939 additional_code.push('\n');
1940 additional_code
1941 .push_str(&SecurityTestGenerator::generate_apply_payload(&target_fields));
1942 additional_code.push('\n');
1943 additional_code.push_str(&SecurityTestGenerator::generate_security_checks());
1944 additional_code.push('\n');
1945
1946 let mode = if self.wafbench_cycle_all {
1947 "cycle-all"
1948 } else {
1949 "random"
1950 };
1951 TerminalReporter::print_success(&format!(
1952 "Security testing enabled ({} payloads, {} mode)",
1953 payload_list.len(),
1954 mode
1955 ));
1956 } else if security_requested {
1957 TerminalReporter::print_warning(
1961 "Security testing was requested but no payloads were loaded. \
1962 Ensure --wafbench-dir points to valid CRS YAML files or add --security-test.",
1963 );
1964 additional_code
1965 .push_str(&SecurityTestGenerator::generate_payload_selection(&[], false));
1966 additional_code.push('\n');
1967 additional_code.push_str(&SecurityTestGenerator::generate_apply_payload(&[]));
1968 additional_code.push('\n');
1969 }
1970
1971 if let Some(config) = self.build_parallel_config() {
1973 TerminalReporter::print_progress("Adding parallel execution support...");
1974 additional_code.push_str(&ParallelRequestGenerator::generate_batch_helper(&config));
1975 additional_code.push('\n');
1976 TerminalReporter::print_success(&format!(
1977 "Parallel execution enabled (count: {})",
1978 config.count
1979 ));
1980 }
1981
1982 if !additional_code.is_empty() {
1984 if let Some(import_end) = enhanced_script.find("export const options") {
1986 enhanced_script.insert_str(
1987 import_end,
1988 &format!("\n// === Advanced Testing Features ===\n{}\n", additional_code),
1989 );
1990 }
1991 }
1992
1993 Ok(enhanced_script)
1994 }
1995
1996 async fn execute_sequential_specs(&self) -> Result<()> {
1998 TerminalReporter::print_progress("Sequential spec mode: Loading specs individually...");
1999
2000 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
2002
2003 if !self.spec.is_empty() {
2004 let specs = load_specs_from_files(self.spec.clone())
2005 .await
2006 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
2007 all_specs.extend(specs);
2008 }
2009
2010 if let Some(spec_dir) = &self.spec_dir {
2011 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
2012 BenchError::Other(format!("Failed to load specs from directory: {}", e))
2013 })?;
2014 all_specs.extend(dir_specs);
2015 }
2016
2017 if all_specs.is_empty() {
2018 return Err(BenchError::Other(
2019 "No spec files found for sequential execution".to_string(),
2020 ));
2021 }
2022
2023 TerminalReporter::print_success(&format!("Loaded {} spec(s)", all_specs.len()));
2024
2025 let execution_order = if let Some(config_path) = &self.dependency_config {
2027 TerminalReporter::print_progress("Loading dependency configuration...");
2028 let config = SpecDependencyConfig::from_file(config_path)?;
2029
2030 if !config.disable_auto_detect && config.execution_order.is_empty() {
2031 self.detect_and_sort_specs(&all_specs)?
2033 } else {
2034 config.execution_order.iter().flat_map(|g| g.specs.clone()).collect()
2036 }
2037 } else {
2038 self.detect_and_sort_specs(&all_specs)?
2040 };
2041
2042 TerminalReporter::print_success(&format!(
2043 "Execution order: {}",
2044 execution_order
2045 .iter()
2046 .map(|p| p.file_name().unwrap_or_default().to_string_lossy().to_string())
2047 .collect::<Vec<_>>()
2048 .join(" → ")
2049 ));
2050
2051 let mut extracted_values = ExtractedValues::new();
2053 let total_specs = execution_order.len();
2054
2055 for (index, spec_path) in execution_order.iter().enumerate() {
2056 let spec_name = spec_path.file_name().unwrap_or_default().to_string_lossy().to_string();
2057
2058 TerminalReporter::print_progress(&format!(
2059 "[{}/{}] Executing spec: {}",
2060 index + 1,
2061 total_specs,
2062 spec_name
2063 ));
2064
2065 let spec = all_specs
2067 .iter()
2068 .find(|(p, _)| {
2069 p == spec_path
2070 || p.file_name() == spec_path.file_name()
2071 || p.file_name() == Some(spec_path.as_os_str())
2072 })
2073 .map(|(_, s)| s.clone())
2074 .ok_or_else(|| {
2075 BenchError::Other(format!("Spec not found: {}", spec_path.display()))
2076 })?;
2077
2078 let new_values = self.execute_single_spec(&spec, &spec_name, &extracted_values).await?;
2080
2081 extracted_values.merge(&new_values);
2083
2084 TerminalReporter::print_success(&format!(
2085 "[{}/{}] Completed: {} (extracted {} values)",
2086 index + 1,
2087 total_specs,
2088 spec_name,
2089 new_values.values.len()
2090 ));
2091 }
2092
2093 TerminalReporter::print_success(&format!(
2094 "Sequential execution complete: {} specs executed",
2095 total_specs
2096 ));
2097
2098 Ok(())
2099 }
2100
2101 fn detect_and_sort_specs(&self, specs: &[(PathBuf, OpenApiSpec)]) -> Result<Vec<PathBuf>> {
2103 TerminalReporter::print_progress("Auto-detecting spec dependencies...");
2104
2105 let mut detector = DependencyDetector::new();
2106 let dependencies = detector.detect_dependencies(specs);
2107
2108 if dependencies.is_empty() {
2109 TerminalReporter::print_progress("No dependencies detected, using file order");
2110 return Ok(specs.iter().map(|(p, _)| p.clone()).collect());
2111 }
2112
2113 TerminalReporter::print_progress(&format!(
2114 "Detected {} cross-spec dependencies",
2115 dependencies.len()
2116 ));
2117
2118 for dep in &dependencies {
2119 TerminalReporter::print_progress(&format!(
2120 " {} → {} (via field '{}')",
2121 dep.dependency_spec.file_name().unwrap_or_default().to_string_lossy(),
2122 dep.dependent_spec.file_name().unwrap_or_default().to_string_lossy(),
2123 dep.field_name
2124 ));
2125 }
2126
2127 topological_sort(specs, &dependencies)
2128 }
2129
2130 async fn execute_single_spec(
2132 &self,
2133 spec: &OpenApiSpec,
2134 spec_name: &str,
2135 _external_values: &ExtractedValues,
2136 ) -> Result<ExtractedValues> {
2137 let parser = SpecParser::from_spec(spec.clone());
2138
2139 if self.crud_flow {
2141 self.execute_crud_flow_with_extraction(&parser, spec_name).await
2143 } else {
2144 self.execute_standard_spec(&parser, spec_name).await?;
2146 Ok(ExtractedValues::new())
2147 }
2148 }
2149
2150 async fn execute_crud_flow_with_extraction(
2152 &self,
2153 parser: &SpecParser,
2154 spec_name: &str,
2155 ) -> Result<ExtractedValues> {
2156 let operations = parser.get_operations();
2157 let flows = CrudFlowDetector::detect_flows(&operations);
2158
2159 if flows.is_empty() {
2160 TerminalReporter::print_warning(&format!("No CRUD flows detected in {}", spec_name));
2161 return Ok(ExtractedValues::new());
2162 }
2163
2164 TerminalReporter::print_progress(&format!(
2165 " {} CRUD flow(s) in {}",
2166 flows.len(),
2167 spec_name
2168 ));
2169
2170 let mut handlebars = handlebars::Handlebars::new();
2172 handlebars.register_helper(
2174 "json",
2175 Box::new(
2176 |h: &handlebars::Helper,
2177 _: &handlebars::Handlebars,
2178 _: &handlebars::Context,
2179 _: &mut handlebars::RenderContext,
2180 out: &mut dyn handlebars::Output|
2181 -> handlebars::HelperResult {
2182 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2183 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2184 Ok(())
2185 },
2186 ),
2187 );
2188 let template = include_str!("templates/k6_crud_flow.hbs");
2189 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
2190
2191 let custom_headers = self.parse_headers()?;
2192 let config = self.build_crud_flow_config().unwrap_or_default();
2193
2194 let param_overrides = if let Some(params_file) = &self.params_file {
2196 let overrides = ParameterOverrides::from_file(params_file)?;
2197 Some(overrides)
2198 } else {
2199 None
2200 };
2201
2202 let duration_secs = Self::parse_duration(&self.duration)?;
2204 let scenario =
2205 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2206 let stages = scenario.generate_stages(duration_secs, self.vus);
2207
2208 let api_base_path = self.resolve_base_path(parser);
2210
2211 let mut all_headers = custom_headers.clone();
2213 if let Some(auth) = &self.auth {
2214 all_headers.insert("Authorization".to_string(), auth.clone());
2215 }
2216 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2217
2218 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2220
2221 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2222 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2226 serde_json::json!({
2227 "name": sanitized_name.clone(),
2228 "display_name": f.name,
2229 "base_path": f.base_path,
2230 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2231 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2233 let method_raw = if !parts.is_empty() {
2234 parts[0].to_uppercase()
2235 } else {
2236 "GET".to_string()
2237 };
2238 let method = if !parts.is_empty() {
2239 let m = parts[0].to_lowercase();
2240 if m == "delete" { "del".to_string() } else { m }
2242 } else {
2243 "get".to_string()
2244 };
2245 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2246 let path = if let Some(ref bp) = api_base_path {
2248 format!("{}{}", bp, raw_path)
2249 } else {
2250 raw_path.to_string()
2251 };
2252 let is_get_or_head = method == "get" || method == "head";
2253 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2255
2256 let body_value = if has_body {
2258 param_overrides.as_ref()
2259 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2260 .and_then(|oo| oo.body)
2261 .unwrap_or_else(|| serde_json::json!({}))
2262 } else {
2263 serde_json::json!({})
2264 };
2265
2266 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2268
2269 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2271 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2272
2273 serde_json::json!({
2274 "operation": s.operation,
2275 "method": method,
2276 "path": path,
2277 "extract": s.extract,
2278 "use_values": s.use_values,
2279 "use_body": s.use_body,
2280 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2281 "inject_attacks": s.inject_attacks,
2282 "attack_types": s.attack_types,
2283 "description": s.description,
2284 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2285 "is_get_or_head": is_get_or_head,
2286 "has_body": has_body,
2287 "body": processed_body.value,
2288 "body_is_dynamic": body_is_dynamic,
2289 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2290 })
2291 }).collect::<Vec<_>>(),
2292 })
2293 }).collect();
2294
2295 for flow_data in &flows_data {
2297 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2298 for step in steps {
2299 if let Some(placeholders_arr) =
2300 step.get("_placeholders").and_then(|p| p.as_array())
2301 {
2302 for p_str in placeholders_arr {
2303 if let Some(p_name) = p_str.as_str() {
2304 match p_name {
2305 "VU" => {
2306 all_placeholders.insert(DynamicPlaceholder::VU);
2307 }
2308 "Iteration" => {
2309 all_placeholders.insert(DynamicPlaceholder::Iteration);
2310 }
2311 "Timestamp" => {
2312 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2313 }
2314 "UUID" => {
2315 all_placeholders.insert(DynamicPlaceholder::UUID);
2316 }
2317 "Random" => {
2318 all_placeholders.insert(DynamicPlaceholder::Random);
2319 }
2320 "Counter" => {
2321 all_placeholders.insert(DynamicPlaceholder::Counter);
2322 }
2323 "Date" => {
2324 all_placeholders.insert(DynamicPlaceholder::Date);
2325 }
2326 "VuIter" => {
2327 all_placeholders.insert(DynamicPlaceholder::VuIter);
2328 }
2329 _ => {}
2330 }
2331 }
2332 }
2333 }
2334 }
2335 }
2336 }
2337
2338 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2340 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2341
2342 let security_testing_enabled = self.security_testing_enabled();
2344
2345 let data = serde_json::json!({
2346 "base_url": self.target,
2347 "flows": flows_data,
2348 "extract_fields": config.default_extract_fields,
2349 "duration_secs": duration_secs,
2350 "max_vus": self.vus,
2351 "auth_header": self.auth,
2352 "custom_headers": custom_headers,
2353 "skip_tls_verify": self.skip_tls_verify,
2354 "stages": stages.iter().map(|s| serde_json::json!({
2356 "duration": s.duration,
2357 "target": s.target,
2358 })).collect::<Vec<_>>(),
2359 "threshold_percentile": self.threshold_percentile,
2360 "threshold_ms": self.threshold_ms,
2361 "max_error_rate": self.max_error_rate,
2362 "abort_on_error": self.abort_on_error,
2363 "abort_on_error_rate": self.abort_on_error_rate,
2364 "headers": headers_json,
2365 "dynamic_imports": required_imports,
2366 "dynamic_globals": required_globals,
2367 "extracted_values_output_path": output_dir.join("extracted_values.json").to_string_lossy(),
2368 "security_testing_enabled": security_testing_enabled,
2370 "has_custom_headers": !custom_headers.is_empty(),
2371 });
2372
2373 let mut script = handlebars
2374 .render_template(template, &data)
2375 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2376
2377 if security_testing_enabled {
2379 script = self.generate_enhanced_script(&script)?;
2380 }
2381
2382 let script_path =
2384 self.output.join(format!("k6-{}-crud-flow.js", spec_name.replace('.', "_")));
2385
2386 std::fs::create_dir_all(self.output.clone())?;
2387 std::fs::write(&script_path, &script)?;
2388
2389 if !self.generate_only {
2390 let executor = K6Executor::new()?
2391 .with_local_ips(self.source_ips.join(","))
2392 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
2393 std::fs::create_dir_all(&output_dir)?;
2394
2395 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
2396
2397 let extracted = Self::parse_extracted_values(&output_dir)?;
2398 TerminalReporter::print_progress(&format!(
2399 " Extracted {} value(s) from {}",
2400 extracted.values.len(),
2401 spec_name
2402 ));
2403 return Ok(extracted);
2404 }
2405
2406 Ok(ExtractedValues::new())
2407 }
2408
2409 async fn execute_standard_spec(&self, parser: &SpecParser, spec_name: &str) -> Result<()> {
2411 let mut operations = if let Some(filter) = &self.operations {
2412 parser.filter_operations(filter)?
2413 } else {
2414 parser.get_operations()
2415 };
2416
2417 if let Some(exclude) = &self.exclude_operations {
2418 operations = parser.exclude_operations(operations, exclude)?;
2419 }
2420
2421 if operations.is_empty() {
2422 TerminalReporter::print_warning(&format!("No operations found in {}", spec_name));
2423 return Ok(());
2424 }
2425
2426 TerminalReporter::print_progress(&format!(
2427 " {} operations in {}",
2428 operations.len(),
2429 spec_name
2430 ));
2431
2432 let templates: Vec<_> = operations
2434 .iter()
2435 .map(RequestGenerator::generate_template)
2436 .collect::<Result<Vec<_>>>()?;
2437
2438 let custom_headers = self.parse_headers()?;
2440
2441 let base_path = self.resolve_base_path(parser);
2443
2444 let scenario =
2446 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2447
2448 let security_testing_enabled = self.security_testing_enabled();
2449
2450 let force_http1 = crate::request_gen::should_force_k6_http1(
2451 self.wafbench_verbatim,
2452 &templates,
2453 &custom_headers,
2454 );
2455
2456 let duration_secs = Self::parse_duration(&self.duration)?;
2457 let (per_op_metrics, per_op_warn) = crate::k6_gen::resolve_per_op_metrics(
2458 self.per_op_metrics,
2459 templates.len(),
2460 duration_secs,
2461 );
2462 if let Some(msg) = per_op_warn {
2463 TerminalReporter::print_warning(&msg);
2464 }
2465
2466 let k6_config = K6Config {
2467 target_url: self.target.clone(),
2468 base_path,
2469 scenario,
2470 duration_secs,
2471 max_vus: self.vus,
2472 threshold_percentile: self.threshold_percentile.clone(),
2473 threshold_ms: self.threshold_ms,
2474 max_error_rate: self.max_error_rate,
2475 auth_header: self.auth.clone(),
2476 custom_headers,
2477 skip_tls_verify: self.skip_tls_verify,
2478 security_testing_enabled,
2479 chunked_request_bodies: self.chunked_request_bodies,
2480 target_rps: self.target_rps,
2481 no_keep_alive: self.no_keep_alive,
2482 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
2484 .into_iter()
2485 .map(|ip| ip.to_string())
2486 .collect(),
2487 geo_source_headers: if self.geo_source_headers.is_empty()
2488 && !self.geo_source_ips.is_empty()
2489 {
2490 crate::conformance::self_test::default_geo_source_headers()
2491 } else {
2492 self.geo_source_headers.clone()
2493 },
2494 };
2495
2496 let generator = K6ScriptGenerator::new(k6_config, templates)
2497 .with_abort_valve(self.abort_on_error, self.abort_on_error_rate)
2498 .with_force_http1(force_http1)
2499 .with_per_op_metrics(per_op_metrics);
2500 let mut script = generator.generate()?;
2501
2502 let has_advanced_features = self.data_file.is_some()
2504 || self.error_rate.is_some()
2505 || self.security_test
2506 || self.parallel_create.is_some()
2507 || self.wafbench_dir.is_some();
2508
2509 if has_advanced_features {
2510 script = self.generate_enhanced_script(&script)?;
2511 }
2512
2513 let script_path = self.output.join(format!("k6-{}.js", spec_name.replace('.', "_")));
2515
2516 std::fs::create_dir_all(self.output.clone())?;
2517 std::fs::write(&script_path, &script)?;
2518
2519 if !self.generate_only {
2520 let executor = K6Executor::new()?
2523 .with_local_ips(self.source_ips.join(","))
2524 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
2525 .with_discard_response_bodies(self.discard_response_bodies)
2526 .with_force_http1(force_http1);
2527 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
2528 std::fs::create_dir_all(&output_dir)?;
2529
2530 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
2531 }
2532
2533 Ok(())
2534 }
2535
2536 async fn execute_crud_flow(&self, parser: &SpecParser) -> Result<()> {
2538 let config = self.build_crud_flow_config().unwrap_or_default();
2540
2541 let flows = if !config.flows.is_empty() {
2543 TerminalReporter::print_progress("Using custom flow configuration...");
2544 config.flows.clone()
2545 } else {
2546 TerminalReporter::print_progress("Detecting CRUD operations...");
2547 let operations = parser.get_operations();
2548 CrudFlowDetector::detect_flows(&operations)
2549 };
2550
2551 if flows.is_empty() {
2552 return Err(BenchError::Other(
2553 "No CRUD flows detected in spec. Ensure spec has POST/GET/PUT/DELETE operations on related paths.".to_string(),
2554 ));
2555 }
2556
2557 if config.flows.is_empty() {
2558 TerminalReporter::print_success(&format!("Detected {} CRUD flow(s)", flows.len()));
2559 } else {
2560 TerminalReporter::print_success(&format!("Loaded {} custom flow(s)", flows.len()));
2561 }
2562
2563 for flow in &flows {
2564 TerminalReporter::print_progress(&format!(
2565 " - {}: {} steps",
2566 flow.name,
2567 flow.steps.len()
2568 ));
2569 }
2570
2571 let mut handlebars = handlebars::Handlebars::new();
2573 handlebars.register_helper(
2575 "json",
2576 Box::new(
2577 |h: &handlebars::Helper,
2578 _: &handlebars::Handlebars,
2579 _: &handlebars::Context,
2580 _: &mut handlebars::RenderContext,
2581 out: &mut dyn handlebars::Output|
2582 -> handlebars::HelperResult {
2583 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2584 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2585 Ok(())
2586 },
2587 ),
2588 );
2589 let template = include_str!("templates/k6_crud_flow.hbs");
2590
2591 let custom_headers = self.parse_headers()?;
2592
2593 let param_overrides = if let Some(params_file) = &self.params_file {
2595 TerminalReporter::print_progress("Loading parameter overrides...");
2596 let overrides = ParameterOverrides::from_file(params_file)?;
2597 TerminalReporter::print_success(&format!(
2598 "Loaded parameter overrides ({} operation-specific, {} defaults)",
2599 overrides.operations.len(),
2600 if overrides.defaults.is_empty() { 0 } else { 1 }
2601 ));
2602 Some(overrides)
2603 } else {
2604 None
2605 };
2606
2607 let duration_secs = Self::parse_duration(&self.duration)?;
2609 let scenario =
2610 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2611 let stages = scenario.generate_stages(duration_secs, self.vus);
2612
2613 let api_base_path = self.resolve_base_path(parser);
2615 if let Some(ref bp) = api_base_path {
2616 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
2617 }
2618
2619 let mut all_headers = custom_headers.clone();
2621 if let Some(auth) = &self.auth {
2622 all_headers.insert("Authorization".to_string(), auth.clone());
2623 }
2624 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2625
2626 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2628
2629 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2630 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2635 serde_json::json!({
2636 "name": sanitized_name.clone(), "display_name": f.name, "base_path": f.base_path,
2639 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2640 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2642 let method_raw = if !parts.is_empty() {
2643 parts[0].to_uppercase()
2644 } else {
2645 "GET".to_string()
2646 };
2647 let method = if !parts.is_empty() {
2648 let m = parts[0].to_lowercase();
2649 if m == "delete" { "del".to_string() } else { m }
2651 } else {
2652 "get".to_string()
2653 };
2654 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2655 let path = if let Some(ref bp) = api_base_path {
2657 format!("{}{}", bp, raw_path)
2658 } else {
2659 raw_path.to_string()
2660 };
2661 let is_get_or_head = method == "get" || method == "head";
2662 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2664
2665 let body_value = if has_body {
2667 param_overrides.as_ref()
2668 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2669 .and_then(|oo| oo.body)
2670 .unwrap_or_else(|| serde_json::json!({}))
2671 } else {
2672 serde_json::json!({})
2673 };
2674
2675 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2677 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2682 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2683
2684 serde_json::json!({
2685 "operation": s.operation,
2686 "method": method,
2687 "path": path,
2688 "extract": s.extract,
2689 "use_values": s.use_values,
2690 "use_body": s.use_body,
2691 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2692 "inject_attacks": s.inject_attacks,
2693 "attack_types": s.attack_types,
2694 "description": s.description,
2695 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2696 "is_get_or_head": is_get_or_head,
2697 "has_body": has_body,
2698 "body": processed_body.value,
2699 "body_is_dynamic": body_is_dynamic,
2700 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2701 })
2702 }).collect::<Vec<_>>(),
2703 })
2704 }).collect();
2705
2706 for flow_data in &flows_data {
2708 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2709 for step in steps {
2710 if let Some(placeholders_arr) =
2711 step.get("_placeholders").and_then(|p| p.as_array())
2712 {
2713 for p_str in placeholders_arr {
2714 if let Some(p_name) = p_str.as_str() {
2715 match p_name {
2717 "VU" => {
2718 all_placeholders.insert(DynamicPlaceholder::VU);
2719 }
2720 "Iteration" => {
2721 all_placeholders.insert(DynamicPlaceholder::Iteration);
2722 }
2723 "Timestamp" => {
2724 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2725 }
2726 "UUID" => {
2727 all_placeholders.insert(DynamicPlaceholder::UUID);
2728 }
2729 "Random" => {
2730 all_placeholders.insert(DynamicPlaceholder::Random);
2731 }
2732 "Counter" => {
2733 all_placeholders.insert(DynamicPlaceholder::Counter);
2734 }
2735 "Date" => {
2736 all_placeholders.insert(DynamicPlaceholder::Date);
2737 }
2738 "VuIter" => {
2739 all_placeholders.insert(DynamicPlaceholder::VuIter);
2740 }
2741 _ => {}
2742 }
2743 }
2744 }
2745 }
2746 }
2747 }
2748 }
2749
2750 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2752 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2753
2754 let invalid_data_config = self.build_invalid_data_config();
2756 let error_injection_enabled = invalid_data_config.is_some();
2757 let error_rate = self.error_rate.unwrap_or(0.0);
2758 let error_types: Vec<String> = invalid_data_config
2759 .as_ref()
2760 .map(|c| c.error_types.iter().map(|t| format!("{:?}", t)).collect())
2761 .unwrap_or_default();
2762
2763 if error_injection_enabled {
2764 TerminalReporter::print_progress(&format!(
2765 "Error injection enabled ({}% rate)",
2766 (error_rate * 100.0) as u32
2767 ));
2768 }
2769
2770 let security_testing_enabled = self.security_testing_enabled();
2772
2773 let data = serde_json::json!({
2774 "base_url": self.target,
2775 "flows": flows_data,
2776 "extract_fields": config.default_extract_fields,
2777 "duration_secs": duration_secs,
2778 "max_vus": self.vus,
2779 "auth_header": self.auth,
2780 "custom_headers": custom_headers,
2781 "skip_tls_verify": self.skip_tls_verify,
2782 "stages": stages.iter().map(|s| serde_json::json!({
2784 "duration": s.duration,
2785 "target": s.target,
2786 })).collect::<Vec<_>>(),
2787 "threshold_percentile": self.threshold_percentile,
2788 "threshold_ms": self.threshold_ms,
2789 "max_error_rate": self.max_error_rate,
2790 "abort_on_error": self.abort_on_error,
2791 "abort_on_error_rate": self.abort_on_error_rate,
2792 "headers": headers_json,
2793 "dynamic_imports": required_imports,
2794 "dynamic_globals": required_globals,
2795 "extracted_values_output_path": self
2796 .output
2797 .join("crud_flow_extracted_values.json")
2798 .to_string_lossy(),
2799 "error_injection_enabled": error_injection_enabled,
2801 "error_rate": error_rate,
2802 "error_types": error_types,
2803 "security_testing_enabled": security_testing_enabled,
2805 "has_custom_headers": !custom_headers.is_empty(),
2806 });
2807
2808 let mut script = handlebars
2809 .render_template(template, &data)
2810 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2811
2812 if security_testing_enabled {
2814 script = self.generate_enhanced_script(&script)?;
2815 }
2816
2817 TerminalReporter::print_progress("Validating CRUD flow script...");
2819 let validation_errors = K6ScriptGenerator::validate_script(&script);
2820 if !validation_errors.is_empty() {
2821 TerminalReporter::print_error("CRUD flow script validation failed");
2822 for error in &validation_errors {
2823 eprintln!(" {}", error);
2824 }
2825 return Err(BenchError::Other(format!(
2826 "CRUD flow script validation failed with {} error(s)",
2827 validation_errors.len()
2828 )));
2829 }
2830
2831 TerminalReporter::print_success("CRUD flow script generated");
2832
2833 let script_path = if let Some(output) = &self.script_output {
2835 output.clone()
2836 } else {
2837 self.output.join("k6-crud-flow-script.js")
2838 };
2839
2840 if let Some(parent) = script_path.parent() {
2841 std::fs::create_dir_all(parent)?;
2842 }
2843 std::fs::write(&script_path, &script)?;
2844 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
2845
2846 if self.generate_only {
2847 println!("\nScript generated successfully. Run it with:");
2848 println!(" k6 run {}", script_path.display());
2849 return Ok(());
2850 }
2851
2852 TerminalReporter::print_progress("Executing CRUD flow test...");
2854 let executor = K6Executor::new()?
2855 .with_local_ips(self.source_ips.join(","))
2856 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
2857 std::fs::create_dir_all(&self.output)?;
2858
2859 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
2860
2861 let duration_secs = Self::parse_duration(&self.duration)?;
2862 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
2863
2864 Ok(())
2865 }
2866
2867 async fn execute_conformance_test(&self) -> Result<()> {
2869 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
2870 use crate::conformance::report::ConformanceReport;
2871 use crate::conformance::spec::ConformanceFeature;
2872
2873 TerminalReporter::print_progress("OpenAPI 3.0.0 Conformance Testing Mode");
2874
2875 TerminalReporter::print_progress(CONFORMANCE_REPLACES_LOAD_ADVISORY);
2876
2877 let categories = self.conformance_categories.as_ref().map(|cats_str| {
2879 cats_str
2880 .split(',')
2881 .filter_map(|s| {
2882 let trimmed = s.trim();
2883 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
2884 Some(canonical.to_string())
2885 } else {
2886 TerminalReporter::print_warning(&format!(
2887 "Unknown conformance category: '{}'. Valid categories: {}",
2888 trimmed,
2889 ConformanceFeature::cli_category_names()
2890 .iter()
2891 .map(|(cli, _)| *cli)
2892 .collect::<Vec<_>>()
2893 .join(", ")
2894 ));
2895 None
2896 }
2897 })
2898 .collect::<Vec<String>>()
2899 });
2900
2901 let custom_headers: Vec<(String, String)> = self
2903 .conformance_headers
2904 .iter()
2905 .filter_map(|h| {
2906 let (name, value) = h.split_once(':')?;
2907 Some((name.trim().to_string(), value.trim().to_string()))
2908 })
2909 .collect();
2910
2911 if !custom_headers.is_empty() {
2912 TerminalReporter::print_progress(&format!(
2913 "Using {} custom header(s) for authentication",
2914 custom_headers.len()
2915 ));
2916 }
2917
2918 if self.conformance_delay_ms > 0 {
2919 TerminalReporter::print_progress(&format!(
2920 "Using {}ms delay between conformance requests",
2921 self.conformance_delay_ms
2922 ));
2923 }
2924
2925 std::fs::create_dir_all(&self.output)?;
2927
2928 let config = ConformanceConfig {
2929 target_url: self.target.clone(),
2930 api_key: self.conformance_api_key.clone(),
2931 basic_auth: self.conformance_basic_auth.clone(),
2932 skip_tls_verify: self.skip_tls_verify,
2933 categories,
2934 base_path: self.base_path.clone(),
2935 custom_headers,
2936 output_dir: Some(self.output.clone()),
2937 all_operations: self.conformance_all_operations,
2938 custom_checks_file: self.conformance_custom.clone(),
2939 request_delay_ms: self.conformance_delay_ms,
2940 custom_filter: self.conformance_custom_filter.clone(),
2941 export_requests: self.export_requests,
2942 validate_requests: self.validate_requests,
2943 };
2944
2945 let mut resolved_base_path: Option<String> = None;
2953 let annotated_ops = if !self.spec.is_empty() {
2954 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
2955 let parser = SpecParser::from_file(&self.spec[0]).await?;
2956 resolved_base_path = self.resolve_base_path(&parser);
2957
2958 let mut operations = if let Some(filter) = &self.operations {
2963 parser.filter_operations(filter)?
2964 } else {
2965 parser.get_operations()
2966 };
2967 if let Some(exclude) = &self.exclude_operations {
2968 let before_count = operations.len();
2969 operations = parser.exclude_operations(operations, exclude)?;
2970 let excluded_count = before_count - operations.len();
2971 if excluded_count > 0 {
2972 TerminalReporter::print_progress(&format!(
2973 "Excluded {} operations matching '{}'",
2974 excluded_count, exclude
2975 ));
2976 }
2977 }
2978
2979 let annotated =
2980 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
2981 &operations,
2982 parser.spec(),
2983 );
2984 TerminalReporter::print_success(&format!(
2985 "Analyzed {} operations, found {} feature annotations",
2986 operations.len(),
2987 annotated.iter().map(|a| a.features.len()).sum::<usize>()
2988 ));
2989 Some(annotated)
2990 } else {
2991 None
2992 };
2993
2994 if self.conformance_self_test {
3001 let Some(ops) = annotated_ops else {
3002 TerminalReporter::print_error(
3003 "--conformance-self-test requires --spec; no operations to test",
3004 );
3005 return Ok(());
3006 };
3007 let cfg = crate::conformance::self_test::SelfTestConfig {
3008 target_url: self.target.clone(),
3009 skip_tls_verify: self.skip_tls_verify,
3010 timeout: std::time::Duration::from_secs(30),
3011 extra_headers: self
3015 .conformance_headers
3016 .iter()
3017 .filter_map(|h| {
3018 let (n, v) = h.split_once(':')?;
3019 Some((n.trim().to_string(), v.trim().to_string()))
3020 })
3021 .collect(),
3022 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
3023 base_path: resolved_base_path.clone(),
3027 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
3031 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
3032 geo_source_headers: if self.geo_source_headers.is_empty() {
3033 crate::conformance::self_test::default_geo_source_headers()
3034 } else {
3035 self.geo_source_headers.clone()
3036 },
3037 capture: if self.conformance_self_test_capture
3041 || self.validate_response_schemas
3042 || self.validate_requests
3043 {
3044 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
3055 } else {
3056 None
3057 },
3058 validate_response_schemas: self.validate_response_schemas,
3059 spec_label: self.spec.first().map(|p| {
3065 p.file_name()
3066 .map(|s| s.to_string_lossy().into_owned())
3067 .unwrap_or_else(|| p.to_string_lossy().into_owned())
3068 }),
3069 network_events: Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))),
3076 current_iteration: 1,
3077 };
3078 let capture_sink = cfg.capture.clone();
3079 let network_events_sink = cfg.network_events.clone();
3080 TerminalReporter::print_progress(&format!(
3081 "Self-test mode: driving {} operations with positive + per-category negative cases",
3082 ops.len()
3083 ));
3084 let target_iterations = self.conformance_self_test_iterations.max(1);
3091 let duration_budget = self
3092 .conformance_self_test_duration
3093 .as_ref()
3094 .map(|s| Self::parse_duration(s))
3095 .transpose()?
3096 .map(std::time::Duration::from_secs);
3097 let start = std::time::Instant::now();
3098 let deadline = duration_budget.map(|d| start + d);
3107 let mut cfg = cfg;
3111 cfg.current_iteration = 1;
3112 let mut report =
3113 crate::conformance::self_test::run_self_test_with_deadline(&ops, &cfg, deadline)
3114 .await
3115 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3116 let mut iter_done: u32 = 1;
3117 loop {
3118 let by_iter = iter_done >= target_iterations;
3119 let by_dur = duration_budget.map(|d| start.elapsed() >= d).unwrap_or(true);
3120 if by_iter && by_dur {
3121 break;
3122 }
3123 cfg.current_iteration = iter_done.saturating_add(1);
3124 let next = crate::conformance::self_test::run_self_test_with_deadline(
3125 &ops, &cfg, deadline,
3126 )
3127 .await
3128 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3129 report.merge_iteration(next);
3130 iter_done = iter_done.saturating_add(1);
3131 }
3132 if iter_done > 1 {
3133 TerminalReporter::print_progress(&format!(
3134 "Self-test repeated {} iteration(s) ({:.1?} elapsed)",
3135 iter_done,
3136 start.elapsed(),
3137 ));
3138 }
3139 let per_endpoint_summary: Vec<
3149 crate::conformance::per_endpoint_summary::PerEndpointSummary,
3150 >;
3151 if let Some(sink) = capture_sink {
3152 if let Ok(guard) = sink.lock() {
3153 let jsonl_path = self.output.join("conformance-self-test-requests.jsonl");
3154 let mut lines = String::with_capacity(guard.len() * 256);
3155 for entry in guard.iter() {
3156 if let Ok(line) = serde_json::to_string(entry) {
3157 lines.push_str(&line);
3158 lines.push('\n');
3159 }
3160 }
3161 let _ = std::fs::write(&jsonl_path, lines);
3162 let html_path = self.output.join("conformance-self-test-requests.html");
3163 let html =
3164 crate::conformance::capture_html::render_capture_html(guard.as_slice());
3165 let _ = std::fs::write(&html_path, html);
3166
3167 per_endpoint_summary =
3171 crate::conformance::per_endpoint_summary::build_summary(guard.as_slice());
3172 let summary_path = self.output.join("conformance-per-endpoint.json");
3173 if let Ok(json) = serde_json::to_string_pretty(&per_endpoint_summary) {
3174 let _ = std::fs::write(&summary_path, json);
3175 TerminalReporter::print_progress(&format!(
3176 "Self-test request/response capture written to {} ({} entries) + {} + {}",
3177 jsonl_path.display(),
3178 guard.len(),
3179 html_path.display(),
3180 summary_path.display(),
3181 ));
3182 } else {
3183 TerminalReporter::print_progress(&format!(
3184 "Self-test request/response capture written to {} ({} entries) + {}",
3185 jsonl_path.display(),
3186 guard.len(),
3187 html_path.display(),
3188 ));
3189 }
3190 } else {
3191 per_endpoint_summary = Vec::new();
3192 }
3193 } else {
3194 per_endpoint_summary = Vec::new();
3195 }
3196 TerminalReporter::print_progress(&report.render_summary());
3197 if let Some(sink) = network_events_sink {
3204 if let Ok(guard) = sink.lock() {
3205 let path = self.output.join("conformance-network-events.json");
3206 if let Ok(json) = serde_json::to_string_pretty(&*guard) {
3207 let _ = std::fs::write(&path, json);
3208 if guard.is_empty() {
3209 TerminalReporter::print_progress(
3210 "No wire-level network failures during self-test (file written empty)",
3211 );
3212 } else {
3213 TerminalReporter::print_warning(&format!(
3214 "Recorded {} wire-level network event(s) to {}",
3215 guard.len(),
3216 path.display()
3217 ));
3218 }
3219 }
3220 }
3221 }
3222 let json_path = self.output.join("conformance-self-test.json");
3226 if let Ok(json) = serde_json::to_string_pretty(&report) {
3227 let _ = std::fs::write(&json_path, json);
3228 TerminalReporter::print_progress(&format!(
3229 "Self-test report written to {}",
3230 json_path.display()
3231 ));
3232 }
3233 let issues = report.definite_issues();
3237 let issues_path = self.output.join("conformance-definite-issues.json");
3238 if let Ok(json) = serde_json::to_string_pretty(&issues) {
3239 if std::fs::write(&issues_path, json).is_ok() && !issues.is_empty() {
3240 TerminalReporter::print_warning(&format!(
3241 "{} definite issue(s) — see {}",
3242 issues.len(),
3243 issues_path.display()
3244 ));
3245 }
3246 }
3247 let owasp_accepted = report.owasp_accepted_probes();
3250 if !owasp_accepted.is_empty() {
3251 let owasp_path = self.output.join("conformance-owasp-accepted.json");
3252 if let Ok(json) = serde_json::to_string_pretty(&owasp_accepted) {
3253 if std::fs::write(&owasp_path, json).is_ok() {
3254 TerminalReporter::print_warning(&format!(
3255 "{} owasp injection probe(s) accepted by the target — see {}",
3256 owasp_accepted.len(),
3257 owasp_path.display()
3258 ));
3259 }
3260 }
3261 }
3262 if let Some(status) = report.detect_target_misconfiguration() {
3271 let hint = match status {
3272 404 => " Likely cause: spec paths don't match deployed routes — check --base-path and the spec's `servers` block.",
3273 401 | 403 => " Likely cause: authentication header is missing or invalid — check --conformance-header.",
3274 _ => "",
3275 };
3276 TerminalReporter::print_warning(&format!(
3277 "Self-test misconfiguration: every positive case returned {status}.{hint} Negative results below are meaningless under this condition."
3278 ));
3279 } else if !report.all_passed() {
3280 TerminalReporter::print_warning(
3281 "Self-test detected gaps — server let through at least one request that should have been a 4xx",
3282 );
3283 } else {
3284 TerminalReporter::print_success(
3285 "Self-test passed — all positive cases accepted and all negative cases rejected",
3286 );
3287 }
3288 let html_path = self.output.join("conformance-report.html");
3295 let audit_path = self.output.join("conformance-spec-audit.json");
3296 let audit_value = std::fs::read_to_string(&audit_path)
3297 .ok()
3298 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok());
3299 let render_opts = crate::conformance::report_html::RenderOptions {
3304 missed_cap: match self.report_missed_cap {
3305 Some(0) => None,
3306 Some(n) => Some(n as usize),
3307 None => Some(200),
3308 },
3309 };
3310 let mut html = crate::conformance::report_html::render_html_with_options(
3311 &report,
3312 audit_value.as_ref(),
3313 &render_opts,
3314 );
3315 let summary_section = crate::conformance::per_endpoint_summary::render_html_section(
3321 &per_endpoint_summary,
3322 );
3323 if !summary_section.is_empty() {
3324 if let Some(idx) = html.rfind("</body>") {
3325 html.insert_str(idx, &summary_section);
3326 } else {
3327 html.push_str(&summary_section);
3328 }
3329 }
3330 if std::fs::write(&html_path, html).is_ok() {
3331 TerminalReporter::print_progress(&format!(
3332 "HTML report written to {}",
3333 html_path.display()
3334 ));
3335 }
3336
3337 if self.validate_requests && !self.spec.is_empty() {
3349 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3350 &self.spec,
3351 &self.output,
3352 self.base_path.as_deref(),
3353 )
3354 .await?;
3355 if n > 0 {
3356 TerminalReporter::print_warning(&format!(
3357 "{} emitted request(s) recorded against the spec — see conformance-request-violations.json",
3358 n
3359 ));
3360 }
3361 }
3362 return Ok(());
3363 }
3364
3365 if self.validate_requests && !self.spec.is_empty() {
3367 TerminalReporter::print_progress("Validating requests against OpenAPI spec...");
3368 let violation_count = crate::conformance::request_validator::run_request_validation(
3369 &self.spec,
3370 self.conformance_custom.as_deref(),
3371 self.base_path.as_deref(),
3372 &self.output,
3373 )
3374 .await?;
3375 if violation_count > 0 {
3376 TerminalReporter::print_warning(&format!(
3377 "{} request validation violation(s) found — see conformance-request-violations.json",
3378 violation_count
3379 ));
3380 } else {
3381 TerminalReporter::print_success("All requests conform to the OpenAPI spec");
3382 }
3383 }
3384
3385 if self.generate_only || self.use_k6 {
3387 let script = if let Some(annotated) = &annotated_ops {
3388 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
3389 config,
3390 annotated.clone(),
3391 );
3392 let op_count = gen.operation_count();
3393 let (script, check_count) = gen.generate()?;
3394 TerminalReporter::print_success(&format!(
3395 "Conformance: {} operations analyzed, {} unique checks generated",
3396 op_count, check_count
3397 ));
3398 script
3399 } else {
3400 let generator = ConformanceGenerator::new(config);
3401 generator.generate()?
3402 };
3403
3404 let script_path = self.output.join("k6-conformance.js");
3405 std::fs::write(&script_path, &script).map_err(|e| {
3406 BenchError::Other(format!("Failed to write conformance script: {}", e))
3407 })?;
3408 TerminalReporter::print_success(&format!(
3409 "Conformance script generated: {}",
3410 script_path.display()
3411 ));
3412
3413 if self.generate_only {
3414 println!("\nScript generated. Run with:");
3415 println!(" k6 run {}", script_path.display());
3416 return Ok(());
3417 }
3418
3419 if !K6Executor::is_k6_installed() {
3421 TerminalReporter::print_error("k6 is not installed");
3422 TerminalReporter::print_warning(
3423 "Install k6 from: https://k6.io/docs/get-started/installation/",
3424 );
3425 return Err(BenchError::K6NotFound);
3426 }
3427
3428 K6Executor::warn_if_pre_v1().await;
3429 TerminalReporter::print_progress("Running conformance tests via k6...");
3430 let executor = K6Executor::new()?
3431 .with_local_ips(self.source_ips.join(","))
3432 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
3433 executor.execute(&script_path, Some(&self.output), self.verbose).await?;
3434
3435 let report_path = self.output.join("conformance-report.json");
3436 if report_path.exists() {
3437 let report = ConformanceReport::from_file(&report_path)?;
3438 report.print_report_with_options(self.conformance_all_operations);
3439 self.save_conformance_report(&report, &report_path)?;
3440 } else {
3441 TerminalReporter::print_warning(
3442 "Conformance report not generated (k6 handleSummary may not have run)",
3443 );
3444 }
3445
3446 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
3458 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3459 &self.spec,
3460 &self.output,
3461 self.base_path.as_deref(),
3462 )
3463 .await?;
3464 if n > 0 {
3465 TerminalReporter::print_warning(&format!(
3466 "{} emitted request(s) violated the spec — see conformance-request-violations.json",
3467 n
3468 ));
3469 }
3470 }
3471
3472 return Ok(());
3473 }
3474
3475 TerminalReporter::print_progress("Running conformance tests (native executor)...");
3477
3478 let mut executor = crate::conformance::executor::NativeConformanceExecutor::new(config)?;
3479
3480 let custom_only = annotated_ops.is_none() && self.conformance_custom.is_some();
3490 executor = if let Some(annotated) = &annotated_ops {
3491 executor.with_spec_driven_checks(annotated)
3492 } else if custom_only {
3493 executor
3494 } else {
3495 executor.with_reference_checks()
3496 };
3497 executor = executor.with_custom_checks()?;
3498
3499 TerminalReporter::print_success(&format!(
3500 "Executing {} conformance checks...",
3501 executor.check_count()
3502 ));
3503
3504 let report = executor.execute().await?;
3505 report.print_report_with_options(self.conformance_all_operations);
3506
3507 let failure_details = report.failure_details();
3509 if !failure_details.is_empty() {
3510 let details_path = self.output.join("conformance-failure-details.json");
3511 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
3512 let _ = std::fs::write(&details_path, json);
3513 TerminalReporter::print_success(&format!(
3514 "Failure details saved to: {}",
3515 details_path.display()
3516 ));
3517 }
3518 }
3519
3520 let report_path = self.output.join("conformance-report.json");
3522 let report_json = serde_json::to_string_pretty(&report.to_json())
3523 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
3524 std::fs::write(&report_path, &report_json)
3525 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
3526 TerminalReporter::print_success(&format!("Report saved to: {}", report_path.display()));
3527
3528 self.save_conformance_report(&report, &report_path)?;
3529
3530 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
3541 let n =
3542 crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3543 &self.spec,
3544 &self.output,
3545 self.base_path.as_deref(),
3546 )
3547 .await?;
3548 if n > 0 {
3549 TerminalReporter::print_warning(&format!(
3550 "{} emitted request(s) violated the spec — see conformance-request-violations.json",
3551 n
3552 ));
3553 }
3554 }
3555
3556 Ok(())
3557 }
3558
3559 fn save_conformance_report(
3561 &self,
3562 report: &crate::conformance::report::ConformanceReport,
3563 report_path: &Path,
3564 ) -> Result<()> {
3565 if self.conformance_report_format == "sarif" {
3566 use crate::conformance::sarif::ConformanceSarifReport;
3567 ConformanceSarifReport::write(report, &self.target, &self.conformance_report)?;
3568 TerminalReporter::print_success(&format!(
3569 "SARIF report saved to: {}",
3570 self.conformance_report.display()
3571 ));
3572 } else if self.conformance_report != *report_path {
3573 std::fs::copy(report_path, &self.conformance_report)?;
3574 TerminalReporter::print_success(&format!(
3575 "Report saved to: {}",
3576 self.conformance_report.display()
3577 ));
3578 }
3579 Ok(())
3580 }
3581
3582 async fn execute_multi_target_self_test(&self, targets_file: &Path) -> Result<()> {
3594 use crate::conformance::self_test::SelfTestConfig;
3595
3596 TerminalReporter::print_progress("Multi-target conformance self-test mode");
3597 let targets = parse_targets_file(targets_file)?;
3598 if targets.is_empty() {
3599 return Err(BenchError::Other("No targets found in file".to_string()));
3600 }
3601 TerminalReporter::print_success(&format!("Loaded {} target(s)", targets.len()));
3602
3603 let annotated_ops = if !self.spec.is_empty() {
3605 let parser = SpecParser::from_file(&self.spec[0]).await?;
3606 let operations = parser.get_operations();
3607 Some(
3608 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
3609 &operations,
3610 parser.spec(),
3611 ),
3612 )
3613 } else {
3614 return Err(BenchError::Other("--conformance-self-test requires --spec".to_string()));
3615 };
3616 let Some(ops) = annotated_ops else {
3617 unreachable!()
3618 };
3619
3620 std::fs::create_dir_all(&self.output)?;
3621 let resolved_base_path = self.base_path.clone();
3622 let target_iterations = self.conformance_self_test_iterations.max(1);
3623 let duration_budget = self
3624 .conformance_self_test_duration
3625 .as_ref()
3626 .map(|s| Self::parse_duration(s))
3627 .transpose()?
3628 .map(std::time::Duration::from_secs);
3629
3630 for (idx, target) in targets.iter().enumerate() {
3631 let target_dir = self.output.join(format!("target_{}", idx));
3632 std::fs::create_dir_all(&target_dir)?;
3633 TerminalReporter::print_progress(&format!(
3634 "[target {}/{}] {}",
3635 idx + 1,
3636 targets.len(),
3637 target.url
3638 ));
3639
3640 let merged_headers: Vec<(String, String)> = self
3641 .conformance_headers
3642 .iter()
3643 .filter_map(|h| {
3644 let (n, v) = h.split_once(':')?;
3645 Some((n.trim().to_string(), v.trim().to_string()))
3646 })
3647 .collect();
3648
3649 let cfg = SelfTestConfig {
3650 target_url: target.url.clone(),
3651 skip_tls_verify: self.skip_tls_verify,
3652 timeout: std::time::Duration::from_secs(30),
3653 extra_headers: merged_headers,
3654 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
3655 base_path: resolved_base_path.clone(),
3656 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
3657 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
3658 geo_source_headers: if self.geo_source_headers.is_empty() {
3659 crate::conformance::self_test::default_geo_source_headers()
3660 } else {
3661 self.geo_source_headers.clone()
3662 },
3663 capture: if self.conformance_self_test_capture
3664 || self.validate_response_schemas
3665 || self.validate_requests
3666 {
3667 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
3671 } else {
3672 None
3673 },
3674 validate_response_schemas: self.validate_response_schemas,
3675 spec_label: self.spec.first().map(|p| {
3676 p.file_name()
3677 .map(|s| s.to_string_lossy().into_owned())
3678 .unwrap_or_else(|| p.to_string_lossy().into_owned())
3679 }),
3680 network_events: Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))),
3681 current_iteration: 1,
3682 };
3683 let capture_sink = cfg.capture.clone();
3684 let network_events_sink = cfg.network_events.clone();
3685
3686 let start = std::time::Instant::now();
3687 let deadline = duration_budget.map(|d| start + d);
3691 let mut cfg = cfg;
3695 cfg.current_iteration = 1;
3696 let mut report =
3697 crate::conformance::self_test::run_self_test_with_deadline(&ops, &cfg, deadline)
3698 .await
3699 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3700 let mut iter_done: u32 = 1;
3701 loop {
3702 let by_iter = iter_done >= target_iterations;
3703 let by_dur = duration_budget.map(|d| start.elapsed() >= d).unwrap_or(true);
3704 if by_iter && by_dur {
3705 break;
3706 }
3707 cfg.current_iteration = iter_done.saturating_add(1);
3708 let next = crate::conformance::self_test::run_self_test_with_deadline(
3709 &ops, &cfg, deadline,
3710 )
3711 .await
3712 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3713 report.merge_iteration(next);
3714 iter_done = iter_done.saturating_add(1);
3715 }
3716 if iter_done > 1 {
3717 TerminalReporter::print_progress(&format!(
3718 " ran {} iteration(s) in {:.1?}",
3719 iter_done,
3720 start.elapsed(),
3721 ));
3722 }
3723
3724 if let Some(sink) = capture_sink {
3726 if let Ok(guard) = sink.lock() {
3727 let jsonl = target_dir.join("conformance-self-test-requests.jsonl");
3728 let mut lines = String::with_capacity(guard.len() * 256);
3729 for entry in guard.iter() {
3730 if let Ok(line) = serde_json::to_string(entry) {
3731 lines.push_str(&line);
3732 lines.push('\n');
3733 }
3734 }
3735 let _ = std::fs::write(&jsonl, lines);
3736 }
3737 }
3738 if let Some(sink) = network_events_sink {
3739 if let Ok(guard) = sink.lock() {
3740 let path = target_dir.join("conformance-network-events.json");
3741 if let Ok(json) = serde_json::to_string_pretty(&*guard) {
3742 let _ = std::fs::write(&path, json);
3743 if !guard.is_empty() {
3744 TerminalReporter::print_warning(&format!(
3745 " recorded {} wire-level network event(s)",
3746 guard.len()
3747 ));
3748 }
3749 }
3750 }
3751 }
3752
3753 let json_path = target_dir.join("conformance-self-test.json");
3754 if let Ok(json) = serde_json::to_string_pretty(&report) {
3755 let _ = std::fs::write(&json_path, json);
3756 }
3757 let issues = report.definite_issues();
3760 if let Ok(json) = serde_json::to_string_pretty(&issues) {
3761 let issues_path = target_dir.join("conformance-definite-issues.json");
3762 if std::fs::write(&issues_path, json).is_ok() && !issues.is_empty() {
3763 TerminalReporter::print_warning(&format!(
3764 " {} definite issue(s) — see {}",
3765 issues.len(),
3766 issues_path.display()
3767 ));
3768 }
3769 }
3770 let owasp_accepted = report.owasp_accepted_probes();
3772 if !owasp_accepted.is_empty() {
3773 if let Ok(json) = serde_json::to_string_pretty(&owasp_accepted) {
3774 let owasp_path = target_dir.join("conformance-owasp-accepted.json");
3775 if std::fs::write(&owasp_path, json).is_ok() {
3776 TerminalReporter::print_warning(&format!(
3777 " {} owasp injection probe(s) accepted by the target — see {}",
3778 owasp_accepted.len(),
3779 owasp_path.display()
3780 ));
3781 }
3782 }
3783 }
3784 TerminalReporter::print_progress(&report.render_summary());
3785
3786 if self.validate_requests {
3795 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3796 &self.spec,
3797 &target_dir,
3798 self.base_path.as_deref(),
3799 )
3800 .await?;
3801 if n > 0 {
3802 TerminalReporter::print_warning(&format!(
3803 " {} emitted request(s) violated the spec — see {}/conformance-request-violations.json",
3804 n,
3805 target_dir.display(),
3806 ));
3807 }
3808 }
3809 }
3810
3811 Ok(())
3812 }
3813
3814 async fn execute_multi_target_conformance(&self, targets_file: &Path) -> Result<()> {
3820 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
3821 use crate::conformance::report::ConformanceReport;
3822 use crate::conformance::spec::ConformanceFeature;
3823
3824 TerminalReporter::print_progress("Multi-target OpenAPI 3.0.0 Conformance Testing Mode");
3825
3826 TerminalReporter::print_progress("Parsing targets file...");
3828 let targets = parse_targets_file(targets_file)?;
3829 let num_targets = targets.len();
3830 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
3831
3832 if targets.is_empty() {
3833 return Err(BenchError::Other("No targets found in file".to_string()));
3834 }
3835
3836 TerminalReporter::print_progress(CONFORMANCE_REPLACES_LOAD_ADVISORY);
3837
3838 let categories = self.conformance_categories.as_ref().map(|cats_str| {
3840 cats_str
3841 .split(',')
3842 .filter_map(|s| {
3843 let trimmed = s.trim();
3844 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
3845 Some(canonical.to_string())
3846 } else {
3847 TerminalReporter::print_warning(&format!(
3848 "Unknown conformance category: '{}'. Valid categories: {}",
3849 trimmed,
3850 ConformanceFeature::cli_category_names()
3851 .iter()
3852 .map(|(cli, _)| *cli)
3853 .collect::<Vec<_>>()
3854 .join(", ")
3855 ));
3856 None
3857 }
3858 })
3859 .collect::<Vec<String>>()
3860 });
3861
3862 let base_custom_headers: Vec<(String, String)> = self
3864 .conformance_headers
3865 .iter()
3866 .filter_map(|h| {
3867 let (name, value) = h.split_once(':')?;
3868 Some((name.trim().to_string(), value.trim().to_string()))
3869 })
3870 .collect();
3871
3872 if !base_custom_headers.is_empty() {
3873 TerminalReporter::print_progress(&format!(
3874 "Using {} base custom header(s) for authentication",
3875 base_custom_headers.len()
3876 ));
3877 }
3878
3879 let annotated_ops = if !self.spec.is_empty() {
3881 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
3882 let parser = SpecParser::from_file(&self.spec[0]).await?;
3883 let operations = parser.get_operations();
3884 let annotated =
3885 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
3886 &operations,
3887 parser.spec(),
3888 );
3889 TerminalReporter::print_success(&format!(
3890 "Analyzed {} operations, found {} feature annotations",
3891 operations.len(),
3892 annotated.iter().map(|a| a.features.len()).sum::<usize>()
3893 ));
3894 Some(annotated)
3895 } else {
3896 None
3897 };
3898
3899 std::fs::create_dir_all(&self.output)?;
3901
3902 struct TargetResult {
3904 url: String,
3905 passed: usize,
3906 failed: usize,
3907 elapsed: std::time::Duration,
3908 report_json: serde_json::Value,
3909 owasp_coverage: Vec<crate::conformance::report::OwaspCoverageEntry>,
3910 }
3911
3912 let mut target_results: Vec<TargetResult> = Vec::with_capacity(num_targets);
3913 let total_start = std::time::Instant::now();
3914
3915 for (idx, target) in targets.iter().enumerate() {
3916 tracing::info!(
3917 "Running conformance tests against target {}/{}: {}",
3918 idx + 1,
3919 num_targets,
3920 target.url
3921 );
3922 TerminalReporter::print_progress(&format!(
3923 "\n--- Target {}/{}: {} ---",
3924 idx + 1,
3925 num_targets,
3926 target.url
3927 ));
3928
3929 let mut merged_headers = base_custom_headers.clone();
3931 if let Some(ref target_headers) = target.headers {
3932 for (name, value) in target_headers {
3933 if let Some(existing) = merged_headers.iter_mut().find(|(n, _)| n == name) {
3935 existing.1 = value.clone();
3936 } else {
3937 merged_headers.push((name.clone(), value.clone()));
3938 }
3939 }
3940 }
3941 if let Some(ref auth) = target.auth {
3943 if let Some(existing) =
3944 merged_headers.iter_mut().find(|(n, _)| n.eq_ignore_ascii_case("Authorization"))
3945 {
3946 existing.1 = auth.clone();
3947 } else {
3948 merged_headers.push(("Authorization".to_string(), auth.clone()));
3949 }
3950 }
3951
3952 let target_dir = self.output.join(format!("target_{}", idx));
3958 std::fs::create_dir_all(&target_dir)?;
3959
3960 let config = ConformanceConfig {
3961 target_url: target.url.clone(),
3962 api_key: self.conformance_api_key.clone(),
3963 basic_auth: self.conformance_basic_auth.clone(),
3964 skip_tls_verify: self.skip_tls_verify,
3965 categories: categories.clone(),
3966 base_path: self.base_path.clone(),
3967 custom_headers: merged_headers,
3968 output_dir: Some(target_dir.clone()),
3969 all_operations: self.conformance_all_operations,
3970 custom_checks_file: self.conformance_custom.clone(),
3971 request_delay_ms: self.conformance_delay_ms,
3972 custom_filter: self.conformance_custom_filter.clone(),
3973 export_requests: self.export_requests,
3974 validate_requests: self.validate_requests,
3975 };
3976
3977 let target_start = std::time::Instant::now();
3978 let report = if self.use_k6 {
3979 if !K6Executor::is_k6_installed() {
3980 TerminalReporter::print_error("k6 is not installed");
3981 TerminalReporter::print_warning(
3982 "Install k6 from: https://k6.io/docs/get-started/installation/",
3983 );
3984 return Err(BenchError::K6NotFound);
3985 }
3986 K6Executor::warn_if_pre_v1().await;
3987
3988 let script = if let Some(ref annotated) = annotated_ops {
3989 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
3990 config.clone(),
3991 annotated.clone(),
3992 );
3993 let (script, _check_count) = gen.generate()?;
3994 script
3995 } else {
3996 let generator = ConformanceGenerator::new(config.clone());
3997 generator.generate()?
3998 };
3999
4000 let script_path = target_dir.join("k6-conformance.js");
4001 std::fs::write(&script_path, &script).map_err(|e| {
4002 BenchError::Other(format!("Failed to write conformance script: {}", e))
4003 })?;
4004 TerminalReporter::print_success(&format!(
4005 "Conformance script generated: {}",
4006 script_path.display()
4007 ));
4008
4009 TerminalReporter::print_progress(&format!(
4010 "Running conformance tests via k6 against {}...",
4011 target.url
4012 ));
4013 let k6 = K6Executor::new()?
4014 .with_local_ips(self.source_ips.join(","))
4015 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
4016 let api_port = 6565u16.saturating_add(idx as u16);
4018 k6.execute_with_port(&script_path, Some(&target_dir), self.verbose, Some(api_port))
4019 .await?;
4020
4021 let report_path = target_dir.join("conformance-report.json");
4022 if report_path.exists() {
4023 ConformanceReport::from_file(&report_path)?
4024 } else {
4025 TerminalReporter::print_warning(&format!(
4026 "Conformance report not generated for target {} (k6 handleSummary may not have run)",
4027 target.url
4028 ));
4029 continue;
4030 }
4031 } else {
4032 let mut executor =
4033 crate::conformance::executor::NativeConformanceExecutor::new(config)?;
4034
4035 let custom_only = annotated_ops.is_none() && self.conformance_custom.is_some();
4038 executor = if let Some(ref annotated) = annotated_ops {
4039 executor.with_spec_driven_checks(annotated)
4040 } else if custom_only {
4041 executor
4042 } else {
4043 executor.with_reference_checks()
4044 };
4045 executor = executor.with_custom_checks()?;
4046
4047 TerminalReporter::print_success(&format!(
4048 "Executing {} conformance checks against {}...",
4049 executor.check_count(),
4050 target.url
4051 ));
4052
4053 executor.execute().await?
4054 };
4055 let target_elapsed = target_start.elapsed();
4056
4057 let report_json = report.to_json();
4058
4059 let passed = report_json["summary"]["passed"].as_u64().unwrap_or(0) as usize;
4061 let failed = report_json["summary"]["failed"].as_u64().unwrap_or(0) as usize;
4062 let total_checks = passed + failed;
4063 let rate = if total_checks == 0 {
4064 0.0
4065 } else {
4066 (passed as f64 / total_checks as f64) * 100.0
4067 };
4068
4069 TerminalReporter::print_success(&format!(
4070 "Target {}: {}/{} passed ({:.1}%) in {:.1}s",
4071 target.url,
4072 passed,
4073 total_checks,
4074 rate,
4075 target_elapsed.as_secs_f64()
4076 ));
4077
4078 let target_report_path = target_dir.join("conformance-report.json");
4080 let report_str = serde_json::to_string_pretty(&report_json)
4081 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
4082 std::fs::write(&target_report_path, &report_str)
4083 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
4084
4085 let failure_details = report.failure_details();
4087 if !failure_details.is_empty() {
4088 let details_path = target_dir.join("conformance-failure-details.json");
4089 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
4090 let _ = std::fs::write(&details_path, json);
4091 }
4092 }
4093
4094 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
4101 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
4102 &self.spec,
4103 &target_dir,
4104 self.base_path.as_deref(),
4105 )
4106 .await?;
4107 if n > 0 {
4108 TerminalReporter::print_warning(&format!(
4109 "Target {}: {} emitted request(s) violated the spec — see {}/conformance-request-violations.json",
4110 target.url,
4111 n,
4112 target_dir.display()
4113 ));
4114 }
4115 }
4116
4117 let owasp_coverage = report.owasp_coverage_data();
4119
4120 target_results.push(TargetResult {
4121 url: target.url.clone(),
4122 passed,
4123 failed,
4124 elapsed: target_elapsed,
4125 report_json,
4126 owasp_coverage,
4127 });
4128 }
4129
4130 let total_elapsed = total_start.elapsed();
4131
4132 println!("\n{}", "=".repeat(80));
4134 println!(" Multi-Target Conformance Summary");
4135 println!("{}", "=".repeat(80));
4136 println!(
4137 " {:<40} {:>8} {:>8} {:>8} {:>8}",
4138 "Target URL", "Passed", "Failed", "Rate", "Time"
4139 );
4140 println!(" {}", "-".repeat(76));
4141
4142 let mut total_passed = 0usize;
4143 let mut total_failed = 0usize;
4144
4145 for result in &target_results {
4146 let total_checks = result.passed + result.failed;
4147 let rate = if total_checks == 0 {
4148 0.0
4149 } else {
4150 (result.passed as f64 / total_checks as f64) * 100.0
4151 };
4152
4153 let display_url = if result.url.len() > 38 {
4155 format!("{}...", &result.url[..35])
4156 } else {
4157 result.url.clone()
4158 };
4159
4160 println!(
4161 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
4162 display_url,
4163 result.passed,
4164 result.failed,
4165 rate,
4166 result.elapsed.as_secs_f64()
4167 );
4168
4169 total_passed += result.passed;
4170 total_failed += result.failed;
4171 }
4172
4173 let grand_total = total_passed + total_failed;
4174 let overall_rate = if grand_total == 0 {
4175 0.0
4176 } else {
4177 (total_passed as f64 / grand_total as f64) * 100.0
4178 };
4179
4180 println!(" {}", "-".repeat(76));
4181 println!(
4182 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
4183 format!("TOTAL ({} targets)", num_targets),
4184 total_passed,
4185 total_failed,
4186 overall_rate,
4187 total_elapsed.as_secs_f64()
4188 );
4189 println!("{}", "=".repeat(80));
4190
4191 for result in &target_results {
4193 println!("\n OWASP API Security Top 10 Coverage for {}:", result.url);
4194 for entry in &result.owasp_coverage {
4195 let status = if !entry.tested {
4196 "-"
4197 } else if entry.all_passed {
4198 "pass"
4199 } else {
4200 "FAIL"
4201 };
4202 let via = if entry.via_categories.is_empty() {
4203 String::new()
4204 } else {
4205 format!(" (via {})", entry.via_categories.join(", "))
4206 };
4207 println!(" {:<12} {:<40} {}{}", entry.id, entry.name, status, via);
4208 }
4209 }
4210
4211 let per_target_summaries: Vec<serde_json::Value> = target_results
4213 .iter()
4214 .enumerate()
4215 .map(|(idx, r)| {
4216 let total_checks = r.passed + r.failed;
4217 let rate = if total_checks == 0 {
4218 0.0
4219 } else {
4220 (r.passed as f64 / total_checks as f64) * 100.0
4221 };
4222 let owasp_json: Vec<serde_json::Value> = r
4223 .owasp_coverage
4224 .iter()
4225 .map(|e| {
4226 serde_json::json!({
4227 "id": e.id,
4228 "name": e.name,
4229 "tested": e.tested,
4230 "all_passed": e.all_passed,
4231 "via_categories": e.via_categories,
4232 })
4233 })
4234 .collect();
4235 serde_json::json!({
4236 "target_url": r.url,
4237 "target_index": idx,
4238 "checks_passed": r.passed,
4239 "checks_failed": r.failed,
4240 "total_checks": total_checks,
4241 "pass_rate": rate,
4242 "elapsed_seconds": r.elapsed.as_secs_f64(),
4243 "report": r.report_json,
4244 "owasp_coverage": owasp_json,
4245 })
4246 })
4247 .collect();
4248
4249 let combined_summary = serde_json::json!({
4250 "total_targets": num_targets,
4251 "total_checks_passed": total_passed,
4252 "total_checks_failed": total_failed,
4253 "overall_pass_rate": overall_rate,
4254 "total_elapsed_seconds": total_elapsed.as_secs_f64(),
4255 "targets": per_target_summaries,
4256 });
4257
4258 let summary_path = self.output.join("multi-target-conformance-summary.json");
4259 let summary_str = serde_json::to_string_pretty(&combined_summary)
4260 .map_err(|e| BenchError::Other(format!("Failed to serialize summary: {}", e)))?;
4261 std::fs::write(&summary_path, &summary_str)
4262 .map_err(|e| BenchError::Other(format!("Failed to write summary: {}", e)))?;
4263 TerminalReporter::print_success(&format!(
4264 "Combined summary saved to: {}",
4265 summary_path.display()
4266 ));
4267
4268 Ok(())
4269 }
4270
4271 async fn execute_owasp_test(&self, parser: &SpecParser) -> Result<()> {
4273 TerminalReporter::print_progress("OWASP API Security Top 10 Testing Mode");
4274
4275 let custom_headers = self.parse_headers()?;
4277
4278 let mut config = OwaspApiConfig::new()
4280 .with_auth_header(&self.owasp_auth_header)
4281 .with_verbose(self.verbose)
4282 .with_insecure(self.skip_tls_verify)
4283 .with_concurrency(self.vus as usize)
4284 .with_iterations(self.owasp_iterations as usize)
4285 .with_base_path(self.base_path.clone())
4286 .with_custom_headers(custom_headers);
4287
4288 if let Some(ref token) = self.owasp_auth_token {
4290 config = config.with_valid_auth_token(token);
4291 }
4292
4293 if let Some(ref cats_str) = self.owasp_categories {
4295 let categories: Vec<OwaspCategory> = cats_str
4296 .split(',')
4297 .filter_map(|s| {
4298 let trimmed = s.trim();
4299 match trimmed.parse::<OwaspCategory>() {
4300 Ok(cat) => Some(cat),
4301 Err(e) => {
4302 TerminalReporter::print_warning(&e);
4303 None
4304 }
4305 }
4306 })
4307 .collect();
4308
4309 if !categories.is_empty() {
4310 config = config.with_categories(categories);
4311 }
4312 }
4313
4314 if let Some(ref admin_paths_file) = self.owasp_admin_paths {
4316 config.admin_paths_file = Some(admin_paths_file.clone());
4317 if let Err(e) = config.load_admin_paths() {
4318 TerminalReporter::print_warning(&format!("Failed to load admin paths file: {}", e));
4319 }
4320 }
4321
4322 if let Some(ref id_fields_str) = self.owasp_id_fields {
4324 let id_fields: Vec<String> = id_fields_str
4325 .split(',')
4326 .map(|s| s.trim().to_string())
4327 .filter(|s| !s.is_empty())
4328 .collect();
4329 if !id_fields.is_empty() {
4330 config = config.with_id_fields(id_fields);
4331 }
4332 }
4333
4334 if let Some(ref report_path) = self.owasp_report {
4336 config = config.with_report_path(report_path);
4337 }
4338 if let Ok(format) = self.owasp_report_format.parse::<ReportFormat>() {
4339 config = config.with_report_format(format);
4340 }
4341
4342 let categories = config.categories_to_test();
4344 TerminalReporter::print_success(&format!(
4345 "Testing {} OWASP categories: {}",
4346 categories.len(),
4347 categories.iter().map(|c| c.cli_name()).collect::<Vec<_>>().join(", ")
4348 ));
4349
4350 if config.valid_auth_token.is_some() {
4351 TerminalReporter::print_progress("Using provided auth token for baseline requests");
4352 }
4353
4354 TerminalReporter::print_progress("Generating OWASP security test script...");
4356 let generator = OwaspApiGenerator::new(config, self.target.clone(), parser);
4357
4358 let script = generator.generate()?;
4360 TerminalReporter::print_success("OWASP security test script generated");
4361
4362 let script_path = if let Some(output) = &self.script_output {
4364 output.clone()
4365 } else {
4366 self.output.join("k6-owasp-security-test.js")
4367 };
4368
4369 if let Some(parent) = script_path.parent() {
4370 std::fs::create_dir_all(parent)?;
4371 }
4372 std::fs::write(&script_path, &script)?;
4373 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
4374
4375 if self.generate_only {
4377 println!("\nOWASP security test script generated. Run it with:");
4378 println!(" k6 run {}", script_path.display());
4379 return Ok(());
4380 }
4381
4382 TerminalReporter::print_progress("Executing OWASP security tests...");
4384 let executor = K6Executor::new()?
4385 .with_local_ips(self.source_ips.join(","))
4386 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
4387 std::fs::create_dir_all(&self.output)?;
4388
4389 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
4390
4391 let duration_secs = Self::parse_duration(&self.duration)?;
4392 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
4393
4394 println!("\nOWASP security test results saved to: {}", self.output.display());
4395
4396 Ok(())
4397 }
4398}
4399
4400#[cfg(test)]
4401mod tests {
4402 use super::*;
4403 use tempfile::tempdir;
4404
4405 #[test]
4406 fn test_parse_duration() {
4407 assert_eq!(BenchCommand::parse_duration("30s").unwrap(), 30);
4408 assert_eq!(BenchCommand::parse_duration("5m").unwrap(), 300);
4409 assert_eq!(BenchCommand::parse_duration("1h").unwrap(), 3600);
4410 assert_eq!(BenchCommand::parse_duration("60").unwrap(), 60);
4411 }
4412
4413 #[test]
4417 fn parse_ip_list_ipv4_range_inclusive() {
4418 let v = parse_ip_list(&["10.0.0.5-10.0.0.27".into()], "source-ip");
4419 assert_eq!(v.len(), 23);
4420 assert_eq!(v.first().unwrap().to_string(), "10.0.0.5");
4421 assert_eq!(v.last().unwrap().to_string(), "10.0.0.27");
4422 }
4423
4424 #[test]
4427 fn parse_ip_list_range_rejects_backwards() {
4428 let v = parse_ip_list(&["10.0.0.10-10.0.0.5".into()], "source-ip");
4429 assert!(v.is_empty(), "backwards range should produce no IPs; got {v:?}");
4430 }
4431
4432 #[test]
4436 fn parse_ip_list_rejects_ipv6_range_syntax() {
4437 let v = parse_ip_list(&["2001:db8::1-2001:db8::5".into()], "geo-source-ip");
4438 assert!(v.is_empty(), "IPv6 range should be rejected; got {v:?}");
4439 }
4440
4441 #[test]
4443 fn parse_ip_list_range_capped_at_256() {
4444 let v = parse_ip_list(&["10.0.0.0-10.0.5.0".into()], "source-ip");
4445 assert_eq!(v.len(), 256);
4446 assert_eq!(v.first().unwrap().to_string(), "10.0.0.0");
4447 }
4448
4449 #[test]
4452 fn parse_ip_list_plain_and_comma() {
4453 let v = parse_ip_list(&["10.0.0.5".into(), "10.0.0.6,10.0.0.7".into()], "source-ip");
4454 assert_eq!(v.len(), 3);
4455 assert_eq!(v[0].to_string(), "10.0.0.5");
4456 assert_eq!(v[2].to_string(), "10.0.0.7");
4457 }
4458
4459 #[test]
4462 fn parse_ip_list_ipv4_cidr_29_expands_to_8() {
4463 let v = parse_ip_list(&["10.0.0.0/29".into()], "source-ip");
4464 assert_eq!(v.len(), 8);
4465 assert_eq!(v[0].to_string(), "10.0.0.0");
4466 assert_eq!(v[7].to_string(), "10.0.0.7");
4467 }
4468
4469 #[test]
4472 fn parse_ip_list_ipv4_cidr_8_capped_at_256() {
4473 let v = parse_ip_list(&["10.0.0.0/8".into()], "source-ip");
4474 assert_eq!(v.len(), 256);
4475 assert_eq!(v[0].to_string(), "10.0.0.0");
4476 assert_eq!(v[255].to_string(), "10.0.0.255");
4477 }
4478
4479 #[test]
4481 fn parse_ip_list_ipv6_cidr_126_expands_to_4() {
4482 let v = parse_ip_list(&["2001:db8::/126".into()], "geo-source-ip");
4483 assert_eq!(v.len(), 4);
4484 assert!(v[0].is_ipv6());
4485 assert_eq!(v[0].to_string(), "2001:db8::");
4486 assert_eq!(v[3].to_string(), "2001:db8::3");
4487 }
4488
4489 #[test]
4491 fn parse_ip_list_mixed_v4_v6_cidr() {
4492 let v = parse_ip_list(&["10.0.0.0/30,2001:db8::1,203.0.113.42".into()], "geo-source-ip");
4493 assert_eq!(v.len(), 6); assert!(v.iter().any(|ip| ip.to_string() == "2001:db8::1"));
4495 assert!(v.iter().any(|ip| ip.to_string() == "203.0.113.42"));
4496 }
4497
4498 #[test]
4501 fn parse_ip_list_skips_malformed() {
4502 let v = parse_ip_list(
4503 &[
4504 "10.0.0.5".into(),
4505 "not-an-ip".into(),
4506 "10.0.0.6".into(),
4507 "/24".into(),
4508 "1.2.3.4/200".into(),
4509 ],
4510 "source-ip",
4511 );
4512 assert_eq!(v.len(), 2);
4513 assert_eq!(v[0].to_string(), "10.0.0.5");
4514 assert_eq!(v[1].to_string(), "10.0.0.6");
4515 }
4516
4517 #[test]
4518 fn test_parse_duration_invalid() {
4519 assert!(BenchCommand::parse_duration("invalid").is_err());
4520 assert!(BenchCommand::parse_duration("30x").is_err());
4521 }
4522
4523 #[test]
4524 fn test_parse_headers() {
4525 let cmd = BenchCommand {
4526 spec: vec![PathBuf::from("test.yaml")],
4527 spec_dir: None,
4528 merge_conflicts: "error".to_string(),
4529 spec_mode: "merge".to_string(),
4530 dependency_config: None,
4531 target: "http://localhost".to_string(),
4532 base_path: None,
4533 duration: "1m".to_string(),
4534 vus: 10,
4535 scenario: "ramp-up".to_string(),
4536 operations: None,
4537 exclude_operations: None,
4538 auth: None,
4539 headers: vec![
4540 "X-API-Key:test123".to_string(),
4541 "X-Client-ID:client456".to_string(),
4542 ],
4543 output: PathBuf::from("output"),
4544 generate_only: false,
4545 script_output: None,
4546 threshold_percentile: "p(95)".to_string(),
4547 threshold_ms: 500,
4548 max_error_rate: 0.05,
4549 abort_on_error: true,
4550 abort_on_error_rate: 0.95,
4551 per_op_metrics: None,
4552 verbose: false,
4553 skip_tls_verify: false,
4554 chunked_request_bodies: false,
4555 target_rps: None,
4556 no_keep_alive: false,
4557 targets_file: None,
4558 max_concurrency: None,
4559 results_format: "both".to_string(),
4560 params_file: None,
4561 crud_flow: false,
4562 flow_config: None,
4563 extract_fields: None,
4564 parallel_create: None,
4565 data_file: None,
4566 data_distribution: "unique-per-vu".to_string(),
4567 data_mappings: None,
4568 per_uri_control: false,
4569 error_rate: None,
4570 error_types: None,
4571 security_test: false,
4572 security_payloads: None,
4573 security_categories: None,
4574 security_target_fields: None,
4575 wafbench_dir: None,
4576 wafbench_cycle_all: false,
4577 wafbench_verbatim: false,
4578 owasp_api_top10: false,
4579 owasp_categories: None,
4580 owasp_auth_header: "Authorization".to_string(),
4581 owasp_auth_token: None,
4582 owasp_admin_paths: None,
4583 owasp_id_fields: None,
4584 owasp_report: None,
4585 owasp_report_format: "json".to_string(),
4586 owasp_iterations: 1,
4587 conformance: false,
4588 conformance_api_key: None,
4589 conformance_basic_auth: None,
4590 conformance_report: PathBuf::from("conformance-report.json"),
4591 conformance_categories: None,
4592 conformance_report_format: "json".to_string(),
4593 conformance_headers: vec![],
4594 conformance_all_operations: false,
4595 conformance_custom: None,
4596 conformance_delay_ms: 0,
4597 use_k6: false,
4598 conformance_custom_filter: None,
4599 export_requests: false,
4600 validate_requests: false,
4601 conformance_self_test: false,
4602 conformance_self_test_capture: false,
4603 conformance_self_test_iterations: 1,
4604 conformance_self_test_duration: None,
4605 validate_response_schemas: false,
4606 source_ips: Vec::new(),
4607 geo_source_ips: Vec::new(),
4608 geo_source_headers: Vec::new(),
4609 report_missed_cap: None,
4610 discard_response_bodies: false,
4611 dns_policy: None,
4612 };
4613
4614 let headers = cmd.parse_headers().unwrap();
4615 assert_eq!(headers.get("X-API-Key"), Some(&"test123".to_string()));
4616 assert_eq!(headers.get("X-Client-ID"), Some(&"client456".to_string()));
4617 }
4618
4619 #[test]
4620 fn test_parse_header_string_preserves_comma_in_value() {
4621 let inputs = vec![
4624 "Cookie:session=abc; expires=Thu, 01 Jan 2099 00:00:00 GMT".to_string(),
4625 "X-Trace:1".to_string(),
4626 ];
4627 let headers = parse_header_string(&inputs).unwrap();
4628 assert_eq!(
4629 headers.get("Cookie"),
4630 Some(&"session=abc; expires=Thu, 01 Jan 2099 00:00:00 GMT".to_string())
4631 );
4632 assert_eq!(headers.get("X-Trace"), Some(&"1".to_string()));
4633 }
4634
4635 #[test]
4643 fn conformance_advisory_names_every_discarded_flag() {
4644 let msg = CONFORMANCE_REPLACES_LOAD_ADVISORY;
4645 for flag in ["--vus", "--rps", "-d"] {
4646 assert!(
4647 msg.contains(flag),
4648 "conformance advisory must name `{flag}` as ignored; it is discarded on that \
4649 path and silently dropping it is how users end up tuning a knob that does \
4650 nothing (#980). Message was: {msg}"
4651 );
4652 }
4653 assert!(
4654 msg.contains("REPLACES"),
4655 "conformance advisory must say the load run is REPLACED, not merely that some \
4656 flags are ignored — `--conformance` returns before the load path runs, so no \
4657 load traffic is generated at all (#980). Message was: {msg}"
4658 );
4659 }
4660
4661 #[test]
4675 fn multi_target_clone_preserves_fields_parse_headers_reads() {
4676 let src = include_str!("command.rs");
4677
4678 let fn_start = src
4679 .find("async fn execute_multi_target(")
4680 .expect("execute_multi_target should exist");
4681 let block_start = src[fn_start..]
4682 .find("ParallelExecutor::new(")
4683 .map(|i| i + fn_start)
4684 .expect("multi-target path should build a ParallelExecutor");
4685 let block_end = src[block_start..]
4687 .find("\n );")
4688 .map(|i| i + block_start)
4689 .expect("ParallelExecutor::new(..) should be closed");
4690 let block = &src[block_start..block_end];
4691
4692 for field in ["conformance_basic_auth", "conformance_headers"] {
4695 for zeroed in [format!("{field}: None"), format!("{field}: vec![]")] {
4696 assert!(
4697 !block.contains(&zeroed),
4698 "execute_multi_target zeroes `{zeroed}`. parse_headers() folds `{field}` \
4699 into the header map, so zeroing it here strips auth from every \
4700 multi-target run while single-target keeps working (#79 round 64)."
4701 );
4702 }
4703 let passthrough = format!("{field}: self.{field}.clone()");
4704 assert!(
4705 block.contains(&passthrough),
4706 "execute_multi_target must carry `{field}` through as `{passthrough}` so \
4707 parse_headers() can fold it (#79 round 64)."
4708 );
4709 }
4710 }
4711
4712 #[test]
4713 fn test_get_spec_display_name() {
4714 let cmd = BenchCommand {
4715 spec: vec![PathBuf::from("test.yaml")],
4716 spec_dir: None,
4717 merge_conflicts: "error".to_string(),
4718 spec_mode: "merge".to_string(),
4719 dependency_config: None,
4720 target: "http://localhost".to_string(),
4721 base_path: None,
4722 duration: "1m".to_string(),
4723 vus: 10,
4724 scenario: "ramp-up".to_string(),
4725 operations: None,
4726 exclude_operations: None,
4727 auth: None,
4728 headers: Vec::new(),
4729 output: PathBuf::from("output"),
4730 generate_only: false,
4731 script_output: None,
4732 threshold_percentile: "p(95)".to_string(),
4733 threshold_ms: 500,
4734 max_error_rate: 0.05,
4735 abort_on_error: true,
4736 abort_on_error_rate: 0.95,
4737 per_op_metrics: None,
4738 verbose: false,
4739 skip_tls_verify: false,
4740 chunked_request_bodies: false,
4741 target_rps: None,
4742 no_keep_alive: false,
4743 targets_file: None,
4744 max_concurrency: None,
4745 results_format: "both".to_string(),
4746 params_file: None,
4747 crud_flow: false,
4748 flow_config: None,
4749 extract_fields: None,
4750 parallel_create: None,
4751 data_file: None,
4752 data_distribution: "unique-per-vu".to_string(),
4753 data_mappings: None,
4754 per_uri_control: false,
4755 error_rate: None,
4756 error_types: None,
4757 security_test: false,
4758 security_payloads: None,
4759 security_categories: None,
4760 security_target_fields: None,
4761 wafbench_dir: None,
4762 wafbench_cycle_all: false,
4763 wafbench_verbatim: false,
4764 owasp_api_top10: false,
4765 owasp_categories: None,
4766 owasp_auth_header: "Authorization".to_string(),
4767 owasp_auth_token: None,
4768 owasp_admin_paths: None,
4769 owasp_id_fields: None,
4770 owasp_report: None,
4771 owasp_report_format: "json".to_string(),
4772 owasp_iterations: 1,
4773 conformance: false,
4774 conformance_api_key: None,
4775 conformance_basic_auth: None,
4776 conformance_report: PathBuf::from("conformance-report.json"),
4777 conformance_categories: None,
4778 conformance_report_format: "json".to_string(),
4779 conformance_headers: vec![],
4780 conformance_all_operations: false,
4781 conformance_custom: None,
4782 conformance_delay_ms: 0,
4783 use_k6: false,
4784 conformance_custom_filter: None,
4785 export_requests: false,
4786 validate_requests: false,
4787 conformance_self_test: false,
4788 conformance_self_test_capture: false,
4789 conformance_self_test_iterations: 1,
4790 conformance_self_test_duration: None,
4791 validate_response_schemas: false,
4792 source_ips: Vec::new(),
4793 geo_source_ips: Vec::new(),
4794 geo_source_headers: Vec::new(),
4795 report_missed_cap: None,
4796 discard_response_bodies: false,
4797 dns_policy: None,
4798 };
4799
4800 assert_eq!(cmd.get_spec_display_name(), "test.yaml");
4801
4802 let cmd_multi = BenchCommand {
4804 spec: vec![PathBuf::from("a.yaml"), PathBuf::from("b.yaml")],
4805 spec_dir: None,
4806 merge_conflicts: "error".to_string(),
4807 spec_mode: "merge".to_string(),
4808 dependency_config: None,
4809 target: "http://localhost".to_string(),
4810 base_path: None,
4811 duration: "1m".to_string(),
4812 vus: 10,
4813 scenario: "ramp-up".to_string(),
4814 operations: None,
4815 exclude_operations: None,
4816 auth: None,
4817 headers: Vec::new(),
4818 output: PathBuf::from("output"),
4819 generate_only: false,
4820 script_output: None,
4821 threshold_percentile: "p(95)".to_string(),
4822 threshold_ms: 500,
4823 max_error_rate: 0.05,
4824 abort_on_error: true,
4825 abort_on_error_rate: 0.95,
4826 per_op_metrics: None,
4827 verbose: false,
4828 skip_tls_verify: false,
4829 chunked_request_bodies: false,
4830 target_rps: None,
4831 no_keep_alive: false,
4832 targets_file: None,
4833 max_concurrency: None,
4834 results_format: "both".to_string(),
4835 params_file: None,
4836 crud_flow: false,
4837 flow_config: None,
4838 extract_fields: None,
4839 parallel_create: None,
4840 data_file: None,
4841 data_distribution: "unique-per-vu".to_string(),
4842 data_mappings: None,
4843 per_uri_control: false,
4844 error_rate: None,
4845 error_types: None,
4846 security_test: false,
4847 security_payloads: None,
4848 security_categories: None,
4849 security_target_fields: None,
4850 wafbench_dir: None,
4851 wafbench_cycle_all: false,
4852 wafbench_verbatim: false,
4853 owasp_api_top10: false,
4854 owasp_categories: None,
4855 owasp_auth_header: "Authorization".to_string(),
4856 owasp_auth_token: None,
4857 owasp_admin_paths: None,
4858 owasp_id_fields: None,
4859 owasp_report: None,
4860 owasp_report_format: "json".to_string(),
4861 owasp_iterations: 1,
4862 conformance: false,
4863 conformance_api_key: None,
4864 conformance_basic_auth: None,
4865 conformance_report: PathBuf::from("conformance-report.json"),
4866 conformance_categories: None,
4867 conformance_report_format: "json".to_string(),
4868 conformance_headers: vec![],
4869 conformance_all_operations: false,
4870 conformance_custom: None,
4871 conformance_delay_ms: 0,
4872 use_k6: false,
4873 conformance_custom_filter: None,
4874 export_requests: false,
4875 validate_requests: false,
4876 conformance_self_test: false,
4877 conformance_self_test_capture: false,
4878 conformance_self_test_iterations: 1,
4879 conformance_self_test_duration: None,
4880 validate_response_schemas: false,
4881 source_ips: Vec::new(),
4882 geo_source_ips: Vec::new(),
4883 geo_source_headers: Vec::new(),
4884 report_missed_cap: None,
4885 discard_response_bodies: false,
4886 dns_policy: None,
4887 };
4888
4889 assert_eq!(cmd_multi.get_spec_display_name(), "2 spec files");
4890 }
4891
4892 #[test]
4893 fn test_parse_extracted_values_from_output_dir() {
4894 let dir = tempdir().unwrap();
4895 let path = dir.path().join("extracted_values.json");
4896 std::fs::write(
4897 &path,
4898 r#"{
4899 "pool_id": "abc123",
4900 "count": 0,
4901 "enabled": false,
4902 "metadata": { "owner": "team-a" }
4903}"#,
4904 )
4905 .unwrap();
4906
4907 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
4908 assert_eq!(extracted.get("pool_id"), Some(&serde_json::json!("abc123")));
4909 assert_eq!(extracted.get("count"), Some(&serde_json::json!(0)));
4910 assert_eq!(extracted.get("enabled"), Some(&serde_json::json!(false)));
4911 assert_eq!(extracted.get("metadata"), Some(&serde_json::json!({"owner": "team-a"})));
4912 }
4913
4914 #[test]
4915 fn test_parse_extracted_values_missing_file() {
4916 let dir = tempdir().unwrap();
4917 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
4918 assert!(extracted.values.is_empty());
4919 }
4920
4921 fn sample_bench_command() -> BenchCommand {
4924 BenchCommand {
4925 spec: vec![PathBuf::from("test.yaml")],
4926 spec_dir: None,
4927 merge_conflicts: "error".to_string(),
4928 spec_mode: "merge".to_string(),
4929 dependency_config: None,
4930 target: "http://localhost".to_string(),
4931 base_path: None,
4932 duration: "1m".to_string(),
4933 vus: 10,
4934 scenario: "ramp-up".to_string(),
4935 operations: None,
4936 exclude_operations: None,
4937 auth: None,
4938 headers: vec![
4939 "X-API-Key:test123".to_string(),
4940 "X-Client-ID:client456".to_string(),
4941 ],
4942 output: PathBuf::from("output"),
4943 generate_only: false,
4944 script_output: None,
4945 threshold_percentile: "p(95)".to_string(),
4946 threshold_ms: 500,
4947 max_error_rate: 0.05,
4948 abort_on_error: true,
4949 abort_on_error_rate: 0.95,
4950 per_op_metrics: None,
4951 verbose: false,
4952 skip_tls_verify: false,
4953 chunked_request_bodies: false,
4954 target_rps: None,
4955 no_keep_alive: false,
4956 targets_file: None,
4957 max_concurrency: None,
4958 results_format: "both".to_string(),
4959 params_file: None,
4960 crud_flow: false,
4961 flow_config: None,
4962 extract_fields: None,
4963 parallel_create: None,
4964 data_file: None,
4965 data_distribution: "unique-per-vu".to_string(),
4966 data_mappings: None,
4967 per_uri_control: false,
4968 error_rate: None,
4969 error_types: None,
4970 security_test: false,
4971 security_payloads: None,
4972 security_categories: None,
4973 security_target_fields: None,
4974 wafbench_dir: None,
4975 wafbench_cycle_all: false,
4976 wafbench_verbatim: false,
4977 owasp_api_top10: false,
4978 owasp_categories: None,
4979 owasp_auth_header: "Authorization".to_string(),
4980 owasp_auth_token: None,
4981 owasp_admin_paths: None,
4982 owasp_id_fields: None,
4983 owasp_report: None,
4984 owasp_report_format: "json".to_string(),
4985 owasp_iterations: 1,
4986 conformance: false,
4987 conformance_api_key: None,
4988 conformance_basic_auth: None,
4989 conformance_report: PathBuf::from("conformance-report.json"),
4990 conformance_categories: None,
4991 conformance_report_format: "json".to_string(),
4992 conformance_headers: vec![],
4993 conformance_all_operations: false,
4994 conformance_custom: None,
4995 conformance_delay_ms: 0,
4996 use_k6: false,
4997 conformance_custom_filter: None,
4998 export_requests: false,
4999 validate_requests: false,
5000 conformance_self_test: false,
5001 conformance_self_test_capture: false,
5002 conformance_self_test_iterations: 1,
5003 conformance_self_test_duration: None,
5004 validate_response_schemas: false,
5005 source_ips: Vec::new(),
5006 geo_source_ips: Vec::new(),
5007 geo_source_headers: Vec::new(),
5008 report_missed_cap: None,
5009 discard_response_bodies: false,
5010 dns_policy: None,
5011 }
5012 }
5013
5014 #[test]
5022 fn verbatim_disables_security_payload_injection() {
5023 let mut cmd = sample_bench_command();
5024 cmd.wafbench_dir = Some("traffic.yaml".to_string());
5025
5026 assert!(
5027 cmd.security_testing_enabled(),
5028 "--wafbench-dir alone must still enable payload injection"
5029 );
5030
5031 cmd.wafbench_verbatim = true;
5032 assert!(
5033 !cmd.security_testing_enabled(),
5034 "verbatim mode must not inject payloads into requests sent as written"
5035 );
5036
5037 cmd.security_test = true;
5040 assert!(
5041 !cmd.security_testing_enabled(),
5042 "--security-test must not re-enable injection under --wafbench-verbatim"
5043 );
5044 }
5045
5046 #[test]
5051 fn security_testing_enabled_has_a_single_definition() {
5052 let src = include_str!("command.rs");
5053 let parallel = include_str!("parallel_executor.rs");
5054 let a = format!("self.{} || self.{}.is_some()", "security_test", "wafbench_dir");
5056 let b = format!("self.{}.is_some() || self.{}", "wafbench_dir", "security_test");
5057 let inline = src.matches(a.as_str()).count() + src.matches(b.as_str()).count();
5058 assert_eq!(
5059 inline, 1,
5060 "expected the security_testing_enabled() method to be the only place this is \
5061 computed, found {inline} inline copies -- collapse them or the render paths drift"
5062 );
5063
5064 let parallel_inline = format!(
5069 "{}.{} || {}.{}.is_some()",
5070 "base_command", "security_test", "self.base_command", "wafbench_dir"
5071 );
5072 assert!(
5073 !parallel.contains(¶llel_inline),
5074 "ParallelExecutor must not recompute the security flag inline"
5075 );
5076 assert!(
5077 parallel.contains("security_testing_enabled()"),
5078 "ParallelExecutor must call security_testing_enabled() so --wafbench-verbatim \
5079 turns injection off on --targets-file runs too"
5080 );
5081 }
5082
5083 #[test]
5087 fn missing_wafbench_dir_is_not_swallowed() {
5088 let src = include_str!("command.rs");
5089 let swallowed = format!("Failed to {} WAFBench tests", "load");
5091 let impl_line = src
5092 .lines()
5093 .filter(|l| !l.trim_start().starts_with("//"))
5094 .any(|l| l.contains(&swallowed));
5095 assert!(!impl_line, "missing --wafbench-dir must not be downgraded to a warning");
5096 assert!(
5097 src.contains("self.load_wafbench_payloads()?"),
5098 "payload load errors must reach generate_enhanced_script"
5099 );
5100 }
5101
5102 #[test]
5107 fn multi_target_path_honors_verbatim_templates() {
5108 let src = include_str!("parallel_executor.rs");
5109 assert!(
5110 src.contains("load_verbatim_templates"),
5111 "ParallelExecutor must load traffic-file requests under --wafbench-verbatim. \
5112 Requiring a spec and generating templates from its operations is how \
5113 --targets-file ignored the flag and fuzzed spec URLs (#79)."
5114 );
5115 }
5116
5117 #[test]
5118 fn single_target_k6_spawn_sets_force_http1() {
5119 let src = include_str!("command.rs");
5120 assert!(
5121 src.contains("with_force_http1(force_http1)"),
5122 "single-target k6 spawn must set GODEBUG=http2client=0 for Connection-header WAF cases"
5123 );
5124 assert!(
5125 src.contains("print_k6_run_hint"),
5126 "generate-only must print GODEBUG=http2client=0 when HTTP/1.1 is required"
5127 );
5128 assert!(
5129 src.contains("with_per_op_metrics(per_op_metrics)"),
5130 "single-target k6 generation must apply Round-65 per-op metrics collapse (#79)"
5131 );
5132 assert!(
5133 src.contains("resolve_per_op_metrics"),
5134 "single-target path must resolve auto/forced per-op metrics (#79)"
5135 );
5136 }
5137
5138 #[test]
5140 fn traffic_breakdown_json_multiplies_unique_by_rps() {
5141 let dir = std::env::temp_dir().join(format!(
5142 "mf-traffic-breakdown-{}-{}",
5143 std::process::id(),
5144 std::time::SystemTime::now()
5145 .duration_since(std::time::UNIX_EPOCH)
5146 .unwrap()
5147 .as_nanos()
5148 ));
5149 let _ = std::fs::create_dir_all(&dir);
5150 let mut cmd = sample_bench_command();
5151 cmd.output = dir.clone();
5152 cmd.target_rps = Some(50);
5153 cmd.duration = "1200s".to_string();
5154 let stats = crate::wafbench::WafBenchStats {
5155 per_file: vec![crate::wafbench::TrafficFileSummary {
5156 file: "apisix_cve-2026-44087.yaml".into(),
5157 sent: 5,
5158 attack: 3,
5159 normal: 2,
5160 omitted: 1,
5161 other: 0,
5162 }],
5163 ..Default::default()
5164 };
5165 cmd.emit_traffic_file_breakdown(&stats, "what to expect in proxy logs");
5166 let raw = std::fs::read_to_string(dir.join("traffic-breakdown.json"))
5167 .expect("traffic-breakdown.json");
5168 let v: serde_json::Value = serde_json::from_str(&raw).unwrap();
5169 assert_eq!(v["rps"], 50);
5170 assert_eq!(v["duration_secs"], 1200);
5171 assert_eq!(v["files"][0]["sent"]["unique_cases"], 5);
5172 assert_eq!(v["files"][0]["sent"]["projected_per_second"], 250);
5173 assert_eq!(v["files"][0]["sent"]["projected_over_run"], 300000);
5174 assert_eq!(v["files"][0]["attack"]["projected_per_second"], 150);
5175 assert_eq!(v["files"][0]["normal"]["projected_per_second"], 100);
5176 for gone in [
5177 "unique",
5178 "total",
5179 "expected_requests",
5180 "expected_requests_unit",
5181 ] {
5182 assert!(
5183 v["files"][0]["sent"].get(gone).is_none(),
5184 "{gone} alias must not appear in traffic-breakdown.json"
5185 );
5186 }
5187 assert!(v["note"].as_str().unwrap().contains("Plan, not k6 counters"));
5188 assert!(v["note"].as_str().unwrap().contains("not traffic on the wire"));
5189 assert_eq!(
5190 BenchCommand::format_unique_total(5, Some(50)),
5191 "unique_cases=5 projected_per_second=250 (5 * 50 RPS)"
5192 );
5193 let _ = std::fs::remove_dir_all(&dir);
5194 }
5195
5196 #[test]
5199 fn traffic_breakdown_json_omits_projected_without_rps() {
5200 let dir = std::env::temp_dir().join(format!(
5201 "mf-traffic-breakdown-norps-{}-{}",
5202 std::process::id(),
5203 std::time::SystemTime::now()
5204 .duration_since(std::time::UNIX_EPOCH)
5205 .unwrap()
5206 .as_nanos()
5207 ));
5208 let _ = std::fs::create_dir_all(&dir);
5209 let mut cmd = sample_bench_command();
5210 cmd.output = dir.clone();
5211 cmd.target_rps = None;
5212 cmd.duration = "60s".to_string();
5213 let stats = crate::wafbench::WafBenchStats {
5214 per_file: vec![crate::wafbench::TrafficFileSummary {
5215 file: "apisix_cve-2026-44087.yaml".into(),
5216 sent: 5,
5217 attack: 3,
5218 normal: 2,
5219 omitted: 1,
5220 other: 0,
5221 }],
5222 ..Default::default()
5223 };
5224 cmd.emit_traffic_file_breakdown(&stats, "what to expect in proxy logs");
5225 let raw = std::fs::read_to_string(dir.join("traffic-breakdown.json"))
5226 .expect("traffic-breakdown.json");
5227 let v: serde_json::Value = serde_json::from_str(&raw).unwrap();
5228 assert!(v["rps"].is_null());
5229 assert_eq!(v["duration_secs"], 60);
5230 assert_eq!(v["files"][0]["sent"]["unique_cases"], 5);
5231 assert!(v["files"][0]["sent"]["projected_per_second"].is_null());
5232 assert!(v["files"][0]["sent"]["projected_over_run"].is_null());
5233 for gone in ["unique", "total", "expected_requests"] {
5234 assert!(
5235 v["files"][0]["sent"].get(gone).is_none(),
5236 "{gone} alias must not appear when --rps is unset"
5237 );
5238 }
5239 let _ = std::fs::remove_dir_all(&dir);
5240 }
5241}