1use crate::crud_flow::{CrudFlowConfig, CrudFlowDetector};
4use crate::data_driven::{DataDistribution, DataDrivenConfig, DataDrivenGenerator, DataMapping};
5use crate::dynamic_params::{DynamicParamProcessor, DynamicPlaceholder};
6use crate::error::{BenchError, Result};
7use crate::executor::K6Executor;
8use crate::invalid_data::{InvalidDataConfig, InvalidDataGenerator};
9use crate::k6_gen::{K6Config, K6ScriptGenerator};
10use crate::mock_integration::{
11 MockIntegrationConfig, MockIntegrationGenerator, MockServerDetector,
12};
13use crate::owasp_api::{OwaspApiConfig, OwaspApiGenerator, OwaspCategory, ReportFormat};
14use crate::parallel_executor::{AggregatedResults, ParallelExecutor};
15use crate::parallel_requests::{ParallelConfig, ParallelRequestGenerator};
16use crate::param_overrides::ParameterOverrides;
17use crate::reporter::TerminalReporter;
18use crate::request_gen::RequestGenerator;
19use crate::scenarios::LoadScenario;
20use crate::security_payloads::{
21 SecurityCategory, SecurityPayload, SecurityPayloads, SecurityTestConfig, SecurityTestGenerator,
22};
23use crate::spec_dependencies::{
24 topological_sort, DependencyDetector, ExtractedValues, SpecDependencyConfig,
25};
26use crate::spec_parser::SpecParser;
27use crate::target_parser::parse_targets_file;
28use crate::wafbench::WafBenchLoader;
29use mockforge_openapi::multi_spec::{
30 load_specs_from_directory, load_specs_from_files, merge_specs, ConflictStrategy,
31};
32use mockforge_openapi::spec::OpenApiSpec;
33use std::collections::{HashMap, HashSet};
34use std::path::{Path, PathBuf};
35use std::str::FromStr;
36
37pub fn parse_header_string(inputs: &[String]) -> Result<HashMap<String, String>> {
45 let mut headers = HashMap::new();
46
47 for pair in inputs {
48 let pair = pair.trim();
49 if pair.is_empty() {
50 continue;
51 }
52 let parts: Vec<&str> = pair.splitn(2, ':').collect();
53 if parts.len() != 2 {
54 return Err(BenchError::Other(format!(
55 "Invalid header format: '{}'. Expected 'Key:Value'",
56 pair
57 )));
58 }
59 headers.insert(parts[0].trim().to_string(), parts[1].trim().to_string());
60 }
61
62 Ok(headers)
63}
64
65const CONFORMANCE_REPLACES_LOAD_ADVISORY: &str =
85 "Conformance mode REPLACES the load run: 1 VU, 1 iteration per endpoint. \
86 --vus, --rps and -d are ignored. Run bench a second time without \
87 --conformance if you also want a load test.";
88
89pub struct BenchCommand {
91 pub spec: Vec<PathBuf>,
93 pub spec_dir: Option<PathBuf>,
95 pub merge_conflicts: String,
97 pub spec_mode: String,
99 pub dependency_config: Option<PathBuf>,
101 pub target: String,
102 pub base_path: Option<String>,
105 pub duration: String,
106 pub vus: u32,
107 pub target_rps: Option<u32>,
113 pub no_keep_alive: bool,
118 pub scenario: String,
119 pub operations: Option<String>,
120 pub exclude_operations: Option<String>,
124 pub auth: Option<String>,
125 pub headers: Vec<String>,
128 pub output: PathBuf,
129 pub generate_only: bool,
130 pub script_output: Option<PathBuf>,
131 pub threshold_percentile: String,
132 pub threshold_ms: u64,
133 pub max_error_rate: f64,
134 pub abort_on_error: bool,
139 pub abort_on_error_rate: f64,
143 pub verbose: bool,
144 pub skip_tls_verify: bool,
145 pub chunked_request_bodies: bool,
150 pub targets_file: Option<PathBuf>,
152 pub max_concurrency: Option<u32>,
154 pub results_format: String,
156 pub params_file: Option<PathBuf>,
161
162 pub crud_flow: bool,
165 pub flow_config: Option<PathBuf>,
167 pub extract_fields: Option<String>,
169
170 pub parallel_create: Option<u32>,
173
174 pub data_file: Option<PathBuf>,
177 pub data_distribution: String,
179 pub data_mappings: Option<String>,
181 pub per_uri_control: bool,
183
184 pub error_rate: Option<f64>,
187 pub error_types: Option<String>,
189
190 pub security_test: bool,
193 pub security_payloads: Option<PathBuf>,
195 pub security_categories: Option<String>,
197 pub security_target_fields: Option<String>,
199
200 pub wafbench_dir: Option<String>,
203 pub wafbench_cycle_all: bool,
205 pub wafbench_verbatim: bool,
208
209 pub conformance: bool,
212 pub conformance_api_key: Option<String>,
214 pub conformance_basic_auth: Option<String>,
216 pub conformance_report: PathBuf,
218 pub conformance_categories: Option<String>,
220 pub conformance_report_format: String,
222 pub conformance_headers: Vec<String>,
225 pub conformance_all_operations: bool,
228 pub conformance_custom: Option<PathBuf>,
230 pub conformance_delay_ms: u64,
233 pub use_k6: bool,
235 pub conformance_custom_filter: Option<String>,
239 pub export_requests: bool,
242 pub validate_requests: bool,
245 pub conformance_self_test: bool,
252 pub conformance_self_test_capture: bool,
256 pub validate_response_schemas: bool,
262 pub conformance_self_test_iterations: u32,
267 pub conformance_self_test_duration: Option<String>,
272
273 pub source_ips: Vec<String>,
278 pub geo_source_ips: Vec<String>,
282 pub geo_source_headers: Vec<String>,
286
287 pub report_missed_cap: Option<u32>,
294
295 pub discard_response_bodies: bool,
302
303 pub dns_policy: Option<String>,
309
310 pub owasp_api_top10: bool,
313 pub owasp_categories: Option<String>,
315 pub owasp_auth_header: String,
317 pub owasp_auth_token: Option<String>,
319 pub owasp_admin_paths: Option<PathBuf>,
321 pub owasp_id_fields: Option<String>,
323 pub owasp_report: Option<PathBuf>,
325 pub owasp_report_format: String,
327 pub owasp_iterations: u32,
329}
330
331fn parse_ip_list(raw: &[String], flag_name: &str) -> Vec<std::net::IpAddr> {
345 use std::net::IpAddr;
346 const MAX_CIDR_EXPANSION: usize = 256;
347 let mut out = Vec::new();
348 for entry in raw {
349 for piece in entry.split(',') {
350 let s = piece.trim();
351 if s.is_empty() {
352 continue;
353 }
354 if let Some((addr_part, prefix_part)) = s.split_once('/') {
356 let prefix: u32 = match prefix_part.parse() {
357 Ok(p) => p,
358 Err(e) => {
359 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR prefix: {e}");
360 continue;
361 }
362 };
363 let net_addr: IpAddr = match addr_part.parse() {
364 Ok(a) => a,
365 Err(e) => {
366 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR address: {e}");
367 continue;
368 }
369 };
370 expand_cidr(net_addr, prefix, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
371 continue;
372 }
373 if let Some((start_str, end_str)) = s.split_once('-') {
379 let start_s = start_str.trim();
380 let end_s = end_str.trim();
381 if start_s.contains(':') || end_s.contains(':') {
385 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{s}': IPv6 range syntax not supported (use CIDR like 2001:db8::/126 instead)");
386 continue;
387 }
388 let start: IpAddr = match start_s.parse() {
389 Ok(a) => a,
390 Err(e) => {
391 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range start: {e}");
392 continue;
393 }
394 };
395 let end: IpAddr = match end_s.parse() {
396 Ok(a) => a,
397 Err(e) => {
398 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range end: {e}");
399 continue;
400 }
401 };
402 expand_range(start, end, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
403 continue;
404 }
405 match s.parse::<IpAddr>() {
407 Ok(ip) => out.push(ip),
408 Err(e) => {
409 tracing::warn!(target: "mockforge::bench", "ignoring malformed --{flag_name} value '{s}': {e}");
410 }
411 }
412 }
413 }
414 out
415}
416
417fn expand_range(
421 start: std::net::IpAddr,
422 end: std::net::IpAddr,
423 cap: usize,
424 flag_name: &str,
425 raw: &str,
426 out: &mut Vec<std::net::IpAddr>,
427) {
428 use std::net::{IpAddr, Ipv4Addr};
429 let (start_v4, end_v4) = match (start, end) {
430 (IpAddr::V4(a), IpAddr::V4(b)) => (a, b),
431 _ => {
432 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range start/end must both be IPv4");
433 return;
434 }
435 };
436 let start_u32 = u32::from(start_v4);
437 let end_u32 = u32::from(end_v4);
438 if end_u32 < start_u32 {
439 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range end {end_v4} is before start {start_v4}");
440 return;
441 }
442 let total = (end_u32 - start_u32).saturating_add(1) as usize;
443 let take = total.min(cap);
444 if total > cap {
445 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range has {total} addresses, capping at {cap}");
446 }
447 for i in 0..take as u32 {
448 out.push(IpAddr::V4(Ipv4Addr::from(start_u32 + i)));
449 }
450}
451
452fn expand_cidr(
456 net: std::net::IpAddr,
457 prefix: u32,
458 cap: usize,
459 flag_name: &str,
460 raw: &str,
461 out: &mut Vec<std::net::IpAddr>,
462) {
463 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
464 match net {
465 IpAddr::V4(ipv4) => {
466 if prefix > 32 {
467 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv4 prefix must be <= 32");
468 return;
469 }
470 let total: u64 = 1u64 << (32 - prefix);
471 let take = total.min(cap as u64) as u32;
472 if total > cap as u64 {
473 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': CIDR has {total} addresses, capping at {cap}");
474 }
475 let mask: u32 = if prefix == 0 {
476 0
477 } else {
478 !0u32 << (32 - prefix)
479 };
480 let net_u32 = u32::from(ipv4) & mask;
481 for i in 0..take {
482 out.push(IpAddr::V4(Ipv4Addr::from(net_u32.wrapping_add(i))));
483 }
484 }
485 IpAddr::V6(ipv6) => {
486 if prefix > 128 {
487 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv6 prefix must be <= 128");
488 return;
489 }
490 let mask: u128 = if prefix == 0 {
494 0
495 } else {
496 !0u128 << (128 - prefix)
497 };
498 let net_u128 = u128::from(ipv6) & mask;
499 let remaining_bits = 128 - prefix;
500 let total_capped = if remaining_bits >= 64 {
503 cap as u128
504 } else {
505 (1u128 << remaining_bits).min(cap as u128)
506 };
507 if remaining_bits < 128 && (1u128 << remaining_bits) > cap as u128 {
508 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': IPv6 CIDR exceeds {cap} addresses, capping");
509 }
510 for i in 0..total_capped {
511 out.push(IpAddr::V6(Ipv6Addr::from(net_u128.wrapping_add(i))));
512 }
513 }
514 }
515}
516
517impl BenchCommand {
518 pub fn security_testing_enabled(&self) -> bool {
534 if self.wafbench_verbatim {
535 return false;
536 }
537 self.security_test || self.wafbench_dir.is_some()
538 }
539
540 pub async fn load_and_merge_specs(&self) -> Result<OpenApiSpec> {
542 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
543
544 if !self.spec.is_empty() {
546 let specs = load_specs_from_files(self.spec.clone())
547 .await
548 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
549 all_specs.extend(specs);
550 }
551
552 if let Some(spec_dir) = &self.spec_dir {
554 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
555 BenchError::Other(format!("Failed to load specs from directory: {}", e))
556 })?;
557 all_specs.extend(dir_specs);
558 }
559
560 if all_specs.is_empty() {
561 return Err(BenchError::Other(
562 "No spec files provided. Use --spec or --spec-dir.".to_string(),
563 ));
564 }
565
566 if all_specs.len() == 1 {
568 return Ok(all_specs.into_iter().next().expect("checked len() == 1 above").1);
570 }
571
572 let conflict_strategy = match self.merge_conflicts.as_str() {
574 "first" => ConflictStrategy::First,
575 "last" => ConflictStrategy::Last,
576 _ => ConflictStrategy::Error,
577 };
578
579 merge_specs(all_specs, conflict_strategy)
580 .map_err(|e| BenchError::Other(format!("Failed to merge specs: {}", e)))
581 }
582
583 fn get_spec_display_name(&self) -> String {
585 if self.spec.len() == 1 {
586 self.spec[0].to_string_lossy().to_string()
587 } else if !self.spec.is_empty() {
588 format!("{} spec files", self.spec.len())
589 } else if let Some(dir) = &self.spec_dir {
590 format!("specs from {}", dir.display())
591 } else {
592 "no specs".to_string()
593 }
594 }
595
596 fn advise_capacity(&self) {
603 let target_count = self
604 .targets_file
605 .as_ref()
606 .and_then(|p| std::fs::read_to_string(p).ok())
607 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
608 .and_then(|v| v.as_array().map(|a| a.len()))
609 .unwrap_or(1);
610 let vus = self.vus.max(1);
611 let rps_total = self.target_rps.unwrap_or(0) as usize * target_count.max(1);
612 let load_product = target_count * vus as usize;
616 if load_product >= 150 {
617 let est_ram_gb =
618 (vus as usize * 50) / 1024 + (target_count * 10 * 2) / 1024 + target_count / 2;
619 let est_cores = ((vus as usize) / 50).max(2);
620 TerminalReporter::print_warning(&format!(
621 "Capacity advisory: targets={target_count}, VUs={vus}, RPS-total≈{rps_total}. \
622 Single-client estimate: ~{est_cores} CPU cores, ~{est_ram_gb} GB RAM. \
623 If your machine is below that, expect OOM hangs partway through the run. \
624 See https://docs.mockforge.dev/reference/bench-capacity-sizing.html \
625 for the sizing table and sharding guide."
626 ));
627 }
628 }
629
630 pub async fn execute(&self) -> Result<()> {
632 if self.conformance_self_test && self.use_k6 {
639 TerminalReporter::print_warning(
640 "--use-k6 has no effect with --conformance-self-test: the self-test driver runs and returns before k6 is invoked. Drop one or the other depending on whether you want the spec-driven self-test or a k6 bench run.",
641 );
642 }
643
644 self.advise_capacity();
650
651 if let Some(targets_file) = &self.targets_file {
653 if self.conformance && self.conformance_self_test {
662 return self.execute_multi_target_self_test(targets_file).await;
663 }
664 if self.conformance {
665 return self.execute_multi_target_conformance(targets_file).await;
666 }
667 return self.execute_multi_target(targets_file).await;
668 }
669
670 if self.spec_mode == "sequential" && (self.spec.len() > 1 || self.spec_dir.is_some()) {
672 return self.execute_sequential_specs().await;
673 }
674
675 TerminalReporter::print_header(
678 &self.get_spec_display_name(),
679 &self.target,
680 0, &self.scenario,
682 Self::parse_duration(&self.duration)?,
683 );
684
685 if !K6Executor::is_k6_installed() {
687 TerminalReporter::print_error("k6 is not installed");
688 TerminalReporter::print_warning(
689 "Install k6 from: https://k6.io/docs/get-started/installation/",
690 );
691 return Err(BenchError::K6NotFound);
692 }
693 K6Executor::warn_if_pre_v1().await;
694
695 if self.conformance {
697 return self.execute_conformance_test().await;
698 }
699
700 let spec_supplied = !self.spec.is_empty() || self.spec_dir.is_some();
707 let merged_spec = if self.wafbench_verbatim && !spec_supplied {
708 tracing::info!(
709 target: "mockforge::bench",
710 "--wafbench-verbatim without --spec: sending only the traffic file's requests"
711 );
712 OpenApiSpec {
713 spec: Default::default(),
714 file_path: None,
715 raw_document: None,
716 }
717 } else {
718 TerminalReporter::print_progress("Loading OpenAPI specification(s)...");
719 self.load_and_merge_specs().await?
720 };
721 let parser = SpecParser::from_spec(merged_spec);
722 if self.spec.len() > 1 || self.spec_dir.is_some() {
723 TerminalReporter::print_success(&format!(
724 "Loaded and merged {} specification(s)",
725 self.spec.len() + self.spec_dir.as_ref().map(|_| 1).unwrap_or(0)
726 ));
727 } else {
728 TerminalReporter::print_success("Specification loaded");
729 }
730
731 let mock_config = self.build_mock_config().await;
733 if mock_config.is_mock_server {
734 TerminalReporter::print_progress("Mock server integration enabled");
735 }
736
737 if self.crud_flow {
739 return self.execute_crud_flow(&parser).await;
740 }
741
742 if self.owasp_api_top10 {
744 return self.execute_owasp_test(&parser).await;
745 }
746
747 TerminalReporter::print_progress("Extracting API operations...");
749 let mut operations = if let Some(filter) = &self.operations {
750 parser.filter_operations(filter)?
751 } else {
752 parser.get_operations()
753 };
754
755 if let Some(exclude) = &self.exclude_operations {
757 let before_count = operations.len();
758 operations = parser.exclude_operations(operations, exclude)?;
759 let excluded_count = before_count - operations.len();
760 if excluded_count > 0 {
761 TerminalReporter::print_progress(&format!(
762 "Excluded {} operations matching '{}'",
763 excluded_count, exclude
764 ));
765 }
766 }
767
768 if operations.is_empty() && !self.wafbench_verbatim {
774 return Err(BenchError::Other("No operations found in spec".to_string()));
775 }
776
777 TerminalReporter::print_success(&format!("Found {} operations", operations.len()));
778
779 let param_overrides = if let Some(params_file) = &self.params_file {
781 TerminalReporter::print_progress("Loading parameter overrides...");
782 let overrides = ParameterOverrides::from_file(params_file)?;
783 TerminalReporter::print_success(&format!(
784 "Loaded parameter overrides ({} operation-specific, {} defaults)",
785 overrides.operations.len(),
786 if overrides.defaults.is_empty() { 0 } else { 1 }
787 ));
788 Some(overrides)
789 } else {
790 None
791 };
792
793 TerminalReporter::print_progress("Generating request templates...");
795 let templates: Vec<_> = operations
796 .iter()
797 .map(|op| {
798 let op_overrides = param_overrides.as_ref().map(|po| {
799 po.get_for_operation(op.operation_id.as_deref(), &op.method, &op.path)
800 });
801 RequestGenerator::generate_template_with_overrides(op, op_overrides.as_ref())
802 })
803 .collect::<Result<Vec<_>>>()?;
804 TerminalReporter::print_success("Request templates generated");
805
806 let templates = if self.wafbench_verbatim {
812 let verbatim = self.load_verbatim_templates()?;
813 if verbatim.is_empty() {
814 return Err(BenchError::Other(
815 "--wafbench-verbatim was set but no traffic cases were loaded. Check \
816 --wafbench-dir points at a file, directory or glob containing cases with \
817 a `request.uri`."
818 .to_string(),
819 ));
820 }
821 TerminalReporter::print_success(&format!(
822 "Verbatim mode: {} request(s) will be sent exactly as written (spec endpoints not used)",
823 verbatim.len()
824 ));
825 verbatim
826 } else {
827 templates
828 };
829
830 let custom_headers = self.parse_headers()?;
832
833 let force_http1 = crate::request_gen::should_force_k6_http1(
836 self.wafbench_verbatim,
837 &templates,
838 &custom_headers,
839 );
840
841 let base_path = self.resolve_base_path(&parser);
843 if let Some(ref bp) = base_path {
844 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
845 }
846
847 TerminalReporter::print_progress("Generating k6 load test script...");
849 let scenario =
850 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
851
852 let security_testing_enabled = self.security_testing_enabled();
853
854 let num_ops = operations.len() as u32;
872 if let Some(rps) = self.target_rps {
873 let probe =
874 crate::preflight::probe_target_latency(&self.target, 3, self.skip_tls_verify).await;
875
876 let (required_vus, basis) = match probe {
877 Some(p) => (
878 p.required_vus(rps, num_ops),
879 format!("avg {:.1}ms (measured)", p.avg_latency.as_secs_f64() * 1000.0),
880 ),
881 None => {
882 let fallback = (rps as u64)
884 .saturating_mul(num_ops.max(1) as u64)
885 .div_ceil(10)
886 .min(u32::MAX as u64) as u32;
887 (fallback, "~100ms (default — probe failed)".to_string())
888 }
889 };
890
891 if self.vus < required_vus {
892 const VU_RECOMMENDATION_CAP: u32 = 1000;
898 let recommendation = required_vus.max(self.vus + 1);
899 if recommendation > VU_RECOMMENDATION_CAP {
900 TerminalReporter::print_warning(&format!(
901 "Workload is very large: --rps {} × {} ops/iteration × {} \
902 baseline ⇒ ~{} VUs needed end-to-end, far beyond what's \
903 practical to drive. Two ways to fix:\n 1. Reduce \
904 operations per iteration with `--operations 'pattern,…'` \
905 (or `--exclude-operations`) to focus the bench on a \
906 representative subset.\n 2. Drop `--rps` and use \
907 `--vus {}` alone — closed-model load runs as fast as \
908 the VU pool allows, bounded by latency, with no per-\
909 iteration deadline. Expect 1-iteration coverage of ~{} \
910 operations in {}s.",
911 rps,
912 num_ops,
913 basis,
914 recommendation,
915 self.vus.max(5),
916 num_ops,
917 Self::parse_duration(&self.duration).unwrap_or(0),
918 ));
919 } else {
920 TerminalReporter::print_warning(&format!(
921 "--vus {} may be insufficient for --rps {} × {} ops/iteration \
922 (baseline latency {}). k6's constant-arrival-rate counts ITERATIONS \
923 and each runs every operation in the spec — required ≈ rps × ops × \
924 latency_secs VUs. Bump --vus to ~{} if you see \"Insufficient VUs\" \
925 warnings.",
926 self.vus, rps, num_ops, basis, recommendation,
927 ));
928 }
929 } else if probe.is_some() {
930 TerminalReporter::print_progress(&format!(
931 "Pre-flight probe: target latency {}, {} ops/iteration — --vus {} \
932 is sufficient for --rps {}",
933 basis, num_ops, self.vus, rps,
934 ));
935 }
936 }
937
938 let k6_config = K6Config {
939 target_url: self.target.clone(),
940 base_path,
941 scenario,
942 duration_secs: Self::parse_duration(&self.duration)?,
943 max_vus: self.vus,
944 threshold_percentile: self.threshold_percentile.clone(),
945 threshold_ms: self.threshold_ms,
946 max_error_rate: self.max_error_rate,
947 auth_header: self.auth.clone(),
948 custom_headers,
949 skip_tls_verify: self.skip_tls_verify,
950 security_testing_enabled,
951 chunked_request_bodies: self.chunked_request_bodies,
952 target_rps: self.target_rps,
953 no_keep_alive: self.no_keep_alive,
954 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
960 .into_iter()
961 .map(|ip| ip.to_string())
962 .collect(),
963 geo_source_headers: if self.geo_source_headers.is_empty()
964 && !self.geo_source_ips.is_empty()
965 {
966 crate::conformance::self_test::default_geo_source_headers()
967 } else {
968 self.geo_source_headers.clone()
969 },
970 };
971
972 let generator = K6ScriptGenerator::new(k6_config, templates)
973 .with_abort_valve(self.abort_on_error, self.abort_on_error_rate)
974 .with_force_http1(force_http1);
975 let mut script = generator.generate()?;
976 TerminalReporter::print_success("k6 script generated");
977
978 let has_advanced_features = self.data_file.is_some()
980 || self.error_rate.is_some()
981 || self.security_test
982 || self.parallel_create.is_some()
983 || self.wafbench_dir.is_some();
984
985 if has_advanced_features {
987 script = self.generate_enhanced_script(&script)?;
988 }
989
990 if mock_config.is_mock_server {
992 let setup_code = MockIntegrationGenerator::generate_setup(&mock_config);
993 let teardown_code = MockIntegrationGenerator::generate_teardown(&mock_config);
994 let helper_code = MockIntegrationGenerator::generate_vu_id_helper();
995
996 if let Some(import_end) = script.find("export const options") {
998 script.insert_str(
999 import_end,
1000 &format!(
1001 "\n// === Mock Server Integration ===\n{}\n{}\n{}\n",
1002 helper_code, setup_code, teardown_code
1003 ),
1004 );
1005 }
1006 }
1007
1008 TerminalReporter::print_progress("Validating k6 script...");
1010 let validation_errors = K6ScriptGenerator::validate_script(&script);
1011 if !validation_errors.is_empty() {
1012 TerminalReporter::print_error("Script validation failed");
1013 for error in &validation_errors {
1014 eprintln!(" {}", error);
1015 }
1016 return Err(BenchError::Other(format!(
1017 "Generated k6 script has {} validation error(s). Please check the output above.",
1018 validation_errors.len()
1019 )));
1020 }
1021 TerminalReporter::print_success("Script validation passed");
1022
1023 let script_path = if let Some(output) = &self.script_output {
1025 output.clone()
1026 } else {
1027 self.output.join("k6-script.js")
1028 };
1029
1030 if let Some(parent) = script_path.parent() {
1031 std::fs::create_dir_all(parent)?;
1032 }
1033 std::fs::write(&script_path, &script)?;
1034 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
1035
1036 if self.generate_only {
1038 Self::print_k6_run_hint(&script_path, force_http1);
1039 return Ok(());
1040 }
1041
1042 TerminalReporter::print_progress("Executing load test...");
1044 if force_http1 {
1045 TerminalReporter::print_progress(
1046 "Forcing HTTP/1.1 (GODEBUG=http2client=0): Connection headers are hop-by-hop and HTTP/2 rejects them. The header stays on the wire.",
1047 );
1048 }
1049 let executor = K6Executor::new()?
1053 .with_local_ips(self.source_ips.join(","))
1054 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
1055 .with_discard_response_bodies(self.discard_response_bodies)
1056 .with_force_http1(force_http1);
1057
1058 std::fs::create_dir_all(&self.output)?;
1059
1060 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
1061
1062 let duration_secs = Self::parse_duration(&self.duration)?;
1064 TerminalReporter::print_summary_full(
1065 &results,
1066 duration_secs,
1067 self.no_keep_alive,
1068 Some(num_ops),
1069 );
1070
1071 self.reprint_traffic_file_breakdown();
1072 println!("\nResults saved to: {}", self.output.display());
1073
1074 Ok(())
1075 }
1076
1077 async fn execute_multi_target(&self, targets_file: &Path) -> Result<()> {
1079 TerminalReporter::print_progress("Parsing targets file...");
1080 let targets = parse_targets_file(targets_file)?;
1081 let num_targets = targets.len();
1082 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
1083
1084 if targets.is_empty() {
1085 return Err(BenchError::Other("No targets found in file".to_string()));
1086 }
1087
1088 let max_concurrency = self.max_concurrency.unwrap_or(10) as usize;
1090 let max_concurrency = max_concurrency.min(num_targets); TerminalReporter::print_header(
1094 &self.get_spec_display_name(),
1095 &format!("{} targets", num_targets),
1096 0,
1097 &self.scenario,
1098 Self::parse_duration(&self.duration)?,
1099 );
1100
1101 let executor = ParallelExecutor::new(
1103 BenchCommand {
1104 spec: self.spec.clone(),
1106 spec_dir: self.spec_dir.clone(),
1107 merge_conflicts: self.merge_conflicts.clone(),
1108 spec_mode: self.spec_mode.clone(),
1109 dependency_config: self.dependency_config.clone(),
1110 target: self.target.clone(), base_path: self.base_path.clone(),
1112 duration: self.duration.clone(),
1113 vus: self.vus,
1114 target_rps: self.target_rps,
1115 no_keep_alive: self.no_keep_alive,
1116 scenario: self.scenario.clone(),
1117 operations: self.operations.clone(),
1118 exclude_operations: self.exclude_operations.clone(),
1119 auth: self.auth.clone(),
1120 headers: self.headers.clone(),
1121 output: self.output.clone(),
1122 generate_only: self.generate_only,
1123 script_output: self.script_output.clone(),
1124 threshold_percentile: self.threshold_percentile.clone(),
1125 threshold_ms: self.threshold_ms,
1126 max_error_rate: self.max_error_rate,
1127 abort_on_error: self.abort_on_error,
1128 abort_on_error_rate: self.abort_on_error_rate,
1129 verbose: self.verbose,
1130 skip_tls_verify: self.skip_tls_verify,
1131 chunked_request_bodies: self.chunked_request_bodies,
1132 targets_file: None,
1133 max_concurrency: None,
1134 results_format: self.results_format.clone(),
1135 params_file: self.params_file.clone(),
1136 crud_flow: self.crud_flow,
1137 flow_config: self.flow_config.clone(),
1138 extract_fields: self.extract_fields.clone(),
1139 parallel_create: self.parallel_create,
1140 data_file: self.data_file.clone(),
1141 data_distribution: self.data_distribution.clone(),
1142 data_mappings: self.data_mappings.clone(),
1143 per_uri_control: self.per_uri_control,
1144 error_rate: self.error_rate,
1145 error_types: self.error_types.clone(),
1146 security_test: self.security_test,
1147 security_payloads: self.security_payloads.clone(),
1148 security_categories: self.security_categories.clone(),
1149 security_target_fields: self.security_target_fields.clone(),
1150 wafbench_dir: self.wafbench_dir.clone(),
1151 wafbench_cycle_all: self.wafbench_cycle_all,
1152 wafbench_verbatim: self.wafbench_verbatim,
1153 owasp_api_top10: self.owasp_api_top10,
1154 owasp_categories: self.owasp_categories.clone(),
1155 owasp_auth_header: self.owasp_auth_header.clone(),
1156 owasp_auth_token: self.owasp_auth_token.clone(),
1157 owasp_admin_paths: self.owasp_admin_paths.clone(),
1158 owasp_id_fields: self.owasp_id_fields.clone(),
1159 owasp_report: self.owasp_report.clone(),
1160 owasp_report_format: self.owasp_report_format.clone(),
1161 owasp_iterations: self.owasp_iterations,
1162 conformance: false,
1163 conformance_api_key: self.conformance_api_key.clone(),
1179 conformance_basic_auth: self.conformance_basic_auth.clone(),
1180 conformance_report: PathBuf::from("conformance-report.json"),
1181 conformance_categories: None,
1182 conformance_report_format: "json".to_string(),
1183 conformance_headers: self.conformance_headers.clone(),
1187 conformance_all_operations: false,
1188 conformance_custom: None,
1189 conformance_delay_ms: 0,
1190 use_k6: false,
1191 conformance_custom_filter: None,
1192 export_requests: false,
1193 validate_requests: false,
1194 conformance_self_test: false,
1195 conformance_self_test_capture: false,
1196 conformance_self_test_iterations: 1,
1197 conformance_self_test_duration: None,
1198 validate_response_schemas: false,
1199 source_ips: self.source_ips.clone(),
1204 geo_source_ips: self.geo_source_ips.clone(),
1205 geo_source_headers: self.geo_source_headers.clone(),
1206 report_missed_cap: None,
1207 discard_response_bodies: self.discard_response_bodies,
1211 dns_policy: self.dns_policy.clone(),
1214 },
1215 targets,
1216 max_concurrency,
1217 );
1218
1219 let start_time = std::time::Instant::now();
1221 let aggregated_results = executor.execute_all().await?;
1222 let elapsed = start_time.elapsed();
1223
1224 self.report_multi_target_results(&aggregated_results, elapsed)?;
1226
1227 Ok(())
1228 }
1229
1230 fn report_multi_target_results(
1232 &self,
1233 results: &AggregatedResults,
1234 elapsed: std::time::Duration,
1235 ) -> Result<()> {
1236 TerminalReporter::print_multi_target_summary(results);
1238
1239 let total_secs = elapsed.as_secs();
1241 let hours = total_secs / 3600;
1242 let minutes = (total_secs % 3600) / 60;
1243 let seconds = total_secs % 60;
1244 if hours > 0 {
1245 println!("\n Total Elapsed Time: {}h {}m {}s", hours, minutes, seconds);
1246 } else if minutes > 0 {
1247 println!("\n Total Elapsed Time: {}m {}s", minutes, seconds);
1248 } else {
1249 println!("\n Total Elapsed Time: {}s", seconds);
1250 }
1251
1252 if self.results_format == "aggregated" || self.results_format == "both" {
1254 let summary_path = self.output.join("aggregated_summary.json");
1255 let summary_json = serde_json::json!({
1256 "total_elapsed_seconds": elapsed.as_secs(),
1257 "total_targets": results.total_targets,
1258 "successful_targets": results.successful_targets,
1259 "failed_targets": results.failed_targets,
1260 "aggregated_metrics": {
1261 "total_requests": results.aggregated_metrics.total_requests,
1262 "total_failed_requests": results.aggregated_metrics.total_failed_requests,
1263 "avg_duration_ms": results.aggregated_metrics.avg_duration_ms,
1264 "p95_duration_ms": results.aggregated_metrics.p95_duration_ms,
1265 "p99_duration_ms": results.aggregated_metrics.p99_duration_ms,
1266 "error_rate": results.aggregated_metrics.error_rate,
1267 "total_rps": results.aggregated_metrics.total_rps,
1268 "avg_rps": results.aggregated_metrics.avg_rps,
1269 "total_vus_max": results.aggregated_metrics.total_vus_max,
1270 },
1271 "target_results": results.target_results.iter().map(|r| {
1272 serde_json::json!({
1273 "target_url": r.target_url,
1274 "target_index": r.target_index,
1275 "success": r.success,
1276 "error": r.error,
1277 "total_requests": r.results.total_requests,
1278 "failed_requests": r.results.failed_requests,
1279 "avg_duration_ms": r.results.avg_duration_ms,
1280 "min_duration_ms": r.results.min_duration_ms,
1281 "med_duration_ms": r.results.med_duration_ms,
1282 "p90_duration_ms": r.results.p90_duration_ms,
1283 "p95_duration_ms": r.results.p95_duration_ms,
1284 "p99_duration_ms": r.results.p99_duration_ms,
1285 "max_duration_ms": r.results.max_duration_ms,
1286 "rps": r.results.rps,
1287 "vus_max": r.results.vus_max,
1288 "output_dir": r.output_dir.to_string_lossy(),
1289 })
1290 }).collect::<Vec<_>>(),
1291 });
1292
1293 std::fs::write(&summary_path, serde_json::to_string_pretty(&summary_json)?)?;
1294 TerminalReporter::print_success(&format!(
1295 "Aggregated summary saved to: {}",
1296 summary_path.display()
1297 ));
1298 }
1299
1300 let csv_path = self.output.join("all_targets.csv");
1302 let mut csv = String::from(
1303 "target_url,success,requests,failed,rps,vus,min_ms,avg_ms,med_ms,p90_ms,p95_ms,p99_ms,max_ms,error\n",
1304 );
1305 for r in &results.target_results {
1306 csv.push_str(&format!(
1307 "{},{},{},{},{:.1},{},{:.1},{:.1},{:.1},{:.1},{:.1},{:.1},{:.1},{}\n",
1308 r.target_url,
1309 r.success,
1310 r.results.total_requests,
1311 r.results.failed_requests,
1312 r.results.rps,
1313 r.results.vus_max,
1314 r.results.min_duration_ms,
1315 r.results.avg_duration_ms,
1316 r.results.med_duration_ms,
1317 r.results.p90_duration_ms,
1318 r.results.p95_duration_ms,
1319 r.results.p99_duration_ms,
1320 r.results.max_duration_ms,
1321 r.error.as_deref().unwrap_or(""),
1322 ));
1323 }
1324 let _ = std::fs::write(&csv_path, &csv);
1325
1326 self.reprint_traffic_file_breakdown();
1327 println!("\nResults saved to: {}", self.output.display());
1328 println!(" - Per-target results: {}", self.output.join("target_*").display());
1329 println!(" - All targets CSV: {}", csv_path.display());
1330 if self.results_format == "aggregated" || self.results_format == "both" {
1331 println!(
1332 " - Aggregated summary: {}",
1333 self.output.join("aggregated_summary.json").display()
1334 );
1335 }
1336
1337 Ok(())
1338 }
1339
1340 pub fn parse_duration(duration: &str) -> Result<u64> {
1342 let duration = duration.trim();
1343
1344 if let Some(secs) = duration.strip_suffix('s') {
1345 secs.parse::<u64>()
1346 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1347 } else if let Some(mins) = duration.strip_suffix('m') {
1348 mins.parse::<u64>()
1349 .map(|m| m * 60)
1350 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1351 } else if let Some(hours) = duration.strip_suffix('h') {
1352 hours
1353 .parse::<u64>()
1354 .map(|h| h * 3600)
1355 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1356 } else {
1357 duration
1359 .parse::<u64>()
1360 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1361 }
1362 }
1363
1364 fn print_k6_run_hint(script_path: &Path, force_http1: bool) {
1368 println!("\nScript generated successfully. Run it with:");
1369 if force_http1 {
1370 println!(" GODEBUG=http2client=0 k6 run {}", script_path.display());
1371 println!(
1372 " (HTTP/1.1: a Connection header is on the wire; HTTP/2 rejects it. mockforge bench sets this automatically when it invokes k6.)"
1373 );
1374 } else {
1375 println!(" k6 run {}", script_path.display());
1376 }
1377 }
1378
1379 pub(crate) fn load_verbatim_templates(
1386 &self,
1387 ) -> Result<Vec<crate::request_gen::RequestTemplate>> {
1388 let Some(pattern) = self.wafbench_dir.as_ref() else {
1389 return Err(BenchError::Other(
1390 "--wafbench-verbatim requires --wafbench-dir pointing at your traffic file(s)"
1391 .to_string(),
1392 ));
1393 };
1394
1395 let mut loader = WafBenchLoader::new();
1396 loader.load_from_pattern(pattern)?;
1397 self.emit_traffic_file_breakdown(loader.stats(), "what to expect in proxy logs");
1398
1399 Ok(crate::wafbench::traffic_cases_to_templates(loader.test_cases()))
1400 }
1401
1402 pub fn parse_headers(&self) -> Result<HashMap<String, String>> {
1404 let mut headers = parse_header_string(&self.headers)?;
1405
1406 let already_has = |hs: &HashMap<String, String>, name: &str| -> bool {
1417 hs.keys().any(|k| k.eq_ignore_ascii_case(name))
1418 };
1419
1420 if !already_has(&headers, "Authorization") {
1421 if let Some(b) = self.conformance_basic_auth.as_ref().filter(|s| !s.is_empty()) {
1422 use base64::Engine as _;
1423 let encoded = base64::engine::general_purpose::STANDARD.encode(b.as_bytes());
1424 headers.insert("Authorization".to_string(), format!("Basic {}", encoded));
1425 }
1426 }
1427
1428 for line in &self.conformance_headers {
1434 let Some((name, value)) = line.split_once(':') else {
1435 continue;
1436 };
1437 let name = name.trim();
1438 let value = value.trim();
1439 if name.is_empty() || already_has(&headers, name) {
1440 continue;
1441 }
1442 headers.insert(name.to_string(), value.to_string());
1443 }
1444
1445 if !self.conformance && self.conformance_api_key.is_some() {
1451 TerminalReporter::print_warning(
1452 "--conformance-api-key only fires under --conformance. For plain bench use --header 'X-API-Key: ...'.",
1453 );
1454 }
1455
1456 Ok(headers)
1457 }
1458
1459 fn parse_extracted_values(output_dir: &Path) -> Result<ExtractedValues> {
1460 let extracted_path = output_dir.join("extracted_values.json");
1461 if !extracted_path.exists() {
1462 return Ok(ExtractedValues::new());
1463 }
1464
1465 let content = std::fs::read_to_string(&extracted_path)
1466 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1467 let parsed: serde_json::Value = serde_json::from_str(&content)
1468 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1469
1470 let mut extracted = ExtractedValues::new();
1471 if let Some(values) = parsed.as_object() {
1472 for (key, value) in values {
1473 extracted.set(key.clone(), value.clone());
1474 }
1475 }
1476
1477 Ok(extracted)
1478 }
1479
1480 fn resolve_base_path(&self, parser: &SpecParser) -> Option<String> {
1489 if let Some(cli_base_path) = &self.base_path {
1491 if cli_base_path.is_empty() {
1492 return None;
1494 }
1495 return Some(cli_base_path.clone());
1496 }
1497
1498 parser.get_base_path()
1500 }
1501
1502 async fn build_mock_config(&self) -> MockIntegrationConfig {
1504 if MockServerDetector::looks_like_mock_server(&self.target) {
1506 if let Ok(info) = MockServerDetector::detect(&self.target).await {
1508 if info.is_mockforge {
1509 TerminalReporter::print_success(&format!(
1510 "Detected MockForge server (version: {})",
1511 info.version.as_deref().unwrap_or("unknown")
1512 ));
1513 return MockIntegrationConfig::mock_server();
1514 }
1515 }
1516 }
1517 MockIntegrationConfig::real_api()
1518 }
1519
1520 fn build_crud_flow_config(&self) -> Option<CrudFlowConfig> {
1522 if !self.crud_flow {
1523 return None;
1524 }
1525
1526 if let Some(config_path) = &self.flow_config {
1528 match CrudFlowConfig::from_file(config_path) {
1529 Ok(config) => return Some(config),
1530 Err(e) => {
1531 TerminalReporter::print_warning(&format!(
1532 "Failed to load flow config: {}. Using auto-detection.",
1533 e
1534 ));
1535 }
1536 }
1537 }
1538
1539 let extract_fields = self
1541 .extract_fields
1542 .as_ref()
1543 .map(|f| f.split(',').map(|s| s.trim().to_string()).collect())
1544 .unwrap_or_else(|| vec!["id".to_string(), "uuid".to_string()]);
1545
1546 Some(CrudFlowConfig {
1547 flows: Vec::new(), default_extract_fields: extract_fields,
1549 })
1550 }
1551
1552 fn build_data_driven_config(&self) -> Option<DataDrivenConfig> {
1554 let data_file = self.data_file.as_ref()?;
1555
1556 let distribution = DataDistribution::from_str(&self.data_distribution)
1557 .unwrap_or(DataDistribution::UniquePerVu);
1558
1559 let mappings = self
1560 .data_mappings
1561 .as_ref()
1562 .map(|m| DataMapping::parse_mappings(m).unwrap_or_default())
1563 .unwrap_or_default();
1564
1565 Some(DataDrivenConfig {
1566 file_path: data_file.to_string_lossy().to_string(),
1567 distribution,
1568 mappings,
1569 csv_has_header: true,
1570 per_uri_control: self.per_uri_control,
1571 per_uri_columns: crate::data_driven::PerUriColumns::default(),
1572 })
1573 }
1574
1575 fn build_invalid_data_config(&self) -> Option<InvalidDataConfig> {
1577 let error_rate = self.error_rate?;
1578
1579 let error_types = self
1580 .error_types
1581 .as_ref()
1582 .map(|types| InvalidDataConfig::parse_error_types(types).unwrap_or_default())
1583 .unwrap_or_default();
1584
1585 Some(InvalidDataConfig {
1586 error_rate,
1587 error_types,
1588 target_fields: Vec::new(),
1589 })
1590 }
1591
1592 fn build_security_config(&self) -> Option<SecurityTestConfig> {
1594 if !self.security_test {
1595 return None;
1596 }
1597
1598 let categories = self
1599 .security_categories
1600 .as_ref()
1601 .map(|cats| SecurityTestConfig::parse_categories(cats).unwrap_or_default())
1602 .unwrap_or_else(|| {
1603 let mut default = HashSet::new();
1604 default.insert(SecurityCategory::SqlInjection);
1605 default.insert(SecurityCategory::Xss);
1606 default
1607 });
1608
1609 let target_fields = self
1610 .security_target_fields
1611 .as_ref()
1612 .map(|fields| fields.split(',').map(|f| f.trim().to_string()).collect())
1613 .unwrap_or_default();
1614
1615 let custom_payloads_file =
1616 self.security_payloads.as_ref().map(|p| p.to_string_lossy().to_string());
1617
1618 Some(SecurityTestConfig {
1619 enabled: true,
1620 categories,
1621 target_fields,
1622 custom_payloads_file,
1623 include_high_risk: false,
1624 })
1625 }
1626
1627 fn build_parallel_config(&self) -> Option<ParallelConfig> {
1629 let count = self.parallel_create?;
1630
1631 Some(ParallelConfig::new(count))
1632 }
1633
1634 fn format_unique_total(unique: usize, rps: Option<u32>) -> String {
1637 match rps {
1638 Some(r) if r > 0 => {
1639 let projected = unique.saturating_mul(r as usize);
1640 format!(
1641 "unique_cases={unique} projected_per_second={projected} ({unique} * {r} RPS)"
1642 )
1643 }
1644 _ => format!("unique_cases={unique}"),
1645 }
1646 }
1647
1648 fn traffic_bucket_json(
1649 unique: usize,
1650 rps: Option<u32>,
1651 duration_secs: Option<u64>,
1652 ) -> serde_json::Value {
1653 let per_second = rps.filter(|&r| r > 0).map(|r| (unique as u64).saturating_mul(r as u64));
1660 let total = per_second.unwrap_or(unique as u64);
1661 let projected_over_run = match (rps.filter(|&r| r > 0), duration_secs) {
1662 (Some(r), Some(d)) => Some((unique as u64).saturating_mul(r as u64).saturating_mul(d)),
1663 _ => None,
1664 };
1665 serde_json::json!({
1666 "unique_cases": unique,
1667 "unique": unique,
1668 "projected_per_second": per_second,
1669 "total": total,
1670 "projected_over_run": projected_over_run,
1671 "expected_requests": projected_over_run,
1672 })
1673 }
1674
1675 fn emit_traffic_file_breakdown(&self, stats: &crate::wafbench::WafBenchStats, phase: &str) {
1678 if stats.per_file.is_empty() {
1679 return;
1680 }
1681 let rps = self.target_rps.filter(|&r| r > 0);
1682 TerminalReporter::print_success(&format!("Traffic file breakdown ({phase}):"));
1683 for file in &stats.per_file {
1684 let other = if file.other > 0 {
1685 format!(" other={}", file.other)
1686 } else {
1687 String::new()
1688 };
1689 TerminalReporter::print_progress(&format!(
1690 " {}: sent {} attack(expected 403) {} normal(expected 200) {} omitted={}{other}",
1691 file.file,
1692 Self::format_unique_total(file.sent, rps),
1693 Self::format_unique_total(file.attack, rps),
1694 Self::format_unique_total(file.normal, rps),
1695 file.omitted
1696 ));
1697 }
1698 self.write_traffic_breakdown_json(stats);
1699 }
1700
1701 fn write_traffic_breakdown_json(&self, stats: &crate::wafbench::WafBenchStats) {
1703 if stats.per_file.is_empty() {
1704 return;
1705 }
1706 let rps = self.target_rps.filter(|&r| r > 0);
1707 let duration_secs = Self::parse_duration(&self.duration).ok();
1708 let files: Vec<serde_json::Value> = stats
1709 .per_file
1710 .iter()
1711 .map(|file| {
1712 serde_json::json!({
1713 "file": file.file,
1714 "sent": Self::traffic_bucket_json(file.sent, rps, duration_secs),
1715 "attack": Self::traffic_bucket_json(file.attack, rps, duration_secs),
1716 "normal": Self::traffic_bucket_json(file.normal, rps, duration_secs),
1717 "omitted": file.omitted,
1718 "other": file.other,
1719 })
1720 })
1721 .collect();
1722 let payload = serde_json::json!({
1723 "rps": rps,
1724 "duration_secs": duration_secs,
1725 "note": "Plan, not k6 counters: unique_cases is YAML case count, projected_per_second is unique_cases * rps, projected_over_run is unique_cases * rps * duration_secs. Assumes each k6 iteration sends every unique case. projected_* are null when --rps is unset. unique/total/expected_requests are aliases for one release.",
1726 "files": files,
1727 });
1728 if let Some(parent) = self.output.parent() {
1729 let _ = std::fs::create_dir_all(parent);
1730 }
1731 let _ = std::fs::create_dir_all(&self.output);
1732 let path = self.output.join("traffic-breakdown.json");
1733 if let Ok(body) = serde_json::to_string_pretty(&payload) {
1734 if std::fs::write(&path, body).is_ok() {
1735 TerminalReporter::print_progress(&format!(
1736 "Traffic breakdown written to: {}",
1737 path.display()
1738 ));
1739 }
1740 }
1741 }
1742
1743 fn reprint_traffic_file_breakdown(&self) {
1746 let path = self.output.join("traffic-breakdown.json");
1747 let Ok(raw) = std::fs::read_to_string(&path) else {
1748 return;
1749 };
1750 let Ok(v) = serde_json::from_str::<serde_json::Value>(&raw) else {
1751 return;
1752 };
1753 let Some(files) = v.get("files").and_then(|f| f.as_array()) else {
1754 return;
1755 };
1756 if files.is_empty() {
1757 return;
1758 }
1759 TerminalReporter::print_success("Traffic file breakdown (end of run):");
1760 for file in files {
1761 let name = file.get("file").and_then(|x| x.as_str()).unwrap_or("?");
1762 let bucket = |key: &str| -> String {
1763 let unique = file
1764 .get(key)
1765 .and_then(|b| b.get("unique"))
1766 .and_then(|u| u.as_u64())
1767 .unwrap_or(0) as usize;
1768 Self::format_unique_total(unique, self.target_rps.filter(|&r| r > 0))
1769 };
1770 let omitted = file.get("omitted").and_then(|o| o.as_u64()).unwrap_or(0);
1771 let other = file.get("other").and_then(|o| o.as_u64()).unwrap_or(0);
1772 let other = if other > 0 {
1773 format!(" other={other}")
1774 } else {
1775 String::new()
1776 };
1777 TerminalReporter::print_progress(&format!(
1778 " {name}: sent {} attack(expected 403) {} normal(expected 200) {} omitted={omitted}{other}",
1779 bucket("sent"),
1780 bucket("attack"),
1781 bucket("normal"),
1782 ));
1783 }
1784 TerminalReporter::print_progress(&format!(" (also in {})", path.display()));
1785 }
1786
1787 fn load_wafbench_payloads(&self) -> Result<Vec<SecurityPayload>> {
1794 let Some(ref wafbench_dir) = self.wafbench_dir else {
1795 return Ok(Vec::new());
1796 };
1797
1798 let mut loader = WafBenchLoader::new();
1799 loader.load_from_pattern(wafbench_dir)?;
1800
1801 let stats = loader.stats();
1802
1803 if stats.files_processed == 0 {
1804 let mut msg = format!(
1805 "No WAFBench YAML files found matching '{wafbench_dir}'. \
1806 --wafbench-dir is a file, a directory or a glob. A missing \
1807 file is an error, not an empty payload pool."
1808 );
1809 if !stats.parse_errors.is_empty() {
1810 msg.push_str(" Parse errors:");
1811 for error in &stats.parse_errors {
1812 msg.push_str(&format!("\n - {error}"));
1813 }
1814 }
1815 return Err(BenchError::Other(msg));
1816 }
1817
1818 TerminalReporter::print_progress(&format!(
1819 "Loaded {} WAFBench files, {} test cases, {} payloads",
1820 stats.files_processed, stats.test_cases_loaded, stats.payloads_extracted
1821 ));
1822 self.emit_traffic_file_breakdown(stats, "what to expect in proxy logs");
1823
1824 for (category, count) in &stats.by_category {
1826 TerminalReporter::print_progress(&format!(" - {}: {} tests", category, count));
1827 }
1828
1829 for error in &stats.parse_errors {
1831 TerminalReporter::print_warning(&format!(" Parse error: {}", error));
1832 }
1833
1834 Ok(loader.to_security_payloads())
1835 }
1836
1837 pub(crate) fn generate_enhanced_script(&self, base_script: &str) -> Result<String> {
1839 let mut enhanced_script = base_script.to_string();
1840 let mut additional_code = String::new();
1841
1842 if let Some(config) = self.build_data_driven_config() {
1844 TerminalReporter::print_progress("Adding data-driven testing support...");
1845 additional_code.push_str(&DataDrivenGenerator::generate_setup(&config));
1846 additional_code.push('\n');
1847 TerminalReporter::print_success("Data-driven testing enabled");
1848 }
1849
1850 if let Some(config) = self.build_invalid_data_config() {
1852 TerminalReporter::print_progress("Adding invalid data testing support...");
1853 additional_code.push_str(&InvalidDataGenerator::generate_invalidation_logic());
1854 additional_code.push('\n');
1855 additional_code
1856 .push_str(&InvalidDataGenerator::generate_should_invalidate(config.error_rate));
1857 additional_code.push('\n');
1858 additional_code
1859 .push_str(&InvalidDataGenerator::generate_type_selection(&config.error_types));
1860 additional_code.push('\n');
1861 TerminalReporter::print_success(&format!(
1862 "Invalid data testing enabled ({}% error rate)",
1863 (self.error_rate.unwrap_or(0.0) * 100.0) as u32
1864 ));
1865 }
1866
1867 let verbatim = self.wafbench_verbatim;
1874 if verbatim && self.security_test {
1875 TerminalReporter::print_warning(
1876 "--security-test is ignored under --wafbench-verbatim: verbatim mode sends your \
1877 traffic cases exactly as written and will not append attack payloads to them. \
1878 Drop --wafbench-verbatim if you want payload injection.",
1879 );
1880 }
1881 let security_config = if verbatim {
1882 None
1883 } else {
1884 self.build_security_config()
1885 };
1886 let wafbench_payloads = if verbatim {
1887 Vec::new()
1888 } else {
1889 self.load_wafbench_payloads()?
1890 };
1891 let security_requested =
1892 !verbatim && (security_config.is_some() || self.wafbench_dir.is_some());
1893
1894 if security_config.is_some() || !wafbench_payloads.is_empty() {
1895 TerminalReporter::print_progress("Adding security testing support...");
1896
1897 let mut payload_list: Vec<SecurityPayload> = Vec::new();
1899
1900 if let Some(ref config) = security_config {
1901 payload_list.extend(SecurityPayloads::get_payloads(config));
1902 }
1903
1904 if !wafbench_payloads.is_empty() {
1906 TerminalReporter::print_progress(&format!(
1907 "Loading {} WAFBench attack patterns...",
1908 wafbench_payloads.len()
1909 ));
1910 payload_list.extend(wafbench_payloads);
1911 }
1912
1913 let target_fields =
1914 security_config.as_ref().map(|c| c.target_fields.clone()).unwrap_or_default();
1915
1916 additional_code.push_str(&SecurityTestGenerator::generate_payload_selection(
1917 &payload_list,
1918 self.wafbench_cycle_all,
1919 ));
1920 additional_code.push('\n');
1921 additional_code
1922 .push_str(&SecurityTestGenerator::generate_apply_payload(&target_fields));
1923 additional_code.push('\n');
1924 additional_code.push_str(&SecurityTestGenerator::generate_security_checks());
1925 additional_code.push('\n');
1926
1927 let mode = if self.wafbench_cycle_all {
1928 "cycle-all"
1929 } else {
1930 "random"
1931 };
1932 TerminalReporter::print_success(&format!(
1933 "Security testing enabled ({} payloads, {} mode)",
1934 payload_list.len(),
1935 mode
1936 ));
1937 } else if security_requested {
1938 TerminalReporter::print_warning(
1942 "Security testing was requested but no payloads were loaded. \
1943 Ensure --wafbench-dir points to valid CRS YAML files or add --security-test.",
1944 );
1945 additional_code
1946 .push_str(&SecurityTestGenerator::generate_payload_selection(&[], false));
1947 additional_code.push('\n');
1948 additional_code.push_str(&SecurityTestGenerator::generate_apply_payload(&[]));
1949 additional_code.push('\n');
1950 }
1951
1952 if let Some(config) = self.build_parallel_config() {
1954 TerminalReporter::print_progress("Adding parallel execution support...");
1955 additional_code.push_str(&ParallelRequestGenerator::generate_batch_helper(&config));
1956 additional_code.push('\n');
1957 TerminalReporter::print_success(&format!(
1958 "Parallel execution enabled (count: {})",
1959 config.count
1960 ));
1961 }
1962
1963 if !additional_code.is_empty() {
1965 if let Some(import_end) = enhanced_script.find("export const options") {
1967 enhanced_script.insert_str(
1968 import_end,
1969 &format!("\n// === Advanced Testing Features ===\n{}\n", additional_code),
1970 );
1971 }
1972 }
1973
1974 Ok(enhanced_script)
1975 }
1976
1977 async fn execute_sequential_specs(&self) -> Result<()> {
1979 TerminalReporter::print_progress("Sequential spec mode: Loading specs individually...");
1980
1981 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
1983
1984 if !self.spec.is_empty() {
1985 let specs = load_specs_from_files(self.spec.clone())
1986 .await
1987 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
1988 all_specs.extend(specs);
1989 }
1990
1991 if let Some(spec_dir) = &self.spec_dir {
1992 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
1993 BenchError::Other(format!("Failed to load specs from directory: {}", e))
1994 })?;
1995 all_specs.extend(dir_specs);
1996 }
1997
1998 if all_specs.is_empty() {
1999 return Err(BenchError::Other(
2000 "No spec files found for sequential execution".to_string(),
2001 ));
2002 }
2003
2004 TerminalReporter::print_success(&format!("Loaded {} spec(s)", all_specs.len()));
2005
2006 let execution_order = if let Some(config_path) = &self.dependency_config {
2008 TerminalReporter::print_progress("Loading dependency configuration...");
2009 let config = SpecDependencyConfig::from_file(config_path)?;
2010
2011 if !config.disable_auto_detect && config.execution_order.is_empty() {
2012 self.detect_and_sort_specs(&all_specs)?
2014 } else {
2015 config.execution_order.iter().flat_map(|g| g.specs.clone()).collect()
2017 }
2018 } else {
2019 self.detect_and_sort_specs(&all_specs)?
2021 };
2022
2023 TerminalReporter::print_success(&format!(
2024 "Execution order: {}",
2025 execution_order
2026 .iter()
2027 .map(|p| p.file_name().unwrap_or_default().to_string_lossy().to_string())
2028 .collect::<Vec<_>>()
2029 .join(" → ")
2030 ));
2031
2032 let mut extracted_values = ExtractedValues::new();
2034 let total_specs = execution_order.len();
2035
2036 for (index, spec_path) in execution_order.iter().enumerate() {
2037 let spec_name = spec_path.file_name().unwrap_or_default().to_string_lossy().to_string();
2038
2039 TerminalReporter::print_progress(&format!(
2040 "[{}/{}] Executing spec: {}",
2041 index + 1,
2042 total_specs,
2043 spec_name
2044 ));
2045
2046 let spec = all_specs
2048 .iter()
2049 .find(|(p, _)| {
2050 p == spec_path
2051 || p.file_name() == spec_path.file_name()
2052 || p.file_name() == Some(spec_path.as_os_str())
2053 })
2054 .map(|(_, s)| s.clone())
2055 .ok_or_else(|| {
2056 BenchError::Other(format!("Spec not found: {}", spec_path.display()))
2057 })?;
2058
2059 let new_values = self.execute_single_spec(&spec, &spec_name, &extracted_values).await?;
2061
2062 extracted_values.merge(&new_values);
2064
2065 TerminalReporter::print_success(&format!(
2066 "[{}/{}] Completed: {} (extracted {} values)",
2067 index + 1,
2068 total_specs,
2069 spec_name,
2070 new_values.values.len()
2071 ));
2072 }
2073
2074 TerminalReporter::print_success(&format!(
2075 "Sequential execution complete: {} specs executed",
2076 total_specs
2077 ));
2078
2079 Ok(())
2080 }
2081
2082 fn detect_and_sort_specs(&self, specs: &[(PathBuf, OpenApiSpec)]) -> Result<Vec<PathBuf>> {
2084 TerminalReporter::print_progress("Auto-detecting spec dependencies...");
2085
2086 let mut detector = DependencyDetector::new();
2087 let dependencies = detector.detect_dependencies(specs);
2088
2089 if dependencies.is_empty() {
2090 TerminalReporter::print_progress("No dependencies detected, using file order");
2091 return Ok(specs.iter().map(|(p, _)| p.clone()).collect());
2092 }
2093
2094 TerminalReporter::print_progress(&format!(
2095 "Detected {} cross-spec dependencies",
2096 dependencies.len()
2097 ));
2098
2099 for dep in &dependencies {
2100 TerminalReporter::print_progress(&format!(
2101 " {} → {} (via field '{}')",
2102 dep.dependency_spec.file_name().unwrap_or_default().to_string_lossy(),
2103 dep.dependent_spec.file_name().unwrap_or_default().to_string_lossy(),
2104 dep.field_name
2105 ));
2106 }
2107
2108 topological_sort(specs, &dependencies)
2109 }
2110
2111 async fn execute_single_spec(
2113 &self,
2114 spec: &OpenApiSpec,
2115 spec_name: &str,
2116 _external_values: &ExtractedValues,
2117 ) -> Result<ExtractedValues> {
2118 let parser = SpecParser::from_spec(spec.clone());
2119
2120 if self.crud_flow {
2122 self.execute_crud_flow_with_extraction(&parser, spec_name).await
2124 } else {
2125 self.execute_standard_spec(&parser, spec_name).await?;
2127 Ok(ExtractedValues::new())
2128 }
2129 }
2130
2131 async fn execute_crud_flow_with_extraction(
2133 &self,
2134 parser: &SpecParser,
2135 spec_name: &str,
2136 ) -> Result<ExtractedValues> {
2137 let operations = parser.get_operations();
2138 let flows = CrudFlowDetector::detect_flows(&operations);
2139
2140 if flows.is_empty() {
2141 TerminalReporter::print_warning(&format!("No CRUD flows detected in {}", spec_name));
2142 return Ok(ExtractedValues::new());
2143 }
2144
2145 TerminalReporter::print_progress(&format!(
2146 " {} CRUD flow(s) in {}",
2147 flows.len(),
2148 spec_name
2149 ));
2150
2151 let mut handlebars = handlebars::Handlebars::new();
2153 handlebars.register_helper(
2155 "json",
2156 Box::new(
2157 |h: &handlebars::Helper,
2158 _: &handlebars::Handlebars,
2159 _: &handlebars::Context,
2160 _: &mut handlebars::RenderContext,
2161 out: &mut dyn handlebars::Output|
2162 -> handlebars::HelperResult {
2163 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2164 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2165 Ok(())
2166 },
2167 ),
2168 );
2169 let template = include_str!("templates/k6_crud_flow.hbs");
2170 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
2171
2172 let custom_headers = self.parse_headers()?;
2173 let config = self.build_crud_flow_config().unwrap_or_default();
2174
2175 let param_overrides = if let Some(params_file) = &self.params_file {
2177 let overrides = ParameterOverrides::from_file(params_file)?;
2178 Some(overrides)
2179 } else {
2180 None
2181 };
2182
2183 let duration_secs = Self::parse_duration(&self.duration)?;
2185 let scenario =
2186 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2187 let stages = scenario.generate_stages(duration_secs, self.vus);
2188
2189 let api_base_path = self.resolve_base_path(parser);
2191
2192 let mut all_headers = custom_headers.clone();
2194 if let Some(auth) = &self.auth {
2195 all_headers.insert("Authorization".to_string(), auth.clone());
2196 }
2197 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2198
2199 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2201
2202 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2203 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2207 serde_json::json!({
2208 "name": sanitized_name.clone(),
2209 "display_name": f.name,
2210 "base_path": f.base_path,
2211 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2212 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2214 let method_raw = if !parts.is_empty() {
2215 parts[0].to_uppercase()
2216 } else {
2217 "GET".to_string()
2218 };
2219 let method = if !parts.is_empty() {
2220 let m = parts[0].to_lowercase();
2221 if m == "delete" { "del".to_string() } else { m }
2223 } else {
2224 "get".to_string()
2225 };
2226 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2227 let path = if let Some(ref bp) = api_base_path {
2229 format!("{}{}", bp, raw_path)
2230 } else {
2231 raw_path.to_string()
2232 };
2233 let is_get_or_head = method == "get" || method == "head";
2234 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2236
2237 let body_value = if has_body {
2239 param_overrides.as_ref()
2240 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2241 .and_then(|oo| oo.body)
2242 .unwrap_or_else(|| serde_json::json!({}))
2243 } else {
2244 serde_json::json!({})
2245 };
2246
2247 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2249
2250 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2252 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2253
2254 serde_json::json!({
2255 "operation": s.operation,
2256 "method": method,
2257 "path": path,
2258 "extract": s.extract,
2259 "use_values": s.use_values,
2260 "use_body": s.use_body,
2261 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2262 "inject_attacks": s.inject_attacks,
2263 "attack_types": s.attack_types,
2264 "description": s.description,
2265 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2266 "is_get_or_head": is_get_or_head,
2267 "has_body": has_body,
2268 "body": processed_body.value,
2269 "body_is_dynamic": body_is_dynamic,
2270 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2271 })
2272 }).collect::<Vec<_>>(),
2273 })
2274 }).collect();
2275
2276 for flow_data in &flows_data {
2278 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2279 for step in steps {
2280 if let Some(placeholders_arr) =
2281 step.get("_placeholders").and_then(|p| p.as_array())
2282 {
2283 for p_str in placeholders_arr {
2284 if let Some(p_name) = p_str.as_str() {
2285 match p_name {
2286 "VU" => {
2287 all_placeholders.insert(DynamicPlaceholder::VU);
2288 }
2289 "Iteration" => {
2290 all_placeholders.insert(DynamicPlaceholder::Iteration);
2291 }
2292 "Timestamp" => {
2293 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2294 }
2295 "UUID" => {
2296 all_placeholders.insert(DynamicPlaceholder::UUID);
2297 }
2298 "Random" => {
2299 all_placeholders.insert(DynamicPlaceholder::Random);
2300 }
2301 "Counter" => {
2302 all_placeholders.insert(DynamicPlaceholder::Counter);
2303 }
2304 "Date" => {
2305 all_placeholders.insert(DynamicPlaceholder::Date);
2306 }
2307 "VuIter" => {
2308 all_placeholders.insert(DynamicPlaceholder::VuIter);
2309 }
2310 _ => {}
2311 }
2312 }
2313 }
2314 }
2315 }
2316 }
2317 }
2318
2319 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2321 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2322
2323 let security_testing_enabled = self.security_testing_enabled();
2325
2326 let data = serde_json::json!({
2327 "base_url": self.target,
2328 "flows": flows_data,
2329 "extract_fields": config.default_extract_fields,
2330 "duration_secs": duration_secs,
2331 "max_vus": self.vus,
2332 "auth_header": self.auth,
2333 "custom_headers": custom_headers,
2334 "skip_tls_verify": self.skip_tls_verify,
2335 "stages": stages.iter().map(|s| serde_json::json!({
2337 "duration": s.duration,
2338 "target": s.target,
2339 })).collect::<Vec<_>>(),
2340 "threshold_percentile": self.threshold_percentile,
2341 "threshold_ms": self.threshold_ms,
2342 "max_error_rate": self.max_error_rate,
2343 "abort_on_error": self.abort_on_error,
2344 "abort_on_error_rate": self.abort_on_error_rate,
2345 "headers": headers_json,
2346 "dynamic_imports": required_imports,
2347 "dynamic_globals": required_globals,
2348 "extracted_values_output_path": output_dir.join("extracted_values.json").to_string_lossy(),
2349 "security_testing_enabled": security_testing_enabled,
2351 "has_custom_headers": !custom_headers.is_empty(),
2352 });
2353
2354 let mut script = handlebars
2355 .render_template(template, &data)
2356 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2357
2358 if security_testing_enabled {
2360 script = self.generate_enhanced_script(&script)?;
2361 }
2362
2363 let script_path =
2365 self.output.join(format!("k6-{}-crud-flow.js", spec_name.replace('.', "_")));
2366
2367 std::fs::create_dir_all(self.output.clone())?;
2368 std::fs::write(&script_path, &script)?;
2369
2370 if !self.generate_only {
2371 let executor = K6Executor::new()?
2372 .with_local_ips(self.source_ips.join(","))
2373 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
2374 std::fs::create_dir_all(&output_dir)?;
2375
2376 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
2377
2378 let extracted = Self::parse_extracted_values(&output_dir)?;
2379 TerminalReporter::print_progress(&format!(
2380 " Extracted {} value(s) from {}",
2381 extracted.values.len(),
2382 spec_name
2383 ));
2384 return Ok(extracted);
2385 }
2386
2387 Ok(ExtractedValues::new())
2388 }
2389
2390 async fn execute_standard_spec(&self, parser: &SpecParser, spec_name: &str) -> Result<()> {
2392 let mut operations = if let Some(filter) = &self.operations {
2393 parser.filter_operations(filter)?
2394 } else {
2395 parser.get_operations()
2396 };
2397
2398 if let Some(exclude) = &self.exclude_operations {
2399 operations = parser.exclude_operations(operations, exclude)?;
2400 }
2401
2402 if operations.is_empty() {
2403 TerminalReporter::print_warning(&format!("No operations found in {}", spec_name));
2404 return Ok(());
2405 }
2406
2407 TerminalReporter::print_progress(&format!(
2408 " {} operations in {}",
2409 operations.len(),
2410 spec_name
2411 ));
2412
2413 let templates: Vec<_> = operations
2415 .iter()
2416 .map(RequestGenerator::generate_template)
2417 .collect::<Result<Vec<_>>>()?;
2418
2419 let custom_headers = self.parse_headers()?;
2421
2422 let base_path = self.resolve_base_path(parser);
2424
2425 let scenario =
2427 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2428
2429 let security_testing_enabled = self.security_testing_enabled();
2430
2431 let force_http1 = crate::request_gen::should_force_k6_http1(
2432 self.wafbench_verbatim,
2433 &templates,
2434 &custom_headers,
2435 );
2436
2437 let k6_config = K6Config {
2438 target_url: self.target.clone(),
2439 base_path,
2440 scenario,
2441 duration_secs: Self::parse_duration(&self.duration)?,
2442 max_vus: self.vus,
2443 threshold_percentile: self.threshold_percentile.clone(),
2444 threshold_ms: self.threshold_ms,
2445 max_error_rate: self.max_error_rate,
2446 auth_header: self.auth.clone(),
2447 custom_headers,
2448 skip_tls_verify: self.skip_tls_verify,
2449 security_testing_enabled,
2450 chunked_request_bodies: self.chunked_request_bodies,
2451 target_rps: self.target_rps,
2452 no_keep_alive: self.no_keep_alive,
2453 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
2455 .into_iter()
2456 .map(|ip| ip.to_string())
2457 .collect(),
2458 geo_source_headers: if self.geo_source_headers.is_empty()
2459 && !self.geo_source_ips.is_empty()
2460 {
2461 crate::conformance::self_test::default_geo_source_headers()
2462 } else {
2463 self.geo_source_headers.clone()
2464 },
2465 };
2466
2467 let generator = K6ScriptGenerator::new(k6_config, templates)
2468 .with_abort_valve(self.abort_on_error, self.abort_on_error_rate)
2469 .with_force_http1(force_http1);
2470 let mut script = generator.generate()?;
2471
2472 let has_advanced_features = self.data_file.is_some()
2474 || self.error_rate.is_some()
2475 || self.security_test
2476 || self.parallel_create.is_some()
2477 || self.wafbench_dir.is_some();
2478
2479 if has_advanced_features {
2480 script = self.generate_enhanced_script(&script)?;
2481 }
2482
2483 let script_path = self.output.join(format!("k6-{}.js", spec_name.replace('.', "_")));
2485
2486 std::fs::create_dir_all(self.output.clone())?;
2487 std::fs::write(&script_path, &script)?;
2488
2489 if !self.generate_only {
2490 let executor = K6Executor::new()?
2493 .with_local_ips(self.source_ips.join(","))
2494 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
2495 .with_discard_response_bodies(self.discard_response_bodies)
2496 .with_force_http1(force_http1);
2497 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
2498 std::fs::create_dir_all(&output_dir)?;
2499
2500 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
2501 }
2502
2503 Ok(())
2504 }
2505
2506 async fn execute_crud_flow(&self, parser: &SpecParser) -> Result<()> {
2508 let config = self.build_crud_flow_config().unwrap_or_default();
2510
2511 let flows = if !config.flows.is_empty() {
2513 TerminalReporter::print_progress("Using custom flow configuration...");
2514 config.flows.clone()
2515 } else {
2516 TerminalReporter::print_progress("Detecting CRUD operations...");
2517 let operations = parser.get_operations();
2518 CrudFlowDetector::detect_flows(&operations)
2519 };
2520
2521 if flows.is_empty() {
2522 return Err(BenchError::Other(
2523 "No CRUD flows detected in spec. Ensure spec has POST/GET/PUT/DELETE operations on related paths.".to_string(),
2524 ));
2525 }
2526
2527 if config.flows.is_empty() {
2528 TerminalReporter::print_success(&format!("Detected {} CRUD flow(s)", flows.len()));
2529 } else {
2530 TerminalReporter::print_success(&format!("Loaded {} custom flow(s)", flows.len()));
2531 }
2532
2533 for flow in &flows {
2534 TerminalReporter::print_progress(&format!(
2535 " - {}: {} steps",
2536 flow.name,
2537 flow.steps.len()
2538 ));
2539 }
2540
2541 let mut handlebars = handlebars::Handlebars::new();
2543 handlebars.register_helper(
2545 "json",
2546 Box::new(
2547 |h: &handlebars::Helper,
2548 _: &handlebars::Handlebars,
2549 _: &handlebars::Context,
2550 _: &mut handlebars::RenderContext,
2551 out: &mut dyn handlebars::Output|
2552 -> handlebars::HelperResult {
2553 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2554 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2555 Ok(())
2556 },
2557 ),
2558 );
2559 let template = include_str!("templates/k6_crud_flow.hbs");
2560
2561 let custom_headers = self.parse_headers()?;
2562
2563 let param_overrides = if let Some(params_file) = &self.params_file {
2565 TerminalReporter::print_progress("Loading parameter overrides...");
2566 let overrides = ParameterOverrides::from_file(params_file)?;
2567 TerminalReporter::print_success(&format!(
2568 "Loaded parameter overrides ({} operation-specific, {} defaults)",
2569 overrides.operations.len(),
2570 if overrides.defaults.is_empty() { 0 } else { 1 }
2571 ));
2572 Some(overrides)
2573 } else {
2574 None
2575 };
2576
2577 let duration_secs = Self::parse_duration(&self.duration)?;
2579 let scenario =
2580 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2581 let stages = scenario.generate_stages(duration_secs, self.vus);
2582
2583 let api_base_path = self.resolve_base_path(parser);
2585 if let Some(ref bp) = api_base_path {
2586 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
2587 }
2588
2589 let mut all_headers = custom_headers.clone();
2591 if let Some(auth) = &self.auth {
2592 all_headers.insert("Authorization".to_string(), auth.clone());
2593 }
2594 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2595
2596 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2598
2599 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2600 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2605 serde_json::json!({
2606 "name": sanitized_name.clone(), "display_name": f.name, "base_path": f.base_path,
2609 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2610 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2612 let method_raw = if !parts.is_empty() {
2613 parts[0].to_uppercase()
2614 } else {
2615 "GET".to_string()
2616 };
2617 let method = if !parts.is_empty() {
2618 let m = parts[0].to_lowercase();
2619 if m == "delete" { "del".to_string() } else { m }
2621 } else {
2622 "get".to_string()
2623 };
2624 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2625 let path = if let Some(ref bp) = api_base_path {
2627 format!("{}{}", bp, raw_path)
2628 } else {
2629 raw_path.to_string()
2630 };
2631 let is_get_or_head = method == "get" || method == "head";
2632 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2634
2635 let body_value = if has_body {
2637 param_overrides.as_ref()
2638 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2639 .and_then(|oo| oo.body)
2640 .unwrap_or_else(|| serde_json::json!({}))
2641 } else {
2642 serde_json::json!({})
2643 };
2644
2645 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2647 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2652 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2653
2654 serde_json::json!({
2655 "operation": s.operation,
2656 "method": method,
2657 "path": path,
2658 "extract": s.extract,
2659 "use_values": s.use_values,
2660 "use_body": s.use_body,
2661 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2662 "inject_attacks": s.inject_attacks,
2663 "attack_types": s.attack_types,
2664 "description": s.description,
2665 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2666 "is_get_or_head": is_get_or_head,
2667 "has_body": has_body,
2668 "body": processed_body.value,
2669 "body_is_dynamic": body_is_dynamic,
2670 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2671 })
2672 }).collect::<Vec<_>>(),
2673 })
2674 }).collect();
2675
2676 for flow_data in &flows_data {
2678 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2679 for step in steps {
2680 if let Some(placeholders_arr) =
2681 step.get("_placeholders").and_then(|p| p.as_array())
2682 {
2683 for p_str in placeholders_arr {
2684 if let Some(p_name) = p_str.as_str() {
2685 match p_name {
2687 "VU" => {
2688 all_placeholders.insert(DynamicPlaceholder::VU);
2689 }
2690 "Iteration" => {
2691 all_placeholders.insert(DynamicPlaceholder::Iteration);
2692 }
2693 "Timestamp" => {
2694 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2695 }
2696 "UUID" => {
2697 all_placeholders.insert(DynamicPlaceholder::UUID);
2698 }
2699 "Random" => {
2700 all_placeholders.insert(DynamicPlaceholder::Random);
2701 }
2702 "Counter" => {
2703 all_placeholders.insert(DynamicPlaceholder::Counter);
2704 }
2705 "Date" => {
2706 all_placeholders.insert(DynamicPlaceholder::Date);
2707 }
2708 "VuIter" => {
2709 all_placeholders.insert(DynamicPlaceholder::VuIter);
2710 }
2711 _ => {}
2712 }
2713 }
2714 }
2715 }
2716 }
2717 }
2718 }
2719
2720 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2722 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2723
2724 let invalid_data_config = self.build_invalid_data_config();
2726 let error_injection_enabled = invalid_data_config.is_some();
2727 let error_rate = self.error_rate.unwrap_or(0.0);
2728 let error_types: Vec<String> = invalid_data_config
2729 .as_ref()
2730 .map(|c| c.error_types.iter().map(|t| format!("{:?}", t)).collect())
2731 .unwrap_or_default();
2732
2733 if error_injection_enabled {
2734 TerminalReporter::print_progress(&format!(
2735 "Error injection enabled ({}% rate)",
2736 (error_rate * 100.0) as u32
2737 ));
2738 }
2739
2740 let security_testing_enabled = self.security_testing_enabled();
2742
2743 let data = serde_json::json!({
2744 "base_url": self.target,
2745 "flows": flows_data,
2746 "extract_fields": config.default_extract_fields,
2747 "duration_secs": duration_secs,
2748 "max_vus": self.vus,
2749 "auth_header": self.auth,
2750 "custom_headers": custom_headers,
2751 "skip_tls_verify": self.skip_tls_verify,
2752 "stages": stages.iter().map(|s| serde_json::json!({
2754 "duration": s.duration,
2755 "target": s.target,
2756 })).collect::<Vec<_>>(),
2757 "threshold_percentile": self.threshold_percentile,
2758 "threshold_ms": self.threshold_ms,
2759 "max_error_rate": self.max_error_rate,
2760 "abort_on_error": self.abort_on_error,
2761 "abort_on_error_rate": self.abort_on_error_rate,
2762 "headers": headers_json,
2763 "dynamic_imports": required_imports,
2764 "dynamic_globals": required_globals,
2765 "extracted_values_output_path": self
2766 .output
2767 .join("crud_flow_extracted_values.json")
2768 .to_string_lossy(),
2769 "error_injection_enabled": error_injection_enabled,
2771 "error_rate": error_rate,
2772 "error_types": error_types,
2773 "security_testing_enabled": security_testing_enabled,
2775 "has_custom_headers": !custom_headers.is_empty(),
2776 });
2777
2778 let mut script = handlebars
2779 .render_template(template, &data)
2780 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2781
2782 if security_testing_enabled {
2784 script = self.generate_enhanced_script(&script)?;
2785 }
2786
2787 TerminalReporter::print_progress("Validating CRUD flow script...");
2789 let validation_errors = K6ScriptGenerator::validate_script(&script);
2790 if !validation_errors.is_empty() {
2791 TerminalReporter::print_error("CRUD flow script validation failed");
2792 for error in &validation_errors {
2793 eprintln!(" {}", error);
2794 }
2795 return Err(BenchError::Other(format!(
2796 "CRUD flow script validation failed with {} error(s)",
2797 validation_errors.len()
2798 )));
2799 }
2800
2801 TerminalReporter::print_success("CRUD flow script generated");
2802
2803 let script_path = if let Some(output) = &self.script_output {
2805 output.clone()
2806 } else {
2807 self.output.join("k6-crud-flow-script.js")
2808 };
2809
2810 if let Some(parent) = script_path.parent() {
2811 std::fs::create_dir_all(parent)?;
2812 }
2813 std::fs::write(&script_path, &script)?;
2814 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
2815
2816 if self.generate_only {
2817 println!("\nScript generated successfully. Run it with:");
2818 println!(" k6 run {}", script_path.display());
2819 return Ok(());
2820 }
2821
2822 TerminalReporter::print_progress("Executing CRUD flow test...");
2824 let executor = K6Executor::new()?
2825 .with_local_ips(self.source_ips.join(","))
2826 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
2827 std::fs::create_dir_all(&self.output)?;
2828
2829 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
2830
2831 let duration_secs = Self::parse_duration(&self.duration)?;
2832 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
2833
2834 Ok(())
2835 }
2836
2837 async fn execute_conformance_test(&self) -> Result<()> {
2839 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
2840 use crate::conformance::report::ConformanceReport;
2841 use crate::conformance::spec::ConformanceFeature;
2842
2843 TerminalReporter::print_progress("OpenAPI 3.0.0 Conformance Testing Mode");
2844
2845 TerminalReporter::print_progress(CONFORMANCE_REPLACES_LOAD_ADVISORY);
2846
2847 let categories = self.conformance_categories.as_ref().map(|cats_str| {
2849 cats_str
2850 .split(',')
2851 .filter_map(|s| {
2852 let trimmed = s.trim();
2853 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
2854 Some(canonical.to_string())
2855 } else {
2856 TerminalReporter::print_warning(&format!(
2857 "Unknown conformance category: '{}'. Valid categories: {}",
2858 trimmed,
2859 ConformanceFeature::cli_category_names()
2860 .iter()
2861 .map(|(cli, _)| *cli)
2862 .collect::<Vec<_>>()
2863 .join(", ")
2864 ));
2865 None
2866 }
2867 })
2868 .collect::<Vec<String>>()
2869 });
2870
2871 let custom_headers: Vec<(String, String)> = self
2873 .conformance_headers
2874 .iter()
2875 .filter_map(|h| {
2876 let (name, value) = h.split_once(':')?;
2877 Some((name.trim().to_string(), value.trim().to_string()))
2878 })
2879 .collect();
2880
2881 if !custom_headers.is_empty() {
2882 TerminalReporter::print_progress(&format!(
2883 "Using {} custom header(s) for authentication",
2884 custom_headers.len()
2885 ));
2886 }
2887
2888 if self.conformance_delay_ms > 0 {
2889 TerminalReporter::print_progress(&format!(
2890 "Using {}ms delay between conformance requests",
2891 self.conformance_delay_ms
2892 ));
2893 }
2894
2895 std::fs::create_dir_all(&self.output)?;
2897
2898 let config = ConformanceConfig {
2899 target_url: self.target.clone(),
2900 api_key: self.conformance_api_key.clone(),
2901 basic_auth: self.conformance_basic_auth.clone(),
2902 skip_tls_verify: self.skip_tls_verify,
2903 categories,
2904 base_path: self.base_path.clone(),
2905 custom_headers,
2906 output_dir: Some(self.output.clone()),
2907 all_operations: self.conformance_all_operations,
2908 custom_checks_file: self.conformance_custom.clone(),
2909 request_delay_ms: self.conformance_delay_ms,
2910 custom_filter: self.conformance_custom_filter.clone(),
2911 export_requests: self.export_requests,
2912 validate_requests: self.validate_requests,
2913 };
2914
2915 let mut resolved_base_path: Option<String> = None;
2923 let annotated_ops = if !self.spec.is_empty() {
2924 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
2925 let parser = SpecParser::from_file(&self.spec[0]).await?;
2926 resolved_base_path = self.resolve_base_path(&parser);
2927
2928 let mut operations = if let Some(filter) = &self.operations {
2933 parser.filter_operations(filter)?
2934 } else {
2935 parser.get_operations()
2936 };
2937 if let Some(exclude) = &self.exclude_operations {
2938 let before_count = operations.len();
2939 operations = parser.exclude_operations(operations, exclude)?;
2940 let excluded_count = before_count - operations.len();
2941 if excluded_count > 0 {
2942 TerminalReporter::print_progress(&format!(
2943 "Excluded {} operations matching '{}'",
2944 excluded_count, exclude
2945 ));
2946 }
2947 }
2948
2949 let annotated =
2950 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
2951 &operations,
2952 parser.spec(),
2953 );
2954 TerminalReporter::print_success(&format!(
2955 "Analyzed {} operations, found {} feature annotations",
2956 operations.len(),
2957 annotated.iter().map(|a| a.features.len()).sum::<usize>()
2958 ));
2959 Some(annotated)
2960 } else {
2961 None
2962 };
2963
2964 if self.conformance_self_test {
2971 let Some(ops) = annotated_ops else {
2972 TerminalReporter::print_error(
2973 "--conformance-self-test requires --spec; no operations to test",
2974 );
2975 return Ok(());
2976 };
2977 let cfg = crate::conformance::self_test::SelfTestConfig {
2978 target_url: self.target.clone(),
2979 skip_tls_verify: self.skip_tls_verify,
2980 timeout: std::time::Duration::from_secs(30),
2981 extra_headers: self
2985 .conformance_headers
2986 .iter()
2987 .filter_map(|h| {
2988 let (n, v) = h.split_once(':')?;
2989 Some((n.trim().to_string(), v.trim().to_string()))
2990 })
2991 .collect(),
2992 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
2993 base_path: resolved_base_path.clone(),
2997 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
3001 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
3002 geo_source_headers: if self.geo_source_headers.is_empty() {
3003 crate::conformance::self_test::default_geo_source_headers()
3004 } else {
3005 self.geo_source_headers.clone()
3006 },
3007 capture: if self.conformance_self_test_capture
3011 || self.validate_response_schemas
3012 || self.validate_requests
3013 {
3014 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
3025 } else {
3026 None
3027 },
3028 validate_response_schemas: self.validate_response_schemas,
3029 spec_label: self.spec.first().map(|p| {
3035 p.file_name()
3036 .map(|s| s.to_string_lossy().into_owned())
3037 .unwrap_or_else(|| p.to_string_lossy().into_owned())
3038 }),
3039 network_events: Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))),
3046 current_iteration: 1,
3047 };
3048 let capture_sink = cfg.capture.clone();
3049 let network_events_sink = cfg.network_events.clone();
3050 TerminalReporter::print_progress(&format!(
3051 "Self-test mode: driving {} operations with positive + per-category negative cases",
3052 ops.len()
3053 ));
3054 let target_iterations = self.conformance_self_test_iterations.max(1);
3061 let duration_budget = self
3062 .conformance_self_test_duration
3063 .as_ref()
3064 .map(|s| Self::parse_duration(s))
3065 .transpose()?
3066 .map(std::time::Duration::from_secs);
3067 let start = std::time::Instant::now();
3068 let deadline = duration_budget.map(|d| start + d);
3077 let mut cfg = cfg;
3081 cfg.current_iteration = 1;
3082 let mut report =
3083 crate::conformance::self_test::run_self_test_with_deadline(&ops, &cfg, deadline)
3084 .await
3085 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3086 let mut iter_done: u32 = 1;
3087 loop {
3088 let by_iter = iter_done >= target_iterations;
3089 let by_dur = duration_budget.map(|d| start.elapsed() >= d).unwrap_or(true);
3090 if by_iter && by_dur {
3091 break;
3092 }
3093 cfg.current_iteration = iter_done.saturating_add(1);
3094 let next = crate::conformance::self_test::run_self_test_with_deadline(
3095 &ops, &cfg, deadline,
3096 )
3097 .await
3098 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3099 report.merge_iteration(next);
3100 iter_done = iter_done.saturating_add(1);
3101 }
3102 if iter_done > 1 {
3103 TerminalReporter::print_progress(&format!(
3104 "Self-test repeated {} iteration(s) ({:.1?} elapsed)",
3105 iter_done,
3106 start.elapsed(),
3107 ));
3108 }
3109 let per_endpoint_summary: Vec<
3119 crate::conformance::per_endpoint_summary::PerEndpointSummary,
3120 >;
3121 if let Some(sink) = capture_sink {
3122 if let Ok(guard) = sink.lock() {
3123 let jsonl_path = self.output.join("conformance-self-test-requests.jsonl");
3124 let mut lines = String::with_capacity(guard.len() * 256);
3125 for entry in guard.iter() {
3126 if let Ok(line) = serde_json::to_string(entry) {
3127 lines.push_str(&line);
3128 lines.push('\n');
3129 }
3130 }
3131 let _ = std::fs::write(&jsonl_path, lines);
3132 let html_path = self.output.join("conformance-self-test-requests.html");
3133 let html =
3134 crate::conformance::capture_html::render_capture_html(guard.as_slice());
3135 let _ = std::fs::write(&html_path, html);
3136
3137 per_endpoint_summary =
3141 crate::conformance::per_endpoint_summary::build_summary(guard.as_slice());
3142 let summary_path = self.output.join("conformance-per-endpoint.json");
3143 if let Ok(json) = serde_json::to_string_pretty(&per_endpoint_summary) {
3144 let _ = std::fs::write(&summary_path, json);
3145 TerminalReporter::print_progress(&format!(
3146 "Self-test request/response capture written to {} ({} entries) + {} + {}",
3147 jsonl_path.display(),
3148 guard.len(),
3149 html_path.display(),
3150 summary_path.display(),
3151 ));
3152 } else {
3153 TerminalReporter::print_progress(&format!(
3154 "Self-test request/response capture written to {} ({} entries) + {}",
3155 jsonl_path.display(),
3156 guard.len(),
3157 html_path.display(),
3158 ));
3159 }
3160 } else {
3161 per_endpoint_summary = Vec::new();
3162 }
3163 } else {
3164 per_endpoint_summary = Vec::new();
3165 }
3166 TerminalReporter::print_progress(&report.render_summary());
3167 if let Some(sink) = network_events_sink {
3174 if let Ok(guard) = sink.lock() {
3175 let path = self.output.join("conformance-network-events.json");
3176 if let Ok(json) = serde_json::to_string_pretty(&*guard) {
3177 let _ = std::fs::write(&path, json);
3178 if guard.is_empty() {
3179 TerminalReporter::print_progress(
3180 "No wire-level network failures during self-test (file written empty)",
3181 );
3182 } else {
3183 TerminalReporter::print_warning(&format!(
3184 "Recorded {} wire-level network event(s) to {}",
3185 guard.len(),
3186 path.display()
3187 ));
3188 }
3189 }
3190 }
3191 }
3192 let json_path = self.output.join("conformance-self-test.json");
3196 if let Ok(json) = serde_json::to_string_pretty(&report) {
3197 let _ = std::fs::write(&json_path, json);
3198 TerminalReporter::print_progress(&format!(
3199 "Self-test report written to {}",
3200 json_path.display()
3201 ));
3202 }
3203 let issues = report.definite_issues();
3207 let issues_path = self.output.join("conformance-definite-issues.json");
3208 if let Ok(json) = serde_json::to_string_pretty(&issues) {
3209 if std::fs::write(&issues_path, json).is_ok() && !issues.is_empty() {
3210 TerminalReporter::print_warning(&format!(
3211 "{} definite issue(s) — see {}",
3212 issues.len(),
3213 issues_path.display()
3214 ));
3215 }
3216 }
3217 let owasp_accepted = report.owasp_accepted_probes();
3220 if !owasp_accepted.is_empty() {
3221 let owasp_path = self.output.join("conformance-owasp-accepted.json");
3222 if let Ok(json) = serde_json::to_string_pretty(&owasp_accepted) {
3223 if std::fs::write(&owasp_path, json).is_ok() {
3224 TerminalReporter::print_warning(&format!(
3225 "{} owasp injection probe(s) accepted by the target — see {}",
3226 owasp_accepted.len(),
3227 owasp_path.display()
3228 ));
3229 }
3230 }
3231 }
3232 if let Some(status) = report.detect_target_misconfiguration() {
3241 let hint = match status {
3242 404 => " Likely cause: spec paths don't match deployed routes — check --base-path and the spec's `servers` block.",
3243 401 | 403 => " Likely cause: authentication header is missing or invalid — check --conformance-header.",
3244 _ => "",
3245 };
3246 TerminalReporter::print_warning(&format!(
3247 "Self-test misconfiguration: every positive case returned {status}.{hint} Negative results below are meaningless under this condition."
3248 ));
3249 } else if !report.all_passed() {
3250 TerminalReporter::print_warning(
3251 "Self-test detected gaps — server let through at least one request that should have been a 4xx",
3252 );
3253 } else {
3254 TerminalReporter::print_success(
3255 "Self-test passed — all positive cases accepted and all negative cases rejected",
3256 );
3257 }
3258 let html_path = self.output.join("conformance-report.html");
3265 let audit_path = self.output.join("conformance-spec-audit.json");
3266 let audit_value = std::fs::read_to_string(&audit_path)
3267 .ok()
3268 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok());
3269 let render_opts = crate::conformance::report_html::RenderOptions {
3274 missed_cap: match self.report_missed_cap {
3275 Some(0) => None,
3276 Some(n) => Some(n as usize),
3277 None => Some(200),
3278 },
3279 };
3280 let mut html = crate::conformance::report_html::render_html_with_options(
3281 &report,
3282 audit_value.as_ref(),
3283 &render_opts,
3284 );
3285 let summary_section = crate::conformance::per_endpoint_summary::render_html_section(
3291 &per_endpoint_summary,
3292 );
3293 if !summary_section.is_empty() {
3294 if let Some(idx) = html.rfind("</body>") {
3295 html.insert_str(idx, &summary_section);
3296 } else {
3297 html.push_str(&summary_section);
3298 }
3299 }
3300 if std::fs::write(&html_path, html).is_ok() {
3301 TerminalReporter::print_progress(&format!(
3302 "HTML report written to {}",
3303 html_path.display()
3304 ));
3305 }
3306
3307 if self.validate_requests && !self.spec.is_empty() {
3319 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3320 &self.spec,
3321 &self.output,
3322 self.base_path.as_deref(),
3323 )
3324 .await?;
3325 if n > 0 {
3326 TerminalReporter::print_warning(&format!(
3327 "{} emitted request(s) recorded against the spec — see conformance-request-violations.json",
3328 n
3329 ));
3330 }
3331 }
3332 return Ok(());
3333 }
3334
3335 if self.validate_requests && !self.spec.is_empty() {
3337 TerminalReporter::print_progress("Validating requests against OpenAPI spec...");
3338 let violation_count = crate::conformance::request_validator::run_request_validation(
3339 &self.spec,
3340 self.conformance_custom.as_deref(),
3341 self.base_path.as_deref(),
3342 &self.output,
3343 )
3344 .await?;
3345 if violation_count > 0 {
3346 TerminalReporter::print_warning(&format!(
3347 "{} request validation violation(s) found — see conformance-request-violations.json",
3348 violation_count
3349 ));
3350 } else {
3351 TerminalReporter::print_success("All requests conform to the OpenAPI spec");
3352 }
3353 }
3354
3355 if self.generate_only || self.use_k6 {
3357 let script = if let Some(annotated) = &annotated_ops {
3358 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
3359 config,
3360 annotated.clone(),
3361 );
3362 let op_count = gen.operation_count();
3363 let (script, check_count) = gen.generate()?;
3364 TerminalReporter::print_success(&format!(
3365 "Conformance: {} operations analyzed, {} unique checks generated",
3366 op_count, check_count
3367 ));
3368 script
3369 } else {
3370 let generator = ConformanceGenerator::new(config);
3371 generator.generate()?
3372 };
3373
3374 let script_path = self.output.join("k6-conformance.js");
3375 std::fs::write(&script_path, &script).map_err(|e| {
3376 BenchError::Other(format!("Failed to write conformance script: {}", e))
3377 })?;
3378 TerminalReporter::print_success(&format!(
3379 "Conformance script generated: {}",
3380 script_path.display()
3381 ));
3382
3383 if self.generate_only {
3384 println!("\nScript generated. Run with:");
3385 println!(" k6 run {}", script_path.display());
3386 return Ok(());
3387 }
3388
3389 if !K6Executor::is_k6_installed() {
3391 TerminalReporter::print_error("k6 is not installed");
3392 TerminalReporter::print_warning(
3393 "Install k6 from: https://k6.io/docs/get-started/installation/",
3394 );
3395 return Err(BenchError::K6NotFound);
3396 }
3397
3398 K6Executor::warn_if_pre_v1().await;
3399 TerminalReporter::print_progress("Running conformance tests via k6...");
3400 let executor = K6Executor::new()?
3401 .with_local_ips(self.source_ips.join(","))
3402 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
3403 executor.execute(&script_path, Some(&self.output), self.verbose).await?;
3404
3405 let report_path = self.output.join("conformance-report.json");
3406 if report_path.exists() {
3407 let report = ConformanceReport::from_file(&report_path)?;
3408 report.print_report_with_options(self.conformance_all_operations);
3409 self.save_conformance_report(&report, &report_path)?;
3410 } else {
3411 TerminalReporter::print_warning(
3412 "Conformance report not generated (k6 handleSummary may not have run)",
3413 );
3414 }
3415
3416 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
3428 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3429 &self.spec,
3430 &self.output,
3431 self.base_path.as_deref(),
3432 )
3433 .await?;
3434 if n > 0 {
3435 TerminalReporter::print_warning(&format!(
3436 "{} emitted request(s) violated the spec — see conformance-request-violations.json",
3437 n
3438 ));
3439 }
3440 }
3441
3442 return Ok(());
3443 }
3444
3445 TerminalReporter::print_progress("Running conformance tests (native executor)...");
3447
3448 let mut executor = crate::conformance::executor::NativeConformanceExecutor::new(config)?;
3449
3450 let custom_only = annotated_ops.is_none() && self.conformance_custom.is_some();
3460 executor = if let Some(annotated) = &annotated_ops {
3461 executor.with_spec_driven_checks(annotated)
3462 } else if custom_only {
3463 executor
3464 } else {
3465 executor.with_reference_checks()
3466 };
3467 executor = executor.with_custom_checks()?;
3468
3469 TerminalReporter::print_success(&format!(
3470 "Executing {} conformance checks...",
3471 executor.check_count()
3472 ));
3473
3474 let report = executor.execute().await?;
3475 report.print_report_with_options(self.conformance_all_operations);
3476
3477 let failure_details = report.failure_details();
3479 if !failure_details.is_empty() {
3480 let details_path = self.output.join("conformance-failure-details.json");
3481 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
3482 let _ = std::fs::write(&details_path, json);
3483 TerminalReporter::print_success(&format!(
3484 "Failure details saved to: {}",
3485 details_path.display()
3486 ));
3487 }
3488 }
3489
3490 let report_path = self.output.join("conformance-report.json");
3492 let report_json = serde_json::to_string_pretty(&report.to_json())
3493 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
3494 std::fs::write(&report_path, &report_json)
3495 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
3496 TerminalReporter::print_success(&format!("Report saved to: {}", report_path.display()));
3497
3498 self.save_conformance_report(&report, &report_path)?;
3499
3500 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
3511 let n =
3512 crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3513 &self.spec,
3514 &self.output,
3515 self.base_path.as_deref(),
3516 )
3517 .await?;
3518 if n > 0 {
3519 TerminalReporter::print_warning(&format!(
3520 "{} emitted request(s) violated the spec — see conformance-request-violations.json",
3521 n
3522 ));
3523 }
3524 }
3525
3526 Ok(())
3527 }
3528
3529 fn save_conformance_report(
3531 &self,
3532 report: &crate::conformance::report::ConformanceReport,
3533 report_path: &Path,
3534 ) -> Result<()> {
3535 if self.conformance_report_format == "sarif" {
3536 use crate::conformance::sarif::ConformanceSarifReport;
3537 ConformanceSarifReport::write(report, &self.target, &self.conformance_report)?;
3538 TerminalReporter::print_success(&format!(
3539 "SARIF report saved to: {}",
3540 self.conformance_report.display()
3541 ));
3542 } else if self.conformance_report != *report_path {
3543 std::fs::copy(report_path, &self.conformance_report)?;
3544 TerminalReporter::print_success(&format!(
3545 "Report saved to: {}",
3546 self.conformance_report.display()
3547 ));
3548 }
3549 Ok(())
3550 }
3551
3552 async fn execute_multi_target_self_test(&self, targets_file: &Path) -> Result<()> {
3564 use crate::conformance::self_test::SelfTestConfig;
3565
3566 TerminalReporter::print_progress("Multi-target conformance self-test mode");
3567 let targets = parse_targets_file(targets_file)?;
3568 if targets.is_empty() {
3569 return Err(BenchError::Other("No targets found in file".to_string()));
3570 }
3571 TerminalReporter::print_success(&format!("Loaded {} target(s)", targets.len()));
3572
3573 let annotated_ops = if !self.spec.is_empty() {
3575 let parser = SpecParser::from_file(&self.spec[0]).await?;
3576 let operations = parser.get_operations();
3577 Some(
3578 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
3579 &operations,
3580 parser.spec(),
3581 ),
3582 )
3583 } else {
3584 return Err(BenchError::Other("--conformance-self-test requires --spec".to_string()));
3585 };
3586 let Some(ops) = annotated_ops else {
3587 unreachable!()
3588 };
3589
3590 std::fs::create_dir_all(&self.output)?;
3591 let resolved_base_path = self.base_path.clone();
3592 let target_iterations = self.conformance_self_test_iterations.max(1);
3593 let duration_budget = self
3594 .conformance_self_test_duration
3595 .as_ref()
3596 .map(|s| Self::parse_duration(s))
3597 .transpose()?
3598 .map(std::time::Duration::from_secs);
3599
3600 for (idx, target) in targets.iter().enumerate() {
3601 let target_dir = self.output.join(format!("target_{}", idx));
3602 std::fs::create_dir_all(&target_dir)?;
3603 TerminalReporter::print_progress(&format!(
3604 "[target {}/{}] {}",
3605 idx + 1,
3606 targets.len(),
3607 target.url
3608 ));
3609
3610 let merged_headers: Vec<(String, String)> = self
3611 .conformance_headers
3612 .iter()
3613 .filter_map(|h| {
3614 let (n, v) = h.split_once(':')?;
3615 Some((n.trim().to_string(), v.trim().to_string()))
3616 })
3617 .collect();
3618
3619 let cfg = SelfTestConfig {
3620 target_url: target.url.clone(),
3621 skip_tls_verify: self.skip_tls_verify,
3622 timeout: std::time::Duration::from_secs(30),
3623 extra_headers: merged_headers,
3624 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
3625 base_path: resolved_base_path.clone(),
3626 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
3627 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
3628 geo_source_headers: if self.geo_source_headers.is_empty() {
3629 crate::conformance::self_test::default_geo_source_headers()
3630 } else {
3631 self.geo_source_headers.clone()
3632 },
3633 capture: if self.conformance_self_test_capture
3634 || self.validate_response_schemas
3635 || self.validate_requests
3636 {
3637 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
3641 } else {
3642 None
3643 },
3644 validate_response_schemas: self.validate_response_schemas,
3645 spec_label: self.spec.first().map(|p| {
3646 p.file_name()
3647 .map(|s| s.to_string_lossy().into_owned())
3648 .unwrap_or_else(|| p.to_string_lossy().into_owned())
3649 }),
3650 network_events: Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))),
3651 current_iteration: 1,
3652 };
3653 let capture_sink = cfg.capture.clone();
3654 let network_events_sink = cfg.network_events.clone();
3655
3656 let start = std::time::Instant::now();
3657 let deadline = duration_budget.map(|d| start + d);
3661 let mut cfg = cfg;
3665 cfg.current_iteration = 1;
3666 let mut report =
3667 crate::conformance::self_test::run_self_test_with_deadline(&ops, &cfg, deadline)
3668 .await
3669 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3670 let mut iter_done: u32 = 1;
3671 loop {
3672 let by_iter = iter_done >= target_iterations;
3673 let by_dur = duration_budget.map(|d| start.elapsed() >= d).unwrap_or(true);
3674 if by_iter && by_dur {
3675 break;
3676 }
3677 cfg.current_iteration = iter_done.saturating_add(1);
3678 let next = crate::conformance::self_test::run_self_test_with_deadline(
3679 &ops, &cfg, deadline,
3680 )
3681 .await
3682 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3683 report.merge_iteration(next);
3684 iter_done = iter_done.saturating_add(1);
3685 }
3686 if iter_done > 1 {
3687 TerminalReporter::print_progress(&format!(
3688 " ran {} iteration(s) in {:.1?}",
3689 iter_done,
3690 start.elapsed(),
3691 ));
3692 }
3693
3694 if let Some(sink) = capture_sink {
3696 if let Ok(guard) = sink.lock() {
3697 let jsonl = target_dir.join("conformance-self-test-requests.jsonl");
3698 let mut lines = String::with_capacity(guard.len() * 256);
3699 for entry in guard.iter() {
3700 if let Ok(line) = serde_json::to_string(entry) {
3701 lines.push_str(&line);
3702 lines.push('\n');
3703 }
3704 }
3705 let _ = std::fs::write(&jsonl, lines);
3706 }
3707 }
3708 if let Some(sink) = network_events_sink {
3709 if let Ok(guard) = sink.lock() {
3710 let path = target_dir.join("conformance-network-events.json");
3711 if let Ok(json) = serde_json::to_string_pretty(&*guard) {
3712 let _ = std::fs::write(&path, json);
3713 if !guard.is_empty() {
3714 TerminalReporter::print_warning(&format!(
3715 " recorded {} wire-level network event(s)",
3716 guard.len()
3717 ));
3718 }
3719 }
3720 }
3721 }
3722
3723 let json_path = target_dir.join("conformance-self-test.json");
3724 if let Ok(json) = serde_json::to_string_pretty(&report) {
3725 let _ = std::fs::write(&json_path, json);
3726 }
3727 let issues = report.definite_issues();
3730 if let Ok(json) = serde_json::to_string_pretty(&issues) {
3731 let issues_path = target_dir.join("conformance-definite-issues.json");
3732 if std::fs::write(&issues_path, json).is_ok() && !issues.is_empty() {
3733 TerminalReporter::print_warning(&format!(
3734 " {} definite issue(s) — see {}",
3735 issues.len(),
3736 issues_path.display()
3737 ));
3738 }
3739 }
3740 let owasp_accepted = report.owasp_accepted_probes();
3742 if !owasp_accepted.is_empty() {
3743 if let Ok(json) = serde_json::to_string_pretty(&owasp_accepted) {
3744 let owasp_path = target_dir.join("conformance-owasp-accepted.json");
3745 if std::fs::write(&owasp_path, json).is_ok() {
3746 TerminalReporter::print_warning(&format!(
3747 " {} owasp injection probe(s) accepted by the target — see {}",
3748 owasp_accepted.len(),
3749 owasp_path.display()
3750 ));
3751 }
3752 }
3753 }
3754 TerminalReporter::print_progress(&report.render_summary());
3755
3756 if self.validate_requests {
3765 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3766 &self.spec,
3767 &target_dir,
3768 self.base_path.as_deref(),
3769 )
3770 .await?;
3771 if n > 0 {
3772 TerminalReporter::print_warning(&format!(
3773 " {} emitted request(s) violated the spec — see {}/conformance-request-violations.json",
3774 n,
3775 target_dir.display(),
3776 ));
3777 }
3778 }
3779 }
3780
3781 Ok(())
3782 }
3783
3784 async fn execute_multi_target_conformance(&self, targets_file: &Path) -> Result<()> {
3790 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
3791 use crate::conformance::report::ConformanceReport;
3792 use crate::conformance::spec::ConformanceFeature;
3793
3794 TerminalReporter::print_progress("Multi-target OpenAPI 3.0.0 Conformance Testing Mode");
3795
3796 TerminalReporter::print_progress("Parsing targets file...");
3798 let targets = parse_targets_file(targets_file)?;
3799 let num_targets = targets.len();
3800 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
3801
3802 if targets.is_empty() {
3803 return Err(BenchError::Other("No targets found in file".to_string()));
3804 }
3805
3806 TerminalReporter::print_progress(CONFORMANCE_REPLACES_LOAD_ADVISORY);
3807
3808 let categories = self.conformance_categories.as_ref().map(|cats_str| {
3810 cats_str
3811 .split(',')
3812 .filter_map(|s| {
3813 let trimmed = s.trim();
3814 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
3815 Some(canonical.to_string())
3816 } else {
3817 TerminalReporter::print_warning(&format!(
3818 "Unknown conformance category: '{}'. Valid categories: {}",
3819 trimmed,
3820 ConformanceFeature::cli_category_names()
3821 .iter()
3822 .map(|(cli, _)| *cli)
3823 .collect::<Vec<_>>()
3824 .join(", ")
3825 ));
3826 None
3827 }
3828 })
3829 .collect::<Vec<String>>()
3830 });
3831
3832 let base_custom_headers: Vec<(String, String)> = self
3834 .conformance_headers
3835 .iter()
3836 .filter_map(|h| {
3837 let (name, value) = h.split_once(':')?;
3838 Some((name.trim().to_string(), value.trim().to_string()))
3839 })
3840 .collect();
3841
3842 if !base_custom_headers.is_empty() {
3843 TerminalReporter::print_progress(&format!(
3844 "Using {} base custom header(s) for authentication",
3845 base_custom_headers.len()
3846 ));
3847 }
3848
3849 let annotated_ops = if !self.spec.is_empty() {
3851 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
3852 let parser = SpecParser::from_file(&self.spec[0]).await?;
3853 let operations = parser.get_operations();
3854 let annotated =
3855 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
3856 &operations,
3857 parser.spec(),
3858 );
3859 TerminalReporter::print_success(&format!(
3860 "Analyzed {} operations, found {} feature annotations",
3861 operations.len(),
3862 annotated.iter().map(|a| a.features.len()).sum::<usize>()
3863 ));
3864 Some(annotated)
3865 } else {
3866 None
3867 };
3868
3869 std::fs::create_dir_all(&self.output)?;
3871
3872 struct TargetResult {
3874 url: String,
3875 passed: usize,
3876 failed: usize,
3877 elapsed: std::time::Duration,
3878 report_json: serde_json::Value,
3879 owasp_coverage: Vec<crate::conformance::report::OwaspCoverageEntry>,
3880 }
3881
3882 let mut target_results: Vec<TargetResult> = Vec::with_capacity(num_targets);
3883 let total_start = std::time::Instant::now();
3884
3885 for (idx, target) in targets.iter().enumerate() {
3886 tracing::info!(
3887 "Running conformance tests against target {}/{}: {}",
3888 idx + 1,
3889 num_targets,
3890 target.url
3891 );
3892 TerminalReporter::print_progress(&format!(
3893 "\n--- Target {}/{}: {} ---",
3894 idx + 1,
3895 num_targets,
3896 target.url
3897 ));
3898
3899 let mut merged_headers = base_custom_headers.clone();
3901 if let Some(ref target_headers) = target.headers {
3902 for (name, value) in target_headers {
3903 if let Some(existing) = merged_headers.iter_mut().find(|(n, _)| n == name) {
3905 existing.1 = value.clone();
3906 } else {
3907 merged_headers.push((name.clone(), value.clone()));
3908 }
3909 }
3910 }
3911 if let Some(ref auth) = target.auth {
3913 if let Some(existing) =
3914 merged_headers.iter_mut().find(|(n, _)| n.eq_ignore_ascii_case("Authorization"))
3915 {
3916 existing.1 = auth.clone();
3917 } else {
3918 merged_headers.push(("Authorization".to_string(), auth.clone()));
3919 }
3920 }
3921
3922 let target_dir = self.output.join(format!("target_{}", idx));
3928 std::fs::create_dir_all(&target_dir)?;
3929
3930 let config = ConformanceConfig {
3931 target_url: target.url.clone(),
3932 api_key: self.conformance_api_key.clone(),
3933 basic_auth: self.conformance_basic_auth.clone(),
3934 skip_tls_verify: self.skip_tls_verify,
3935 categories: categories.clone(),
3936 base_path: self.base_path.clone(),
3937 custom_headers: merged_headers,
3938 output_dir: Some(target_dir.clone()),
3939 all_operations: self.conformance_all_operations,
3940 custom_checks_file: self.conformance_custom.clone(),
3941 request_delay_ms: self.conformance_delay_ms,
3942 custom_filter: self.conformance_custom_filter.clone(),
3943 export_requests: self.export_requests,
3944 validate_requests: self.validate_requests,
3945 };
3946
3947 let target_start = std::time::Instant::now();
3948 let report = if self.use_k6 {
3949 if !K6Executor::is_k6_installed() {
3950 TerminalReporter::print_error("k6 is not installed");
3951 TerminalReporter::print_warning(
3952 "Install k6 from: https://k6.io/docs/get-started/installation/",
3953 );
3954 return Err(BenchError::K6NotFound);
3955 }
3956 K6Executor::warn_if_pre_v1().await;
3957
3958 let script = if let Some(ref annotated) = annotated_ops {
3959 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
3960 config.clone(),
3961 annotated.clone(),
3962 );
3963 let (script, _check_count) = gen.generate()?;
3964 script
3965 } else {
3966 let generator = ConformanceGenerator::new(config.clone());
3967 generator.generate()?
3968 };
3969
3970 let script_path = target_dir.join("k6-conformance.js");
3971 std::fs::write(&script_path, &script).map_err(|e| {
3972 BenchError::Other(format!("Failed to write conformance script: {}", e))
3973 })?;
3974 TerminalReporter::print_success(&format!(
3975 "Conformance script generated: {}",
3976 script_path.display()
3977 ));
3978
3979 TerminalReporter::print_progress(&format!(
3980 "Running conformance tests via k6 against {}...",
3981 target.url
3982 ));
3983 let k6 = K6Executor::new()?
3984 .with_local_ips(self.source_ips.join(","))
3985 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
3986 let api_port = 6565u16.saturating_add(idx as u16);
3988 k6.execute_with_port(&script_path, Some(&target_dir), self.verbose, Some(api_port))
3989 .await?;
3990
3991 let report_path = target_dir.join("conformance-report.json");
3992 if report_path.exists() {
3993 ConformanceReport::from_file(&report_path)?
3994 } else {
3995 TerminalReporter::print_warning(&format!(
3996 "Conformance report not generated for target {} (k6 handleSummary may not have run)",
3997 target.url
3998 ));
3999 continue;
4000 }
4001 } else {
4002 let mut executor =
4003 crate::conformance::executor::NativeConformanceExecutor::new(config)?;
4004
4005 let custom_only = annotated_ops.is_none() && self.conformance_custom.is_some();
4008 executor = if let Some(ref annotated) = annotated_ops {
4009 executor.with_spec_driven_checks(annotated)
4010 } else if custom_only {
4011 executor
4012 } else {
4013 executor.with_reference_checks()
4014 };
4015 executor = executor.with_custom_checks()?;
4016
4017 TerminalReporter::print_success(&format!(
4018 "Executing {} conformance checks against {}...",
4019 executor.check_count(),
4020 target.url
4021 ));
4022
4023 executor.execute().await?
4024 };
4025 let target_elapsed = target_start.elapsed();
4026
4027 let report_json = report.to_json();
4028
4029 let passed = report_json["summary"]["passed"].as_u64().unwrap_or(0) as usize;
4031 let failed = report_json["summary"]["failed"].as_u64().unwrap_or(0) as usize;
4032 let total_checks = passed + failed;
4033 let rate = if total_checks == 0 {
4034 0.0
4035 } else {
4036 (passed as f64 / total_checks as f64) * 100.0
4037 };
4038
4039 TerminalReporter::print_success(&format!(
4040 "Target {}: {}/{} passed ({:.1}%) in {:.1}s",
4041 target.url,
4042 passed,
4043 total_checks,
4044 rate,
4045 target_elapsed.as_secs_f64()
4046 ));
4047
4048 let target_report_path = target_dir.join("conformance-report.json");
4050 let report_str = serde_json::to_string_pretty(&report_json)
4051 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
4052 std::fs::write(&target_report_path, &report_str)
4053 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
4054
4055 let failure_details = report.failure_details();
4057 if !failure_details.is_empty() {
4058 let details_path = target_dir.join("conformance-failure-details.json");
4059 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
4060 let _ = std::fs::write(&details_path, json);
4061 }
4062 }
4063
4064 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
4071 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
4072 &self.spec,
4073 &target_dir,
4074 self.base_path.as_deref(),
4075 )
4076 .await?;
4077 if n > 0 {
4078 TerminalReporter::print_warning(&format!(
4079 "Target {}: {} emitted request(s) violated the spec — see {}/conformance-request-violations.json",
4080 target.url,
4081 n,
4082 target_dir.display()
4083 ));
4084 }
4085 }
4086
4087 let owasp_coverage = report.owasp_coverage_data();
4089
4090 target_results.push(TargetResult {
4091 url: target.url.clone(),
4092 passed,
4093 failed,
4094 elapsed: target_elapsed,
4095 report_json,
4096 owasp_coverage,
4097 });
4098 }
4099
4100 let total_elapsed = total_start.elapsed();
4101
4102 println!("\n{}", "=".repeat(80));
4104 println!(" Multi-Target Conformance Summary");
4105 println!("{}", "=".repeat(80));
4106 println!(
4107 " {:<40} {:>8} {:>8} {:>8} {:>8}",
4108 "Target URL", "Passed", "Failed", "Rate", "Time"
4109 );
4110 println!(" {}", "-".repeat(76));
4111
4112 let mut total_passed = 0usize;
4113 let mut total_failed = 0usize;
4114
4115 for result in &target_results {
4116 let total_checks = result.passed + result.failed;
4117 let rate = if total_checks == 0 {
4118 0.0
4119 } else {
4120 (result.passed as f64 / total_checks as f64) * 100.0
4121 };
4122
4123 let display_url = if result.url.len() > 38 {
4125 format!("{}...", &result.url[..35])
4126 } else {
4127 result.url.clone()
4128 };
4129
4130 println!(
4131 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
4132 display_url,
4133 result.passed,
4134 result.failed,
4135 rate,
4136 result.elapsed.as_secs_f64()
4137 );
4138
4139 total_passed += result.passed;
4140 total_failed += result.failed;
4141 }
4142
4143 let grand_total = total_passed + total_failed;
4144 let overall_rate = if grand_total == 0 {
4145 0.0
4146 } else {
4147 (total_passed as f64 / grand_total as f64) * 100.0
4148 };
4149
4150 println!(" {}", "-".repeat(76));
4151 println!(
4152 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
4153 format!("TOTAL ({} targets)", num_targets),
4154 total_passed,
4155 total_failed,
4156 overall_rate,
4157 total_elapsed.as_secs_f64()
4158 );
4159 println!("{}", "=".repeat(80));
4160
4161 for result in &target_results {
4163 println!("\n OWASP API Security Top 10 Coverage for {}:", result.url);
4164 for entry in &result.owasp_coverage {
4165 let status = if !entry.tested {
4166 "-"
4167 } else if entry.all_passed {
4168 "pass"
4169 } else {
4170 "FAIL"
4171 };
4172 let via = if entry.via_categories.is_empty() {
4173 String::new()
4174 } else {
4175 format!(" (via {})", entry.via_categories.join(", "))
4176 };
4177 println!(" {:<12} {:<40} {}{}", entry.id, entry.name, status, via);
4178 }
4179 }
4180
4181 let per_target_summaries: Vec<serde_json::Value> = target_results
4183 .iter()
4184 .enumerate()
4185 .map(|(idx, r)| {
4186 let total_checks = r.passed + r.failed;
4187 let rate = if total_checks == 0 {
4188 0.0
4189 } else {
4190 (r.passed as f64 / total_checks as f64) * 100.0
4191 };
4192 let owasp_json: Vec<serde_json::Value> = r
4193 .owasp_coverage
4194 .iter()
4195 .map(|e| {
4196 serde_json::json!({
4197 "id": e.id,
4198 "name": e.name,
4199 "tested": e.tested,
4200 "all_passed": e.all_passed,
4201 "via_categories": e.via_categories,
4202 })
4203 })
4204 .collect();
4205 serde_json::json!({
4206 "target_url": r.url,
4207 "target_index": idx,
4208 "checks_passed": r.passed,
4209 "checks_failed": r.failed,
4210 "total_checks": total_checks,
4211 "pass_rate": rate,
4212 "elapsed_seconds": r.elapsed.as_secs_f64(),
4213 "report": r.report_json,
4214 "owasp_coverage": owasp_json,
4215 })
4216 })
4217 .collect();
4218
4219 let combined_summary = serde_json::json!({
4220 "total_targets": num_targets,
4221 "total_checks_passed": total_passed,
4222 "total_checks_failed": total_failed,
4223 "overall_pass_rate": overall_rate,
4224 "total_elapsed_seconds": total_elapsed.as_secs_f64(),
4225 "targets": per_target_summaries,
4226 });
4227
4228 let summary_path = self.output.join("multi-target-conformance-summary.json");
4229 let summary_str = serde_json::to_string_pretty(&combined_summary)
4230 .map_err(|e| BenchError::Other(format!("Failed to serialize summary: {}", e)))?;
4231 std::fs::write(&summary_path, &summary_str)
4232 .map_err(|e| BenchError::Other(format!("Failed to write summary: {}", e)))?;
4233 TerminalReporter::print_success(&format!(
4234 "Combined summary saved to: {}",
4235 summary_path.display()
4236 ));
4237
4238 Ok(())
4239 }
4240
4241 async fn execute_owasp_test(&self, parser: &SpecParser) -> Result<()> {
4243 TerminalReporter::print_progress("OWASP API Security Top 10 Testing Mode");
4244
4245 let custom_headers = self.parse_headers()?;
4247
4248 let mut config = OwaspApiConfig::new()
4250 .with_auth_header(&self.owasp_auth_header)
4251 .with_verbose(self.verbose)
4252 .with_insecure(self.skip_tls_verify)
4253 .with_concurrency(self.vus as usize)
4254 .with_iterations(self.owasp_iterations as usize)
4255 .with_base_path(self.base_path.clone())
4256 .with_custom_headers(custom_headers);
4257
4258 if let Some(ref token) = self.owasp_auth_token {
4260 config = config.with_valid_auth_token(token);
4261 }
4262
4263 if let Some(ref cats_str) = self.owasp_categories {
4265 let categories: Vec<OwaspCategory> = cats_str
4266 .split(',')
4267 .filter_map(|s| {
4268 let trimmed = s.trim();
4269 match trimmed.parse::<OwaspCategory>() {
4270 Ok(cat) => Some(cat),
4271 Err(e) => {
4272 TerminalReporter::print_warning(&e);
4273 None
4274 }
4275 }
4276 })
4277 .collect();
4278
4279 if !categories.is_empty() {
4280 config = config.with_categories(categories);
4281 }
4282 }
4283
4284 if let Some(ref admin_paths_file) = self.owasp_admin_paths {
4286 config.admin_paths_file = Some(admin_paths_file.clone());
4287 if let Err(e) = config.load_admin_paths() {
4288 TerminalReporter::print_warning(&format!("Failed to load admin paths file: {}", e));
4289 }
4290 }
4291
4292 if let Some(ref id_fields_str) = self.owasp_id_fields {
4294 let id_fields: Vec<String> = id_fields_str
4295 .split(',')
4296 .map(|s| s.trim().to_string())
4297 .filter(|s| !s.is_empty())
4298 .collect();
4299 if !id_fields.is_empty() {
4300 config = config.with_id_fields(id_fields);
4301 }
4302 }
4303
4304 if let Some(ref report_path) = self.owasp_report {
4306 config = config.with_report_path(report_path);
4307 }
4308 if let Ok(format) = self.owasp_report_format.parse::<ReportFormat>() {
4309 config = config.with_report_format(format);
4310 }
4311
4312 let categories = config.categories_to_test();
4314 TerminalReporter::print_success(&format!(
4315 "Testing {} OWASP categories: {}",
4316 categories.len(),
4317 categories.iter().map(|c| c.cli_name()).collect::<Vec<_>>().join(", ")
4318 ));
4319
4320 if config.valid_auth_token.is_some() {
4321 TerminalReporter::print_progress("Using provided auth token for baseline requests");
4322 }
4323
4324 TerminalReporter::print_progress("Generating OWASP security test script...");
4326 let generator = OwaspApiGenerator::new(config, self.target.clone(), parser);
4327
4328 let script = generator.generate()?;
4330 TerminalReporter::print_success("OWASP security test script generated");
4331
4332 let script_path = if let Some(output) = &self.script_output {
4334 output.clone()
4335 } else {
4336 self.output.join("k6-owasp-security-test.js")
4337 };
4338
4339 if let Some(parent) = script_path.parent() {
4340 std::fs::create_dir_all(parent)?;
4341 }
4342 std::fs::write(&script_path, &script)?;
4343 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
4344
4345 if self.generate_only {
4347 println!("\nOWASP security test script generated. Run it with:");
4348 println!(" k6 run {}", script_path.display());
4349 return Ok(());
4350 }
4351
4352 TerminalReporter::print_progress("Executing OWASP security tests...");
4354 let executor = K6Executor::new()?
4355 .with_local_ips(self.source_ips.join(","))
4356 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
4357 std::fs::create_dir_all(&self.output)?;
4358
4359 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
4360
4361 let duration_secs = Self::parse_duration(&self.duration)?;
4362 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
4363
4364 println!("\nOWASP security test results saved to: {}", self.output.display());
4365
4366 Ok(())
4367 }
4368}
4369
4370#[cfg(test)]
4371mod tests {
4372 use super::*;
4373 use tempfile::tempdir;
4374
4375 #[test]
4376 fn test_parse_duration() {
4377 assert_eq!(BenchCommand::parse_duration("30s").unwrap(), 30);
4378 assert_eq!(BenchCommand::parse_duration("5m").unwrap(), 300);
4379 assert_eq!(BenchCommand::parse_duration("1h").unwrap(), 3600);
4380 assert_eq!(BenchCommand::parse_duration("60").unwrap(), 60);
4381 }
4382
4383 #[test]
4387 fn parse_ip_list_ipv4_range_inclusive() {
4388 let v = parse_ip_list(&["10.0.0.5-10.0.0.27".into()], "source-ip");
4389 assert_eq!(v.len(), 23);
4390 assert_eq!(v.first().unwrap().to_string(), "10.0.0.5");
4391 assert_eq!(v.last().unwrap().to_string(), "10.0.0.27");
4392 }
4393
4394 #[test]
4397 fn parse_ip_list_range_rejects_backwards() {
4398 let v = parse_ip_list(&["10.0.0.10-10.0.0.5".into()], "source-ip");
4399 assert!(v.is_empty(), "backwards range should produce no IPs; got {v:?}");
4400 }
4401
4402 #[test]
4406 fn parse_ip_list_rejects_ipv6_range_syntax() {
4407 let v = parse_ip_list(&["2001:db8::1-2001:db8::5".into()], "geo-source-ip");
4408 assert!(v.is_empty(), "IPv6 range should be rejected; got {v:?}");
4409 }
4410
4411 #[test]
4413 fn parse_ip_list_range_capped_at_256() {
4414 let v = parse_ip_list(&["10.0.0.0-10.0.5.0".into()], "source-ip");
4415 assert_eq!(v.len(), 256);
4416 assert_eq!(v.first().unwrap().to_string(), "10.0.0.0");
4417 }
4418
4419 #[test]
4422 fn parse_ip_list_plain_and_comma() {
4423 let v = parse_ip_list(&["10.0.0.5".into(), "10.0.0.6,10.0.0.7".into()], "source-ip");
4424 assert_eq!(v.len(), 3);
4425 assert_eq!(v[0].to_string(), "10.0.0.5");
4426 assert_eq!(v[2].to_string(), "10.0.0.7");
4427 }
4428
4429 #[test]
4432 fn parse_ip_list_ipv4_cidr_29_expands_to_8() {
4433 let v = parse_ip_list(&["10.0.0.0/29".into()], "source-ip");
4434 assert_eq!(v.len(), 8);
4435 assert_eq!(v[0].to_string(), "10.0.0.0");
4436 assert_eq!(v[7].to_string(), "10.0.0.7");
4437 }
4438
4439 #[test]
4442 fn parse_ip_list_ipv4_cidr_8_capped_at_256() {
4443 let v = parse_ip_list(&["10.0.0.0/8".into()], "source-ip");
4444 assert_eq!(v.len(), 256);
4445 assert_eq!(v[0].to_string(), "10.0.0.0");
4446 assert_eq!(v[255].to_string(), "10.0.0.255");
4447 }
4448
4449 #[test]
4451 fn parse_ip_list_ipv6_cidr_126_expands_to_4() {
4452 let v = parse_ip_list(&["2001:db8::/126".into()], "geo-source-ip");
4453 assert_eq!(v.len(), 4);
4454 assert!(v[0].is_ipv6());
4455 assert_eq!(v[0].to_string(), "2001:db8::");
4456 assert_eq!(v[3].to_string(), "2001:db8::3");
4457 }
4458
4459 #[test]
4461 fn parse_ip_list_mixed_v4_v6_cidr() {
4462 let v = parse_ip_list(&["10.0.0.0/30,2001:db8::1,203.0.113.42".into()], "geo-source-ip");
4463 assert_eq!(v.len(), 6); assert!(v.iter().any(|ip| ip.to_string() == "2001:db8::1"));
4465 assert!(v.iter().any(|ip| ip.to_string() == "203.0.113.42"));
4466 }
4467
4468 #[test]
4471 fn parse_ip_list_skips_malformed() {
4472 let v = parse_ip_list(
4473 &[
4474 "10.0.0.5".into(),
4475 "not-an-ip".into(),
4476 "10.0.0.6".into(),
4477 "/24".into(),
4478 "1.2.3.4/200".into(),
4479 ],
4480 "source-ip",
4481 );
4482 assert_eq!(v.len(), 2);
4483 assert_eq!(v[0].to_string(), "10.0.0.5");
4484 assert_eq!(v[1].to_string(), "10.0.0.6");
4485 }
4486
4487 #[test]
4488 fn test_parse_duration_invalid() {
4489 assert!(BenchCommand::parse_duration("invalid").is_err());
4490 assert!(BenchCommand::parse_duration("30x").is_err());
4491 }
4492
4493 #[test]
4494 fn test_parse_headers() {
4495 let cmd = BenchCommand {
4496 spec: vec![PathBuf::from("test.yaml")],
4497 spec_dir: None,
4498 merge_conflicts: "error".to_string(),
4499 spec_mode: "merge".to_string(),
4500 dependency_config: None,
4501 target: "http://localhost".to_string(),
4502 base_path: None,
4503 duration: "1m".to_string(),
4504 vus: 10,
4505 scenario: "ramp-up".to_string(),
4506 operations: None,
4507 exclude_operations: None,
4508 auth: None,
4509 headers: vec![
4510 "X-API-Key:test123".to_string(),
4511 "X-Client-ID:client456".to_string(),
4512 ],
4513 output: PathBuf::from("output"),
4514 generate_only: false,
4515 script_output: None,
4516 threshold_percentile: "p(95)".to_string(),
4517 threshold_ms: 500,
4518 max_error_rate: 0.05,
4519 abort_on_error: true,
4520 abort_on_error_rate: 0.95,
4521 verbose: false,
4522 skip_tls_verify: false,
4523 chunked_request_bodies: false,
4524 target_rps: None,
4525 no_keep_alive: false,
4526 targets_file: None,
4527 max_concurrency: None,
4528 results_format: "both".to_string(),
4529 params_file: None,
4530 crud_flow: false,
4531 flow_config: None,
4532 extract_fields: None,
4533 parallel_create: None,
4534 data_file: None,
4535 data_distribution: "unique-per-vu".to_string(),
4536 data_mappings: None,
4537 per_uri_control: false,
4538 error_rate: None,
4539 error_types: None,
4540 security_test: false,
4541 security_payloads: None,
4542 security_categories: None,
4543 security_target_fields: None,
4544 wafbench_dir: None,
4545 wafbench_cycle_all: false,
4546 wafbench_verbatim: false,
4547 owasp_api_top10: false,
4548 owasp_categories: None,
4549 owasp_auth_header: "Authorization".to_string(),
4550 owasp_auth_token: None,
4551 owasp_admin_paths: None,
4552 owasp_id_fields: None,
4553 owasp_report: None,
4554 owasp_report_format: "json".to_string(),
4555 owasp_iterations: 1,
4556 conformance: false,
4557 conformance_api_key: None,
4558 conformance_basic_auth: None,
4559 conformance_report: PathBuf::from("conformance-report.json"),
4560 conformance_categories: None,
4561 conformance_report_format: "json".to_string(),
4562 conformance_headers: vec![],
4563 conformance_all_operations: false,
4564 conformance_custom: None,
4565 conformance_delay_ms: 0,
4566 use_k6: false,
4567 conformance_custom_filter: None,
4568 export_requests: false,
4569 validate_requests: false,
4570 conformance_self_test: false,
4571 conformance_self_test_capture: false,
4572 conformance_self_test_iterations: 1,
4573 conformance_self_test_duration: None,
4574 validate_response_schemas: false,
4575 source_ips: Vec::new(),
4576 geo_source_ips: Vec::new(),
4577 geo_source_headers: Vec::new(),
4578 report_missed_cap: None,
4579 discard_response_bodies: false,
4580 dns_policy: None,
4581 };
4582
4583 let headers = cmd.parse_headers().unwrap();
4584 assert_eq!(headers.get("X-API-Key"), Some(&"test123".to_string()));
4585 assert_eq!(headers.get("X-Client-ID"), Some(&"client456".to_string()));
4586 }
4587
4588 #[test]
4589 fn test_parse_header_string_preserves_comma_in_value() {
4590 let inputs = vec![
4593 "Cookie:session=abc; expires=Thu, 01 Jan 2099 00:00:00 GMT".to_string(),
4594 "X-Trace:1".to_string(),
4595 ];
4596 let headers = parse_header_string(&inputs).unwrap();
4597 assert_eq!(
4598 headers.get("Cookie"),
4599 Some(&"session=abc; expires=Thu, 01 Jan 2099 00:00:00 GMT".to_string())
4600 );
4601 assert_eq!(headers.get("X-Trace"), Some(&"1".to_string()));
4602 }
4603
4604 #[test]
4612 fn conformance_advisory_names_every_discarded_flag() {
4613 let msg = CONFORMANCE_REPLACES_LOAD_ADVISORY;
4614 for flag in ["--vus", "--rps", "-d"] {
4615 assert!(
4616 msg.contains(flag),
4617 "conformance advisory must name `{flag}` as ignored; it is discarded on that \
4618 path and silently dropping it is how users end up tuning a knob that does \
4619 nothing (#980). Message was: {msg}"
4620 );
4621 }
4622 assert!(
4623 msg.contains("REPLACES"),
4624 "conformance advisory must say the load run is REPLACED, not merely that some \
4625 flags are ignored — `--conformance` returns before the load path runs, so no \
4626 load traffic is generated at all (#980). Message was: {msg}"
4627 );
4628 }
4629
4630 #[test]
4644 fn multi_target_clone_preserves_fields_parse_headers_reads() {
4645 let src = include_str!("command.rs");
4646
4647 let fn_start = src
4648 .find("async fn execute_multi_target(")
4649 .expect("execute_multi_target should exist");
4650 let block_start = src[fn_start..]
4651 .find("ParallelExecutor::new(")
4652 .map(|i| i + fn_start)
4653 .expect("multi-target path should build a ParallelExecutor");
4654 let block_end = src[block_start..]
4656 .find("\n );")
4657 .map(|i| i + block_start)
4658 .expect("ParallelExecutor::new(..) should be closed");
4659 let block = &src[block_start..block_end];
4660
4661 for field in ["conformance_basic_auth", "conformance_headers"] {
4664 for zeroed in [format!("{field}: None"), format!("{field}: vec![]")] {
4665 assert!(
4666 !block.contains(&zeroed),
4667 "execute_multi_target zeroes `{zeroed}`. parse_headers() folds `{field}` \
4668 into the header map, so zeroing it here strips auth from every \
4669 multi-target run while single-target keeps working (#79 round 64)."
4670 );
4671 }
4672 let passthrough = format!("{field}: self.{field}.clone()");
4673 assert!(
4674 block.contains(&passthrough),
4675 "execute_multi_target must carry `{field}` through as `{passthrough}` so \
4676 parse_headers() can fold it (#79 round 64)."
4677 );
4678 }
4679 }
4680
4681 #[test]
4682 fn test_get_spec_display_name() {
4683 let cmd = BenchCommand {
4684 spec: vec![PathBuf::from("test.yaml")],
4685 spec_dir: None,
4686 merge_conflicts: "error".to_string(),
4687 spec_mode: "merge".to_string(),
4688 dependency_config: None,
4689 target: "http://localhost".to_string(),
4690 base_path: None,
4691 duration: "1m".to_string(),
4692 vus: 10,
4693 scenario: "ramp-up".to_string(),
4694 operations: None,
4695 exclude_operations: None,
4696 auth: None,
4697 headers: Vec::new(),
4698 output: PathBuf::from("output"),
4699 generate_only: false,
4700 script_output: None,
4701 threshold_percentile: "p(95)".to_string(),
4702 threshold_ms: 500,
4703 max_error_rate: 0.05,
4704 abort_on_error: true,
4705 abort_on_error_rate: 0.95,
4706 verbose: false,
4707 skip_tls_verify: false,
4708 chunked_request_bodies: false,
4709 target_rps: None,
4710 no_keep_alive: false,
4711 targets_file: None,
4712 max_concurrency: None,
4713 results_format: "both".to_string(),
4714 params_file: None,
4715 crud_flow: false,
4716 flow_config: None,
4717 extract_fields: None,
4718 parallel_create: None,
4719 data_file: None,
4720 data_distribution: "unique-per-vu".to_string(),
4721 data_mappings: None,
4722 per_uri_control: false,
4723 error_rate: None,
4724 error_types: None,
4725 security_test: false,
4726 security_payloads: None,
4727 security_categories: None,
4728 security_target_fields: None,
4729 wafbench_dir: None,
4730 wafbench_cycle_all: false,
4731 wafbench_verbatim: false,
4732 owasp_api_top10: false,
4733 owasp_categories: None,
4734 owasp_auth_header: "Authorization".to_string(),
4735 owasp_auth_token: None,
4736 owasp_admin_paths: None,
4737 owasp_id_fields: None,
4738 owasp_report: None,
4739 owasp_report_format: "json".to_string(),
4740 owasp_iterations: 1,
4741 conformance: false,
4742 conformance_api_key: None,
4743 conformance_basic_auth: None,
4744 conformance_report: PathBuf::from("conformance-report.json"),
4745 conformance_categories: None,
4746 conformance_report_format: "json".to_string(),
4747 conformance_headers: vec![],
4748 conformance_all_operations: false,
4749 conformance_custom: None,
4750 conformance_delay_ms: 0,
4751 use_k6: false,
4752 conformance_custom_filter: None,
4753 export_requests: false,
4754 validate_requests: false,
4755 conformance_self_test: false,
4756 conformance_self_test_capture: false,
4757 conformance_self_test_iterations: 1,
4758 conformance_self_test_duration: None,
4759 validate_response_schemas: false,
4760 source_ips: Vec::new(),
4761 geo_source_ips: Vec::new(),
4762 geo_source_headers: Vec::new(),
4763 report_missed_cap: None,
4764 discard_response_bodies: false,
4765 dns_policy: None,
4766 };
4767
4768 assert_eq!(cmd.get_spec_display_name(), "test.yaml");
4769
4770 let cmd_multi = BenchCommand {
4772 spec: vec![PathBuf::from("a.yaml"), PathBuf::from("b.yaml")],
4773 spec_dir: None,
4774 merge_conflicts: "error".to_string(),
4775 spec_mode: "merge".to_string(),
4776 dependency_config: None,
4777 target: "http://localhost".to_string(),
4778 base_path: None,
4779 duration: "1m".to_string(),
4780 vus: 10,
4781 scenario: "ramp-up".to_string(),
4782 operations: None,
4783 exclude_operations: None,
4784 auth: None,
4785 headers: Vec::new(),
4786 output: PathBuf::from("output"),
4787 generate_only: false,
4788 script_output: None,
4789 threshold_percentile: "p(95)".to_string(),
4790 threshold_ms: 500,
4791 max_error_rate: 0.05,
4792 abort_on_error: true,
4793 abort_on_error_rate: 0.95,
4794 verbose: false,
4795 skip_tls_verify: false,
4796 chunked_request_bodies: false,
4797 target_rps: None,
4798 no_keep_alive: false,
4799 targets_file: None,
4800 max_concurrency: None,
4801 results_format: "both".to_string(),
4802 params_file: None,
4803 crud_flow: false,
4804 flow_config: None,
4805 extract_fields: None,
4806 parallel_create: None,
4807 data_file: None,
4808 data_distribution: "unique-per-vu".to_string(),
4809 data_mappings: None,
4810 per_uri_control: false,
4811 error_rate: None,
4812 error_types: None,
4813 security_test: false,
4814 security_payloads: None,
4815 security_categories: None,
4816 security_target_fields: None,
4817 wafbench_dir: None,
4818 wafbench_cycle_all: false,
4819 wafbench_verbatim: false,
4820 owasp_api_top10: false,
4821 owasp_categories: None,
4822 owasp_auth_header: "Authorization".to_string(),
4823 owasp_auth_token: None,
4824 owasp_admin_paths: None,
4825 owasp_id_fields: None,
4826 owasp_report: None,
4827 owasp_report_format: "json".to_string(),
4828 owasp_iterations: 1,
4829 conformance: false,
4830 conformance_api_key: None,
4831 conformance_basic_auth: None,
4832 conformance_report: PathBuf::from("conformance-report.json"),
4833 conformance_categories: None,
4834 conformance_report_format: "json".to_string(),
4835 conformance_headers: vec![],
4836 conformance_all_operations: false,
4837 conformance_custom: None,
4838 conformance_delay_ms: 0,
4839 use_k6: false,
4840 conformance_custom_filter: None,
4841 export_requests: false,
4842 validate_requests: false,
4843 conformance_self_test: false,
4844 conformance_self_test_capture: false,
4845 conformance_self_test_iterations: 1,
4846 conformance_self_test_duration: None,
4847 validate_response_schemas: false,
4848 source_ips: Vec::new(),
4849 geo_source_ips: Vec::new(),
4850 geo_source_headers: Vec::new(),
4851 report_missed_cap: None,
4852 discard_response_bodies: false,
4853 dns_policy: None,
4854 };
4855
4856 assert_eq!(cmd_multi.get_spec_display_name(), "2 spec files");
4857 }
4858
4859 #[test]
4860 fn test_parse_extracted_values_from_output_dir() {
4861 let dir = tempdir().unwrap();
4862 let path = dir.path().join("extracted_values.json");
4863 std::fs::write(
4864 &path,
4865 r#"{
4866 "pool_id": "abc123",
4867 "count": 0,
4868 "enabled": false,
4869 "metadata": { "owner": "team-a" }
4870}"#,
4871 )
4872 .unwrap();
4873
4874 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
4875 assert_eq!(extracted.get("pool_id"), Some(&serde_json::json!("abc123")));
4876 assert_eq!(extracted.get("count"), Some(&serde_json::json!(0)));
4877 assert_eq!(extracted.get("enabled"), Some(&serde_json::json!(false)));
4878 assert_eq!(extracted.get("metadata"), Some(&serde_json::json!({"owner": "team-a"})));
4879 }
4880
4881 #[test]
4882 fn test_parse_extracted_values_missing_file() {
4883 let dir = tempdir().unwrap();
4884 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
4885 assert!(extracted.values.is_empty());
4886 }
4887
4888 fn sample_bench_command() -> BenchCommand {
4891 BenchCommand {
4892 spec: vec![PathBuf::from("test.yaml")],
4893 spec_dir: None,
4894 merge_conflicts: "error".to_string(),
4895 spec_mode: "merge".to_string(),
4896 dependency_config: None,
4897 target: "http://localhost".to_string(),
4898 base_path: None,
4899 duration: "1m".to_string(),
4900 vus: 10,
4901 scenario: "ramp-up".to_string(),
4902 operations: None,
4903 exclude_operations: None,
4904 auth: None,
4905 headers: vec![
4906 "X-API-Key:test123".to_string(),
4907 "X-Client-ID:client456".to_string(),
4908 ],
4909 output: PathBuf::from("output"),
4910 generate_only: false,
4911 script_output: None,
4912 threshold_percentile: "p(95)".to_string(),
4913 threshold_ms: 500,
4914 max_error_rate: 0.05,
4915 abort_on_error: true,
4916 abort_on_error_rate: 0.95,
4917 verbose: false,
4918 skip_tls_verify: false,
4919 chunked_request_bodies: false,
4920 target_rps: None,
4921 no_keep_alive: false,
4922 targets_file: None,
4923 max_concurrency: None,
4924 results_format: "both".to_string(),
4925 params_file: None,
4926 crud_flow: false,
4927 flow_config: None,
4928 extract_fields: None,
4929 parallel_create: None,
4930 data_file: None,
4931 data_distribution: "unique-per-vu".to_string(),
4932 data_mappings: None,
4933 per_uri_control: false,
4934 error_rate: None,
4935 error_types: None,
4936 security_test: false,
4937 security_payloads: None,
4938 security_categories: None,
4939 security_target_fields: None,
4940 wafbench_dir: None,
4941 wafbench_cycle_all: false,
4942 wafbench_verbatim: false,
4943 owasp_api_top10: false,
4944 owasp_categories: None,
4945 owasp_auth_header: "Authorization".to_string(),
4946 owasp_auth_token: None,
4947 owasp_admin_paths: None,
4948 owasp_id_fields: None,
4949 owasp_report: None,
4950 owasp_report_format: "json".to_string(),
4951 owasp_iterations: 1,
4952 conformance: false,
4953 conformance_api_key: None,
4954 conformance_basic_auth: None,
4955 conformance_report: PathBuf::from("conformance-report.json"),
4956 conformance_categories: None,
4957 conformance_report_format: "json".to_string(),
4958 conformance_headers: vec![],
4959 conformance_all_operations: false,
4960 conformance_custom: None,
4961 conformance_delay_ms: 0,
4962 use_k6: false,
4963 conformance_custom_filter: None,
4964 export_requests: false,
4965 validate_requests: false,
4966 conformance_self_test: false,
4967 conformance_self_test_capture: false,
4968 conformance_self_test_iterations: 1,
4969 conformance_self_test_duration: None,
4970 validate_response_schemas: false,
4971 source_ips: Vec::new(),
4972 geo_source_ips: Vec::new(),
4973 geo_source_headers: Vec::new(),
4974 report_missed_cap: None,
4975 discard_response_bodies: false,
4976 dns_policy: None,
4977 }
4978 }
4979
4980 #[test]
4988 fn verbatim_disables_security_payload_injection() {
4989 let mut cmd = sample_bench_command();
4990 cmd.wafbench_dir = Some("traffic.yaml".to_string());
4991
4992 assert!(
4993 cmd.security_testing_enabled(),
4994 "--wafbench-dir alone must still enable payload injection"
4995 );
4996
4997 cmd.wafbench_verbatim = true;
4998 assert!(
4999 !cmd.security_testing_enabled(),
5000 "verbatim mode must not inject payloads into requests sent as written"
5001 );
5002
5003 cmd.security_test = true;
5006 assert!(
5007 !cmd.security_testing_enabled(),
5008 "--security-test must not re-enable injection under --wafbench-verbatim"
5009 );
5010 }
5011
5012 #[test]
5017 fn security_testing_enabled_has_a_single_definition() {
5018 let src = include_str!("command.rs");
5019 let parallel = include_str!("parallel_executor.rs");
5020 let a = format!("self.{} || self.{}.is_some()", "security_test", "wafbench_dir");
5022 let b = format!("self.{}.is_some() || self.{}", "wafbench_dir", "security_test");
5023 let inline = src.matches(a.as_str()).count() + src.matches(b.as_str()).count();
5024 assert_eq!(
5025 inline, 1,
5026 "expected the security_testing_enabled() method to be the only place this is \
5027 computed, found {inline} inline copies -- collapse them or the render paths drift"
5028 );
5029
5030 let parallel_inline = format!(
5035 "{}.{} || {}.{}.is_some()",
5036 "base_command", "security_test", "self.base_command", "wafbench_dir"
5037 );
5038 assert!(
5039 !parallel.contains(¶llel_inline),
5040 "ParallelExecutor must not recompute the security flag inline"
5041 );
5042 assert!(
5043 parallel.contains("security_testing_enabled()"),
5044 "ParallelExecutor must call security_testing_enabled() so --wafbench-verbatim \
5045 turns injection off on --targets-file runs too"
5046 );
5047 }
5048
5049 #[test]
5053 fn missing_wafbench_dir_is_not_swallowed() {
5054 let src = include_str!("command.rs");
5055 let swallowed = format!("Failed to {} WAFBench tests", "load");
5057 let impl_line = src
5058 .lines()
5059 .filter(|l| !l.trim_start().starts_with("//"))
5060 .any(|l| l.contains(&swallowed));
5061 assert!(!impl_line, "missing --wafbench-dir must not be downgraded to a warning");
5062 assert!(
5063 src.contains("self.load_wafbench_payloads()?"),
5064 "payload load errors must reach generate_enhanced_script"
5065 );
5066 }
5067
5068 #[test]
5073 fn multi_target_path_honors_verbatim_templates() {
5074 let src = include_str!("parallel_executor.rs");
5075 assert!(
5076 src.contains("load_verbatim_templates"),
5077 "ParallelExecutor must load traffic-file requests under --wafbench-verbatim. \
5078 Requiring a spec and generating templates from its operations is how \
5079 --targets-file ignored the flag and fuzzed spec URLs (#79)."
5080 );
5081 }
5082
5083 #[test]
5084 fn single_target_k6_spawn_sets_force_http1() {
5085 let src = include_str!("command.rs");
5086 assert!(
5087 src.contains("with_force_http1(force_http1)"),
5088 "single-target k6 spawn must set GODEBUG=http2client=0 for Connection-header WAF cases"
5089 );
5090 assert!(
5091 src.contains("print_k6_run_hint"),
5092 "generate-only must print GODEBUG=http2client=0 when HTTP/1.1 is required"
5093 );
5094 }
5095
5096 #[test]
5098 fn traffic_breakdown_json_multiplies_unique_by_rps() {
5099 let dir = std::env::temp_dir().join(format!(
5100 "mf-traffic-breakdown-{}-{}",
5101 std::process::id(),
5102 std::time::SystemTime::now()
5103 .duration_since(std::time::UNIX_EPOCH)
5104 .unwrap()
5105 .as_nanos()
5106 ));
5107 let _ = std::fs::create_dir_all(&dir);
5108 let mut cmd = sample_bench_command();
5109 cmd.output = dir.clone();
5110 cmd.target_rps = Some(50);
5111 cmd.duration = "1200s".to_string();
5112 let stats = crate::wafbench::WafBenchStats {
5113 per_file: vec![crate::wafbench::TrafficFileSummary {
5114 file: "apisix_cve-2026-44087.yaml".into(),
5115 sent: 5,
5116 attack: 3,
5117 normal: 2,
5118 omitted: 1,
5119 other: 0,
5120 }],
5121 ..Default::default()
5122 };
5123 cmd.emit_traffic_file_breakdown(&stats, "what to expect in proxy logs");
5124 let raw = std::fs::read_to_string(dir.join("traffic-breakdown.json"))
5125 .expect("traffic-breakdown.json");
5126 let v: serde_json::Value = serde_json::from_str(&raw).unwrap();
5127 assert_eq!(v["rps"], 50);
5128 assert_eq!(v["duration_secs"], 1200);
5129 assert_eq!(v["files"][0]["sent"]["unique_cases"], 5);
5130 assert_eq!(v["files"][0]["sent"]["unique"], 5);
5131 assert_eq!(v["files"][0]["sent"]["projected_per_second"], 250);
5132 assert_eq!(v["files"][0]["sent"]["total"], 250);
5133 assert_eq!(v["files"][0]["sent"]["projected_over_run"], 300000);
5134 assert_eq!(v["files"][0]["sent"]["expected_requests"], 300000);
5135 assert!(v["files"][0]["sent"].get("expected_requests_unit").is_none());
5136 assert_eq!(v["files"][0]["attack"]["total"], 150);
5137 assert_eq!(v["files"][0]["normal"]["total"], 100);
5138 assert!(v["note"].as_str().unwrap().contains("Plan, not k6 counters"));
5139 assert_eq!(
5140 BenchCommand::format_unique_total(5, Some(50)),
5141 "unique_cases=5 projected_per_second=250 (5 * 50 RPS)"
5142 );
5143 let _ = std::fs::remove_dir_all(&dir);
5144 }
5145
5146 #[test]
5149 fn traffic_breakdown_json_omits_expected_requests_without_rps() {
5150 let dir = std::env::temp_dir().join(format!(
5151 "mf-traffic-breakdown-norps-{}-{}",
5152 std::process::id(),
5153 std::time::SystemTime::now()
5154 .duration_since(std::time::UNIX_EPOCH)
5155 .unwrap()
5156 .as_nanos()
5157 ));
5158 let _ = std::fs::create_dir_all(&dir);
5159 let mut cmd = sample_bench_command();
5160 cmd.output = dir.clone();
5161 cmd.target_rps = None;
5162 cmd.duration = "60s".to_string();
5163 let stats = crate::wafbench::WafBenchStats {
5164 per_file: vec![crate::wafbench::TrafficFileSummary {
5165 file: "apisix_cve-2026-44087.yaml".into(),
5166 sent: 5,
5167 attack: 3,
5168 normal: 2,
5169 omitted: 1,
5170 other: 0,
5171 }],
5172 ..Default::default()
5173 };
5174 cmd.emit_traffic_file_breakdown(&stats, "what to expect in proxy logs");
5175 let raw = std::fs::read_to_string(dir.join("traffic-breakdown.json"))
5176 .expect("traffic-breakdown.json");
5177 let v: serde_json::Value = serde_json::from_str(&raw).unwrap();
5178 assert!(v["rps"].is_null());
5179 assert_eq!(v["duration_secs"], 60);
5180 assert_eq!(v["files"][0]["sent"]["unique_cases"], 5);
5181 assert_eq!(v["files"][0]["sent"]["unique"], 5);
5182 assert!(v["files"][0]["sent"]["projected_per_second"].is_null());
5183 assert_eq!(v["files"][0]["sent"]["total"], 5);
5184 assert!(v["files"][0]["sent"]["projected_over_run"].is_null());
5185 assert!(v["files"][0]["sent"]["expected_requests"].is_null());
5186 let _ = std::fs::remove_dir_all(&dir);
5187 }
5188}