1use crate::crud_flow::{CrudFlowConfig, CrudFlowDetector};
4use crate::data_driven::{DataDistribution, DataDrivenConfig, DataDrivenGenerator, DataMapping};
5use crate::dynamic_params::{DynamicParamProcessor, DynamicPlaceholder};
6use crate::error::{BenchError, Result};
7use crate::executor::K6Executor;
8use crate::invalid_data::{InvalidDataConfig, InvalidDataGenerator};
9use crate::k6_gen::{K6Config, K6ScriptGenerator};
10use crate::mock_integration::{
11 MockIntegrationConfig, MockIntegrationGenerator, MockServerDetector,
12};
13use crate::owasp_api::{OwaspApiConfig, OwaspApiGenerator, OwaspCategory, ReportFormat};
14use crate::parallel_executor::{AggregatedResults, ParallelExecutor};
15use crate::parallel_requests::{ParallelConfig, ParallelRequestGenerator};
16use crate::param_overrides::ParameterOverrides;
17use crate::reporter::TerminalReporter;
18use crate::request_gen::RequestGenerator;
19use crate::scenarios::LoadScenario;
20use crate::security_payloads::{
21 SecurityCategory, SecurityPayload, SecurityPayloads, SecurityTestConfig, SecurityTestGenerator,
22};
23use crate::spec_dependencies::{
24 topological_sort, DependencyDetector, ExtractedValues, SpecDependencyConfig,
25};
26use crate::spec_parser::SpecParser;
27use crate::target_parser::parse_targets_file;
28use crate::wafbench::WafBenchLoader;
29use mockforge_openapi::multi_spec::{
30 load_specs_from_directory, load_specs_from_files, merge_specs, ConflictStrategy,
31};
32use mockforge_openapi::spec::OpenApiSpec;
33use std::collections::{HashMap, HashSet};
34use std::path::{Path, PathBuf};
35use std::str::FromStr;
36
37pub fn parse_header_string(inputs: &[String]) -> Result<HashMap<String, String>> {
45 let mut headers = HashMap::new();
46
47 for pair in inputs {
48 let pair = pair.trim();
49 if pair.is_empty() {
50 continue;
51 }
52 let parts: Vec<&str> = pair.splitn(2, ':').collect();
53 if parts.len() != 2 {
54 return Err(BenchError::Other(format!(
55 "Invalid header format: '{}'. Expected 'Key:Value'",
56 pair
57 )));
58 }
59 headers.insert(parts[0].trim().to_string(), parts[1].trim().to_string());
60 }
61
62 Ok(headers)
63}
64
65const CONFORMANCE_REPLACES_LOAD_ADVISORY: &str =
85 "Conformance mode REPLACES the load run: 1 VU, 1 iteration per endpoint. \
86 --vus, --rps and -d are ignored. Run bench a second time without \
87 --conformance if you also want a load test.";
88
89pub struct BenchCommand {
91 pub spec: Vec<PathBuf>,
93 pub spec_dir: Option<PathBuf>,
95 pub merge_conflicts: String,
97 pub spec_mode: String,
99 pub dependency_config: Option<PathBuf>,
101 pub target: String,
102 pub base_path: Option<String>,
105 pub duration: String,
106 pub vus: u32,
107 pub target_rps: Option<u32>,
113 pub no_keep_alive: bool,
118 pub scenario: String,
119 pub operations: Option<String>,
120 pub exclude_operations: Option<String>,
124 pub auth: Option<String>,
125 pub headers: Vec<String>,
128 pub output: PathBuf,
129 pub generate_only: bool,
130 pub script_output: Option<PathBuf>,
131 pub threshold_percentile: String,
132 pub threshold_ms: u64,
133 pub max_error_rate: f64,
134 pub abort_on_error: bool,
139 pub abort_on_error_rate: f64,
143 pub verbose: bool,
144 pub skip_tls_verify: bool,
145 pub chunked_request_bodies: bool,
150 pub targets_file: Option<PathBuf>,
152 pub max_concurrency: Option<u32>,
154 pub results_format: String,
156 pub params_file: Option<PathBuf>,
161
162 pub crud_flow: bool,
165 pub flow_config: Option<PathBuf>,
167 pub extract_fields: Option<String>,
169
170 pub parallel_create: Option<u32>,
173
174 pub data_file: Option<PathBuf>,
177 pub data_distribution: String,
179 pub data_mappings: Option<String>,
181 pub per_uri_control: bool,
183
184 pub error_rate: Option<f64>,
187 pub error_types: Option<String>,
189
190 pub security_test: bool,
193 pub security_payloads: Option<PathBuf>,
195 pub security_categories: Option<String>,
197 pub security_target_fields: Option<String>,
199
200 pub wafbench_dir: Option<String>,
203 pub wafbench_cycle_all: bool,
205 pub wafbench_verbatim: bool,
208
209 pub conformance: bool,
212 pub conformance_api_key: Option<String>,
214 pub conformance_basic_auth: Option<String>,
216 pub conformance_report: PathBuf,
218 pub conformance_categories: Option<String>,
220 pub conformance_report_format: String,
222 pub conformance_headers: Vec<String>,
225 pub conformance_all_operations: bool,
228 pub conformance_custom: Option<PathBuf>,
230 pub conformance_delay_ms: u64,
233 pub use_k6: bool,
235 pub conformance_custom_filter: Option<String>,
239 pub export_requests: bool,
242 pub validate_requests: bool,
245 pub conformance_self_test: bool,
252 pub conformance_self_test_capture: bool,
256 pub validate_response_schemas: bool,
262 pub conformance_self_test_iterations: u32,
267 pub conformance_self_test_duration: Option<String>,
272
273 pub source_ips: Vec<String>,
278 pub geo_source_ips: Vec<String>,
282 pub geo_source_headers: Vec<String>,
286
287 pub report_missed_cap: Option<u32>,
294
295 pub discard_response_bodies: bool,
302
303 pub dns_policy: Option<String>,
309
310 pub owasp_api_top10: bool,
313 pub owasp_categories: Option<String>,
315 pub owasp_auth_header: String,
317 pub owasp_auth_token: Option<String>,
319 pub owasp_admin_paths: Option<PathBuf>,
321 pub owasp_id_fields: Option<String>,
323 pub owasp_report: Option<PathBuf>,
325 pub owasp_report_format: String,
327 pub owasp_iterations: u32,
329}
330
331fn parse_ip_list(raw: &[String], flag_name: &str) -> Vec<std::net::IpAddr> {
345 use std::net::IpAddr;
346 const MAX_CIDR_EXPANSION: usize = 256;
347 let mut out = Vec::new();
348 for entry in raw {
349 for piece in entry.split(',') {
350 let s = piece.trim();
351 if s.is_empty() {
352 continue;
353 }
354 if let Some((addr_part, prefix_part)) = s.split_once('/') {
356 let prefix: u32 = match prefix_part.parse() {
357 Ok(p) => p,
358 Err(e) => {
359 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR prefix: {e}");
360 continue;
361 }
362 };
363 let net_addr: IpAddr = match addr_part.parse() {
364 Ok(a) => a,
365 Err(e) => {
366 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR address: {e}");
367 continue;
368 }
369 };
370 expand_cidr(net_addr, prefix, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
371 continue;
372 }
373 if let Some((start_str, end_str)) = s.split_once('-') {
379 let start_s = start_str.trim();
380 let end_s = end_str.trim();
381 if start_s.contains(':') || end_s.contains(':') {
385 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{s}': IPv6 range syntax not supported (use CIDR like 2001:db8::/126 instead)");
386 continue;
387 }
388 let start: IpAddr = match start_s.parse() {
389 Ok(a) => a,
390 Err(e) => {
391 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range start: {e}");
392 continue;
393 }
394 };
395 let end: IpAddr = match end_s.parse() {
396 Ok(a) => a,
397 Err(e) => {
398 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range end: {e}");
399 continue;
400 }
401 };
402 expand_range(start, end, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
403 continue;
404 }
405 match s.parse::<IpAddr>() {
407 Ok(ip) => out.push(ip),
408 Err(e) => {
409 tracing::warn!(target: "mockforge::bench", "ignoring malformed --{flag_name} value '{s}': {e}");
410 }
411 }
412 }
413 }
414 out
415}
416
417fn expand_range(
421 start: std::net::IpAddr,
422 end: std::net::IpAddr,
423 cap: usize,
424 flag_name: &str,
425 raw: &str,
426 out: &mut Vec<std::net::IpAddr>,
427) {
428 use std::net::{IpAddr, Ipv4Addr};
429 let (start_v4, end_v4) = match (start, end) {
430 (IpAddr::V4(a), IpAddr::V4(b)) => (a, b),
431 _ => {
432 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range start/end must both be IPv4");
433 return;
434 }
435 };
436 let start_u32 = u32::from(start_v4);
437 let end_u32 = u32::from(end_v4);
438 if end_u32 < start_u32 {
439 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range end {end_v4} is before start {start_v4}");
440 return;
441 }
442 let total = (end_u32 - start_u32).saturating_add(1) as usize;
443 let take = total.min(cap);
444 if total > cap {
445 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range has {total} addresses, capping at {cap}");
446 }
447 for i in 0..take as u32 {
448 out.push(IpAddr::V4(Ipv4Addr::from(start_u32 + i)));
449 }
450}
451
452fn expand_cidr(
456 net: std::net::IpAddr,
457 prefix: u32,
458 cap: usize,
459 flag_name: &str,
460 raw: &str,
461 out: &mut Vec<std::net::IpAddr>,
462) {
463 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
464 match net {
465 IpAddr::V4(ipv4) => {
466 if prefix > 32 {
467 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv4 prefix must be <= 32");
468 return;
469 }
470 let total: u64 = 1u64 << (32 - prefix);
471 let take = total.min(cap as u64) as u32;
472 if total > cap as u64 {
473 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': CIDR has {total} addresses, capping at {cap}");
474 }
475 let mask: u32 = if prefix == 0 {
476 0
477 } else {
478 !0u32 << (32 - prefix)
479 };
480 let net_u32 = u32::from(ipv4) & mask;
481 for i in 0..take {
482 out.push(IpAddr::V4(Ipv4Addr::from(net_u32.wrapping_add(i))));
483 }
484 }
485 IpAddr::V6(ipv6) => {
486 if prefix > 128 {
487 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv6 prefix must be <= 128");
488 return;
489 }
490 let mask: u128 = if prefix == 0 {
494 0
495 } else {
496 !0u128 << (128 - prefix)
497 };
498 let net_u128 = u128::from(ipv6) & mask;
499 let remaining_bits = 128 - prefix;
500 let total_capped = if remaining_bits >= 64 {
503 cap as u128
504 } else {
505 (1u128 << remaining_bits).min(cap as u128)
506 };
507 if remaining_bits < 128 && (1u128 << remaining_bits) > cap as u128 {
508 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': IPv6 CIDR exceeds {cap} addresses, capping");
509 }
510 for i in 0..total_capped {
511 out.push(IpAddr::V6(Ipv6Addr::from(net_u128.wrapping_add(i))));
512 }
513 }
514 }
515}
516
517impl BenchCommand {
518 pub fn security_testing_enabled(&self) -> bool {
534 if self.wafbench_verbatim {
535 return false;
536 }
537 self.security_test || self.wafbench_dir.is_some()
538 }
539
540 pub async fn load_and_merge_specs(&self) -> Result<OpenApiSpec> {
542 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
543
544 if !self.spec.is_empty() {
546 let specs = load_specs_from_files(self.spec.clone())
547 .await
548 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
549 all_specs.extend(specs);
550 }
551
552 if let Some(spec_dir) = &self.spec_dir {
554 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
555 BenchError::Other(format!("Failed to load specs from directory: {}", e))
556 })?;
557 all_specs.extend(dir_specs);
558 }
559
560 if all_specs.is_empty() {
561 return Err(BenchError::Other(
562 "No spec files provided. Use --spec or --spec-dir.".to_string(),
563 ));
564 }
565
566 if all_specs.len() == 1 {
568 return Ok(all_specs.into_iter().next().expect("checked len() == 1 above").1);
570 }
571
572 let conflict_strategy = match self.merge_conflicts.as_str() {
574 "first" => ConflictStrategy::First,
575 "last" => ConflictStrategy::Last,
576 _ => ConflictStrategy::Error,
577 };
578
579 merge_specs(all_specs, conflict_strategy)
580 .map_err(|e| BenchError::Other(format!("Failed to merge specs: {}", e)))
581 }
582
583 fn get_spec_display_name(&self) -> String {
585 if self.spec.len() == 1 {
586 self.spec[0].to_string_lossy().to_string()
587 } else if !self.spec.is_empty() {
588 format!("{} spec files", self.spec.len())
589 } else if let Some(dir) = &self.spec_dir {
590 format!("specs from {}", dir.display())
591 } else {
592 "no specs".to_string()
593 }
594 }
595
596 fn advise_capacity(&self) {
603 let target_count = self
604 .targets_file
605 .as_ref()
606 .and_then(|p| std::fs::read_to_string(p).ok())
607 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
608 .and_then(|v| v.as_array().map(|a| a.len()))
609 .unwrap_or(1);
610 let vus = self.vus.max(1);
611 let rps_total = self.target_rps.unwrap_or(0) as usize * target_count.max(1);
612 let load_product = target_count * vus as usize;
616 if load_product >= 150 {
617 let est_ram_gb =
618 (vus as usize * 50) / 1024 + (target_count * 10 * 2) / 1024 + target_count / 2;
619 let est_cores = ((vus as usize) / 50).max(2);
620 TerminalReporter::print_warning(&format!(
621 "Capacity advisory: targets={target_count}, VUs={vus}, RPS-total≈{rps_total}. \
622 Single-client estimate: ~{est_cores} CPU cores, ~{est_ram_gb} GB RAM. \
623 If your machine is below that, expect OOM hangs partway through the run. \
624 See https://docs.mockforge.dev/reference/bench-capacity-sizing.html \
625 for the sizing table and sharding guide."
626 ));
627 }
628 }
629
630 pub async fn execute(&self) -> Result<()> {
632 if self.conformance_self_test && self.use_k6 {
639 TerminalReporter::print_warning(
640 "--use-k6 has no effect with --conformance-self-test: the self-test driver runs and returns before k6 is invoked. Drop one or the other depending on whether you want the spec-driven self-test or a k6 bench run.",
641 );
642 }
643
644 self.advise_capacity();
650
651 if let Some(targets_file) = &self.targets_file {
653 if self.conformance && self.conformance_self_test {
662 return self.execute_multi_target_self_test(targets_file).await;
663 }
664 if self.conformance {
665 return self.execute_multi_target_conformance(targets_file).await;
666 }
667 return self.execute_multi_target(targets_file).await;
668 }
669
670 if self.spec_mode == "sequential" && (self.spec.len() > 1 || self.spec_dir.is_some()) {
672 return self.execute_sequential_specs().await;
673 }
674
675 TerminalReporter::print_header(
678 &self.get_spec_display_name(),
679 &self.target,
680 0, &self.scenario,
682 Self::parse_duration(&self.duration)?,
683 );
684
685 if !K6Executor::is_k6_installed() {
687 TerminalReporter::print_error("k6 is not installed");
688 TerminalReporter::print_warning(
689 "Install k6 from: https://k6.io/docs/get-started/installation/",
690 );
691 return Err(BenchError::K6NotFound);
692 }
693 K6Executor::warn_if_pre_v1().await;
694
695 if self.conformance {
697 return self.execute_conformance_test().await;
698 }
699
700 let spec_supplied = !self.spec.is_empty() || self.spec_dir.is_some();
707 let merged_spec = if self.wafbench_verbatim && !spec_supplied {
708 tracing::info!(
709 target: "mockforge::bench",
710 "--wafbench-verbatim without --spec: sending only the traffic file's requests"
711 );
712 OpenApiSpec {
713 spec: Default::default(),
714 file_path: None,
715 raw_document: None,
716 }
717 } else {
718 TerminalReporter::print_progress("Loading OpenAPI specification(s)...");
719 self.load_and_merge_specs().await?
720 };
721 let parser = SpecParser::from_spec(merged_spec);
722 if self.spec.len() > 1 || self.spec_dir.is_some() {
723 TerminalReporter::print_success(&format!(
724 "Loaded and merged {} specification(s)",
725 self.spec.len() + self.spec_dir.as_ref().map(|_| 1).unwrap_or(0)
726 ));
727 } else {
728 TerminalReporter::print_success("Specification loaded");
729 }
730
731 let mock_config = self.build_mock_config().await;
733 if mock_config.is_mock_server {
734 TerminalReporter::print_progress("Mock server integration enabled");
735 }
736
737 if self.crud_flow {
739 return self.execute_crud_flow(&parser).await;
740 }
741
742 if self.owasp_api_top10 {
744 return self.execute_owasp_test(&parser).await;
745 }
746
747 TerminalReporter::print_progress("Extracting API operations...");
749 let mut operations = if let Some(filter) = &self.operations {
750 parser.filter_operations(filter)?
751 } else {
752 parser.get_operations()
753 };
754
755 if let Some(exclude) = &self.exclude_operations {
757 let before_count = operations.len();
758 operations = parser.exclude_operations(operations, exclude)?;
759 let excluded_count = before_count - operations.len();
760 if excluded_count > 0 {
761 TerminalReporter::print_progress(&format!(
762 "Excluded {} operations matching '{}'",
763 excluded_count, exclude
764 ));
765 }
766 }
767
768 if operations.is_empty() && !self.wafbench_verbatim {
774 return Err(BenchError::Other("No operations found in spec".to_string()));
775 }
776
777 TerminalReporter::print_success(&format!("Found {} operations", operations.len()));
778
779 let param_overrides = if let Some(params_file) = &self.params_file {
781 TerminalReporter::print_progress("Loading parameter overrides...");
782 let overrides = ParameterOverrides::from_file(params_file)?;
783 TerminalReporter::print_success(&format!(
784 "Loaded parameter overrides ({} operation-specific, {} defaults)",
785 overrides.operations.len(),
786 if overrides.defaults.is_empty() { 0 } else { 1 }
787 ));
788 Some(overrides)
789 } else {
790 None
791 };
792
793 TerminalReporter::print_progress("Generating request templates...");
795 let templates: Vec<_> = operations
796 .iter()
797 .map(|op| {
798 let op_overrides = param_overrides.as_ref().map(|po| {
799 po.get_for_operation(op.operation_id.as_deref(), &op.method, &op.path)
800 });
801 RequestGenerator::generate_template_with_overrides(op, op_overrides.as_ref())
802 })
803 .collect::<Result<Vec<_>>>()?;
804 TerminalReporter::print_success("Request templates generated");
805
806 let templates = if self.wafbench_verbatim {
812 let verbatim = self.load_verbatim_templates()?;
813 if verbatim.is_empty() {
814 return Err(BenchError::Other(
815 "--wafbench-verbatim was set but no traffic cases were loaded. Check \
816 --wafbench-dir points at a file, directory or glob containing cases with \
817 a `request.uri`."
818 .to_string(),
819 ));
820 }
821 TerminalReporter::print_success(&format!(
822 "Verbatim mode: {} request(s) will be sent exactly as written (spec endpoints not used)",
823 verbatim.len()
824 ));
825 verbatim
826 } else {
827 templates
828 };
829
830 let custom_headers = self.parse_headers()?;
832
833 let base_path = self.resolve_base_path(&parser);
835 if let Some(ref bp) = base_path {
836 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
837 }
838
839 TerminalReporter::print_progress("Generating k6 load test script...");
841 let scenario =
842 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
843
844 let security_testing_enabled = self.security_testing_enabled();
845
846 let num_ops = operations.len() as u32;
864 if let Some(rps) = self.target_rps {
865 let probe =
866 crate::preflight::probe_target_latency(&self.target, 3, self.skip_tls_verify).await;
867
868 let (required_vus, basis) = match probe {
869 Some(p) => (
870 p.required_vus(rps, num_ops),
871 format!("avg {:.1}ms (measured)", p.avg_latency.as_secs_f64() * 1000.0),
872 ),
873 None => {
874 let fallback = (rps as u64)
876 .saturating_mul(num_ops.max(1) as u64)
877 .div_ceil(10)
878 .min(u32::MAX as u64) as u32;
879 (fallback, "~100ms (default — probe failed)".to_string())
880 }
881 };
882
883 if self.vus < required_vus {
884 const VU_RECOMMENDATION_CAP: u32 = 1000;
890 let recommendation = required_vus.max(self.vus + 1);
891 if recommendation > VU_RECOMMENDATION_CAP {
892 TerminalReporter::print_warning(&format!(
893 "Workload is very large: --rps {} × {} ops/iteration × {} \
894 baseline ⇒ ~{} VUs needed end-to-end, far beyond what's \
895 practical to drive. Two ways to fix:\n 1. Reduce \
896 operations per iteration with `--operations 'pattern,…'` \
897 (or `--exclude-operations`) to focus the bench on a \
898 representative subset.\n 2. Drop `--rps` and use \
899 `--vus {}` alone — closed-model load runs as fast as \
900 the VU pool allows, bounded by latency, with no per-\
901 iteration deadline. Expect 1-iteration coverage of ~{} \
902 operations in {}s.",
903 rps,
904 num_ops,
905 basis,
906 recommendation,
907 self.vus.max(5),
908 num_ops,
909 Self::parse_duration(&self.duration).unwrap_or(0),
910 ));
911 } else {
912 TerminalReporter::print_warning(&format!(
913 "--vus {} may be insufficient for --rps {} × {} ops/iteration \
914 (baseline latency {}). k6's constant-arrival-rate counts ITERATIONS \
915 and each runs every operation in the spec — required ≈ rps × ops × \
916 latency_secs VUs. Bump --vus to ~{} if you see \"Insufficient VUs\" \
917 warnings.",
918 self.vus, rps, num_ops, basis, recommendation,
919 ));
920 }
921 } else if probe.is_some() {
922 TerminalReporter::print_progress(&format!(
923 "Pre-flight probe: target latency {}, {} ops/iteration — --vus {} \
924 is sufficient for --rps {}",
925 basis, num_ops, self.vus, rps,
926 ));
927 }
928 }
929
930 let k6_config = K6Config {
931 target_url: self.target.clone(),
932 base_path,
933 scenario,
934 duration_secs: Self::parse_duration(&self.duration)?,
935 max_vus: self.vus,
936 threshold_percentile: self.threshold_percentile.clone(),
937 threshold_ms: self.threshold_ms,
938 max_error_rate: self.max_error_rate,
939 auth_header: self.auth.clone(),
940 custom_headers,
941 skip_tls_verify: self.skip_tls_verify,
942 security_testing_enabled,
943 chunked_request_bodies: self.chunked_request_bodies,
944 target_rps: self.target_rps,
945 no_keep_alive: self.no_keep_alive,
946 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
952 .into_iter()
953 .map(|ip| ip.to_string())
954 .collect(),
955 geo_source_headers: if self.geo_source_headers.is_empty()
956 && !self.geo_source_ips.is_empty()
957 {
958 crate::conformance::self_test::default_geo_source_headers()
959 } else {
960 self.geo_source_headers.clone()
961 },
962 };
963
964 let generator = K6ScriptGenerator::new(k6_config, templates)
965 .with_abort_valve(self.abort_on_error, self.abort_on_error_rate);
966 let mut script = generator.generate()?;
967 TerminalReporter::print_success("k6 script generated");
968
969 let has_advanced_features = self.data_file.is_some()
971 || self.error_rate.is_some()
972 || self.security_test
973 || self.parallel_create.is_some()
974 || self.wafbench_dir.is_some();
975
976 if has_advanced_features {
978 script = self.generate_enhanced_script(&script)?;
979 }
980
981 if mock_config.is_mock_server {
983 let setup_code = MockIntegrationGenerator::generate_setup(&mock_config);
984 let teardown_code = MockIntegrationGenerator::generate_teardown(&mock_config);
985 let helper_code = MockIntegrationGenerator::generate_vu_id_helper();
986
987 if let Some(import_end) = script.find("export const options") {
989 script.insert_str(
990 import_end,
991 &format!(
992 "\n// === Mock Server Integration ===\n{}\n{}\n{}\n",
993 helper_code, setup_code, teardown_code
994 ),
995 );
996 }
997 }
998
999 TerminalReporter::print_progress("Validating k6 script...");
1001 let validation_errors = K6ScriptGenerator::validate_script(&script);
1002 if !validation_errors.is_empty() {
1003 TerminalReporter::print_error("Script validation failed");
1004 for error in &validation_errors {
1005 eprintln!(" {}", error);
1006 }
1007 return Err(BenchError::Other(format!(
1008 "Generated k6 script has {} validation error(s). Please check the output above.",
1009 validation_errors.len()
1010 )));
1011 }
1012 TerminalReporter::print_success("Script validation passed");
1013
1014 let script_path = if let Some(output) = &self.script_output {
1016 output.clone()
1017 } else {
1018 self.output.join("k6-script.js")
1019 };
1020
1021 if let Some(parent) = script_path.parent() {
1022 std::fs::create_dir_all(parent)?;
1023 }
1024 std::fs::write(&script_path, &script)?;
1025 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
1026
1027 if self.generate_only {
1029 println!("\nScript generated successfully. Run it with:");
1030 println!(" k6 run {}", script_path.display());
1031 return Ok(());
1032 }
1033
1034 TerminalReporter::print_progress("Executing load test...");
1036 let executor = K6Executor::new()?
1040 .with_local_ips(self.source_ips.join(","))
1041 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
1042 .with_discard_response_bodies(self.discard_response_bodies);
1043
1044 std::fs::create_dir_all(&self.output)?;
1045
1046 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
1047
1048 let duration_secs = Self::parse_duration(&self.duration)?;
1050 TerminalReporter::print_summary_full(
1051 &results,
1052 duration_secs,
1053 self.no_keep_alive,
1054 Some(num_ops),
1055 );
1056
1057 self.reprint_traffic_file_breakdown();
1058 println!("\nResults saved to: {}", self.output.display());
1059
1060 Ok(())
1061 }
1062
1063 async fn execute_multi_target(&self, targets_file: &Path) -> Result<()> {
1065 TerminalReporter::print_progress("Parsing targets file...");
1066 let targets = parse_targets_file(targets_file)?;
1067 let num_targets = targets.len();
1068 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
1069
1070 if targets.is_empty() {
1071 return Err(BenchError::Other("No targets found in file".to_string()));
1072 }
1073
1074 let max_concurrency = self.max_concurrency.unwrap_or(10) as usize;
1076 let max_concurrency = max_concurrency.min(num_targets); TerminalReporter::print_header(
1080 &self.get_spec_display_name(),
1081 &format!("{} targets", num_targets),
1082 0,
1083 &self.scenario,
1084 Self::parse_duration(&self.duration)?,
1085 );
1086
1087 let executor = ParallelExecutor::new(
1089 BenchCommand {
1090 spec: self.spec.clone(),
1092 spec_dir: self.spec_dir.clone(),
1093 merge_conflicts: self.merge_conflicts.clone(),
1094 spec_mode: self.spec_mode.clone(),
1095 dependency_config: self.dependency_config.clone(),
1096 target: self.target.clone(), base_path: self.base_path.clone(),
1098 duration: self.duration.clone(),
1099 vus: self.vus,
1100 target_rps: self.target_rps,
1101 no_keep_alive: self.no_keep_alive,
1102 scenario: self.scenario.clone(),
1103 operations: self.operations.clone(),
1104 exclude_operations: self.exclude_operations.clone(),
1105 auth: self.auth.clone(),
1106 headers: self.headers.clone(),
1107 output: self.output.clone(),
1108 generate_only: self.generate_only,
1109 script_output: self.script_output.clone(),
1110 threshold_percentile: self.threshold_percentile.clone(),
1111 threshold_ms: self.threshold_ms,
1112 max_error_rate: self.max_error_rate,
1113 abort_on_error: self.abort_on_error,
1114 abort_on_error_rate: self.abort_on_error_rate,
1115 verbose: self.verbose,
1116 skip_tls_verify: self.skip_tls_verify,
1117 chunked_request_bodies: self.chunked_request_bodies,
1118 targets_file: None,
1119 max_concurrency: None,
1120 results_format: self.results_format.clone(),
1121 params_file: self.params_file.clone(),
1122 crud_flow: self.crud_flow,
1123 flow_config: self.flow_config.clone(),
1124 extract_fields: self.extract_fields.clone(),
1125 parallel_create: self.parallel_create,
1126 data_file: self.data_file.clone(),
1127 data_distribution: self.data_distribution.clone(),
1128 data_mappings: self.data_mappings.clone(),
1129 per_uri_control: self.per_uri_control,
1130 error_rate: self.error_rate,
1131 error_types: self.error_types.clone(),
1132 security_test: self.security_test,
1133 security_payloads: self.security_payloads.clone(),
1134 security_categories: self.security_categories.clone(),
1135 security_target_fields: self.security_target_fields.clone(),
1136 wafbench_dir: self.wafbench_dir.clone(),
1137 wafbench_cycle_all: self.wafbench_cycle_all,
1138 wafbench_verbatim: self.wafbench_verbatim,
1139 owasp_api_top10: self.owasp_api_top10,
1140 owasp_categories: self.owasp_categories.clone(),
1141 owasp_auth_header: self.owasp_auth_header.clone(),
1142 owasp_auth_token: self.owasp_auth_token.clone(),
1143 owasp_admin_paths: self.owasp_admin_paths.clone(),
1144 owasp_id_fields: self.owasp_id_fields.clone(),
1145 owasp_report: self.owasp_report.clone(),
1146 owasp_report_format: self.owasp_report_format.clone(),
1147 owasp_iterations: self.owasp_iterations,
1148 conformance: false,
1149 conformance_api_key: self.conformance_api_key.clone(),
1165 conformance_basic_auth: self.conformance_basic_auth.clone(),
1166 conformance_report: PathBuf::from("conformance-report.json"),
1167 conformance_categories: None,
1168 conformance_report_format: "json".to_string(),
1169 conformance_headers: self.conformance_headers.clone(),
1173 conformance_all_operations: false,
1174 conformance_custom: None,
1175 conformance_delay_ms: 0,
1176 use_k6: false,
1177 conformance_custom_filter: None,
1178 export_requests: false,
1179 validate_requests: false,
1180 conformance_self_test: false,
1181 conformance_self_test_capture: false,
1182 conformance_self_test_iterations: 1,
1183 conformance_self_test_duration: None,
1184 validate_response_schemas: false,
1185 source_ips: self.source_ips.clone(),
1190 geo_source_ips: self.geo_source_ips.clone(),
1191 geo_source_headers: self.geo_source_headers.clone(),
1192 report_missed_cap: None,
1193 discard_response_bodies: self.discard_response_bodies,
1197 dns_policy: self.dns_policy.clone(),
1200 },
1201 targets,
1202 max_concurrency,
1203 );
1204
1205 let start_time = std::time::Instant::now();
1207 let aggregated_results = executor.execute_all().await?;
1208 let elapsed = start_time.elapsed();
1209
1210 self.report_multi_target_results(&aggregated_results, elapsed)?;
1212
1213 Ok(())
1214 }
1215
1216 fn report_multi_target_results(
1218 &self,
1219 results: &AggregatedResults,
1220 elapsed: std::time::Duration,
1221 ) -> Result<()> {
1222 TerminalReporter::print_multi_target_summary(results);
1224
1225 let total_secs = elapsed.as_secs();
1227 let hours = total_secs / 3600;
1228 let minutes = (total_secs % 3600) / 60;
1229 let seconds = total_secs % 60;
1230 if hours > 0 {
1231 println!("\n Total Elapsed Time: {}h {}m {}s", hours, minutes, seconds);
1232 } else if minutes > 0 {
1233 println!("\n Total Elapsed Time: {}m {}s", minutes, seconds);
1234 } else {
1235 println!("\n Total Elapsed Time: {}s", seconds);
1236 }
1237
1238 if self.results_format == "aggregated" || self.results_format == "both" {
1240 let summary_path = self.output.join("aggregated_summary.json");
1241 let summary_json = serde_json::json!({
1242 "total_elapsed_seconds": elapsed.as_secs(),
1243 "total_targets": results.total_targets,
1244 "successful_targets": results.successful_targets,
1245 "failed_targets": results.failed_targets,
1246 "aggregated_metrics": {
1247 "total_requests": results.aggregated_metrics.total_requests,
1248 "total_failed_requests": results.aggregated_metrics.total_failed_requests,
1249 "avg_duration_ms": results.aggregated_metrics.avg_duration_ms,
1250 "p95_duration_ms": results.aggregated_metrics.p95_duration_ms,
1251 "p99_duration_ms": results.aggregated_metrics.p99_duration_ms,
1252 "error_rate": results.aggregated_metrics.error_rate,
1253 "total_rps": results.aggregated_metrics.total_rps,
1254 "avg_rps": results.aggregated_metrics.avg_rps,
1255 "total_vus_max": results.aggregated_metrics.total_vus_max,
1256 },
1257 "target_results": results.target_results.iter().map(|r| {
1258 serde_json::json!({
1259 "target_url": r.target_url,
1260 "target_index": r.target_index,
1261 "success": r.success,
1262 "error": r.error,
1263 "total_requests": r.results.total_requests,
1264 "failed_requests": r.results.failed_requests,
1265 "avg_duration_ms": r.results.avg_duration_ms,
1266 "min_duration_ms": r.results.min_duration_ms,
1267 "med_duration_ms": r.results.med_duration_ms,
1268 "p90_duration_ms": r.results.p90_duration_ms,
1269 "p95_duration_ms": r.results.p95_duration_ms,
1270 "p99_duration_ms": r.results.p99_duration_ms,
1271 "max_duration_ms": r.results.max_duration_ms,
1272 "rps": r.results.rps,
1273 "vus_max": r.results.vus_max,
1274 "output_dir": r.output_dir.to_string_lossy(),
1275 })
1276 }).collect::<Vec<_>>(),
1277 });
1278
1279 std::fs::write(&summary_path, serde_json::to_string_pretty(&summary_json)?)?;
1280 TerminalReporter::print_success(&format!(
1281 "Aggregated summary saved to: {}",
1282 summary_path.display()
1283 ));
1284 }
1285
1286 let csv_path = self.output.join("all_targets.csv");
1288 let mut csv = String::from(
1289 "target_url,success,requests,failed,rps,vus,min_ms,avg_ms,med_ms,p90_ms,p95_ms,p99_ms,max_ms,error\n",
1290 );
1291 for r in &results.target_results {
1292 csv.push_str(&format!(
1293 "{},{},{},{},{:.1},{},{:.1},{:.1},{:.1},{:.1},{:.1},{:.1},{:.1},{}\n",
1294 r.target_url,
1295 r.success,
1296 r.results.total_requests,
1297 r.results.failed_requests,
1298 r.results.rps,
1299 r.results.vus_max,
1300 r.results.min_duration_ms,
1301 r.results.avg_duration_ms,
1302 r.results.med_duration_ms,
1303 r.results.p90_duration_ms,
1304 r.results.p95_duration_ms,
1305 r.results.p99_duration_ms,
1306 r.results.max_duration_ms,
1307 r.error.as_deref().unwrap_or(""),
1308 ));
1309 }
1310 let _ = std::fs::write(&csv_path, &csv);
1311
1312 self.reprint_traffic_file_breakdown();
1313 println!("\nResults saved to: {}", self.output.display());
1314 println!(" - Per-target results: {}", self.output.join("target_*").display());
1315 println!(" - All targets CSV: {}", csv_path.display());
1316 if self.results_format == "aggregated" || self.results_format == "both" {
1317 println!(
1318 " - Aggregated summary: {}",
1319 self.output.join("aggregated_summary.json").display()
1320 );
1321 }
1322
1323 Ok(())
1324 }
1325
1326 pub fn parse_duration(duration: &str) -> Result<u64> {
1328 let duration = duration.trim();
1329
1330 if let Some(secs) = duration.strip_suffix('s') {
1331 secs.parse::<u64>()
1332 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1333 } else if let Some(mins) = duration.strip_suffix('m') {
1334 mins.parse::<u64>()
1335 .map(|m| m * 60)
1336 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1337 } else if let Some(hours) = duration.strip_suffix('h') {
1338 hours
1339 .parse::<u64>()
1340 .map(|h| h * 3600)
1341 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1342 } else {
1343 duration
1345 .parse::<u64>()
1346 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1347 }
1348 }
1349
1350 pub(crate) fn load_verbatim_templates(
1357 &self,
1358 ) -> Result<Vec<crate::request_gen::RequestTemplate>> {
1359 let Some(pattern) = self.wafbench_dir.as_ref() else {
1360 return Err(BenchError::Other(
1361 "--wafbench-verbatim requires --wafbench-dir pointing at your traffic file(s)"
1362 .to_string(),
1363 ));
1364 };
1365
1366 let mut loader = WafBenchLoader::new();
1367 loader.load_from_pattern(pattern)?;
1368 self.emit_traffic_file_breakdown(loader.stats(), "what to expect in proxy logs");
1369
1370 Ok(crate::wafbench::traffic_cases_to_templates(loader.test_cases()))
1371 }
1372
1373 pub fn parse_headers(&self) -> Result<HashMap<String, String>> {
1375 let mut headers = parse_header_string(&self.headers)?;
1376
1377 let already_has = |hs: &HashMap<String, String>, name: &str| -> bool {
1388 hs.keys().any(|k| k.eq_ignore_ascii_case(name))
1389 };
1390
1391 if !already_has(&headers, "Authorization") {
1392 if let Some(b) = self.conformance_basic_auth.as_ref().filter(|s| !s.is_empty()) {
1393 use base64::Engine as _;
1394 let encoded = base64::engine::general_purpose::STANDARD.encode(b.as_bytes());
1395 headers.insert("Authorization".to_string(), format!("Basic {}", encoded));
1396 }
1397 }
1398
1399 for line in &self.conformance_headers {
1405 let Some((name, value)) = line.split_once(':') else {
1406 continue;
1407 };
1408 let name = name.trim();
1409 let value = value.trim();
1410 if name.is_empty() || already_has(&headers, name) {
1411 continue;
1412 }
1413 headers.insert(name.to_string(), value.to_string());
1414 }
1415
1416 if !self.conformance && self.conformance_api_key.is_some() {
1422 TerminalReporter::print_warning(
1423 "--conformance-api-key only fires under --conformance. For plain bench use --header 'X-API-Key: ...'.",
1424 );
1425 }
1426
1427 Ok(headers)
1428 }
1429
1430 fn parse_extracted_values(output_dir: &Path) -> Result<ExtractedValues> {
1431 let extracted_path = output_dir.join("extracted_values.json");
1432 if !extracted_path.exists() {
1433 return Ok(ExtractedValues::new());
1434 }
1435
1436 let content = std::fs::read_to_string(&extracted_path)
1437 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1438 let parsed: serde_json::Value = serde_json::from_str(&content)
1439 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1440
1441 let mut extracted = ExtractedValues::new();
1442 if let Some(values) = parsed.as_object() {
1443 for (key, value) in values {
1444 extracted.set(key.clone(), value.clone());
1445 }
1446 }
1447
1448 Ok(extracted)
1449 }
1450
1451 fn resolve_base_path(&self, parser: &SpecParser) -> Option<String> {
1460 if let Some(cli_base_path) = &self.base_path {
1462 if cli_base_path.is_empty() {
1463 return None;
1465 }
1466 return Some(cli_base_path.clone());
1467 }
1468
1469 parser.get_base_path()
1471 }
1472
1473 async fn build_mock_config(&self) -> MockIntegrationConfig {
1475 if MockServerDetector::looks_like_mock_server(&self.target) {
1477 if let Ok(info) = MockServerDetector::detect(&self.target).await {
1479 if info.is_mockforge {
1480 TerminalReporter::print_success(&format!(
1481 "Detected MockForge server (version: {})",
1482 info.version.as_deref().unwrap_or("unknown")
1483 ));
1484 return MockIntegrationConfig::mock_server();
1485 }
1486 }
1487 }
1488 MockIntegrationConfig::real_api()
1489 }
1490
1491 fn build_crud_flow_config(&self) -> Option<CrudFlowConfig> {
1493 if !self.crud_flow {
1494 return None;
1495 }
1496
1497 if let Some(config_path) = &self.flow_config {
1499 match CrudFlowConfig::from_file(config_path) {
1500 Ok(config) => return Some(config),
1501 Err(e) => {
1502 TerminalReporter::print_warning(&format!(
1503 "Failed to load flow config: {}. Using auto-detection.",
1504 e
1505 ));
1506 }
1507 }
1508 }
1509
1510 let extract_fields = self
1512 .extract_fields
1513 .as_ref()
1514 .map(|f| f.split(',').map(|s| s.trim().to_string()).collect())
1515 .unwrap_or_else(|| vec!["id".to_string(), "uuid".to_string()]);
1516
1517 Some(CrudFlowConfig {
1518 flows: Vec::new(), default_extract_fields: extract_fields,
1520 })
1521 }
1522
1523 fn build_data_driven_config(&self) -> Option<DataDrivenConfig> {
1525 let data_file = self.data_file.as_ref()?;
1526
1527 let distribution = DataDistribution::from_str(&self.data_distribution)
1528 .unwrap_or(DataDistribution::UniquePerVu);
1529
1530 let mappings = self
1531 .data_mappings
1532 .as_ref()
1533 .map(|m| DataMapping::parse_mappings(m).unwrap_or_default())
1534 .unwrap_or_default();
1535
1536 Some(DataDrivenConfig {
1537 file_path: data_file.to_string_lossy().to_string(),
1538 distribution,
1539 mappings,
1540 csv_has_header: true,
1541 per_uri_control: self.per_uri_control,
1542 per_uri_columns: crate::data_driven::PerUriColumns::default(),
1543 })
1544 }
1545
1546 fn build_invalid_data_config(&self) -> Option<InvalidDataConfig> {
1548 let error_rate = self.error_rate?;
1549
1550 let error_types = self
1551 .error_types
1552 .as_ref()
1553 .map(|types| InvalidDataConfig::parse_error_types(types).unwrap_or_default())
1554 .unwrap_or_default();
1555
1556 Some(InvalidDataConfig {
1557 error_rate,
1558 error_types,
1559 target_fields: Vec::new(),
1560 })
1561 }
1562
1563 fn build_security_config(&self) -> Option<SecurityTestConfig> {
1565 if !self.security_test {
1566 return None;
1567 }
1568
1569 let categories = self
1570 .security_categories
1571 .as_ref()
1572 .map(|cats| SecurityTestConfig::parse_categories(cats).unwrap_or_default())
1573 .unwrap_or_else(|| {
1574 let mut default = HashSet::new();
1575 default.insert(SecurityCategory::SqlInjection);
1576 default.insert(SecurityCategory::Xss);
1577 default
1578 });
1579
1580 let target_fields = self
1581 .security_target_fields
1582 .as_ref()
1583 .map(|fields| fields.split(',').map(|f| f.trim().to_string()).collect())
1584 .unwrap_or_default();
1585
1586 let custom_payloads_file =
1587 self.security_payloads.as_ref().map(|p| p.to_string_lossy().to_string());
1588
1589 Some(SecurityTestConfig {
1590 enabled: true,
1591 categories,
1592 target_fields,
1593 custom_payloads_file,
1594 include_high_risk: false,
1595 })
1596 }
1597
1598 fn build_parallel_config(&self) -> Option<ParallelConfig> {
1600 let count = self.parallel_create?;
1601
1602 Some(ParallelConfig::new(count))
1603 }
1604
1605 fn format_unique_total(unique: usize, rps: Option<u32>) -> String {
1608 match rps {
1609 Some(r) if r > 0 => {
1610 let total = unique.saturating_mul(r as usize);
1611 format!("unique={unique} total={total} ({unique} * {r} RPS)")
1612 }
1613 _ => format!("unique={unique}"),
1614 }
1615 }
1616
1617 fn traffic_bucket_json(
1618 unique: usize,
1619 rps: Option<u32>,
1620 duration_secs: Option<u64>,
1621 ) -> serde_json::Value {
1622 let per_second = rps.filter(|&r| r > 0).map(|r| (unique as u64).saturating_mul(r as u64));
1628 let total = per_second.unwrap_or(unique as u64);
1629 let expected_requests = match (rps.filter(|&r| r > 0), duration_secs) {
1630 (Some(r), Some(d)) => Some((unique as u64).saturating_mul(r as u64).saturating_mul(d)),
1631 _ => None,
1632 };
1633 serde_json::json!({
1634 "unique": unique,
1635 "total": total,
1636 "expected_requests": expected_requests,
1637 "expected_requests_unit": "http",
1638 })
1639 }
1640
1641 fn emit_traffic_file_breakdown(&self, stats: &crate::wafbench::WafBenchStats, phase: &str) {
1644 if stats.per_file.is_empty() {
1645 return;
1646 }
1647 let rps = self.target_rps.filter(|&r| r > 0);
1648 TerminalReporter::print_success(&format!("Traffic file breakdown ({phase}):"));
1649 for file in &stats.per_file {
1650 let other = if file.other > 0 {
1651 format!(" other={}", file.other)
1652 } else {
1653 String::new()
1654 };
1655 TerminalReporter::print_progress(&format!(
1656 " {}: sent {} attack(expected 403) {} normal(expected 200) {} omitted={}{other}",
1657 file.file,
1658 Self::format_unique_total(file.sent, rps),
1659 Self::format_unique_total(file.attack, rps),
1660 Self::format_unique_total(file.normal, rps),
1661 file.omitted
1662 ));
1663 }
1664 self.write_traffic_breakdown_json(stats);
1665 }
1666
1667 fn write_traffic_breakdown_json(&self, stats: &crate::wafbench::WafBenchStats) {
1669 if stats.per_file.is_empty() {
1670 return;
1671 }
1672 let rps = self.target_rps.filter(|&r| r > 0);
1673 let duration_secs = Self::parse_duration(&self.duration).ok();
1674 let files: Vec<serde_json::Value> = stats
1675 .per_file
1676 .iter()
1677 .map(|file| {
1678 serde_json::json!({
1679 "file": file.file,
1680 "sent": Self::traffic_bucket_json(file.sent, rps, duration_secs),
1681 "attack": Self::traffic_bucket_json(file.attack, rps, duration_secs),
1682 "normal": Self::traffic_bucket_json(file.normal, rps, duration_secs),
1683 "omitted": file.omitted,
1684 "other": file.other,
1685 })
1686 })
1687 .collect();
1688 let payload = serde_json::json!({
1689 "rps": rps,
1690 "duration_secs": duration_secs,
1691 "expected_requests_note": "HTTP requests over the run, not seconds. unique * rps * duration_secs, assuming each k6 iteration sends every unique case. null when --rps is unset.",
1692 "files": files,
1693 });
1694 if let Some(parent) = self.output.parent() {
1695 let _ = std::fs::create_dir_all(parent);
1696 }
1697 let _ = std::fs::create_dir_all(&self.output);
1698 let path = self.output.join("traffic-breakdown.json");
1699 if let Ok(body) = serde_json::to_string_pretty(&payload) {
1700 if std::fs::write(&path, body).is_ok() {
1701 TerminalReporter::print_progress(&format!(
1702 "Traffic breakdown written to: {}",
1703 path.display()
1704 ));
1705 }
1706 }
1707 }
1708
1709 fn reprint_traffic_file_breakdown(&self) {
1712 let path = self.output.join("traffic-breakdown.json");
1713 let Ok(raw) = std::fs::read_to_string(&path) else {
1714 return;
1715 };
1716 let Ok(v) = serde_json::from_str::<serde_json::Value>(&raw) else {
1717 return;
1718 };
1719 let Some(files) = v.get("files").and_then(|f| f.as_array()) else {
1720 return;
1721 };
1722 if files.is_empty() {
1723 return;
1724 }
1725 TerminalReporter::print_success("Traffic file breakdown (end of run):");
1726 for file in files {
1727 let name = file.get("file").and_then(|x| x.as_str()).unwrap_or("?");
1728 let bucket = |key: &str| -> String {
1729 let unique = file
1730 .get(key)
1731 .and_then(|b| b.get("unique"))
1732 .and_then(|u| u.as_u64())
1733 .unwrap_or(0) as usize;
1734 Self::format_unique_total(unique, self.target_rps.filter(|&r| r > 0))
1735 };
1736 let omitted = file.get("omitted").and_then(|o| o.as_u64()).unwrap_or(0);
1737 let other = file.get("other").and_then(|o| o.as_u64()).unwrap_or(0);
1738 let other = if other > 0 {
1739 format!(" other={other}")
1740 } else {
1741 String::new()
1742 };
1743 TerminalReporter::print_progress(&format!(
1744 " {name}: sent {} attack(expected 403) {} normal(expected 200) {} omitted={omitted}{other}",
1745 bucket("sent"),
1746 bucket("attack"),
1747 bucket("normal"),
1748 ));
1749 }
1750 TerminalReporter::print_progress(&format!(" (also in {})", path.display()));
1751 }
1752
1753 fn load_wafbench_payloads(&self) -> Result<Vec<SecurityPayload>> {
1760 let Some(ref wafbench_dir) = self.wafbench_dir else {
1761 return Ok(Vec::new());
1762 };
1763
1764 let mut loader = WafBenchLoader::new();
1765 loader.load_from_pattern(wafbench_dir)?;
1766
1767 let stats = loader.stats();
1768
1769 if stats.files_processed == 0 {
1770 let mut msg = format!(
1771 "No WAFBench YAML files found matching '{wafbench_dir}'. \
1772 --wafbench-dir is a file, a directory or a glob. A missing \
1773 file is an error, not an empty payload pool."
1774 );
1775 if !stats.parse_errors.is_empty() {
1776 msg.push_str(" Parse errors:");
1777 for error in &stats.parse_errors {
1778 msg.push_str(&format!("\n - {error}"));
1779 }
1780 }
1781 return Err(BenchError::Other(msg));
1782 }
1783
1784 TerminalReporter::print_progress(&format!(
1785 "Loaded {} WAFBench files, {} test cases, {} payloads",
1786 stats.files_processed, stats.test_cases_loaded, stats.payloads_extracted
1787 ));
1788 self.emit_traffic_file_breakdown(stats, "what to expect in proxy logs");
1789
1790 for (category, count) in &stats.by_category {
1792 TerminalReporter::print_progress(&format!(" - {}: {} tests", category, count));
1793 }
1794
1795 for error in &stats.parse_errors {
1797 TerminalReporter::print_warning(&format!(" Parse error: {}", error));
1798 }
1799
1800 Ok(loader.to_security_payloads())
1801 }
1802
1803 pub(crate) fn generate_enhanced_script(&self, base_script: &str) -> Result<String> {
1805 let mut enhanced_script = base_script.to_string();
1806 let mut additional_code = String::new();
1807
1808 if let Some(config) = self.build_data_driven_config() {
1810 TerminalReporter::print_progress("Adding data-driven testing support...");
1811 additional_code.push_str(&DataDrivenGenerator::generate_setup(&config));
1812 additional_code.push('\n');
1813 TerminalReporter::print_success("Data-driven testing enabled");
1814 }
1815
1816 if let Some(config) = self.build_invalid_data_config() {
1818 TerminalReporter::print_progress("Adding invalid data testing support...");
1819 additional_code.push_str(&InvalidDataGenerator::generate_invalidation_logic());
1820 additional_code.push('\n');
1821 additional_code
1822 .push_str(&InvalidDataGenerator::generate_should_invalidate(config.error_rate));
1823 additional_code.push('\n');
1824 additional_code
1825 .push_str(&InvalidDataGenerator::generate_type_selection(&config.error_types));
1826 additional_code.push('\n');
1827 TerminalReporter::print_success(&format!(
1828 "Invalid data testing enabled ({}% error rate)",
1829 (self.error_rate.unwrap_or(0.0) * 100.0) as u32
1830 ));
1831 }
1832
1833 let verbatim = self.wafbench_verbatim;
1840 if verbatim && self.security_test {
1841 TerminalReporter::print_warning(
1842 "--security-test is ignored under --wafbench-verbatim: verbatim mode sends your \
1843 traffic cases exactly as written and will not append attack payloads to them. \
1844 Drop --wafbench-verbatim if you want payload injection.",
1845 );
1846 }
1847 let security_config = if verbatim {
1848 None
1849 } else {
1850 self.build_security_config()
1851 };
1852 let wafbench_payloads = if verbatim {
1853 Vec::new()
1854 } else {
1855 self.load_wafbench_payloads()?
1856 };
1857 let security_requested =
1858 !verbatim && (security_config.is_some() || self.wafbench_dir.is_some());
1859
1860 if security_config.is_some() || !wafbench_payloads.is_empty() {
1861 TerminalReporter::print_progress("Adding security testing support...");
1862
1863 let mut payload_list: Vec<SecurityPayload> = Vec::new();
1865
1866 if let Some(ref config) = security_config {
1867 payload_list.extend(SecurityPayloads::get_payloads(config));
1868 }
1869
1870 if !wafbench_payloads.is_empty() {
1872 TerminalReporter::print_progress(&format!(
1873 "Loading {} WAFBench attack patterns...",
1874 wafbench_payloads.len()
1875 ));
1876 payload_list.extend(wafbench_payloads);
1877 }
1878
1879 let target_fields =
1880 security_config.as_ref().map(|c| c.target_fields.clone()).unwrap_or_default();
1881
1882 additional_code.push_str(&SecurityTestGenerator::generate_payload_selection(
1883 &payload_list,
1884 self.wafbench_cycle_all,
1885 ));
1886 additional_code.push('\n');
1887 additional_code
1888 .push_str(&SecurityTestGenerator::generate_apply_payload(&target_fields));
1889 additional_code.push('\n');
1890 additional_code.push_str(&SecurityTestGenerator::generate_security_checks());
1891 additional_code.push('\n');
1892
1893 let mode = if self.wafbench_cycle_all {
1894 "cycle-all"
1895 } else {
1896 "random"
1897 };
1898 TerminalReporter::print_success(&format!(
1899 "Security testing enabled ({} payloads, {} mode)",
1900 payload_list.len(),
1901 mode
1902 ));
1903 } else if security_requested {
1904 TerminalReporter::print_warning(
1908 "Security testing was requested but no payloads were loaded. \
1909 Ensure --wafbench-dir points to valid CRS YAML files or add --security-test.",
1910 );
1911 additional_code
1912 .push_str(&SecurityTestGenerator::generate_payload_selection(&[], false));
1913 additional_code.push('\n');
1914 additional_code.push_str(&SecurityTestGenerator::generate_apply_payload(&[]));
1915 additional_code.push('\n');
1916 }
1917
1918 if let Some(config) = self.build_parallel_config() {
1920 TerminalReporter::print_progress("Adding parallel execution support...");
1921 additional_code.push_str(&ParallelRequestGenerator::generate_batch_helper(&config));
1922 additional_code.push('\n');
1923 TerminalReporter::print_success(&format!(
1924 "Parallel execution enabled (count: {})",
1925 config.count
1926 ));
1927 }
1928
1929 if !additional_code.is_empty() {
1931 if let Some(import_end) = enhanced_script.find("export const options") {
1933 enhanced_script.insert_str(
1934 import_end,
1935 &format!("\n// === Advanced Testing Features ===\n{}\n", additional_code),
1936 );
1937 }
1938 }
1939
1940 Ok(enhanced_script)
1941 }
1942
1943 async fn execute_sequential_specs(&self) -> Result<()> {
1945 TerminalReporter::print_progress("Sequential spec mode: Loading specs individually...");
1946
1947 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
1949
1950 if !self.spec.is_empty() {
1951 let specs = load_specs_from_files(self.spec.clone())
1952 .await
1953 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
1954 all_specs.extend(specs);
1955 }
1956
1957 if let Some(spec_dir) = &self.spec_dir {
1958 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
1959 BenchError::Other(format!("Failed to load specs from directory: {}", e))
1960 })?;
1961 all_specs.extend(dir_specs);
1962 }
1963
1964 if all_specs.is_empty() {
1965 return Err(BenchError::Other(
1966 "No spec files found for sequential execution".to_string(),
1967 ));
1968 }
1969
1970 TerminalReporter::print_success(&format!("Loaded {} spec(s)", all_specs.len()));
1971
1972 let execution_order = if let Some(config_path) = &self.dependency_config {
1974 TerminalReporter::print_progress("Loading dependency configuration...");
1975 let config = SpecDependencyConfig::from_file(config_path)?;
1976
1977 if !config.disable_auto_detect && config.execution_order.is_empty() {
1978 self.detect_and_sort_specs(&all_specs)?
1980 } else {
1981 config.execution_order.iter().flat_map(|g| g.specs.clone()).collect()
1983 }
1984 } else {
1985 self.detect_and_sort_specs(&all_specs)?
1987 };
1988
1989 TerminalReporter::print_success(&format!(
1990 "Execution order: {}",
1991 execution_order
1992 .iter()
1993 .map(|p| p.file_name().unwrap_or_default().to_string_lossy().to_string())
1994 .collect::<Vec<_>>()
1995 .join(" → ")
1996 ));
1997
1998 let mut extracted_values = ExtractedValues::new();
2000 let total_specs = execution_order.len();
2001
2002 for (index, spec_path) in execution_order.iter().enumerate() {
2003 let spec_name = spec_path.file_name().unwrap_or_default().to_string_lossy().to_string();
2004
2005 TerminalReporter::print_progress(&format!(
2006 "[{}/{}] Executing spec: {}",
2007 index + 1,
2008 total_specs,
2009 spec_name
2010 ));
2011
2012 let spec = all_specs
2014 .iter()
2015 .find(|(p, _)| {
2016 p == spec_path
2017 || p.file_name() == spec_path.file_name()
2018 || p.file_name() == Some(spec_path.as_os_str())
2019 })
2020 .map(|(_, s)| s.clone())
2021 .ok_or_else(|| {
2022 BenchError::Other(format!("Spec not found: {}", spec_path.display()))
2023 })?;
2024
2025 let new_values = self.execute_single_spec(&spec, &spec_name, &extracted_values).await?;
2027
2028 extracted_values.merge(&new_values);
2030
2031 TerminalReporter::print_success(&format!(
2032 "[{}/{}] Completed: {} (extracted {} values)",
2033 index + 1,
2034 total_specs,
2035 spec_name,
2036 new_values.values.len()
2037 ));
2038 }
2039
2040 TerminalReporter::print_success(&format!(
2041 "Sequential execution complete: {} specs executed",
2042 total_specs
2043 ));
2044
2045 Ok(())
2046 }
2047
2048 fn detect_and_sort_specs(&self, specs: &[(PathBuf, OpenApiSpec)]) -> Result<Vec<PathBuf>> {
2050 TerminalReporter::print_progress("Auto-detecting spec dependencies...");
2051
2052 let mut detector = DependencyDetector::new();
2053 let dependencies = detector.detect_dependencies(specs);
2054
2055 if dependencies.is_empty() {
2056 TerminalReporter::print_progress("No dependencies detected, using file order");
2057 return Ok(specs.iter().map(|(p, _)| p.clone()).collect());
2058 }
2059
2060 TerminalReporter::print_progress(&format!(
2061 "Detected {} cross-spec dependencies",
2062 dependencies.len()
2063 ));
2064
2065 for dep in &dependencies {
2066 TerminalReporter::print_progress(&format!(
2067 " {} → {} (via field '{}')",
2068 dep.dependency_spec.file_name().unwrap_or_default().to_string_lossy(),
2069 dep.dependent_spec.file_name().unwrap_or_default().to_string_lossy(),
2070 dep.field_name
2071 ));
2072 }
2073
2074 topological_sort(specs, &dependencies)
2075 }
2076
2077 async fn execute_single_spec(
2079 &self,
2080 spec: &OpenApiSpec,
2081 spec_name: &str,
2082 _external_values: &ExtractedValues,
2083 ) -> Result<ExtractedValues> {
2084 let parser = SpecParser::from_spec(spec.clone());
2085
2086 if self.crud_flow {
2088 self.execute_crud_flow_with_extraction(&parser, spec_name).await
2090 } else {
2091 self.execute_standard_spec(&parser, spec_name).await?;
2093 Ok(ExtractedValues::new())
2094 }
2095 }
2096
2097 async fn execute_crud_flow_with_extraction(
2099 &self,
2100 parser: &SpecParser,
2101 spec_name: &str,
2102 ) -> Result<ExtractedValues> {
2103 let operations = parser.get_operations();
2104 let flows = CrudFlowDetector::detect_flows(&operations);
2105
2106 if flows.is_empty() {
2107 TerminalReporter::print_warning(&format!("No CRUD flows detected in {}", spec_name));
2108 return Ok(ExtractedValues::new());
2109 }
2110
2111 TerminalReporter::print_progress(&format!(
2112 " {} CRUD flow(s) in {}",
2113 flows.len(),
2114 spec_name
2115 ));
2116
2117 let mut handlebars = handlebars::Handlebars::new();
2119 handlebars.register_helper(
2121 "json",
2122 Box::new(
2123 |h: &handlebars::Helper,
2124 _: &handlebars::Handlebars,
2125 _: &handlebars::Context,
2126 _: &mut handlebars::RenderContext,
2127 out: &mut dyn handlebars::Output|
2128 -> handlebars::HelperResult {
2129 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2130 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2131 Ok(())
2132 },
2133 ),
2134 );
2135 let template = include_str!("templates/k6_crud_flow.hbs");
2136 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
2137
2138 let custom_headers = self.parse_headers()?;
2139 let config = self.build_crud_flow_config().unwrap_or_default();
2140
2141 let param_overrides = if let Some(params_file) = &self.params_file {
2143 let overrides = ParameterOverrides::from_file(params_file)?;
2144 Some(overrides)
2145 } else {
2146 None
2147 };
2148
2149 let duration_secs = Self::parse_duration(&self.duration)?;
2151 let scenario =
2152 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2153 let stages = scenario.generate_stages(duration_secs, self.vus);
2154
2155 let api_base_path = self.resolve_base_path(parser);
2157
2158 let mut all_headers = custom_headers.clone();
2160 if let Some(auth) = &self.auth {
2161 all_headers.insert("Authorization".to_string(), auth.clone());
2162 }
2163 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2164
2165 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2167
2168 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2169 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2173 serde_json::json!({
2174 "name": sanitized_name.clone(),
2175 "display_name": f.name,
2176 "base_path": f.base_path,
2177 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2178 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2180 let method_raw = if !parts.is_empty() {
2181 parts[0].to_uppercase()
2182 } else {
2183 "GET".to_string()
2184 };
2185 let method = if !parts.is_empty() {
2186 let m = parts[0].to_lowercase();
2187 if m == "delete" { "del".to_string() } else { m }
2189 } else {
2190 "get".to_string()
2191 };
2192 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2193 let path = if let Some(ref bp) = api_base_path {
2195 format!("{}{}", bp, raw_path)
2196 } else {
2197 raw_path.to_string()
2198 };
2199 let is_get_or_head = method == "get" || method == "head";
2200 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2202
2203 let body_value = if has_body {
2205 param_overrides.as_ref()
2206 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2207 .and_then(|oo| oo.body)
2208 .unwrap_or_else(|| serde_json::json!({}))
2209 } else {
2210 serde_json::json!({})
2211 };
2212
2213 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2215
2216 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2218 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2219
2220 serde_json::json!({
2221 "operation": s.operation,
2222 "method": method,
2223 "path": path,
2224 "extract": s.extract,
2225 "use_values": s.use_values,
2226 "use_body": s.use_body,
2227 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2228 "inject_attacks": s.inject_attacks,
2229 "attack_types": s.attack_types,
2230 "description": s.description,
2231 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2232 "is_get_or_head": is_get_or_head,
2233 "has_body": has_body,
2234 "body": processed_body.value,
2235 "body_is_dynamic": body_is_dynamic,
2236 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2237 })
2238 }).collect::<Vec<_>>(),
2239 })
2240 }).collect();
2241
2242 for flow_data in &flows_data {
2244 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2245 for step in steps {
2246 if let Some(placeholders_arr) =
2247 step.get("_placeholders").and_then(|p| p.as_array())
2248 {
2249 for p_str in placeholders_arr {
2250 if let Some(p_name) = p_str.as_str() {
2251 match p_name {
2252 "VU" => {
2253 all_placeholders.insert(DynamicPlaceholder::VU);
2254 }
2255 "Iteration" => {
2256 all_placeholders.insert(DynamicPlaceholder::Iteration);
2257 }
2258 "Timestamp" => {
2259 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2260 }
2261 "UUID" => {
2262 all_placeholders.insert(DynamicPlaceholder::UUID);
2263 }
2264 "Random" => {
2265 all_placeholders.insert(DynamicPlaceholder::Random);
2266 }
2267 "Counter" => {
2268 all_placeholders.insert(DynamicPlaceholder::Counter);
2269 }
2270 "Date" => {
2271 all_placeholders.insert(DynamicPlaceholder::Date);
2272 }
2273 "VuIter" => {
2274 all_placeholders.insert(DynamicPlaceholder::VuIter);
2275 }
2276 _ => {}
2277 }
2278 }
2279 }
2280 }
2281 }
2282 }
2283 }
2284
2285 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2287 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2288
2289 let security_testing_enabled = self.security_testing_enabled();
2291
2292 let data = serde_json::json!({
2293 "base_url": self.target,
2294 "flows": flows_data,
2295 "extract_fields": config.default_extract_fields,
2296 "duration_secs": duration_secs,
2297 "max_vus": self.vus,
2298 "auth_header": self.auth,
2299 "custom_headers": custom_headers,
2300 "skip_tls_verify": self.skip_tls_verify,
2301 "stages": stages.iter().map(|s| serde_json::json!({
2303 "duration": s.duration,
2304 "target": s.target,
2305 })).collect::<Vec<_>>(),
2306 "threshold_percentile": self.threshold_percentile,
2307 "threshold_ms": self.threshold_ms,
2308 "max_error_rate": self.max_error_rate,
2309 "abort_on_error": self.abort_on_error,
2310 "abort_on_error_rate": self.abort_on_error_rate,
2311 "headers": headers_json,
2312 "dynamic_imports": required_imports,
2313 "dynamic_globals": required_globals,
2314 "extracted_values_output_path": output_dir.join("extracted_values.json").to_string_lossy(),
2315 "security_testing_enabled": security_testing_enabled,
2317 "has_custom_headers": !custom_headers.is_empty(),
2318 });
2319
2320 let mut script = handlebars
2321 .render_template(template, &data)
2322 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2323
2324 if security_testing_enabled {
2326 script = self.generate_enhanced_script(&script)?;
2327 }
2328
2329 let script_path =
2331 self.output.join(format!("k6-{}-crud-flow.js", spec_name.replace('.', "_")));
2332
2333 std::fs::create_dir_all(self.output.clone())?;
2334 std::fs::write(&script_path, &script)?;
2335
2336 if !self.generate_only {
2337 let executor = K6Executor::new()?
2338 .with_local_ips(self.source_ips.join(","))
2339 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
2340 std::fs::create_dir_all(&output_dir)?;
2341
2342 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
2343
2344 let extracted = Self::parse_extracted_values(&output_dir)?;
2345 TerminalReporter::print_progress(&format!(
2346 " Extracted {} value(s) from {}",
2347 extracted.values.len(),
2348 spec_name
2349 ));
2350 return Ok(extracted);
2351 }
2352
2353 Ok(ExtractedValues::new())
2354 }
2355
2356 async fn execute_standard_spec(&self, parser: &SpecParser, spec_name: &str) -> Result<()> {
2358 let mut operations = if let Some(filter) = &self.operations {
2359 parser.filter_operations(filter)?
2360 } else {
2361 parser.get_operations()
2362 };
2363
2364 if let Some(exclude) = &self.exclude_operations {
2365 operations = parser.exclude_operations(operations, exclude)?;
2366 }
2367
2368 if operations.is_empty() {
2369 TerminalReporter::print_warning(&format!("No operations found in {}", spec_name));
2370 return Ok(());
2371 }
2372
2373 TerminalReporter::print_progress(&format!(
2374 " {} operations in {}",
2375 operations.len(),
2376 spec_name
2377 ));
2378
2379 let templates: Vec<_> = operations
2381 .iter()
2382 .map(RequestGenerator::generate_template)
2383 .collect::<Result<Vec<_>>>()?;
2384
2385 let custom_headers = self.parse_headers()?;
2387
2388 let base_path = self.resolve_base_path(parser);
2390
2391 let scenario =
2393 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2394
2395 let security_testing_enabled = self.security_testing_enabled();
2396
2397 let k6_config = K6Config {
2398 target_url: self.target.clone(),
2399 base_path,
2400 scenario,
2401 duration_secs: Self::parse_duration(&self.duration)?,
2402 max_vus: self.vus,
2403 threshold_percentile: self.threshold_percentile.clone(),
2404 threshold_ms: self.threshold_ms,
2405 max_error_rate: self.max_error_rate,
2406 auth_header: self.auth.clone(),
2407 custom_headers,
2408 skip_tls_verify: self.skip_tls_verify,
2409 security_testing_enabled,
2410 chunked_request_bodies: self.chunked_request_bodies,
2411 target_rps: self.target_rps,
2412 no_keep_alive: self.no_keep_alive,
2413 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
2415 .into_iter()
2416 .map(|ip| ip.to_string())
2417 .collect(),
2418 geo_source_headers: if self.geo_source_headers.is_empty()
2419 && !self.geo_source_ips.is_empty()
2420 {
2421 crate::conformance::self_test::default_geo_source_headers()
2422 } else {
2423 self.geo_source_headers.clone()
2424 },
2425 };
2426
2427 let generator = K6ScriptGenerator::new(k6_config, templates)
2428 .with_abort_valve(self.abort_on_error, self.abort_on_error_rate);
2429 let mut script = generator.generate()?;
2430
2431 let has_advanced_features = self.data_file.is_some()
2433 || self.error_rate.is_some()
2434 || self.security_test
2435 || self.parallel_create.is_some()
2436 || self.wafbench_dir.is_some();
2437
2438 if has_advanced_features {
2439 script = self.generate_enhanced_script(&script)?;
2440 }
2441
2442 let script_path = self.output.join(format!("k6-{}.js", spec_name.replace('.', "_")));
2444
2445 std::fs::create_dir_all(self.output.clone())?;
2446 std::fs::write(&script_path, &script)?;
2447
2448 if !self.generate_only {
2449 let executor = K6Executor::new()?
2452 .with_local_ips(self.source_ips.join(","))
2453 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
2454 .with_discard_response_bodies(self.discard_response_bodies);
2455 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
2456 std::fs::create_dir_all(&output_dir)?;
2457
2458 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
2459 }
2460
2461 Ok(())
2462 }
2463
2464 async fn execute_crud_flow(&self, parser: &SpecParser) -> Result<()> {
2466 let config = self.build_crud_flow_config().unwrap_or_default();
2468
2469 let flows = if !config.flows.is_empty() {
2471 TerminalReporter::print_progress("Using custom flow configuration...");
2472 config.flows.clone()
2473 } else {
2474 TerminalReporter::print_progress("Detecting CRUD operations...");
2475 let operations = parser.get_operations();
2476 CrudFlowDetector::detect_flows(&operations)
2477 };
2478
2479 if flows.is_empty() {
2480 return Err(BenchError::Other(
2481 "No CRUD flows detected in spec. Ensure spec has POST/GET/PUT/DELETE operations on related paths.".to_string(),
2482 ));
2483 }
2484
2485 if config.flows.is_empty() {
2486 TerminalReporter::print_success(&format!("Detected {} CRUD flow(s)", flows.len()));
2487 } else {
2488 TerminalReporter::print_success(&format!("Loaded {} custom flow(s)", flows.len()));
2489 }
2490
2491 for flow in &flows {
2492 TerminalReporter::print_progress(&format!(
2493 " - {}: {} steps",
2494 flow.name,
2495 flow.steps.len()
2496 ));
2497 }
2498
2499 let mut handlebars = handlebars::Handlebars::new();
2501 handlebars.register_helper(
2503 "json",
2504 Box::new(
2505 |h: &handlebars::Helper,
2506 _: &handlebars::Handlebars,
2507 _: &handlebars::Context,
2508 _: &mut handlebars::RenderContext,
2509 out: &mut dyn handlebars::Output|
2510 -> handlebars::HelperResult {
2511 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2512 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2513 Ok(())
2514 },
2515 ),
2516 );
2517 let template = include_str!("templates/k6_crud_flow.hbs");
2518
2519 let custom_headers = self.parse_headers()?;
2520
2521 let param_overrides = if let Some(params_file) = &self.params_file {
2523 TerminalReporter::print_progress("Loading parameter overrides...");
2524 let overrides = ParameterOverrides::from_file(params_file)?;
2525 TerminalReporter::print_success(&format!(
2526 "Loaded parameter overrides ({} operation-specific, {} defaults)",
2527 overrides.operations.len(),
2528 if overrides.defaults.is_empty() { 0 } else { 1 }
2529 ));
2530 Some(overrides)
2531 } else {
2532 None
2533 };
2534
2535 let duration_secs = Self::parse_duration(&self.duration)?;
2537 let scenario =
2538 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2539 let stages = scenario.generate_stages(duration_secs, self.vus);
2540
2541 let api_base_path = self.resolve_base_path(parser);
2543 if let Some(ref bp) = api_base_path {
2544 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
2545 }
2546
2547 let mut all_headers = custom_headers.clone();
2549 if let Some(auth) = &self.auth {
2550 all_headers.insert("Authorization".to_string(), auth.clone());
2551 }
2552 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2553
2554 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2556
2557 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2558 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2563 serde_json::json!({
2564 "name": sanitized_name.clone(), "display_name": f.name, "base_path": f.base_path,
2567 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2568 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2570 let method_raw = if !parts.is_empty() {
2571 parts[0].to_uppercase()
2572 } else {
2573 "GET".to_string()
2574 };
2575 let method = if !parts.is_empty() {
2576 let m = parts[0].to_lowercase();
2577 if m == "delete" { "del".to_string() } else { m }
2579 } else {
2580 "get".to_string()
2581 };
2582 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2583 let path = if let Some(ref bp) = api_base_path {
2585 format!("{}{}", bp, raw_path)
2586 } else {
2587 raw_path.to_string()
2588 };
2589 let is_get_or_head = method == "get" || method == "head";
2590 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2592
2593 let body_value = if has_body {
2595 param_overrides.as_ref()
2596 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2597 .and_then(|oo| oo.body)
2598 .unwrap_or_else(|| serde_json::json!({}))
2599 } else {
2600 serde_json::json!({})
2601 };
2602
2603 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2605 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2610 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2611
2612 serde_json::json!({
2613 "operation": s.operation,
2614 "method": method,
2615 "path": path,
2616 "extract": s.extract,
2617 "use_values": s.use_values,
2618 "use_body": s.use_body,
2619 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2620 "inject_attacks": s.inject_attacks,
2621 "attack_types": s.attack_types,
2622 "description": s.description,
2623 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2624 "is_get_or_head": is_get_or_head,
2625 "has_body": has_body,
2626 "body": processed_body.value,
2627 "body_is_dynamic": body_is_dynamic,
2628 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2629 })
2630 }).collect::<Vec<_>>(),
2631 })
2632 }).collect();
2633
2634 for flow_data in &flows_data {
2636 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2637 for step in steps {
2638 if let Some(placeholders_arr) =
2639 step.get("_placeholders").and_then(|p| p.as_array())
2640 {
2641 for p_str in placeholders_arr {
2642 if let Some(p_name) = p_str.as_str() {
2643 match p_name {
2645 "VU" => {
2646 all_placeholders.insert(DynamicPlaceholder::VU);
2647 }
2648 "Iteration" => {
2649 all_placeholders.insert(DynamicPlaceholder::Iteration);
2650 }
2651 "Timestamp" => {
2652 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2653 }
2654 "UUID" => {
2655 all_placeholders.insert(DynamicPlaceholder::UUID);
2656 }
2657 "Random" => {
2658 all_placeholders.insert(DynamicPlaceholder::Random);
2659 }
2660 "Counter" => {
2661 all_placeholders.insert(DynamicPlaceholder::Counter);
2662 }
2663 "Date" => {
2664 all_placeholders.insert(DynamicPlaceholder::Date);
2665 }
2666 "VuIter" => {
2667 all_placeholders.insert(DynamicPlaceholder::VuIter);
2668 }
2669 _ => {}
2670 }
2671 }
2672 }
2673 }
2674 }
2675 }
2676 }
2677
2678 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2680 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2681
2682 let invalid_data_config = self.build_invalid_data_config();
2684 let error_injection_enabled = invalid_data_config.is_some();
2685 let error_rate = self.error_rate.unwrap_or(0.0);
2686 let error_types: Vec<String> = invalid_data_config
2687 .as_ref()
2688 .map(|c| c.error_types.iter().map(|t| format!("{:?}", t)).collect())
2689 .unwrap_or_default();
2690
2691 if error_injection_enabled {
2692 TerminalReporter::print_progress(&format!(
2693 "Error injection enabled ({}% rate)",
2694 (error_rate * 100.0) as u32
2695 ));
2696 }
2697
2698 let security_testing_enabled = self.security_testing_enabled();
2700
2701 let data = serde_json::json!({
2702 "base_url": self.target,
2703 "flows": flows_data,
2704 "extract_fields": config.default_extract_fields,
2705 "duration_secs": duration_secs,
2706 "max_vus": self.vus,
2707 "auth_header": self.auth,
2708 "custom_headers": custom_headers,
2709 "skip_tls_verify": self.skip_tls_verify,
2710 "stages": stages.iter().map(|s| serde_json::json!({
2712 "duration": s.duration,
2713 "target": s.target,
2714 })).collect::<Vec<_>>(),
2715 "threshold_percentile": self.threshold_percentile,
2716 "threshold_ms": self.threshold_ms,
2717 "max_error_rate": self.max_error_rate,
2718 "abort_on_error": self.abort_on_error,
2719 "abort_on_error_rate": self.abort_on_error_rate,
2720 "headers": headers_json,
2721 "dynamic_imports": required_imports,
2722 "dynamic_globals": required_globals,
2723 "extracted_values_output_path": self
2724 .output
2725 .join("crud_flow_extracted_values.json")
2726 .to_string_lossy(),
2727 "error_injection_enabled": error_injection_enabled,
2729 "error_rate": error_rate,
2730 "error_types": error_types,
2731 "security_testing_enabled": security_testing_enabled,
2733 "has_custom_headers": !custom_headers.is_empty(),
2734 });
2735
2736 let mut script = handlebars
2737 .render_template(template, &data)
2738 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2739
2740 if security_testing_enabled {
2742 script = self.generate_enhanced_script(&script)?;
2743 }
2744
2745 TerminalReporter::print_progress("Validating CRUD flow script...");
2747 let validation_errors = K6ScriptGenerator::validate_script(&script);
2748 if !validation_errors.is_empty() {
2749 TerminalReporter::print_error("CRUD flow script validation failed");
2750 for error in &validation_errors {
2751 eprintln!(" {}", error);
2752 }
2753 return Err(BenchError::Other(format!(
2754 "CRUD flow script validation failed with {} error(s)",
2755 validation_errors.len()
2756 )));
2757 }
2758
2759 TerminalReporter::print_success("CRUD flow script generated");
2760
2761 let script_path = if let Some(output) = &self.script_output {
2763 output.clone()
2764 } else {
2765 self.output.join("k6-crud-flow-script.js")
2766 };
2767
2768 if let Some(parent) = script_path.parent() {
2769 std::fs::create_dir_all(parent)?;
2770 }
2771 std::fs::write(&script_path, &script)?;
2772 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
2773
2774 if self.generate_only {
2775 println!("\nScript generated successfully. Run it with:");
2776 println!(" k6 run {}", script_path.display());
2777 return Ok(());
2778 }
2779
2780 TerminalReporter::print_progress("Executing CRUD flow test...");
2782 let executor = K6Executor::new()?
2783 .with_local_ips(self.source_ips.join(","))
2784 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
2785 std::fs::create_dir_all(&self.output)?;
2786
2787 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
2788
2789 let duration_secs = Self::parse_duration(&self.duration)?;
2790 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
2791
2792 Ok(())
2793 }
2794
2795 async fn execute_conformance_test(&self) -> Result<()> {
2797 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
2798 use crate::conformance::report::ConformanceReport;
2799 use crate::conformance::spec::ConformanceFeature;
2800
2801 TerminalReporter::print_progress("OpenAPI 3.0.0 Conformance Testing Mode");
2802
2803 TerminalReporter::print_progress(CONFORMANCE_REPLACES_LOAD_ADVISORY);
2804
2805 let categories = self.conformance_categories.as_ref().map(|cats_str| {
2807 cats_str
2808 .split(',')
2809 .filter_map(|s| {
2810 let trimmed = s.trim();
2811 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
2812 Some(canonical.to_string())
2813 } else {
2814 TerminalReporter::print_warning(&format!(
2815 "Unknown conformance category: '{}'. Valid categories: {}",
2816 trimmed,
2817 ConformanceFeature::cli_category_names()
2818 .iter()
2819 .map(|(cli, _)| *cli)
2820 .collect::<Vec<_>>()
2821 .join(", ")
2822 ));
2823 None
2824 }
2825 })
2826 .collect::<Vec<String>>()
2827 });
2828
2829 let custom_headers: Vec<(String, String)> = self
2831 .conformance_headers
2832 .iter()
2833 .filter_map(|h| {
2834 let (name, value) = h.split_once(':')?;
2835 Some((name.trim().to_string(), value.trim().to_string()))
2836 })
2837 .collect();
2838
2839 if !custom_headers.is_empty() {
2840 TerminalReporter::print_progress(&format!(
2841 "Using {} custom header(s) for authentication",
2842 custom_headers.len()
2843 ));
2844 }
2845
2846 if self.conformance_delay_ms > 0 {
2847 TerminalReporter::print_progress(&format!(
2848 "Using {}ms delay between conformance requests",
2849 self.conformance_delay_ms
2850 ));
2851 }
2852
2853 std::fs::create_dir_all(&self.output)?;
2855
2856 let config = ConformanceConfig {
2857 target_url: self.target.clone(),
2858 api_key: self.conformance_api_key.clone(),
2859 basic_auth: self.conformance_basic_auth.clone(),
2860 skip_tls_verify: self.skip_tls_verify,
2861 categories,
2862 base_path: self.base_path.clone(),
2863 custom_headers,
2864 output_dir: Some(self.output.clone()),
2865 all_operations: self.conformance_all_operations,
2866 custom_checks_file: self.conformance_custom.clone(),
2867 request_delay_ms: self.conformance_delay_ms,
2868 custom_filter: self.conformance_custom_filter.clone(),
2869 export_requests: self.export_requests,
2870 validate_requests: self.validate_requests,
2871 };
2872
2873 let mut resolved_base_path: Option<String> = None;
2881 let annotated_ops = if !self.spec.is_empty() {
2882 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
2883 let parser = SpecParser::from_file(&self.spec[0]).await?;
2884 resolved_base_path = self.resolve_base_path(&parser);
2885
2886 let mut operations = if let Some(filter) = &self.operations {
2891 parser.filter_operations(filter)?
2892 } else {
2893 parser.get_operations()
2894 };
2895 if let Some(exclude) = &self.exclude_operations {
2896 let before_count = operations.len();
2897 operations = parser.exclude_operations(operations, exclude)?;
2898 let excluded_count = before_count - operations.len();
2899 if excluded_count > 0 {
2900 TerminalReporter::print_progress(&format!(
2901 "Excluded {} operations matching '{}'",
2902 excluded_count, exclude
2903 ));
2904 }
2905 }
2906
2907 let annotated =
2908 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
2909 &operations,
2910 parser.spec(),
2911 );
2912 TerminalReporter::print_success(&format!(
2913 "Analyzed {} operations, found {} feature annotations",
2914 operations.len(),
2915 annotated.iter().map(|a| a.features.len()).sum::<usize>()
2916 ));
2917 Some(annotated)
2918 } else {
2919 None
2920 };
2921
2922 if self.conformance_self_test {
2929 let Some(ops) = annotated_ops else {
2930 TerminalReporter::print_error(
2931 "--conformance-self-test requires --spec; no operations to test",
2932 );
2933 return Ok(());
2934 };
2935 let cfg = crate::conformance::self_test::SelfTestConfig {
2936 target_url: self.target.clone(),
2937 skip_tls_verify: self.skip_tls_verify,
2938 timeout: std::time::Duration::from_secs(30),
2939 extra_headers: self
2943 .conformance_headers
2944 .iter()
2945 .filter_map(|h| {
2946 let (n, v) = h.split_once(':')?;
2947 Some((n.trim().to_string(), v.trim().to_string()))
2948 })
2949 .collect(),
2950 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
2951 base_path: resolved_base_path.clone(),
2955 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
2959 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
2960 geo_source_headers: if self.geo_source_headers.is_empty() {
2961 crate::conformance::self_test::default_geo_source_headers()
2962 } else {
2963 self.geo_source_headers.clone()
2964 },
2965 capture: if self.conformance_self_test_capture
2969 || self.validate_response_schemas
2970 || self.validate_requests
2971 {
2972 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
2983 } else {
2984 None
2985 },
2986 validate_response_schemas: self.validate_response_schemas,
2987 spec_label: self.spec.first().map(|p| {
2993 p.file_name()
2994 .map(|s| s.to_string_lossy().into_owned())
2995 .unwrap_or_else(|| p.to_string_lossy().into_owned())
2996 }),
2997 network_events: Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))),
3004 current_iteration: 1,
3005 };
3006 let capture_sink = cfg.capture.clone();
3007 let network_events_sink = cfg.network_events.clone();
3008 TerminalReporter::print_progress(&format!(
3009 "Self-test mode: driving {} operations with positive + per-category negative cases",
3010 ops.len()
3011 ));
3012 let target_iterations = self.conformance_self_test_iterations.max(1);
3019 let duration_budget = self
3020 .conformance_self_test_duration
3021 .as_ref()
3022 .map(|s| Self::parse_duration(s))
3023 .transpose()?
3024 .map(std::time::Duration::from_secs);
3025 let start = std::time::Instant::now();
3026 let deadline = duration_budget.map(|d| start + d);
3035 let mut cfg = cfg;
3039 cfg.current_iteration = 1;
3040 let mut report =
3041 crate::conformance::self_test::run_self_test_with_deadline(&ops, &cfg, deadline)
3042 .await
3043 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3044 let mut iter_done: u32 = 1;
3045 loop {
3046 let by_iter = iter_done >= target_iterations;
3047 let by_dur = duration_budget.map(|d| start.elapsed() >= d).unwrap_or(true);
3048 if by_iter && by_dur {
3049 break;
3050 }
3051 cfg.current_iteration = iter_done.saturating_add(1);
3052 let next = crate::conformance::self_test::run_self_test_with_deadline(
3053 &ops, &cfg, deadline,
3054 )
3055 .await
3056 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3057 report.merge_iteration(next);
3058 iter_done = iter_done.saturating_add(1);
3059 }
3060 if iter_done > 1 {
3061 TerminalReporter::print_progress(&format!(
3062 "Self-test repeated {} iteration(s) ({:.1?} elapsed)",
3063 iter_done,
3064 start.elapsed(),
3065 ));
3066 }
3067 let per_endpoint_summary: Vec<
3077 crate::conformance::per_endpoint_summary::PerEndpointSummary,
3078 >;
3079 if let Some(sink) = capture_sink {
3080 if let Ok(guard) = sink.lock() {
3081 let jsonl_path = self.output.join("conformance-self-test-requests.jsonl");
3082 let mut lines = String::with_capacity(guard.len() * 256);
3083 for entry in guard.iter() {
3084 if let Ok(line) = serde_json::to_string(entry) {
3085 lines.push_str(&line);
3086 lines.push('\n');
3087 }
3088 }
3089 let _ = std::fs::write(&jsonl_path, lines);
3090 let html_path = self.output.join("conformance-self-test-requests.html");
3091 let html =
3092 crate::conformance::capture_html::render_capture_html(guard.as_slice());
3093 let _ = std::fs::write(&html_path, html);
3094
3095 per_endpoint_summary =
3099 crate::conformance::per_endpoint_summary::build_summary(guard.as_slice());
3100 let summary_path = self.output.join("conformance-per-endpoint.json");
3101 if let Ok(json) = serde_json::to_string_pretty(&per_endpoint_summary) {
3102 let _ = std::fs::write(&summary_path, json);
3103 TerminalReporter::print_progress(&format!(
3104 "Self-test request/response capture written to {} ({} entries) + {} + {}",
3105 jsonl_path.display(),
3106 guard.len(),
3107 html_path.display(),
3108 summary_path.display(),
3109 ));
3110 } else {
3111 TerminalReporter::print_progress(&format!(
3112 "Self-test request/response capture written to {} ({} entries) + {}",
3113 jsonl_path.display(),
3114 guard.len(),
3115 html_path.display(),
3116 ));
3117 }
3118 } else {
3119 per_endpoint_summary = Vec::new();
3120 }
3121 } else {
3122 per_endpoint_summary = Vec::new();
3123 }
3124 TerminalReporter::print_progress(&report.render_summary());
3125 if let Some(sink) = network_events_sink {
3132 if let Ok(guard) = sink.lock() {
3133 let path = self.output.join("conformance-network-events.json");
3134 if let Ok(json) = serde_json::to_string_pretty(&*guard) {
3135 let _ = std::fs::write(&path, json);
3136 if guard.is_empty() {
3137 TerminalReporter::print_progress(
3138 "No wire-level network failures during self-test (file written empty)",
3139 );
3140 } else {
3141 TerminalReporter::print_warning(&format!(
3142 "Recorded {} wire-level network event(s) to {}",
3143 guard.len(),
3144 path.display()
3145 ));
3146 }
3147 }
3148 }
3149 }
3150 let json_path = self.output.join("conformance-self-test.json");
3154 if let Ok(json) = serde_json::to_string_pretty(&report) {
3155 let _ = std::fs::write(&json_path, json);
3156 TerminalReporter::print_progress(&format!(
3157 "Self-test report written to {}",
3158 json_path.display()
3159 ));
3160 }
3161 let issues = report.definite_issues();
3165 let issues_path = self.output.join("conformance-definite-issues.json");
3166 if let Ok(json) = serde_json::to_string_pretty(&issues) {
3167 if std::fs::write(&issues_path, json).is_ok() && !issues.is_empty() {
3168 TerminalReporter::print_warning(&format!(
3169 "{} definite issue(s) — see {}",
3170 issues.len(),
3171 issues_path.display()
3172 ));
3173 }
3174 }
3175 let owasp_accepted = report.owasp_accepted_probes();
3178 if !owasp_accepted.is_empty() {
3179 let owasp_path = self.output.join("conformance-owasp-accepted.json");
3180 if let Ok(json) = serde_json::to_string_pretty(&owasp_accepted) {
3181 if std::fs::write(&owasp_path, json).is_ok() {
3182 TerminalReporter::print_warning(&format!(
3183 "{} owasp injection probe(s) accepted by the target — see {}",
3184 owasp_accepted.len(),
3185 owasp_path.display()
3186 ));
3187 }
3188 }
3189 }
3190 if let Some(status) = report.detect_target_misconfiguration() {
3199 let hint = match status {
3200 404 => " Likely cause: spec paths don't match deployed routes — check --base-path and the spec's `servers` block.",
3201 401 | 403 => " Likely cause: authentication header is missing or invalid — check --conformance-header.",
3202 _ => "",
3203 };
3204 TerminalReporter::print_warning(&format!(
3205 "Self-test misconfiguration: every positive case returned {status}.{hint} Negative results below are meaningless under this condition."
3206 ));
3207 } else if !report.all_passed() {
3208 TerminalReporter::print_warning(
3209 "Self-test detected gaps — server let through at least one request that should have been a 4xx",
3210 );
3211 } else {
3212 TerminalReporter::print_success(
3213 "Self-test passed — all positive cases accepted and all negative cases rejected",
3214 );
3215 }
3216 let html_path = self.output.join("conformance-report.html");
3223 let audit_path = self.output.join("conformance-spec-audit.json");
3224 let audit_value = std::fs::read_to_string(&audit_path)
3225 .ok()
3226 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok());
3227 let render_opts = crate::conformance::report_html::RenderOptions {
3232 missed_cap: match self.report_missed_cap {
3233 Some(0) => None,
3234 Some(n) => Some(n as usize),
3235 None => Some(200),
3236 },
3237 };
3238 let mut html = crate::conformance::report_html::render_html_with_options(
3239 &report,
3240 audit_value.as_ref(),
3241 &render_opts,
3242 );
3243 let summary_section = crate::conformance::per_endpoint_summary::render_html_section(
3249 &per_endpoint_summary,
3250 );
3251 if !summary_section.is_empty() {
3252 if let Some(idx) = html.rfind("</body>") {
3253 html.insert_str(idx, &summary_section);
3254 } else {
3255 html.push_str(&summary_section);
3256 }
3257 }
3258 if std::fs::write(&html_path, html).is_ok() {
3259 TerminalReporter::print_progress(&format!(
3260 "HTML report written to {}",
3261 html_path.display()
3262 ));
3263 }
3264
3265 if self.validate_requests && !self.spec.is_empty() {
3277 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3278 &self.spec,
3279 &self.output,
3280 self.base_path.as_deref(),
3281 )
3282 .await?;
3283 if n > 0 {
3284 TerminalReporter::print_warning(&format!(
3285 "{} emitted request(s) recorded against the spec — see conformance-request-violations.json",
3286 n
3287 ));
3288 }
3289 }
3290 return Ok(());
3291 }
3292
3293 if self.validate_requests && !self.spec.is_empty() {
3295 TerminalReporter::print_progress("Validating requests against OpenAPI spec...");
3296 let violation_count = crate::conformance::request_validator::run_request_validation(
3297 &self.spec,
3298 self.conformance_custom.as_deref(),
3299 self.base_path.as_deref(),
3300 &self.output,
3301 )
3302 .await?;
3303 if violation_count > 0 {
3304 TerminalReporter::print_warning(&format!(
3305 "{} request validation violation(s) found — see conformance-request-violations.json",
3306 violation_count
3307 ));
3308 } else {
3309 TerminalReporter::print_success("All requests conform to the OpenAPI spec");
3310 }
3311 }
3312
3313 if self.generate_only || self.use_k6 {
3315 let script = if let Some(annotated) = &annotated_ops {
3316 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
3317 config,
3318 annotated.clone(),
3319 );
3320 let op_count = gen.operation_count();
3321 let (script, check_count) = gen.generate()?;
3322 TerminalReporter::print_success(&format!(
3323 "Conformance: {} operations analyzed, {} unique checks generated",
3324 op_count, check_count
3325 ));
3326 script
3327 } else {
3328 let generator = ConformanceGenerator::new(config);
3329 generator.generate()?
3330 };
3331
3332 let script_path = self.output.join("k6-conformance.js");
3333 std::fs::write(&script_path, &script).map_err(|e| {
3334 BenchError::Other(format!("Failed to write conformance script: {}", e))
3335 })?;
3336 TerminalReporter::print_success(&format!(
3337 "Conformance script generated: {}",
3338 script_path.display()
3339 ));
3340
3341 if self.generate_only {
3342 println!("\nScript generated. Run with:");
3343 println!(" k6 run {}", script_path.display());
3344 return Ok(());
3345 }
3346
3347 if !K6Executor::is_k6_installed() {
3349 TerminalReporter::print_error("k6 is not installed");
3350 TerminalReporter::print_warning(
3351 "Install k6 from: https://k6.io/docs/get-started/installation/",
3352 );
3353 return Err(BenchError::K6NotFound);
3354 }
3355
3356 K6Executor::warn_if_pre_v1().await;
3357 TerminalReporter::print_progress("Running conformance tests via k6...");
3358 let executor = K6Executor::new()?
3359 .with_local_ips(self.source_ips.join(","))
3360 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
3361 executor.execute(&script_path, Some(&self.output), self.verbose).await?;
3362
3363 let report_path = self.output.join("conformance-report.json");
3364 if report_path.exists() {
3365 let report = ConformanceReport::from_file(&report_path)?;
3366 report.print_report_with_options(self.conformance_all_operations);
3367 self.save_conformance_report(&report, &report_path)?;
3368 } else {
3369 TerminalReporter::print_warning(
3370 "Conformance report not generated (k6 handleSummary may not have run)",
3371 );
3372 }
3373
3374 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
3386 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3387 &self.spec,
3388 &self.output,
3389 self.base_path.as_deref(),
3390 )
3391 .await?;
3392 if n > 0 {
3393 TerminalReporter::print_warning(&format!(
3394 "{} emitted request(s) violated the spec — see conformance-request-violations.json",
3395 n
3396 ));
3397 }
3398 }
3399
3400 return Ok(());
3401 }
3402
3403 TerminalReporter::print_progress("Running conformance tests (native executor)...");
3405
3406 let mut executor = crate::conformance::executor::NativeConformanceExecutor::new(config)?;
3407
3408 let custom_only = annotated_ops.is_none() && self.conformance_custom.is_some();
3418 executor = if let Some(annotated) = &annotated_ops {
3419 executor.with_spec_driven_checks(annotated)
3420 } else if custom_only {
3421 executor
3422 } else {
3423 executor.with_reference_checks()
3424 };
3425 executor = executor.with_custom_checks()?;
3426
3427 TerminalReporter::print_success(&format!(
3428 "Executing {} conformance checks...",
3429 executor.check_count()
3430 ));
3431
3432 let report = executor.execute().await?;
3433 report.print_report_with_options(self.conformance_all_operations);
3434
3435 let failure_details = report.failure_details();
3437 if !failure_details.is_empty() {
3438 let details_path = self.output.join("conformance-failure-details.json");
3439 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
3440 let _ = std::fs::write(&details_path, json);
3441 TerminalReporter::print_success(&format!(
3442 "Failure details saved to: {}",
3443 details_path.display()
3444 ));
3445 }
3446 }
3447
3448 let report_path = self.output.join("conformance-report.json");
3450 let report_json = serde_json::to_string_pretty(&report.to_json())
3451 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
3452 std::fs::write(&report_path, &report_json)
3453 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
3454 TerminalReporter::print_success(&format!("Report saved to: {}", report_path.display()));
3455
3456 self.save_conformance_report(&report, &report_path)?;
3457
3458 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
3469 let n =
3470 crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3471 &self.spec,
3472 &self.output,
3473 self.base_path.as_deref(),
3474 )
3475 .await?;
3476 if n > 0 {
3477 TerminalReporter::print_warning(&format!(
3478 "{} emitted request(s) violated the spec — see conformance-request-violations.json",
3479 n
3480 ));
3481 }
3482 }
3483
3484 Ok(())
3485 }
3486
3487 fn save_conformance_report(
3489 &self,
3490 report: &crate::conformance::report::ConformanceReport,
3491 report_path: &Path,
3492 ) -> Result<()> {
3493 if self.conformance_report_format == "sarif" {
3494 use crate::conformance::sarif::ConformanceSarifReport;
3495 ConformanceSarifReport::write(report, &self.target, &self.conformance_report)?;
3496 TerminalReporter::print_success(&format!(
3497 "SARIF report saved to: {}",
3498 self.conformance_report.display()
3499 ));
3500 } else if self.conformance_report != *report_path {
3501 std::fs::copy(report_path, &self.conformance_report)?;
3502 TerminalReporter::print_success(&format!(
3503 "Report saved to: {}",
3504 self.conformance_report.display()
3505 ));
3506 }
3507 Ok(())
3508 }
3509
3510 async fn execute_multi_target_self_test(&self, targets_file: &Path) -> Result<()> {
3522 use crate::conformance::self_test::SelfTestConfig;
3523
3524 TerminalReporter::print_progress("Multi-target conformance self-test mode");
3525 let targets = parse_targets_file(targets_file)?;
3526 if targets.is_empty() {
3527 return Err(BenchError::Other("No targets found in file".to_string()));
3528 }
3529 TerminalReporter::print_success(&format!("Loaded {} target(s)", targets.len()));
3530
3531 let annotated_ops = if !self.spec.is_empty() {
3533 let parser = SpecParser::from_file(&self.spec[0]).await?;
3534 let operations = parser.get_operations();
3535 Some(
3536 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
3537 &operations,
3538 parser.spec(),
3539 ),
3540 )
3541 } else {
3542 return Err(BenchError::Other("--conformance-self-test requires --spec".to_string()));
3543 };
3544 let Some(ops) = annotated_ops else {
3545 unreachable!()
3546 };
3547
3548 std::fs::create_dir_all(&self.output)?;
3549 let resolved_base_path = self.base_path.clone();
3550 let target_iterations = self.conformance_self_test_iterations.max(1);
3551 let duration_budget = self
3552 .conformance_self_test_duration
3553 .as_ref()
3554 .map(|s| Self::parse_duration(s))
3555 .transpose()?
3556 .map(std::time::Duration::from_secs);
3557
3558 for (idx, target) in targets.iter().enumerate() {
3559 let target_dir = self.output.join(format!("target_{}", idx));
3560 std::fs::create_dir_all(&target_dir)?;
3561 TerminalReporter::print_progress(&format!(
3562 "[target {}/{}] {}",
3563 idx + 1,
3564 targets.len(),
3565 target.url
3566 ));
3567
3568 let merged_headers: Vec<(String, String)> = self
3569 .conformance_headers
3570 .iter()
3571 .filter_map(|h| {
3572 let (n, v) = h.split_once(':')?;
3573 Some((n.trim().to_string(), v.trim().to_string()))
3574 })
3575 .collect();
3576
3577 let cfg = SelfTestConfig {
3578 target_url: target.url.clone(),
3579 skip_tls_verify: self.skip_tls_verify,
3580 timeout: std::time::Duration::from_secs(30),
3581 extra_headers: merged_headers,
3582 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
3583 base_path: resolved_base_path.clone(),
3584 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
3585 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
3586 geo_source_headers: if self.geo_source_headers.is_empty() {
3587 crate::conformance::self_test::default_geo_source_headers()
3588 } else {
3589 self.geo_source_headers.clone()
3590 },
3591 capture: if self.conformance_self_test_capture
3592 || self.validate_response_schemas
3593 || self.validate_requests
3594 {
3595 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
3599 } else {
3600 None
3601 },
3602 validate_response_schemas: self.validate_response_schemas,
3603 spec_label: self.spec.first().map(|p| {
3604 p.file_name()
3605 .map(|s| s.to_string_lossy().into_owned())
3606 .unwrap_or_else(|| p.to_string_lossy().into_owned())
3607 }),
3608 network_events: Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))),
3609 current_iteration: 1,
3610 };
3611 let capture_sink = cfg.capture.clone();
3612 let network_events_sink = cfg.network_events.clone();
3613
3614 let start = std::time::Instant::now();
3615 let deadline = duration_budget.map(|d| start + d);
3619 let mut cfg = cfg;
3623 cfg.current_iteration = 1;
3624 let mut report =
3625 crate::conformance::self_test::run_self_test_with_deadline(&ops, &cfg, deadline)
3626 .await
3627 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3628 let mut iter_done: u32 = 1;
3629 loop {
3630 let by_iter = iter_done >= target_iterations;
3631 let by_dur = duration_budget.map(|d| start.elapsed() >= d).unwrap_or(true);
3632 if by_iter && by_dur {
3633 break;
3634 }
3635 cfg.current_iteration = iter_done.saturating_add(1);
3636 let next = crate::conformance::self_test::run_self_test_with_deadline(
3637 &ops, &cfg, deadline,
3638 )
3639 .await
3640 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3641 report.merge_iteration(next);
3642 iter_done = iter_done.saturating_add(1);
3643 }
3644 if iter_done > 1 {
3645 TerminalReporter::print_progress(&format!(
3646 " ran {} iteration(s) in {:.1?}",
3647 iter_done,
3648 start.elapsed(),
3649 ));
3650 }
3651
3652 if let Some(sink) = capture_sink {
3654 if let Ok(guard) = sink.lock() {
3655 let jsonl = target_dir.join("conformance-self-test-requests.jsonl");
3656 let mut lines = String::with_capacity(guard.len() * 256);
3657 for entry in guard.iter() {
3658 if let Ok(line) = serde_json::to_string(entry) {
3659 lines.push_str(&line);
3660 lines.push('\n');
3661 }
3662 }
3663 let _ = std::fs::write(&jsonl, lines);
3664 }
3665 }
3666 if let Some(sink) = network_events_sink {
3667 if let Ok(guard) = sink.lock() {
3668 let path = target_dir.join("conformance-network-events.json");
3669 if let Ok(json) = serde_json::to_string_pretty(&*guard) {
3670 let _ = std::fs::write(&path, json);
3671 if !guard.is_empty() {
3672 TerminalReporter::print_warning(&format!(
3673 " recorded {} wire-level network event(s)",
3674 guard.len()
3675 ));
3676 }
3677 }
3678 }
3679 }
3680
3681 let json_path = target_dir.join("conformance-self-test.json");
3682 if let Ok(json) = serde_json::to_string_pretty(&report) {
3683 let _ = std::fs::write(&json_path, json);
3684 }
3685 let issues = report.definite_issues();
3688 if let Ok(json) = serde_json::to_string_pretty(&issues) {
3689 let issues_path = target_dir.join("conformance-definite-issues.json");
3690 if std::fs::write(&issues_path, json).is_ok() && !issues.is_empty() {
3691 TerminalReporter::print_warning(&format!(
3692 " {} definite issue(s) — see {}",
3693 issues.len(),
3694 issues_path.display()
3695 ));
3696 }
3697 }
3698 let owasp_accepted = report.owasp_accepted_probes();
3700 if !owasp_accepted.is_empty() {
3701 if let Ok(json) = serde_json::to_string_pretty(&owasp_accepted) {
3702 let owasp_path = target_dir.join("conformance-owasp-accepted.json");
3703 if std::fs::write(&owasp_path, json).is_ok() {
3704 TerminalReporter::print_warning(&format!(
3705 " {} owasp injection probe(s) accepted by the target — see {}",
3706 owasp_accepted.len(),
3707 owasp_path.display()
3708 ));
3709 }
3710 }
3711 }
3712 TerminalReporter::print_progress(&report.render_summary());
3713
3714 if self.validate_requests {
3723 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3724 &self.spec,
3725 &target_dir,
3726 self.base_path.as_deref(),
3727 )
3728 .await?;
3729 if n > 0 {
3730 TerminalReporter::print_warning(&format!(
3731 " {} emitted request(s) violated the spec — see {}/conformance-request-violations.json",
3732 n,
3733 target_dir.display(),
3734 ));
3735 }
3736 }
3737 }
3738
3739 Ok(())
3740 }
3741
3742 async fn execute_multi_target_conformance(&self, targets_file: &Path) -> Result<()> {
3748 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
3749 use crate::conformance::report::ConformanceReport;
3750 use crate::conformance::spec::ConformanceFeature;
3751
3752 TerminalReporter::print_progress("Multi-target OpenAPI 3.0.0 Conformance Testing Mode");
3753
3754 TerminalReporter::print_progress("Parsing targets file...");
3756 let targets = parse_targets_file(targets_file)?;
3757 let num_targets = targets.len();
3758 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
3759
3760 if targets.is_empty() {
3761 return Err(BenchError::Other("No targets found in file".to_string()));
3762 }
3763
3764 TerminalReporter::print_progress(CONFORMANCE_REPLACES_LOAD_ADVISORY);
3765
3766 let categories = self.conformance_categories.as_ref().map(|cats_str| {
3768 cats_str
3769 .split(',')
3770 .filter_map(|s| {
3771 let trimmed = s.trim();
3772 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
3773 Some(canonical.to_string())
3774 } else {
3775 TerminalReporter::print_warning(&format!(
3776 "Unknown conformance category: '{}'. Valid categories: {}",
3777 trimmed,
3778 ConformanceFeature::cli_category_names()
3779 .iter()
3780 .map(|(cli, _)| *cli)
3781 .collect::<Vec<_>>()
3782 .join(", ")
3783 ));
3784 None
3785 }
3786 })
3787 .collect::<Vec<String>>()
3788 });
3789
3790 let base_custom_headers: Vec<(String, String)> = self
3792 .conformance_headers
3793 .iter()
3794 .filter_map(|h| {
3795 let (name, value) = h.split_once(':')?;
3796 Some((name.trim().to_string(), value.trim().to_string()))
3797 })
3798 .collect();
3799
3800 if !base_custom_headers.is_empty() {
3801 TerminalReporter::print_progress(&format!(
3802 "Using {} base custom header(s) for authentication",
3803 base_custom_headers.len()
3804 ));
3805 }
3806
3807 let annotated_ops = if !self.spec.is_empty() {
3809 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
3810 let parser = SpecParser::from_file(&self.spec[0]).await?;
3811 let operations = parser.get_operations();
3812 let annotated =
3813 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
3814 &operations,
3815 parser.spec(),
3816 );
3817 TerminalReporter::print_success(&format!(
3818 "Analyzed {} operations, found {} feature annotations",
3819 operations.len(),
3820 annotated.iter().map(|a| a.features.len()).sum::<usize>()
3821 ));
3822 Some(annotated)
3823 } else {
3824 None
3825 };
3826
3827 std::fs::create_dir_all(&self.output)?;
3829
3830 struct TargetResult {
3832 url: String,
3833 passed: usize,
3834 failed: usize,
3835 elapsed: std::time::Duration,
3836 report_json: serde_json::Value,
3837 owasp_coverage: Vec<crate::conformance::report::OwaspCoverageEntry>,
3838 }
3839
3840 let mut target_results: Vec<TargetResult> = Vec::with_capacity(num_targets);
3841 let total_start = std::time::Instant::now();
3842
3843 for (idx, target) in targets.iter().enumerate() {
3844 tracing::info!(
3845 "Running conformance tests against target {}/{}: {}",
3846 idx + 1,
3847 num_targets,
3848 target.url
3849 );
3850 TerminalReporter::print_progress(&format!(
3851 "\n--- Target {}/{}: {} ---",
3852 idx + 1,
3853 num_targets,
3854 target.url
3855 ));
3856
3857 let mut merged_headers = base_custom_headers.clone();
3859 if let Some(ref target_headers) = target.headers {
3860 for (name, value) in target_headers {
3861 if let Some(existing) = merged_headers.iter_mut().find(|(n, _)| n == name) {
3863 existing.1 = value.clone();
3864 } else {
3865 merged_headers.push((name.clone(), value.clone()));
3866 }
3867 }
3868 }
3869 if let Some(ref auth) = target.auth {
3871 if let Some(existing) =
3872 merged_headers.iter_mut().find(|(n, _)| n.eq_ignore_ascii_case("Authorization"))
3873 {
3874 existing.1 = auth.clone();
3875 } else {
3876 merged_headers.push(("Authorization".to_string(), auth.clone()));
3877 }
3878 }
3879
3880 let target_dir = self.output.join(format!("target_{}", idx));
3886 std::fs::create_dir_all(&target_dir)?;
3887
3888 let config = ConformanceConfig {
3889 target_url: target.url.clone(),
3890 api_key: self.conformance_api_key.clone(),
3891 basic_auth: self.conformance_basic_auth.clone(),
3892 skip_tls_verify: self.skip_tls_verify,
3893 categories: categories.clone(),
3894 base_path: self.base_path.clone(),
3895 custom_headers: merged_headers,
3896 output_dir: Some(target_dir.clone()),
3897 all_operations: self.conformance_all_operations,
3898 custom_checks_file: self.conformance_custom.clone(),
3899 request_delay_ms: self.conformance_delay_ms,
3900 custom_filter: self.conformance_custom_filter.clone(),
3901 export_requests: self.export_requests,
3902 validate_requests: self.validate_requests,
3903 };
3904
3905 let target_start = std::time::Instant::now();
3906 let report = if self.use_k6 {
3907 if !K6Executor::is_k6_installed() {
3908 TerminalReporter::print_error("k6 is not installed");
3909 TerminalReporter::print_warning(
3910 "Install k6 from: https://k6.io/docs/get-started/installation/",
3911 );
3912 return Err(BenchError::K6NotFound);
3913 }
3914 K6Executor::warn_if_pre_v1().await;
3915
3916 let script = if let Some(ref annotated) = annotated_ops {
3917 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
3918 config.clone(),
3919 annotated.clone(),
3920 );
3921 let (script, _check_count) = gen.generate()?;
3922 script
3923 } else {
3924 let generator = ConformanceGenerator::new(config.clone());
3925 generator.generate()?
3926 };
3927
3928 let script_path = target_dir.join("k6-conformance.js");
3929 std::fs::write(&script_path, &script).map_err(|e| {
3930 BenchError::Other(format!("Failed to write conformance script: {}", e))
3931 })?;
3932 TerminalReporter::print_success(&format!(
3933 "Conformance script generated: {}",
3934 script_path.display()
3935 ));
3936
3937 TerminalReporter::print_progress(&format!(
3938 "Running conformance tests via k6 against {}...",
3939 target.url
3940 ));
3941 let k6 = K6Executor::new()?
3942 .with_local_ips(self.source_ips.join(","))
3943 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
3944 let api_port = 6565u16.saturating_add(idx as u16);
3946 k6.execute_with_port(&script_path, Some(&target_dir), self.verbose, Some(api_port))
3947 .await?;
3948
3949 let report_path = target_dir.join("conformance-report.json");
3950 if report_path.exists() {
3951 ConformanceReport::from_file(&report_path)?
3952 } else {
3953 TerminalReporter::print_warning(&format!(
3954 "Conformance report not generated for target {} (k6 handleSummary may not have run)",
3955 target.url
3956 ));
3957 continue;
3958 }
3959 } else {
3960 let mut executor =
3961 crate::conformance::executor::NativeConformanceExecutor::new(config)?;
3962
3963 let custom_only = annotated_ops.is_none() && self.conformance_custom.is_some();
3966 executor = if let Some(ref annotated) = annotated_ops {
3967 executor.with_spec_driven_checks(annotated)
3968 } else if custom_only {
3969 executor
3970 } else {
3971 executor.with_reference_checks()
3972 };
3973 executor = executor.with_custom_checks()?;
3974
3975 TerminalReporter::print_success(&format!(
3976 "Executing {} conformance checks against {}...",
3977 executor.check_count(),
3978 target.url
3979 ));
3980
3981 executor.execute().await?
3982 };
3983 let target_elapsed = target_start.elapsed();
3984
3985 let report_json = report.to_json();
3986
3987 let passed = report_json["summary"]["passed"].as_u64().unwrap_or(0) as usize;
3989 let failed = report_json["summary"]["failed"].as_u64().unwrap_or(0) as usize;
3990 let total_checks = passed + failed;
3991 let rate = if total_checks == 0 {
3992 0.0
3993 } else {
3994 (passed as f64 / total_checks as f64) * 100.0
3995 };
3996
3997 TerminalReporter::print_success(&format!(
3998 "Target {}: {}/{} passed ({:.1}%) in {:.1}s",
3999 target.url,
4000 passed,
4001 total_checks,
4002 rate,
4003 target_elapsed.as_secs_f64()
4004 ));
4005
4006 let target_report_path = target_dir.join("conformance-report.json");
4008 let report_str = serde_json::to_string_pretty(&report_json)
4009 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
4010 std::fs::write(&target_report_path, &report_str)
4011 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
4012
4013 let failure_details = report.failure_details();
4015 if !failure_details.is_empty() {
4016 let details_path = target_dir.join("conformance-failure-details.json");
4017 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
4018 let _ = std::fs::write(&details_path, json);
4019 }
4020 }
4021
4022 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
4029 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
4030 &self.spec,
4031 &target_dir,
4032 self.base_path.as_deref(),
4033 )
4034 .await?;
4035 if n > 0 {
4036 TerminalReporter::print_warning(&format!(
4037 "Target {}: {} emitted request(s) violated the spec — see {}/conformance-request-violations.json",
4038 target.url,
4039 n,
4040 target_dir.display()
4041 ));
4042 }
4043 }
4044
4045 let owasp_coverage = report.owasp_coverage_data();
4047
4048 target_results.push(TargetResult {
4049 url: target.url.clone(),
4050 passed,
4051 failed,
4052 elapsed: target_elapsed,
4053 report_json,
4054 owasp_coverage,
4055 });
4056 }
4057
4058 let total_elapsed = total_start.elapsed();
4059
4060 println!("\n{}", "=".repeat(80));
4062 println!(" Multi-Target Conformance Summary");
4063 println!("{}", "=".repeat(80));
4064 println!(
4065 " {:<40} {:>8} {:>8} {:>8} {:>8}",
4066 "Target URL", "Passed", "Failed", "Rate", "Time"
4067 );
4068 println!(" {}", "-".repeat(76));
4069
4070 let mut total_passed = 0usize;
4071 let mut total_failed = 0usize;
4072
4073 for result in &target_results {
4074 let total_checks = result.passed + result.failed;
4075 let rate = if total_checks == 0 {
4076 0.0
4077 } else {
4078 (result.passed as f64 / total_checks as f64) * 100.0
4079 };
4080
4081 let display_url = if result.url.len() > 38 {
4083 format!("{}...", &result.url[..35])
4084 } else {
4085 result.url.clone()
4086 };
4087
4088 println!(
4089 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
4090 display_url,
4091 result.passed,
4092 result.failed,
4093 rate,
4094 result.elapsed.as_secs_f64()
4095 );
4096
4097 total_passed += result.passed;
4098 total_failed += result.failed;
4099 }
4100
4101 let grand_total = total_passed + total_failed;
4102 let overall_rate = if grand_total == 0 {
4103 0.0
4104 } else {
4105 (total_passed as f64 / grand_total as f64) * 100.0
4106 };
4107
4108 println!(" {}", "-".repeat(76));
4109 println!(
4110 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
4111 format!("TOTAL ({} targets)", num_targets),
4112 total_passed,
4113 total_failed,
4114 overall_rate,
4115 total_elapsed.as_secs_f64()
4116 );
4117 println!("{}", "=".repeat(80));
4118
4119 for result in &target_results {
4121 println!("\n OWASP API Security Top 10 Coverage for {}:", result.url);
4122 for entry in &result.owasp_coverage {
4123 let status = if !entry.tested {
4124 "-"
4125 } else if entry.all_passed {
4126 "pass"
4127 } else {
4128 "FAIL"
4129 };
4130 let via = if entry.via_categories.is_empty() {
4131 String::new()
4132 } else {
4133 format!(" (via {})", entry.via_categories.join(", "))
4134 };
4135 println!(" {:<12} {:<40} {}{}", entry.id, entry.name, status, via);
4136 }
4137 }
4138
4139 let per_target_summaries: Vec<serde_json::Value> = target_results
4141 .iter()
4142 .enumerate()
4143 .map(|(idx, r)| {
4144 let total_checks = r.passed + r.failed;
4145 let rate = if total_checks == 0 {
4146 0.0
4147 } else {
4148 (r.passed as f64 / total_checks as f64) * 100.0
4149 };
4150 let owasp_json: Vec<serde_json::Value> = r
4151 .owasp_coverage
4152 .iter()
4153 .map(|e| {
4154 serde_json::json!({
4155 "id": e.id,
4156 "name": e.name,
4157 "tested": e.tested,
4158 "all_passed": e.all_passed,
4159 "via_categories": e.via_categories,
4160 })
4161 })
4162 .collect();
4163 serde_json::json!({
4164 "target_url": r.url,
4165 "target_index": idx,
4166 "checks_passed": r.passed,
4167 "checks_failed": r.failed,
4168 "total_checks": total_checks,
4169 "pass_rate": rate,
4170 "elapsed_seconds": r.elapsed.as_secs_f64(),
4171 "report": r.report_json,
4172 "owasp_coverage": owasp_json,
4173 })
4174 })
4175 .collect();
4176
4177 let combined_summary = serde_json::json!({
4178 "total_targets": num_targets,
4179 "total_checks_passed": total_passed,
4180 "total_checks_failed": total_failed,
4181 "overall_pass_rate": overall_rate,
4182 "total_elapsed_seconds": total_elapsed.as_secs_f64(),
4183 "targets": per_target_summaries,
4184 });
4185
4186 let summary_path = self.output.join("multi-target-conformance-summary.json");
4187 let summary_str = serde_json::to_string_pretty(&combined_summary)
4188 .map_err(|e| BenchError::Other(format!("Failed to serialize summary: {}", e)))?;
4189 std::fs::write(&summary_path, &summary_str)
4190 .map_err(|e| BenchError::Other(format!("Failed to write summary: {}", e)))?;
4191 TerminalReporter::print_success(&format!(
4192 "Combined summary saved to: {}",
4193 summary_path.display()
4194 ));
4195
4196 Ok(())
4197 }
4198
4199 async fn execute_owasp_test(&self, parser: &SpecParser) -> Result<()> {
4201 TerminalReporter::print_progress("OWASP API Security Top 10 Testing Mode");
4202
4203 let custom_headers = self.parse_headers()?;
4205
4206 let mut config = OwaspApiConfig::new()
4208 .with_auth_header(&self.owasp_auth_header)
4209 .with_verbose(self.verbose)
4210 .with_insecure(self.skip_tls_verify)
4211 .with_concurrency(self.vus as usize)
4212 .with_iterations(self.owasp_iterations as usize)
4213 .with_base_path(self.base_path.clone())
4214 .with_custom_headers(custom_headers);
4215
4216 if let Some(ref token) = self.owasp_auth_token {
4218 config = config.with_valid_auth_token(token);
4219 }
4220
4221 if let Some(ref cats_str) = self.owasp_categories {
4223 let categories: Vec<OwaspCategory> = cats_str
4224 .split(',')
4225 .filter_map(|s| {
4226 let trimmed = s.trim();
4227 match trimmed.parse::<OwaspCategory>() {
4228 Ok(cat) => Some(cat),
4229 Err(e) => {
4230 TerminalReporter::print_warning(&e);
4231 None
4232 }
4233 }
4234 })
4235 .collect();
4236
4237 if !categories.is_empty() {
4238 config = config.with_categories(categories);
4239 }
4240 }
4241
4242 if let Some(ref admin_paths_file) = self.owasp_admin_paths {
4244 config.admin_paths_file = Some(admin_paths_file.clone());
4245 if let Err(e) = config.load_admin_paths() {
4246 TerminalReporter::print_warning(&format!("Failed to load admin paths file: {}", e));
4247 }
4248 }
4249
4250 if let Some(ref id_fields_str) = self.owasp_id_fields {
4252 let id_fields: Vec<String> = id_fields_str
4253 .split(',')
4254 .map(|s| s.trim().to_string())
4255 .filter(|s| !s.is_empty())
4256 .collect();
4257 if !id_fields.is_empty() {
4258 config = config.with_id_fields(id_fields);
4259 }
4260 }
4261
4262 if let Some(ref report_path) = self.owasp_report {
4264 config = config.with_report_path(report_path);
4265 }
4266 if let Ok(format) = self.owasp_report_format.parse::<ReportFormat>() {
4267 config = config.with_report_format(format);
4268 }
4269
4270 let categories = config.categories_to_test();
4272 TerminalReporter::print_success(&format!(
4273 "Testing {} OWASP categories: {}",
4274 categories.len(),
4275 categories.iter().map(|c| c.cli_name()).collect::<Vec<_>>().join(", ")
4276 ));
4277
4278 if config.valid_auth_token.is_some() {
4279 TerminalReporter::print_progress("Using provided auth token for baseline requests");
4280 }
4281
4282 TerminalReporter::print_progress("Generating OWASP security test script...");
4284 let generator = OwaspApiGenerator::new(config, self.target.clone(), parser);
4285
4286 let script = generator.generate()?;
4288 TerminalReporter::print_success("OWASP security test script generated");
4289
4290 let script_path = if let Some(output) = &self.script_output {
4292 output.clone()
4293 } else {
4294 self.output.join("k6-owasp-security-test.js")
4295 };
4296
4297 if let Some(parent) = script_path.parent() {
4298 std::fs::create_dir_all(parent)?;
4299 }
4300 std::fs::write(&script_path, &script)?;
4301 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
4302
4303 if self.generate_only {
4305 println!("\nOWASP security test script generated. Run it with:");
4306 println!(" k6 run {}", script_path.display());
4307 return Ok(());
4308 }
4309
4310 TerminalReporter::print_progress("Executing OWASP security tests...");
4312 let executor = K6Executor::new()?
4313 .with_local_ips(self.source_ips.join(","))
4314 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
4315 std::fs::create_dir_all(&self.output)?;
4316
4317 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
4318
4319 let duration_secs = Self::parse_duration(&self.duration)?;
4320 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
4321
4322 println!("\nOWASP security test results saved to: {}", self.output.display());
4323
4324 Ok(())
4325 }
4326}
4327
4328#[cfg(test)]
4329mod tests {
4330 use super::*;
4331 use tempfile::tempdir;
4332
4333 #[test]
4334 fn test_parse_duration() {
4335 assert_eq!(BenchCommand::parse_duration("30s").unwrap(), 30);
4336 assert_eq!(BenchCommand::parse_duration("5m").unwrap(), 300);
4337 assert_eq!(BenchCommand::parse_duration("1h").unwrap(), 3600);
4338 assert_eq!(BenchCommand::parse_duration("60").unwrap(), 60);
4339 }
4340
4341 #[test]
4345 fn parse_ip_list_ipv4_range_inclusive() {
4346 let v = parse_ip_list(&["10.0.0.5-10.0.0.27".into()], "source-ip");
4347 assert_eq!(v.len(), 23);
4348 assert_eq!(v.first().unwrap().to_string(), "10.0.0.5");
4349 assert_eq!(v.last().unwrap().to_string(), "10.0.0.27");
4350 }
4351
4352 #[test]
4355 fn parse_ip_list_range_rejects_backwards() {
4356 let v = parse_ip_list(&["10.0.0.10-10.0.0.5".into()], "source-ip");
4357 assert!(v.is_empty(), "backwards range should produce no IPs; got {v:?}");
4358 }
4359
4360 #[test]
4364 fn parse_ip_list_rejects_ipv6_range_syntax() {
4365 let v = parse_ip_list(&["2001:db8::1-2001:db8::5".into()], "geo-source-ip");
4366 assert!(v.is_empty(), "IPv6 range should be rejected; got {v:?}");
4367 }
4368
4369 #[test]
4371 fn parse_ip_list_range_capped_at_256() {
4372 let v = parse_ip_list(&["10.0.0.0-10.0.5.0".into()], "source-ip");
4373 assert_eq!(v.len(), 256);
4374 assert_eq!(v.first().unwrap().to_string(), "10.0.0.0");
4375 }
4376
4377 #[test]
4380 fn parse_ip_list_plain_and_comma() {
4381 let v = parse_ip_list(&["10.0.0.5".into(), "10.0.0.6,10.0.0.7".into()], "source-ip");
4382 assert_eq!(v.len(), 3);
4383 assert_eq!(v[0].to_string(), "10.0.0.5");
4384 assert_eq!(v[2].to_string(), "10.0.0.7");
4385 }
4386
4387 #[test]
4390 fn parse_ip_list_ipv4_cidr_29_expands_to_8() {
4391 let v = parse_ip_list(&["10.0.0.0/29".into()], "source-ip");
4392 assert_eq!(v.len(), 8);
4393 assert_eq!(v[0].to_string(), "10.0.0.0");
4394 assert_eq!(v[7].to_string(), "10.0.0.7");
4395 }
4396
4397 #[test]
4400 fn parse_ip_list_ipv4_cidr_8_capped_at_256() {
4401 let v = parse_ip_list(&["10.0.0.0/8".into()], "source-ip");
4402 assert_eq!(v.len(), 256);
4403 assert_eq!(v[0].to_string(), "10.0.0.0");
4404 assert_eq!(v[255].to_string(), "10.0.0.255");
4405 }
4406
4407 #[test]
4409 fn parse_ip_list_ipv6_cidr_126_expands_to_4() {
4410 let v = parse_ip_list(&["2001:db8::/126".into()], "geo-source-ip");
4411 assert_eq!(v.len(), 4);
4412 assert!(v[0].is_ipv6());
4413 assert_eq!(v[0].to_string(), "2001:db8::");
4414 assert_eq!(v[3].to_string(), "2001:db8::3");
4415 }
4416
4417 #[test]
4419 fn parse_ip_list_mixed_v4_v6_cidr() {
4420 let v = parse_ip_list(&["10.0.0.0/30,2001:db8::1,203.0.113.42".into()], "geo-source-ip");
4421 assert_eq!(v.len(), 6); assert!(v.iter().any(|ip| ip.to_string() == "2001:db8::1"));
4423 assert!(v.iter().any(|ip| ip.to_string() == "203.0.113.42"));
4424 }
4425
4426 #[test]
4429 fn parse_ip_list_skips_malformed() {
4430 let v = parse_ip_list(
4431 &[
4432 "10.0.0.5".into(),
4433 "not-an-ip".into(),
4434 "10.0.0.6".into(),
4435 "/24".into(),
4436 "1.2.3.4/200".into(),
4437 ],
4438 "source-ip",
4439 );
4440 assert_eq!(v.len(), 2);
4441 assert_eq!(v[0].to_string(), "10.0.0.5");
4442 assert_eq!(v[1].to_string(), "10.0.0.6");
4443 }
4444
4445 #[test]
4446 fn test_parse_duration_invalid() {
4447 assert!(BenchCommand::parse_duration("invalid").is_err());
4448 assert!(BenchCommand::parse_duration("30x").is_err());
4449 }
4450
4451 #[test]
4452 fn test_parse_headers() {
4453 let cmd = BenchCommand {
4454 spec: vec![PathBuf::from("test.yaml")],
4455 spec_dir: None,
4456 merge_conflicts: "error".to_string(),
4457 spec_mode: "merge".to_string(),
4458 dependency_config: None,
4459 target: "http://localhost".to_string(),
4460 base_path: None,
4461 duration: "1m".to_string(),
4462 vus: 10,
4463 scenario: "ramp-up".to_string(),
4464 operations: None,
4465 exclude_operations: None,
4466 auth: None,
4467 headers: vec![
4468 "X-API-Key:test123".to_string(),
4469 "X-Client-ID:client456".to_string(),
4470 ],
4471 output: PathBuf::from("output"),
4472 generate_only: false,
4473 script_output: None,
4474 threshold_percentile: "p(95)".to_string(),
4475 threshold_ms: 500,
4476 max_error_rate: 0.05,
4477 abort_on_error: true,
4478 abort_on_error_rate: 0.95,
4479 verbose: false,
4480 skip_tls_verify: false,
4481 chunked_request_bodies: false,
4482 target_rps: None,
4483 no_keep_alive: false,
4484 targets_file: None,
4485 max_concurrency: None,
4486 results_format: "both".to_string(),
4487 params_file: None,
4488 crud_flow: false,
4489 flow_config: None,
4490 extract_fields: None,
4491 parallel_create: None,
4492 data_file: None,
4493 data_distribution: "unique-per-vu".to_string(),
4494 data_mappings: None,
4495 per_uri_control: false,
4496 error_rate: None,
4497 error_types: None,
4498 security_test: false,
4499 security_payloads: None,
4500 security_categories: None,
4501 security_target_fields: None,
4502 wafbench_dir: None,
4503 wafbench_cycle_all: false,
4504 wafbench_verbatim: false,
4505 owasp_api_top10: false,
4506 owasp_categories: None,
4507 owasp_auth_header: "Authorization".to_string(),
4508 owasp_auth_token: None,
4509 owasp_admin_paths: None,
4510 owasp_id_fields: None,
4511 owasp_report: None,
4512 owasp_report_format: "json".to_string(),
4513 owasp_iterations: 1,
4514 conformance: false,
4515 conformance_api_key: None,
4516 conformance_basic_auth: None,
4517 conformance_report: PathBuf::from("conformance-report.json"),
4518 conformance_categories: None,
4519 conformance_report_format: "json".to_string(),
4520 conformance_headers: vec![],
4521 conformance_all_operations: false,
4522 conformance_custom: None,
4523 conformance_delay_ms: 0,
4524 use_k6: false,
4525 conformance_custom_filter: None,
4526 export_requests: false,
4527 validate_requests: false,
4528 conformance_self_test: false,
4529 conformance_self_test_capture: false,
4530 conformance_self_test_iterations: 1,
4531 conformance_self_test_duration: None,
4532 validate_response_schemas: false,
4533 source_ips: Vec::new(),
4534 geo_source_ips: Vec::new(),
4535 geo_source_headers: Vec::new(),
4536 report_missed_cap: None,
4537 discard_response_bodies: false,
4538 dns_policy: None,
4539 };
4540
4541 let headers = cmd.parse_headers().unwrap();
4542 assert_eq!(headers.get("X-API-Key"), Some(&"test123".to_string()));
4543 assert_eq!(headers.get("X-Client-ID"), Some(&"client456".to_string()));
4544 }
4545
4546 #[test]
4547 fn test_parse_header_string_preserves_comma_in_value() {
4548 let inputs = vec![
4551 "Cookie:session=abc; expires=Thu, 01 Jan 2099 00:00:00 GMT".to_string(),
4552 "X-Trace:1".to_string(),
4553 ];
4554 let headers = parse_header_string(&inputs).unwrap();
4555 assert_eq!(
4556 headers.get("Cookie"),
4557 Some(&"session=abc; expires=Thu, 01 Jan 2099 00:00:00 GMT".to_string())
4558 );
4559 assert_eq!(headers.get("X-Trace"), Some(&"1".to_string()));
4560 }
4561
4562 #[test]
4570 fn conformance_advisory_names_every_discarded_flag() {
4571 let msg = CONFORMANCE_REPLACES_LOAD_ADVISORY;
4572 for flag in ["--vus", "--rps", "-d"] {
4573 assert!(
4574 msg.contains(flag),
4575 "conformance advisory must name `{flag}` as ignored; it is discarded on that \
4576 path and silently dropping it is how users end up tuning a knob that does \
4577 nothing (#980). Message was: {msg}"
4578 );
4579 }
4580 assert!(
4581 msg.contains("REPLACES"),
4582 "conformance advisory must say the load run is REPLACED, not merely that some \
4583 flags are ignored — `--conformance` returns before the load path runs, so no \
4584 load traffic is generated at all (#980). Message was: {msg}"
4585 );
4586 }
4587
4588 #[test]
4602 fn multi_target_clone_preserves_fields_parse_headers_reads() {
4603 let src = include_str!("command.rs");
4604
4605 let fn_start = src
4606 .find("async fn execute_multi_target(")
4607 .expect("execute_multi_target should exist");
4608 let block_start = src[fn_start..]
4609 .find("ParallelExecutor::new(")
4610 .map(|i| i + fn_start)
4611 .expect("multi-target path should build a ParallelExecutor");
4612 let block_end = src[block_start..]
4614 .find("\n );")
4615 .map(|i| i + block_start)
4616 .expect("ParallelExecutor::new(..) should be closed");
4617 let block = &src[block_start..block_end];
4618
4619 for field in ["conformance_basic_auth", "conformance_headers"] {
4622 for zeroed in [format!("{field}: None"), format!("{field}: vec![]")] {
4623 assert!(
4624 !block.contains(&zeroed),
4625 "execute_multi_target zeroes `{zeroed}`. parse_headers() folds `{field}` \
4626 into the header map, so zeroing it here strips auth from every \
4627 multi-target run while single-target keeps working (#79 round 64)."
4628 );
4629 }
4630 let passthrough = format!("{field}: self.{field}.clone()");
4631 assert!(
4632 block.contains(&passthrough),
4633 "execute_multi_target must carry `{field}` through as `{passthrough}` so \
4634 parse_headers() can fold it (#79 round 64)."
4635 );
4636 }
4637 }
4638
4639 #[test]
4640 fn test_get_spec_display_name() {
4641 let cmd = BenchCommand {
4642 spec: vec![PathBuf::from("test.yaml")],
4643 spec_dir: None,
4644 merge_conflicts: "error".to_string(),
4645 spec_mode: "merge".to_string(),
4646 dependency_config: None,
4647 target: "http://localhost".to_string(),
4648 base_path: None,
4649 duration: "1m".to_string(),
4650 vus: 10,
4651 scenario: "ramp-up".to_string(),
4652 operations: None,
4653 exclude_operations: None,
4654 auth: None,
4655 headers: Vec::new(),
4656 output: PathBuf::from("output"),
4657 generate_only: false,
4658 script_output: None,
4659 threshold_percentile: "p(95)".to_string(),
4660 threshold_ms: 500,
4661 max_error_rate: 0.05,
4662 abort_on_error: true,
4663 abort_on_error_rate: 0.95,
4664 verbose: false,
4665 skip_tls_verify: false,
4666 chunked_request_bodies: false,
4667 target_rps: None,
4668 no_keep_alive: false,
4669 targets_file: None,
4670 max_concurrency: None,
4671 results_format: "both".to_string(),
4672 params_file: None,
4673 crud_flow: false,
4674 flow_config: None,
4675 extract_fields: None,
4676 parallel_create: None,
4677 data_file: None,
4678 data_distribution: "unique-per-vu".to_string(),
4679 data_mappings: None,
4680 per_uri_control: false,
4681 error_rate: None,
4682 error_types: None,
4683 security_test: false,
4684 security_payloads: None,
4685 security_categories: None,
4686 security_target_fields: None,
4687 wafbench_dir: None,
4688 wafbench_cycle_all: false,
4689 wafbench_verbatim: false,
4690 owasp_api_top10: false,
4691 owasp_categories: None,
4692 owasp_auth_header: "Authorization".to_string(),
4693 owasp_auth_token: None,
4694 owasp_admin_paths: None,
4695 owasp_id_fields: None,
4696 owasp_report: None,
4697 owasp_report_format: "json".to_string(),
4698 owasp_iterations: 1,
4699 conformance: false,
4700 conformance_api_key: None,
4701 conformance_basic_auth: None,
4702 conformance_report: PathBuf::from("conformance-report.json"),
4703 conformance_categories: None,
4704 conformance_report_format: "json".to_string(),
4705 conformance_headers: vec![],
4706 conformance_all_operations: false,
4707 conformance_custom: None,
4708 conformance_delay_ms: 0,
4709 use_k6: false,
4710 conformance_custom_filter: None,
4711 export_requests: false,
4712 validate_requests: false,
4713 conformance_self_test: false,
4714 conformance_self_test_capture: false,
4715 conformance_self_test_iterations: 1,
4716 conformance_self_test_duration: None,
4717 validate_response_schemas: false,
4718 source_ips: Vec::new(),
4719 geo_source_ips: Vec::new(),
4720 geo_source_headers: Vec::new(),
4721 report_missed_cap: None,
4722 discard_response_bodies: false,
4723 dns_policy: None,
4724 };
4725
4726 assert_eq!(cmd.get_spec_display_name(), "test.yaml");
4727
4728 let cmd_multi = BenchCommand {
4730 spec: vec![PathBuf::from("a.yaml"), PathBuf::from("b.yaml")],
4731 spec_dir: None,
4732 merge_conflicts: "error".to_string(),
4733 spec_mode: "merge".to_string(),
4734 dependency_config: None,
4735 target: "http://localhost".to_string(),
4736 base_path: None,
4737 duration: "1m".to_string(),
4738 vus: 10,
4739 scenario: "ramp-up".to_string(),
4740 operations: None,
4741 exclude_operations: None,
4742 auth: None,
4743 headers: Vec::new(),
4744 output: PathBuf::from("output"),
4745 generate_only: false,
4746 script_output: None,
4747 threshold_percentile: "p(95)".to_string(),
4748 threshold_ms: 500,
4749 max_error_rate: 0.05,
4750 abort_on_error: true,
4751 abort_on_error_rate: 0.95,
4752 verbose: false,
4753 skip_tls_verify: false,
4754 chunked_request_bodies: false,
4755 target_rps: None,
4756 no_keep_alive: false,
4757 targets_file: None,
4758 max_concurrency: None,
4759 results_format: "both".to_string(),
4760 params_file: None,
4761 crud_flow: false,
4762 flow_config: None,
4763 extract_fields: None,
4764 parallel_create: None,
4765 data_file: None,
4766 data_distribution: "unique-per-vu".to_string(),
4767 data_mappings: None,
4768 per_uri_control: false,
4769 error_rate: None,
4770 error_types: None,
4771 security_test: false,
4772 security_payloads: None,
4773 security_categories: None,
4774 security_target_fields: None,
4775 wafbench_dir: None,
4776 wafbench_cycle_all: false,
4777 wafbench_verbatim: false,
4778 owasp_api_top10: false,
4779 owasp_categories: None,
4780 owasp_auth_header: "Authorization".to_string(),
4781 owasp_auth_token: None,
4782 owasp_admin_paths: None,
4783 owasp_id_fields: None,
4784 owasp_report: None,
4785 owasp_report_format: "json".to_string(),
4786 owasp_iterations: 1,
4787 conformance: false,
4788 conformance_api_key: None,
4789 conformance_basic_auth: None,
4790 conformance_report: PathBuf::from("conformance-report.json"),
4791 conformance_categories: None,
4792 conformance_report_format: "json".to_string(),
4793 conformance_headers: vec![],
4794 conformance_all_operations: false,
4795 conformance_custom: None,
4796 conformance_delay_ms: 0,
4797 use_k6: false,
4798 conformance_custom_filter: None,
4799 export_requests: false,
4800 validate_requests: false,
4801 conformance_self_test: false,
4802 conformance_self_test_capture: false,
4803 conformance_self_test_iterations: 1,
4804 conformance_self_test_duration: None,
4805 validate_response_schemas: false,
4806 source_ips: Vec::new(),
4807 geo_source_ips: Vec::new(),
4808 geo_source_headers: Vec::new(),
4809 report_missed_cap: None,
4810 discard_response_bodies: false,
4811 dns_policy: None,
4812 };
4813
4814 assert_eq!(cmd_multi.get_spec_display_name(), "2 spec files");
4815 }
4816
4817 #[test]
4818 fn test_parse_extracted_values_from_output_dir() {
4819 let dir = tempdir().unwrap();
4820 let path = dir.path().join("extracted_values.json");
4821 std::fs::write(
4822 &path,
4823 r#"{
4824 "pool_id": "abc123",
4825 "count": 0,
4826 "enabled": false,
4827 "metadata": { "owner": "team-a" }
4828}"#,
4829 )
4830 .unwrap();
4831
4832 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
4833 assert_eq!(extracted.get("pool_id"), Some(&serde_json::json!("abc123")));
4834 assert_eq!(extracted.get("count"), Some(&serde_json::json!(0)));
4835 assert_eq!(extracted.get("enabled"), Some(&serde_json::json!(false)));
4836 assert_eq!(extracted.get("metadata"), Some(&serde_json::json!({"owner": "team-a"})));
4837 }
4838
4839 #[test]
4840 fn test_parse_extracted_values_missing_file() {
4841 let dir = tempdir().unwrap();
4842 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
4843 assert!(extracted.values.is_empty());
4844 }
4845
4846 fn sample_bench_command() -> BenchCommand {
4849 BenchCommand {
4850 spec: vec![PathBuf::from("test.yaml")],
4851 spec_dir: None,
4852 merge_conflicts: "error".to_string(),
4853 spec_mode: "merge".to_string(),
4854 dependency_config: None,
4855 target: "http://localhost".to_string(),
4856 base_path: None,
4857 duration: "1m".to_string(),
4858 vus: 10,
4859 scenario: "ramp-up".to_string(),
4860 operations: None,
4861 exclude_operations: None,
4862 auth: None,
4863 headers: vec![
4864 "X-API-Key:test123".to_string(),
4865 "X-Client-ID:client456".to_string(),
4866 ],
4867 output: PathBuf::from("output"),
4868 generate_only: false,
4869 script_output: None,
4870 threshold_percentile: "p(95)".to_string(),
4871 threshold_ms: 500,
4872 max_error_rate: 0.05,
4873 abort_on_error: true,
4874 abort_on_error_rate: 0.95,
4875 verbose: false,
4876 skip_tls_verify: false,
4877 chunked_request_bodies: false,
4878 target_rps: None,
4879 no_keep_alive: false,
4880 targets_file: None,
4881 max_concurrency: None,
4882 results_format: "both".to_string(),
4883 params_file: None,
4884 crud_flow: false,
4885 flow_config: None,
4886 extract_fields: None,
4887 parallel_create: None,
4888 data_file: None,
4889 data_distribution: "unique-per-vu".to_string(),
4890 data_mappings: None,
4891 per_uri_control: false,
4892 error_rate: None,
4893 error_types: None,
4894 security_test: false,
4895 security_payloads: None,
4896 security_categories: None,
4897 security_target_fields: None,
4898 wafbench_dir: None,
4899 wafbench_cycle_all: false,
4900 wafbench_verbatim: false,
4901 owasp_api_top10: false,
4902 owasp_categories: None,
4903 owasp_auth_header: "Authorization".to_string(),
4904 owasp_auth_token: None,
4905 owasp_admin_paths: None,
4906 owasp_id_fields: None,
4907 owasp_report: None,
4908 owasp_report_format: "json".to_string(),
4909 owasp_iterations: 1,
4910 conformance: false,
4911 conformance_api_key: None,
4912 conformance_basic_auth: None,
4913 conformance_report: PathBuf::from("conformance-report.json"),
4914 conformance_categories: None,
4915 conformance_report_format: "json".to_string(),
4916 conformance_headers: vec![],
4917 conformance_all_operations: false,
4918 conformance_custom: None,
4919 conformance_delay_ms: 0,
4920 use_k6: false,
4921 conformance_custom_filter: None,
4922 export_requests: false,
4923 validate_requests: false,
4924 conformance_self_test: false,
4925 conformance_self_test_capture: false,
4926 conformance_self_test_iterations: 1,
4927 conformance_self_test_duration: None,
4928 validate_response_schemas: false,
4929 source_ips: Vec::new(),
4930 geo_source_ips: Vec::new(),
4931 geo_source_headers: Vec::new(),
4932 report_missed_cap: None,
4933 discard_response_bodies: false,
4934 dns_policy: None,
4935 }
4936 }
4937
4938 #[test]
4946 fn verbatim_disables_security_payload_injection() {
4947 let mut cmd = sample_bench_command();
4948 cmd.wafbench_dir = Some("traffic.yaml".to_string());
4949
4950 assert!(
4951 cmd.security_testing_enabled(),
4952 "--wafbench-dir alone must still enable payload injection"
4953 );
4954
4955 cmd.wafbench_verbatim = true;
4956 assert!(
4957 !cmd.security_testing_enabled(),
4958 "verbatim mode must not inject payloads into requests sent as written"
4959 );
4960
4961 cmd.security_test = true;
4964 assert!(
4965 !cmd.security_testing_enabled(),
4966 "--security-test must not re-enable injection under --wafbench-verbatim"
4967 );
4968 }
4969
4970 #[test]
4975 fn security_testing_enabled_has_a_single_definition() {
4976 let src = include_str!("command.rs");
4977 let parallel = include_str!("parallel_executor.rs");
4978 let a = format!("self.{} || self.{}.is_some()", "security_test", "wafbench_dir");
4980 let b = format!("self.{}.is_some() || self.{}", "wafbench_dir", "security_test");
4981 let inline = src.matches(a.as_str()).count() + src.matches(b.as_str()).count();
4982 assert_eq!(
4983 inline, 1,
4984 "expected the security_testing_enabled() method to be the only place this is \
4985 computed, found {inline} inline copies -- collapse them or the render paths drift"
4986 );
4987
4988 let parallel_inline = format!(
4993 "{}.{} || {}.{}.is_some()",
4994 "base_command", "security_test", "self.base_command", "wafbench_dir"
4995 );
4996 assert!(
4997 !parallel.contains(¶llel_inline),
4998 "ParallelExecutor must not recompute the security flag inline"
4999 );
5000 assert!(
5001 parallel.contains("security_testing_enabled()"),
5002 "ParallelExecutor must call security_testing_enabled() so --wafbench-verbatim \
5003 turns injection off on --targets-file runs too"
5004 );
5005 }
5006
5007 #[test]
5011 fn missing_wafbench_dir_is_not_swallowed() {
5012 let src = include_str!("command.rs");
5013 let swallowed = format!("Failed to {} WAFBench tests", "load");
5015 let impl_line = src
5016 .lines()
5017 .filter(|l| !l.trim_start().starts_with("//"))
5018 .any(|l| l.contains(&swallowed));
5019 assert!(!impl_line, "missing --wafbench-dir must not be downgraded to a warning");
5020 assert!(
5021 src.contains("self.load_wafbench_payloads()?"),
5022 "payload load errors must reach generate_enhanced_script"
5023 );
5024 }
5025
5026 #[test]
5031 fn multi_target_path_honors_verbatim_templates() {
5032 let src = include_str!("parallel_executor.rs");
5033 assert!(
5034 src.contains("load_verbatim_templates"),
5035 "ParallelExecutor must load traffic-file requests under --wafbench-verbatim. \
5036 Requiring a spec and generating templates from its operations is how \
5037 --targets-file ignored the flag and fuzzed spec URLs (#79)."
5038 );
5039 }
5040
5041 #[test]
5043 fn traffic_breakdown_json_multiplies_unique_by_rps() {
5044 let dir = std::env::temp_dir().join(format!(
5045 "mf-traffic-breakdown-{}-{}",
5046 std::process::id(),
5047 std::time::SystemTime::now()
5048 .duration_since(std::time::UNIX_EPOCH)
5049 .unwrap()
5050 .as_nanos()
5051 ));
5052 let _ = std::fs::create_dir_all(&dir);
5053 let mut cmd = sample_bench_command();
5054 cmd.output = dir.clone();
5055 cmd.target_rps = Some(50);
5056 cmd.duration = "1200s".to_string();
5057 let stats = crate::wafbench::WafBenchStats {
5058 per_file: vec![crate::wafbench::TrafficFileSummary {
5059 file: "apisix_cve-2026-44087.yaml".into(),
5060 sent: 5,
5061 attack: 3,
5062 normal: 2,
5063 omitted: 1,
5064 other: 0,
5065 }],
5066 ..Default::default()
5067 };
5068 cmd.emit_traffic_file_breakdown(&stats, "what to expect in proxy logs");
5069 let raw = std::fs::read_to_string(dir.join("traffic-breakdown.json"))
5070 .expect("traffic-breakdown.json");
5071 let v: serde_json::Value = serde_json::from_str(&raw).unwrap();
5072 assert_eq!(v["rps"], 50);
5073 assert_eq!(v["duration_secs"], 1200);
5074 assert_eq!(v["files"][0]["sent"]["unique"], 5);
5075 assert_eq!(v["files"][0]["sent"]["total"], 250);
5076 assert_eq!(v["files"][0]["sent"]["expected_requests"], 300000);
5077 assert_eq!(v["files"][0]["sent"]["expected_requests_unit"], "http");
5078 assert_eq!(v["files"][0]["attack"]["total"], 150);
5079 assert_eq!(v["files"][0]["normal"]["total"], 100);
5080 assert!(v["expected_requests_note"].as_str().unwrap().contains("HTTP requests"));
5081 assert_eq!(
5082 BenchCommand::format_unique_total(5, Some(50)),
5083 "unique=5 total=250 (5 * 50 RPS)"
5084 );
5085 let _ = std::fs::remove_dir_all(&dir);
5086 }
5087
5088 #[test]
5091 fn traffic_breakdown_json_omits_expected_requests_without_rps() {
5092 let dir = std::env::temp_dir().join(format!(
5093 "mf-traffic-breakdown-norps-{}-{}",
5094 std::process::id(),
5095 std::time::SystemTime::now()
5096 .duration_since(std::time::UNIX_EPOCH)
5097 .unwrap()
5098 .as_nanos()
5099 ));
5100 let _ = std::fs::create_dir_all(&dir);
5101 let mut cmd = sample_bench_command();
5102 cmd.output = dir.clone();
5103 cmd.target_rps = None;
5104 cmd.duration = "60s".to_string();
5105 let stats = crate::wafbench::WafBenchStats {
5106 per_file: vec![crate::wafbench::TrafficFileSummary {
5107 file: "apisix_cve-2026-44087.yaml".into(),
5108 sent: 5,
5109 attack: 3,
5110 normal: 2,
5111 omitted: 1,
5112 other: 0,
5113 }],
5114 ..Default::default()
5115 };
5116 cmd.emit_traffic_file_breakdown(&stats, "what to expect in proxy logs");
5117 let raw = std::fs::read_to_string(dir.join("traffic-breakdown.json"))
5118 .expect("traffic-breakdown.json");
5119 let v: serde_json::Value = serde_json::from_str(&raw).unwrap();
5120 assert!(v["rps"].is_null());
5121 assert_eq!(v["duration_secs"], 60);
5122 assert_eq!(v["files"][0]["sent"]["unique"], 5);
5123 assert_eq!(v["files"][0]["sent"]["total"], 5);
5124 assert!(v["files"][0]["sent"]["expected_requests"].is_null());
5125 let _ = std::fs::remove_dir_all(&dir);
5126 }
5127}