1use crate::crud_flow::{CrudFlowConfig, CrudFlowDetector};
4use crate::data_driven::{DataDistribution, DataDrivenConfig, DataDrivenGenerator, DataMapping};
5use crate::dynamic_params::{DynamicParamProcessor, DynamicPlaceholder};
6use crate::error::{BenchError, Result};
7use crate::executor::K6Executor;
8use crate::invalid_data::{InvalidDataConfig, InvalidDataGenerator};
9use crate::k6_gen::{K6Config, K6ScriptGenerator};
10use crate::mock_integration::{
11 MockIntegrationConfig, MockIntegrationGenerator, MockServerDetector,
12};
13use crate::owasp_api::{OwaspApiConfig, OwaspApiGenerator, OwaspCategory, ReportFormat};
14use crate::parallel_executor::{AggregatedResults, ParallelExecutor};
15use crate::parallel_requests::{ParallelConfig, ParallelRequestGenerator};
16use crate::param_overrides::ParameterOverrides;
17use crate::reporter::TerminalReporter;
18use crate::request_gen::RequestGenerator;
19use crate::scenarios::LoadScenario;
20use crate::security_payloads::{
21 SecurityCategory, SecurityPayload, SecurityPayloads, SecurityTestConfig, SecurityTestGenerator,
22};
23use crate::spec_dependencies::{
24 topological_sort, DependencyDetector, ExtractedValues, SpecDependencyConfig,
25};
26use crate::spec_parser::SpecParser;
27use crate::target_parser::parse_targets_file;
28use crate::wafbench::WafBenchLoader;
29use mockforge_openapi::multi_spec::{
30 load_specs_from_directory, load_specs_from_files, merge_specs, ConflictStrategy,
31};
32use mockforge_openapi::spec::OpenApiSpec;
33use std::collections::{HashMap, HashSet};
34use std::path::{Path, PathBuf};
35use std::str::FromStr;
36
37pub fn parse_header_string(inputs: &[String]) -> Result<HashMap<String, String>> {
45 let mut headers = HashMap::new();
46
47 for pair in inputs {
48 let pair = pair.trim();
49 if pair.is_empty() {
50 continue;
51 }
52 let parts: Vec<&str> = pair.splitn(2, ':').collect();
53 if parts.len() != 2 {
54 return Err(BenchError::Other(format!(
55 "Invalid header format: '{}'. Expected 'Key:Value'",
56 pair
57 )));
58 }
59 headers.insert(parts[0].trim().to_string(), parts[1].trim().to_string());
60 }
61
62 Ok(headers)
63}
64
65const CONFORMANCE_REPLACES_LOAD_ADVISORY: &str =
85 "Conformance mode REPLACES the load run: 1 VU, 1 iteration per endpoint. \
86 --vus, --rps and -d are ignored. Run bench a second time without \
87 --conformance if you also want a load test.";
88
89pub struct BenchCommand {
91 pub spec: Vec<PathBuf>,
93 pub spec_dir: Option<PathBuf>,
95 pub merge_conflicts: String,
97 pub spec_mode: String,
99 pub dependency_config: Option<PathBuf>,
101 pub target: String,
102 pub base_path: Option<String>,
105 pub duration: String,
106 pub vus: u32,
107 pub target_rps: Option<u32>,
113 pub no_keep_alive: bool,
118 pub scenario: String,
119 pub operations: Option<String>,
120 pub exclude_operations: Option<String>,
124 pub auth: Option<String>,
125 pub headers: Vec<String>,
128 pub output: PathBuf,
129 pub generate_only: bool,
130 pub script_output: Option<PathBuf>,
131 pub threshold_percentile: String,
132 pub threshold_ms: u64,
133 pub max_error_rate: f64,
134 pub abort_on_error: bool,
139 pub abort_on_error_rate: f64,
143 pub verbose: bool,
144 pub skip_tls_verify: bool,
145 pub chunked_request_bodies: bool,
150 pub targets_file: Option<PathBuf>,
152 pub max_concurrency: Option<u32>,
154 pub results_format: String,
156 pub params_file: Option<PathBuf>,
161
162 pub crud_flow: bool,
165 pub flow_config: Option<PathBuf>,
167 pub extract_fields: Option<String>,
169
170 pub parallel_create: Option<u32>,
173
174 pub data_file: Option<PathBuf>,
177 pub data_distribution: String,
179 pub data_mappings: Option<String>,
181 pub per_uri_control: bool,
183
184 pub error_rate: Option<f64>,
187 pub error_types: Option<String>,
189
190 pub security_test: bool,
193 pub security_payloads: Option<PathBuf>,
195 pub security_categories: Option<String>,
197 pub security_target_fields: Option<String>,
199
200 pub wafbench_dir: Option<String>,
203 pub wafbench_cycle_all: bool,
205 pub wafbench_verbatim: bool,
208
209 pub conformance: bool,
212 pub conformance_api_key: Option<String>,
214 pub conformance_basic_auth: Option<String>,
216 pub conformance_report: PathBuf,
218 pub conformance_categories: Option<String>,
220 pub conformance_report_format: String,
222 pub conformance_headers: Vec<String>,
225 pub conformance_all_operations: bool,
228 pub conformance_custom: Option<PathBuf>,
230 pub conformance_delay_ms: u64,
233 pub use_k6: bool,
235 pub conformance_custom_filter: Option<String>,
239 pub export_requests: bool,
242 pub validate_requests: bool,
245 pub conformance_self_test: bool,
252 pub conformance_self_test_capture: bool,
256 pub validate_response_schemas: bool,
262 pub conformance_self_test_iterations: u32,
267 pub conformance_self_test_duration: Option<String>,
272
273 pub source_ips: Vec<String>,
278 pub geo_source_ips: Vec<String>,
282 pub geo_source_headers: Vec<String>,
286
287 pub report_missed_cap: Option<u32>,
294
295 pub discard_response_bodies: bool,
302
303 pub dns_policy: Option<String>,
309
310 pub owasp_api_top10: bool,
313 pub owasp_categories: Option<String>,
315 pub owasp_auth_header: String,
317 pub owasp_auth_token: Option<String>,
319 pub owasp_admin_paths: Option<PathBuf>,
321 pub owasp_id_fields: Option<String>,
323 pub owasp_report: Option<PathBuf>,
325 pub owasp_report_format: String,
327 pub owasp_iterations: u32,
329}
330
331fn parse_ip_list(raw: &[String], flag_name: &str) -> Vec<std::net::IpAddr> {
345 use std::net::IpAddr;
346 const MAX_CIDR_EXPANSION: usize = 256;
347 let mut out = Vec::new();
348 for entry in raw {
349 for piece in entry.split(',') {
350 let s = piece.trim();
351 if s.is_empty() {
352 continue;
353 }
354 if let Some((addr_part, prefix_part)) = s.split_once('/') {
356 let prefix: u32 = match prefix_part.parse() {
357 Ok(p) => p,
358 Err(e) => {
359 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR prefix: {e}");
360 continue;
361 }
362 };
363 let net_addr: IpAddr = match addr_part.parse() {
364 Ok(a) => a,
365 Err(e) => {
366 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR address: {e}");
367 continue;
368 }
369 };
370 expand_cidr(net_addr, prefix, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
371 continue;
372 }
373 if let Some((start_str, end_str)) = s.split_once('-') {
379 let start_s = start_str.trim();
380 let end_s = end_str.trim();
381 if start_s.contains(':') || end_s.contains(':') {
385 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{s}': IPv6 range syntax not supported (use CIDR like 2001:db8::/126 instead)");
386 continue;
387 }
388 let start: IpAddr = match start_s.parse() {
389 Ok(a) => a,
390 Err(e) => {
391 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range start: {e}");
392 continue;
393 }
394 };
395 let end: IpAddr = match end_s.parse() {
396 Ok(a) => a,
397 Err(e) => {
398 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range end: {e}");
399 continue;
400 }
401 };
402 expand_range(start, end, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
403 continue;
404 }
405 match s.parse::<IpAddr>() {
407 Ok(ip) => out.push(ip),
408 Err(e) => {
409 tracing::warn!(target: "mockforge::bench", "ignoring malformed --{flag_name} value '{s}': {e}");
410 }
411 }
412 }
413 }
414 out
415}
416
417fn expand_range(
421 start: std::net::IpAddr,
422 end: std::net::IpAddr,
423 cap: usize,
424 flag_name: &str,
425 raw: &str,
426 out: &mut Vec<std::net::IpAddr>,
427) {
428 use std::net::{IpAddr, Ipv4Addr};
429 let (start_v4, end_v4) = match (start, end) {
430 (IpAddr::V4(a), IpAddr::V4(b)) => (a, b),
431 _ => {
432 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range start/end must both be IPv4");
433 return;
434 }
435 };
436 let start_u32 = u32::from(start_v4);
437 let end_u32 = u32::from(end_v4);
438 if end_u32 < start_u32 {
439 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range end {end_v4} is before start {start_v4}");
440 return;
441 }
442 let total = (end_u32 - start_u32).saturating_add(1) as usize;
443 let take = total.min(cap);
444 if total > cap {
445 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range has {total} addresses, capping at {cap}");
446 }
447 for i in 0..take as u32 {
448 out.push(IpAddr::V4(Ipv4Addr::from(start_u32 + i)));
449 }
450}
451
452fn expand_cidr(
456 net: std::net::IpAddr,
457 prefix: u32,
458 cap: usize,
459 flag_name: &str,
460 raw: &str,
461 out: &mut Vec<std::net::IpAddr>,
462) {
463 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
464 match net {
465 IpAddr::V4(ipv4) => {
466 if prefix > 32 {
467 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv4 prefix must be <= 32");
468 return;
469 }
470 let total: u64 = 1u64 << (32 - prefix);
471 let take = total.min(cap as u64) as u32;
472 if total > cap as u64 {
473 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': CIDR has {total} addresses, capping at {cap}");
474 }
475 let mask: u32 = if prefix == 0 {
476 0
477 } else {
478 !0u32 << (32 - prefix)
479 };
480 let net_u32 = u32::from(ipv4) & mask;
481 for i in 0..take {
482 out.push(IpAddr::V4(Ipv4Addr::from(net_u32.wrapping_add(i))));
483 }
484 }
485 IpAddr::V6(ipv6) => {
486 if prefix > 128 {
487 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv6 prefix must be <= 128");
488 return;
489 }
490 let mask: u128 = if prefix == 0 {
494 0
495 } else {
496 !0u128 << (128 - prefix)
497 };
498 let net_u128 = u128::from(ipv6) & mask;
499 let remaining_bits = 128 - prefix;
500 let total_capped = if remaining_bits >= 64 {
503 cap as u128
504 } else {
505 (1u128 << remaining_bits).min(cap as u128)
506 };
507 if remaining_bits < 128 && (1u128 << remaining_bits) > cap as u128 {
508 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': IPv6 CIDR exceeds {cap} addresses, capping");
509 }
510 for i in 0..total_capped {
511 out.push(IpAddr::V6(Ipv6Addr::from(net_u128.wrapping_add(i))));
512 }
513 }
514 }
515}
516
517impl BenchCommand {
518 pub fn security_testing_enabled(&self) -> bool {
534 if self.wafbench_verbatim {
535 return false;
536 }
537 self.security_test || self.wafbench_dir.is_some()
538 }
539
540 pub async fn load_and_merge_specs(&self) -> Result<OpenApiSpec> {
542 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
543
544 if !self.spec.is_empty() {
546 let specs = load_specs_from_files(self.spec.clone())
547 .await
548 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
549 all_specs.extend(specs);
550 }
551
552 if let Some(spec_dir) = &self.spec_dir {
554 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
555 BenchError::Other(format!("Failed to load specs from directory: {}", e))
556 })?;
557 all_specs.extend(dir_specs);
558 }
559
560 if all_specs.is_empty() {
561 return Err(BenchError::Other(
562 "No spec files provided. Use --spec or --spec-dir.".to_string(),
563 ));
564 }
565
566 if all_specs.len() == 1 {
568 return Ok(all_specs.into_iter().next().expect("checked len() == 1 above").1);
570 }
571
572 let conflict_strategy = match self.merge_conflicts.as_str() {
574 "first" => ConflictStrategy::First,
575 "last" => ConflictStrategy::Last,
576 _ => ConflictStrategy::Error,
577 };
578
579 merge_specs(all_specs, conflict_strategy)
580 .map_err(|e| BenchError::Other(format!("Failed to merge specs: {}", e)))
581 }
582
583 fn get_spec_display_name(&self) -> String {
585 if self.spec.len() == 1 {
586 self.spec[0].to_string_lossy().to_string()
587 } else if !self.spec.is_empty() {
588 format!("{} spec files", self.spec.len())
589 } else if let Some(dir) = &self.spec_dir {
590 format!("specs from {}", dir.display())
591 } else {
592 "no specs".to_string()
593 }
594 }
595
596 fn advise_capacity(&self) {
603 let target_count = self
604 .targets_file
605 .as_ref()
606 .and_then(|p| std::fs::read_to_string(p).ok())
607 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
608 .and_then(|v| v.as_array().map(|a| a.len()))
609 .unwrap_or(1);
610 let vus = self.vus.max(1);
611 let rps_total = self.target_rps.unwrap_or(0) as usize * target_count.max(1);
612 let load_product = target_count * vus as usize;
616 if load_product >= 150 {
617 let est_ram_gb =
618 (vus as usize * 50) / 1024 + (target_count * 10 * 2) / 1024 + target_count / 2;
619 let est_cores = ((vus as usize) / 50).max(2);
620 TerminalReporter::print_warning(&format!(
621 "Capacity advisory: targets={target_count}, VUs={vus}, RPS-total≈{rps_total}. \
622 Single-client estimate: ~{est_cores} CPU cores, ~{est_ram_gb} GB RAM. \
623 If your machine is below that, expect OOM hangs partway through the run. \
624 See https://docs.mockforge.dev/reference/bench-capacity-sizing.html \
625 for the sizing table and sharding guide."
626 ));
627 }
628 }
629
630 pub async fn execute(&self) -> Result<()> {
632 if self.conformance_self_test && self.use_k6 {
639 TerminalReporter::print_warning(
640 "--use-k6 has no effect with --conformance-self-test: the self-test driver runs and returns before k6 is invoked. Drop one or the other depending on whether you want the spec-driven self-test or a k6 bench run.",
641 );
642 }
643
644 self.advise_capacity();
650
651 if let Some(targets_file) = &self.targets_file {
653 if self.conformance && self.conformance_self_test {
662 return self.execute_multi_target_self_test(targets_file).await;
663 }
664 if self.conformance {
665 return self.execute_multi_target_conformance(targets_file).await;
666 }
667 return self.execute_multi_target(targets_file).await;
668 }
669
670 if self.spec_mode == "sequential" && (self.spec.len() > 1 || self.spec_dir.is_some()) {
672 return self.execute_sequential_specs().await;
673 }
674
675 TerminalReporter::print_header(
678 &self.get_spec_display_name(),
679 &self.target,
680 0, &self.scenario,
682 Self::parse_duration(&self.duration)?,
683 );
684
685 if !K6Executor::is_k6_installed() {
687 TerminalReporter::print_error("k6 is not installed");
688 TerminalReporter::print_warning(
689 "Install k6 from: https://k6.io/docs/get-started/installation/",
690 );
691 return Err(BenchError::K6NotFound);
692 }
693 K6Executor::warn_if_pre_v1().await;
694
695 if self.conformance {
697 return self.execute_conformance_test().await;
698 }
699
700 let spec_supplied = !self.spec.is_empty() || self.spec_dir.is_some();
707 let merged_spec = if self.wafbench_verbatim && !spec_supplied {
708 tracing::info!(
709 target: "mockforge::bench",
710 "--wafbench-verbatim without --spec: sending only the traffic file's requests"
711 );
712 OpenApiSpec {
713 spec: Default::default(),
714 file_path: None,
715 raw_document: None,
716 }
717 } else {
718 TerminalReporter::print_progress("Loading OpenAPI specification(s)...");
719 self.load_and_merge_specs().await?
720 };
721 let parser = SpecParser::from_spec(merged_spec);
722 if self.spec.len() > 1 || self.spec_dir.is_some() {
723 TerminalReporter::print_success(&format!(
724 "Loaded and merged {} specification(s)",
725 self.spec.len() + self.spec_dir.as_ref().map(|_| 1).unwrap_or(0)
726 ));
727 } else {
728 TerminalReporter::print_success("Specification loaded");
729 }
730
731 let mock_config = self.build_mock_config().await;
733 if mock_config.is_mock_server {
734 TerminalReporter::print_progress("Mock server integration enabled");
735 }
736
737 if self.crud_flow {
739 return self.execute_crud_flow(&parser).await;
740 }
741
742 if self.owasp_api_top10 {
744 return self.execute_owasp_test(&parser).await;
745 }
746
747 TerminalReporter::print_progress("Extracting API operations...");
749 let mut operations = if let Some(filter) = &self.operations {
750 parser.filter_operations(filter)?
751 } else {
752 parser.get_operations()
753 };
754
755 if let Some(exclude) = &self.exclude_operations {
757 let before_count = operations.len();
758 operations = parser.exclude_operations(operations, exclude)?;
759 let excluded_count = before_count - operations.len();
760 if excluded_count > 0 {
761 TerminalReporter::print_progress(&format!(
762 "Excluded {} operations matching '{}'",
763 excluded_count, exclude
764 ));
765 }
766 }
767
768 if operations.is_empty() && !self.wafbench_verbatim {
774 return Err(BenchError::Other("No operations found in spec".to_string()));
775 }
776
777 TerminalReporter::print_success(&format!("Found {} operations", operations.len()));
778
779 let param_overrides = if let Some(params_file) = &self.params_file {
781 TerminalReporter::print_progress("Loading parameter overrides...");
782 let overrides = ParameterOverrides::from_file(params_file)?;
783 TerminalReporter::print_success(&format!(
784 "Loaded parameter overrides ({} operation-specific, {} defaults)",
785 overrides.operations.len(),
786 if overrides.defaults.is_empty() { 0 } else { 1 }
787 ));
788 Some(overrides)
789 } else {
790 None
791 };
792
793 TerminalReporter::print_progress("Generating request templates...");
795 let templates: Vec<_> = operations
796 .iter()
797 .map(|op| {
798 let op_overrides = param_overrides.as_ref().map(|po| {
799 po.get_for_operation(op.operation_id.as_deref(), &op.method, &op.path)
800 });
801 RequestGenerator::generate_template_with_overrides(op, op_overrides.as_ref())
802 })
803 .collect::<Result<Vec<_>>>()?;
804 TerminalReporter::print_success("Request templates generated");
805
806 let templates = if self.wafbench_verbatim {
812 let verbatim = self.load_verbatim_templates()?;
813 if verbatim.is_empty() {
814 return Err(BenchError::Other(
815 "--wafbench-verbatim was set but no traffic cases were loaded. Check \
816 --wafbench-dir points at a file, directory or glob containing cases with \
817 a `request.uri`."
818 .to_string(),
819 ));
820 }
821 TerminalReporter::print_success(&format!(
822 "Verbatim mode: {} request(s) will be sent exactly as written (spec endpoints not used)",
823 verbatim.len()
824 ));
825 verbatim
826 } else {
827 templates
828 };
829
830 let custom_headers = self.parse_headers()?;
832
833 let base_path = self.resolve_base_path(&parser);
835 if let Some(ref bp) = base_path {
836 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
837 }
838
839 TerminalReporter::print_progress("Generating k6 load test script...");
841 let scenario =
842 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
843
844 let security_testing_enabled = self.security_testing_enabled();
845
846 let num_ops = operations.len() as u32;
864 if let Some(rps) = self.target_rps {
865 let probe =
866 crate::preflight::probe_target_latency(&self.target, 3, self.skip_tls_verify).await;
867
868 let (required_vus, basis) = match probe {
869 Some(p) => (
870 p.required_vus(rps, num_ops),
871 format!("avg {:.1}ms (measured)", p.avg_latency.as_secs_f64() * 1000.0),
872 ),
873 None => {
874 let fallback = (rps as u64)
876 .saturating_mul(num_ops.max(1) as u64)
877 .div_ceil(10)
878 .min(u32::MAX as u64) as u32;
879 (fallback, "~100ms (default — probe failed)".to_string())
880 }
881 };
882
883 if self.vus < required_vus {
884 const VU_RECOMMENDATION_CAP: u32 = 1000;
890 let recommendation = required_vus.max(self.vus + 1);
891 if recommendation > VU_RECOMMENDATION_CAP {
892 TerminalReporter::print_warning(&format!(
893 "Workload is very large: --rps {} × {} ops/iteration × {} \
894 baseline ⇒ ~{} VUs needed end-to-end, far beyond what's \
895 practical to drive. Two ways to fix:\n 1. Reduce \
896 operations per iteration with `--operations 'pattern,…'` \
897 (or `--exclude-operations`) to focus the bench on a \
898 representative subset.\n 2. Drop `--rps` and use \
899 `--vus {}` alone — closed-model load runs as fast as \
900 the VU pool allows, bounded by latency, with no per-\
901 iteration deadline. Expect 1-iteration coverage of ~{} \
902 operations in {}s.",
903 rps,
904 num_ops,
905 basis,
906 recommendation,
907 self.vus.max(5),
908 num_ops,
909 Self::parse_duration(&self.duration).unwrap_or(0),
910 ));
911 } else {
912 TerminalReporter::print_warning(&format!(
913 "--vus {} may be insufficient for --rps {} × {} ops/iteration \
914 (baseline latency {}). k6's constant-arrival-rate counts ITERATIONS \
915 and each runs every operation in the spec — required ≈ rps × ops × \
916 latency_secs VUs. Bump --vus to ~{} if you see \"Insufficient VUs\" \
917 warnings.",
918 self.vus, rps, num_ops, basis, recommendation,
919 ));
920 }
921 } else if probe.is_some() {
922 TerminalReporter::print_progress(&format!(
923 "Pre-flight probe: target latency {}, {} ops/iteration — --vus {} \
924 is sufficient for --rps {}",
925 basis, num_ops, self.vus, rps,
926 ));
927 }
928 }
929
930 let k6_config = K6Config {
931 target_url: self.target.clone(),
932 base_path,
933 scenario,
934 duration_secs: Self::parse_duration(&self.duration)?,
935 max_vus: self.vus,
936 threshold_percentile: self.threshold_percentile.clone(),
937 threshold_ms: self.threshold_ms,
938 max_error_rate: self.max_error_rate,
939 auth_header: self.auth.clone(),
940 custom_headers,
941 skip_tls_verify: self.skip_tls_verify,
942 security_testing_enabled,
943 chunked_request_bodies: self.chunked_request_bodies,
944 target_rps: self.target_rps,
945 no_keep_alive: self.no_keep_alive,
946 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
952 .into_iter()
953 .map(|ip| ip.to_string())
954 .collect(),
955 geo_source_headers: if self.geo_source_headers.is_empty()
956 && !self.geo_source_ips.is_empty()
957 {
958 crate::conformance::self_test::default_geo_source_headers()
959 } else {
960 self.geo_source_headers.clone()
961 },
962 };
963
964 let generator = K6ScriptGenerator::new(k6_config, templates)
965 .with_abort_valve(self.abort_on_error, self.abort_on_error_rate);
966 let mut script = generator.generate()?;
967 TerminalReporter::print_success("k6 script generated");
968
969 let has_advanced_features = self.data_file.is_some()
971 || self.error_rate.is_some()
972 || self.security_test
973 || self.parallel_create.is_some()
974 || self.wafbench_dir.is_some();
975
976 if has_advanced_features {
978 script = self.generate_enhanced_script(&script)?;
979 }
980
981 if mock_config.is_mock_server {
983 let setup_code = MockIntegrationGenerator::generate_setup(&mock_config);
984 let teardown_code = MockIntegrationGenerator::generate_teardown(&mock_config);
985 let helper_code = MockIntegrationGenerator::generate_vu_id_helper();
986
987 if let Some(import_end) = script.find("export const options") {
989 script.insert_str(
990 import_end,
991 &format!(
992 "\n// === Mock Server Integration ===\n{}\n{}\n{}\n",
993 helper_code, setup_code, teardown_code
994 ),
995 );
996 }
997 }
998
999 TerminalReporter::print_progress("Validating k6 script...");
1001 let validation_errors = K6ScriptGenerator::validate_script(&script);
1002 if !validation_errors.is_empty() {
1003 TerminalReporter::print_error("Script validation failed");
1004 for error in &validation_errors {
1005 eprintln!(" {}", error);
1006 }
1007 return Err(BenchError::Other(format!(
1008 "Generated k6 script has {} validation error(s). Please check the output above.",
1009 validation_errors.len()
1010 )));
1011 }
1012 TerminalReporter::print_success("Script validation passed");
1013
1014 let script_path = if let Some(output) = &self.script_output {
1016 output.clone()
1017 } else {
1018 self.output.join("k6-script.js")
1019 };
1020
1021 if let Some(parent) = script_path.parent() {
1022 std::fs::create_dir_all(parent)?;
1023 }
1024 std::fs::write(&script_path, &script)?;
1025 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
1026
1027 if self.generate_only {
1029 println!("\nScript generated successfully. Run it with:");
1030 println!(" k6 run {}", script_path.display());
1031 return Ok(());
1032 }
1033
1034 TerminalReporter::print_progress("Executing load test...");
1036 let executor = K6Executor::new()?
1040 .with_local_ips(self.source_ips.join(","))
1041 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
1042 .with_discard_response_bodies(self.discard_response_bodies);
1043
1044 std::fs::create_dir_all(&self.output)?;
1045
1046 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
1047
1048 let duration_secs = Self::parse_duration(&self.duration)?;
1050 TerminalReporter::print_summary_full(
1051 &results,
1052 duration_secs,
1053 self.no_keep_alive,
1054 Some(num_ops),
1055 );
1056
1057 self.reprint_traffic_file_breakdown();
1058 println!("\nResults saved to: {}", self.output.display());
1059
1060 Ok(())
1061 }
1062
1063 async fn execute_multi_target(&self, targets_file: &Path) -> Result<()> {
1065 TerminalReporter::print_progress("Parsing targets file...");
1066 let targets = parse_targets_file(targets_file)?;
1067 let num_targets = targets.len();
1068 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
1069
1070 if targets.is_empty() {
1071 return Err(BenchError::Other("No targets found in file".to_string()));
1072 }
1073
1074 let max_concurrency = self.max_concurrency.unwrap_or(10) as usize;
1076 let max_concurrency = max_concurrency.min(num_targets); TerminalReporter::print_header(
1080 &self.get_spec_display_name(),
1081 &format!("{} targets", num_targets),
1082 0,
1083 &self.scenario,
1084 Self::parse_duration(&self.duration)?,
1085 );
1086
1087 let executor = ParallelExecutor::new(
1089 BenchCommand {
1090 spec: self.spec.clone(),
1092 spec_dir: self.spec_dir.clone(),
1093 merge_conflicts: self.merge_conflicts.clone(),
1094 spec_mode: self.spec_mode.clone(),
1095 dependency_config: self.dependency_config.clone(),
1096 target: self.target.clone(), base_path: self.base_path.clone(),
1098 duration: self.duration.clone(),
1099 vus: self.vus,
1100 target_rps: self.target_rps,
1101 no_keep_alive: self.no_keep_alive,
1102 scenario: self.scenario.clone(),
1103 operations: self.operations.clone(),
1104 exclude_operations: self.exclude_operations.clone(),
1105 auth: self.auth.clone(),
1106 headers: self.headers.clone(),
1107 output: self.output.clone(),
1108 generate_only: self.generate_only,
1109 script_output: self.script_output.clone(),
1110 threshold_percentile: self.threshold_percentile.clone(),
1111 threshold_ms: self.threshold_ms,
1112 max_error_rate: self.max_error_rate,
1113 abort_on_error: self.abort_on_error,
1114 abort_on_error_rate: self.abort_on_error_rate,
1115 verbose: self.verbose,
1116 skip_tls_verify: self.skip_tls_verify,
1117 chunked_request_bodies: self.chunked_request_bodies,
1118 targets_file: None,
1119 max_concurrency: None,
1120 results_format: self.results_format.clone(),
1121 params_file: self.params_file.clone(),
1122 crud_flow: self.crud_flow,
1123 flow_config: self.flow_config.clone(),
1124 extract_fields: self.extract_fields.clone(),
1125 parallel_create: self.parallel_create,
1126 data_file: self.data_file.clone(),
1127 data_distribution: self.data_distribution.clone(),
1128 data_mappings: self.data_mappings.clone(),
1129 per_uri_control: self.per_uri_control,
1130 error_rate: self.error_rate,
1131 error_types: self.error_types.clone(),
1132 security_test: self.security_test,
1133 security_payloads: self.security_payloads.clone(),
1134 security_categories: self.security_categories.clone(),
1135 security_target_fields: self.security_target_fields.clone(),
1136 wafbench_dir: self.wafbench_dir.clone(),
1137 wafbench_cycle_all: self.wafbench_cycle_all,
1138 wafbench_verbatim: self.wafbench_verbatim,
1139 owasp_api_top10: self.owasp_api_top10,
1140 owasp_categories: self.owasp_categories.clone(),
1141 owasp_auth_header: self.owasp_auth_header.clone(),
1142 owasp_auth_token: self.owasp_auth_token.clone(),
1143 owasp_admin_paths: self.owasp_admin_paths.clone(),
1144 owasp_id_fields: self.owasp_id_fields.clone(),
1145 owasp_report: self.owasp_report.clone(),
1146 owasp_report_format: self.owasp_report_format.clone(),
1147 owasp_iterations: self.owasp_iterations,
1148 conformance: false,
1149 conformance_api_key: self.conformance_api_key.clone(),
1165 conformance_basic_auth: self.conformance_basic_auth.clone(),
1166 conformance_report: PathBuf::from("conformance-report.json"),
1167 conformance_categories: None,
1168 conformance_report_format: "json".to_string(),
1169 conformance_headers: self.conformance_headers.clone(),
1173 conformance_all_operations: false,
1174 conformance_custom: None,
1175 conformance_delay_ms: 0,
1176 use_k6: false,
1177 conformance_custom_filter: None,
1178 export_requests: false,
1179 validate_requests: false,
1180 conformance_self_test: false,
1181 conformance_self_test_capture: false,
1182 conformance_self_test_iterations: 1,
1183 conformance_self_test_duration: None,
1184 validate_response_schemas: false,
1185 source_ips: self.source_ips.clone(),
1190 geo_source_ips: self.geo_source_ips.clone(),
1191 geo_source_headers: self.geo_source_headers.clone(),
1192 report_missed_cap: None,
1193 discard_response_bodies: self.discard_response_bodies,
1197 dns_policy: self.dns_policy.clone(),
1200 },
1201 targets,
1202 max_concurrency,
1203 );
1204
1205 let start_time = std::time::Instant::now();
1207 let aggregated_results = executor.execute_all().await?;
1208 let elapsed = start_time.elapsed();
1209
1210 self.report_multi_target_results(&aggregated_results, elapsed)?;
1212
1213 Ok(())
1214 }
1215
1216 fn report_multi_target_results(
1218 &self,
1219 results: &AggregatedResults,
1220 elapsed: std::time::Duration,
1221 ) -> Result<()> {
1222 TerminalReporter::print_multi_target_summary(results);
1224
1225 let total_secs = elapsed.as_secs();
1227 let hours = total_secs / 3600;
1228 let minutes = (total_secs % 3600) / 60;
1229 let seconds = total_secs % 60;
1230 if hours > 0 {
1231 println!("\n Total Elapsed Time: {}h {}m {}s", hours, minutes, seconds);
1232 } else if minutes > 0 {
1233 println!("\n Total Elapsed Time: {}m {}s", minutes, seconds);
1234 } else {
1235 println!("\n Total Elapsed Time: {}s", seconds);
1236 }
1237
1238 if self.results_format == "aggregated" || self.results_format == "both" {
1240 let summary_path = self.output.join("aggregated_summary.json");
1241 let summary_json = serde_json::json!({
1242 "total_elapsed_seconds": elapsed.as_secs(),
1243 "total_targets": results.total_targets,
1244 "successful_targets": results.successful_targets,
1245 "failed_targets": results.failed_targets,
1246 "aggregated_metrics": {
1247 "total_requests": results.aggregated_metrics.total_requests,
1248 "total_failed_requests": results.aggregated_metrics.total_failed_requests,
1249 "avg_duration_ms": results.aggregated_metrics.avg_duration_ms,
1250 "p95_duration_ms": results.aggregated_metrics.p95_duration_ms,
1251 "p99_duration_ms": results.aggregated_metrics.p99_duration_ms,
1252 "error_rate": results.aggregated_metrics.error_rate,
1253 "total_rps": results.aggregated_metrics.total_rps,
1254 "avg_rps": results.aggregated_metrics.avg_rps,
1255 "total_vus_max": results.aggregated_metrics.total_vus_max,
1256 },
1257 "target_results": results.target_results.iter().map(|r| {
1258 serde_json::json!({
1259 "target_url": r.target_url,
1260 "target_index": r.target_index,
1261 "success": r.success,
1262 "error": r.error,
1263 "total_requests": r.results.total_requests,
1264 "failed_requests": r.results.failed_requests,
1265 "avg_duration_ms": r.results.avg_duration_ms,
1266 "min_duration_ms": r.results.min_duration_ms,
1267 "med_duration_ms": r.results.med_duration_ms,
1268 "p90_duration_ms": r.results.p90_duration_ms,
1269 "p95_duration_ms": r.results.p95_duration_ms,
1270 "p99_duration_ms": r.results.p99_duration_ms,
1271 "max_duration_ms": r.results.max_duration_ms,
1272 "rps": r.results.rps,
1273 "vus_max": r.results.vus_max,
1274 "output_dir": r.output_dir.to_string_lossy(),
1275 })
1276 }).collect::<Vec<_>>(),
1277 });
1278
1279 std::fs::write(&summary_path, serde_json::to_string_pretty(&summary_json)?)?;
1280 TerminalReporter::print_success(&format!(
1281 "Aggregated summary saved to: {}",
1282 summary_path.display()
1283 ));
1284 }
1285
1286 let csv_path = self.output.join("all_targets.csv");
1288 let mut csv = String::from(
1289 "target_url,success,requests,failed,rps,vus,min_ms,avg_ms,med_ms,p90_ms,p95_ms,p99_ms,max_ms,error\n",
1290 );
1291 for r in &results.target_results {
1292 csv.push_str(&format!(
1293 "{},{},{},{},{:.1},{},{:.1},{:.1},{:.1},{:.1},{:.1},{:.1},{:.1},{}\n",
1294 r.target_url,
1295 r.success,
1296 r.results.total_requests,
1297 r.results.failed_requests,
1298 r.results.rps,
1299 r.results.vus_max,
1300 r.results.min_duration_ms,
1301 r.results.avg_duration_ms,
1302 r.results.med_duration_ms,
1303 r.results.p90_duration_ms,
1304 r.results.p95_duration_ms,
1305 r.results.p99_duration_ms,
1306 r.results.max_duration_ms,
1307 r.error.as_deref().unwrap_or(""),
1308 ));
1309 }
1310 let _ = std::fs::write(&csv_path, &csv);
1311
1312 self.reprint_traffic_file_breakdown();
1313 println!("\nResults saved to: {}", self.output.display());
1314 println!(" - Per-target results: {}", self.output.join("target_*").display());
1315 println!(" - All targets CSV: {}", csv_path.display());
1316 if self.results_format == "aggregated" || self.results_format == "both" {
1317 println!(
1318 " - Aggregated summary: {}",
1319 self.output.join("aggregated_summary.json").display()
1320 );
1321 }
1322
1323 Ok(())
1324 }
1325
1326 pub fn parse_duration(duration: &str) -> Result<u64> {
1328 let duration = duration.trim();
1329
1330 if let Some(secs) = duration.strip_suffix('s') {
1331 secs.parse::<u64>()
1332 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1333 } else if let Some(mins) = duration.strip_suffix('m') {
1334 mins.parse::<u64>()
1335 .map(|m| m * 60)
1336 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1337 } else if let Some(hours) = duration.strip_suffix('h') {
1338 hours
1339 .parse::<u64>()
1340 .map(|h| h * 3600)
1341 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1342 } else {
1343 duration
1345 .parse::<u64>()
1346 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1347 }
1348 }
1349
1350 pub(crate) fn load_verbatim_templates(
1357 &self,
1358 ) -> Result<Vec<crate::request_gen::RequestTemplate>> {
1359 let Some(pattern) = self.wafbench_dir.as_ref() else {
1360 return Err(BenchError::Other(
1361 "--wafbench-verbatim requires --wafbench-dir pointing at your traffic file(s)"
1362 .to_string(),
1363 ));
1364 };
1365
1366 let mut loader = WafBenchLoader::new();
1367 loader.load_from_pattern(pattern)?;
1368 self.emit_traffic_file_breakdown(loader.stats(), "what to expect in proxy logs");
1369
1370 Ok(crate::wafbench::traffic_cases_to_templates(loader.test_cases()))
1371 }
1372
1373 pub fn parse_headers(&self) -> Result<HashMap<String, String>> {
1375 let mut headers = parse_header_string(&self.headers)?;
1376
1377 let already_has = |hs: &HashMap<String, String>, name: &str| -> bool {
1388 hs.keys().any(|k| k.eq_ignore_ascii_case(name))
1389 };
1390
1391 if !already_has(&headers, "Authorization") {
1392 if let Some(b) = self.conformance_basic_auth.as_ref().filter(|s| !s.is_empty()) {
1393 use base64::Engine as _;
1394 let encoded = base64::engine::general_purpose::STANDARD.encode(b.as_bytes());
1395 headers.insert("Authorization".to_string(), format!("Basic {}", encoded));
1396 }
1397 }
1398
1399 for line in &self.conformance_headers {
1405 let Some((name, value)) = line.split_once(':') else {
1406 continue;
1407 };
1408 let name = name.trim();
1409 let value = value.trim();
1410 if name.is_empty() || already_has(&headers, name) {
1411 continue;
1412 }
1413 headers.insert(name.to_string(), value.to_string());
1414 }
1415
1416 if !self.conformance && self.conformance_api_key.is_some() {
1422 TerminalReporter::print_warning(
1423 "--conformance-api-key only fires under --conformance. For plain bench use --header 'X-API-Key: ...'.",
1424 );
1425 }
1426
1427 Ok(headers)
1428 }
1429
1430 fn parse_extracted_values(output_dir: &Path) -> Result<ExtractedValues> {
1431 let extracted_path = output_dir.join("extracted_values.json");
1432 if !extracted_path.exists() {
1433 return Ok(ExtractedValues::new());
1434 }
1435
1436 let content = std::fs::read_to_string(&extracted_path)
1437 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1438 let parsed: serde_json::Value = serde_json::from_str(&content)
1439 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1440
1441 let mut extracted = ExtractedValues::new();
1442 if let Some(values) = parsed.as_object() {
1443 for (key, value) in values {
1444 extracted.set(key.clone(), value.clone());
1445 }
1446 }
1447
1448 Ok(extracted)
1449 }
1450
1451 fn resolve_base_path(&self, parser: &SpecParser) -> Option<String> {
1460 if let Some(cli_base_path) = &self.base_path {
1462 if cli_base_path.is_empty() {
1463 return None;
1465 }
1466 return Some(cli_base_path.clone());
1467 }
1468
1469 parser.get_base_path()
1471 }
1472
1473 async fn build_mock_config(&self) -> MockIntegrationConfig {
1475 if MockServerDetector::looks_like_mock_server(&self.target) {
1477 if let Ok(info) = MockServerDetector::detect(&self.target).await {
1479 if info.is_mockforge {
1480 TerminalReporter::print_success(&format!(
1481 "Detected MockForge server (version: {})",
1482 info.version.as_deref().unwrap_or("unknown")
1483 ));
1484 return MockIntegrationConfig::mock_server();
1485 }
1486 }
1487 }
1488 MockIntegrationConfig::real_api()
1489 }
1490
1491 fn build_crud_flow_config(&self) -> Option<CrudFlowConfig> {
1493 if !self.crud_flow {
1494 return None;
1495 }
1496
1497 if let Some(config_path) = &self.flow_config {
1499 match CrudFlowConfig::from_file(config_path) {
1500 Ok(config) => return Some(config),
1501 Err(e) => {
1502 TerminalReporter::print_warning(&format!(
1503 "Failed to load flow config: {}. Using auto-detection.",
1504 e
1505 ));
1506 }
1507 }
1508 }
1509
1510 let extract_fields = self
1512 .extract_fields
1513 .as_ref()
1514 .map(|f| f.split(',').map(|s| s.trim().to_string()).collect())
1515 .unwrap_or_else(|| vec!["id".to_string(), "uuid".to_string()]);
1516
1517 Some(CrudFlowConfig {
1518 flows: Vec::new(), default_extract_fields: extract_fields,
1520 })
1521 }
1522
1523 fn build_data_driven_config(&self) -> Option<DataDrivenConfig> {
1525 let data_file = self.data_file.as_ref()?;
1526
1527 let distribution = DataDistribution::from_str(&self.data_distribution)
1528 .unwrap_or(DataDistribution::UniquePerVu);
1529
1530 let mappings = self
1531 .data_mappings
1532 .as_ref()
1533 .map(|m| DataMapping::parse_mappings(m).unwrap_or_default())
1534 .unwrap_or_default();
1535
1536 Some(DataDrivenConfig {
1537 file_path: data_file.to_string_lossy().to_string(),
1538 distribution,
1539 mappings,
1540 csv_has_header: true,
1541 per_uri_control: self.per_uri_control,
1542 per_uri_columns: crate::data_driven::PerUriColumns::default(),
1543 })
1544 }
1545
1546 fn build_invalid_data_config(&self) -> Option<InvalidDataConfig> {
1548 let error_rate = self.error_rate?;
1549
1550 let error_types = self
1551 .error_types
1552 .as_ref()
1553 .map(|types| InvalidDataConfig::parse_error_types(types).unwrap_or_default())
1554 .unwrap_or_default();
1555
1556 Some(InvalidDataConfig {
1557 error_rate,
1558 error_types,
1559 target_fields: Vec::new(),
1560 })
1561 }
1562
1563 fn build_security_config(&self) -> Option<SecurityTestConfig> {
1565 if !self.security_test {
1566 return None;
1567 }
1568
1569 let categories = self
1570 .security_categories
1571 .as_ref()
1572 .map(|cats| SecurityTestConfig::parse_categories(cats).unwrap_or_default())
1573 .unwrap_or_else(|| {
1574 let mut default = HashSet::new();
1575 default.insert(SecurityCategory::SqlInjection);
1576 default.insert(SecurityCategory::Xss);
1577 default
1578 });
1579
1580 let target_fields = self
1581 .security_target_fields
1582 .as_ref()
1583 .map(|fields| fields.split(',').map(|f| f.trim().to_string()).collect())
1584 .unwrap_or_default();
1585
1586 let custom_payloads_file =
1587 self.security_payloads.as_ref().map(|p| p.to_string_lossy().to_string());
1588
1589 Some(SecurityTestConfig {
1590 enabled: true,
1591 categories,
1592 target_fields,
1593 custom_payloads_file,
1594 include_high_risk: false,
1595 })
1596 }
1597
1598 fn build_parallel_config(&self) -> Option<ParallelConfig> {
1600 let count = self.parallel_create?;
1601
1602 Some(ParallelConfig::new(count))
1603 }
1604
1605 fn format_unique_total(unique: usize, rps: Option<u32>) -> String {
1608 match rps {
1609 Some(r) if r > 0 => {
1610 let total = unique.saturating_mul(r as usize);
1611 format!("unique={unique} total={total} ({unique} * {r} RPS)")
1612 }
1613 _ => format!("unique={unique}"),
1614 }
1615 }
1616
1617 fn traffic_bucket_json(
1618 unique: usize,
1619 rps: Option<u32>,
1620 duration_secs: Option<u64>,
1621 ) -> serde_json::Value {
1622 let multiplier = rps.filter(|&r| r > 0).unwrap_or(1) as u64;
1623 let total = (unique as u64).saturating_mul(multiplier);
1624 let expected_over_duration =
1625 duration_secs.map(|d| (unique as u64).saturating_mul(multiplier).saturating_mul(d));
1626 serde_json::json!({
1627 "unique": unique,
1628 "total": total,
1629 "expected_over_duration": expected_over_duration,
1630 })
1631 }
1632
1633 fn emit_traffic_file_breakdown(&self, stats: &crate::wafbench::WafBenchStats, phase: &str) {
1636 if stats.per_file.is_empty() {
1637 return;
1638 }
1639 let rps = self.target_rps.filter(|&r| r > 0);
1640 TerminalReporter::print_success(&format!("Traffic file breakdown ({phase}):"));
1641 for file in &stats.per_file {
1642 let other = if file.other > 0 {
1643 format!(" other={}", file.other)
1644 } else {
1645 String::new()
1646 };
1647 TerminalReporter::print_progress(&format!(
1648 " {}: sent {} attack(expected 403) {} normal(expected 200) {} omitted={}{other}",
1649 file.file,
1650 Self::format_unique_total(file.sent, rps),
1651 Self::format_unique_total(file.attack, rps),
1652 Self::format_unique_total(file.normal, rps),
1653 file.omitted
1654 ));
1655 }
1656 self.write_traffic_breakdown_json(stats);
1657 }
1658
1659 fn write_traffic_breakdown_json(&self, stats: &crate::wafbench::WafBenchStats) {
1661 if stats.per_file.is_empty() {
1662 return;
1663 }
1664 let rps = self.target_rps.filter(|&r| r > 0);
1665 let duration_secs = Self::parse_duration(&self.duration).ok();
1666 let files: Vec<serde_json::Value> = stats
1667 .per_file
1668 .iter()
1669 .map(|file| {
1670 serde_json::json!({
1671 "file": file.file,
1672 "sent": Self::traffic_bucket_json(file.sent, rps, duration_secs),
1673 "attack": Self::traffic_bucket_json(file.attack, rps, duration_secs),
1674 "normal": Self::traffic_bucket_json(file.normal, rps, duration_secs),
1675 "omitted": file.omitted,
1676 "other": file.other,
1677 })
1678 })
1679 .collect();
1680 let payload = serde_json::json!({
1681 "rps": rps,
1682 "duration_secs": duration_secs,
1683 "files": files,
1684 });
1685 if let Some(parent) = self.output.parent() {
1686 let _ = std::fs::create_dir_all(parent);
1687 }
1688 let _ = std::fs::create_dir_all(&self.output);
1689 let path = self.output.join("traffic-breakdown.json");
1690 if let Ok(body) = serde_json::to_string_pretty(&payload) {
1691 if std::fs::write(&path, body).is_ok() {
1692 TerminalReporter::print_progress(&format!(
1693 "Traffic breakdown written to: {}",
1694 path.display()
1695 ));
1696 }
1697 }
1698 }
1699
1700 fn reprint_traffic_file_breakdown(&self) {
1703 let path = self.output.join("traffic-breakdown.json");
1704 let Ok(raw) = std::fs::read_to_string(&path) else {
1705 return;
1706 };
1707 let Ok(v) = serde_json::from_str::<serde_json::Value>(&raw) else {
1708 return;
1709 };
1710 let Some(files) = v.get("files").and_then(|f| f.as_array()) else {
1711 return;
1712 };
1713 if files.is_empty() {
1714 return;
1715 }
1716 TerminalReporter::print_success("Traffic file breakdown (end of run):");
1717 for file in files {
1718 let name = file.get("file").and_then(|x| x.as_str()).unwrap_or("?");
1719 let bucket = |key: &str| -> String {
1720 let unique = file
1721 .get(key)
1722 .and_then(|b| b.get("unique"))
1723 .and_then(|u| u.as_u64())
1724 .unwrap_or(0) as usize;
1725 Self::format_unique_total(unique, self.target_rps.filter(|&r| r > 0))
1726 };
1727 let omitted = file.get("omitted").and_then(|o| o.as_u64()).unwrap_or(0);
1728 let other = file.get("other").and_then(|o| o.as_u64()).unwrap_or(0);
1729 let other = if other > 0 {
1730 format!(" other={other}")
1731 } else {
1732 String::new()
1733 };
1734 TerminalReporter::print_progress(&format!(
1735 " {name}: sent {} attack(expected 403) {} normal(expected 200) {} omitted={omitted}{other}",
1736 bucket("sent"),
1737 bucket("attack"),
1738 bucket("normal"),
1739 ));
1740 }
1741 TerminalReporter::print_progress(&format!(" (also in {})", path.display()));
1742 }
1743
1744 fn load_wafbench_payloads(&self) -> Result<Vec<SecurityPayload>> {
1751 let Some(ref wafbench_dir) = self.wafbench_dir else {
1752 return Ok(Vec::new());
1753 };
1754
1755 let mut loader = WafBenchLoader::new();
1756 loader.load_from_pattern(wafbench_dir)?;
1757
1758 let stats = loader.stats();
1759
1760 if stats.files_processed == 0 {
1761 let mut msg = format!(
1762 "No WAFBench YAML files found matching '{wafbench_dir}'. \
1763 --wafbench-dir is a file, a directory or a glob. A missing \
1764 file is an error, not an empty payload pool."
1765 );
1766 if !stats.parse_errors.is_empty() {
1767 msg.push_str(" Parse errors:");
1768 for error in &stats.parse_errors {
1769 msg.push_str(&format!("\n - {error}"));
1770 }
1771 }
1772 return Err(BenchError::Other(msg));
1773 }
1774
1775 TerminalReporter::print_progress(&format!(
1776 "Loaded {} WAFBench files, {} test cases, {} payloads",
1777 stats.files_processed, stats.test_cases_loaded, stats.payloads_extracted
1778 ));
1779 self.emit_traffic_file_breakdown(stats, "what to expect in proxy logs");
1780
1781 for (category, count) in &stats.by_category {
1783 TerminalReporter::print_progress(&format!(" - {}: {} tests", category, count));
1784 }
1785
1786 for error in &stats.parse_errors {
1788 TerminalReporter::print_warning(&format!(" Parse error: {}", error));
1789 }
1790
1791 Ok(loader.to_security_payloads())
1792 }
1793
1794 pub(crate) fn generate_enhanced_script(&self, base_script: &str) -> Result<String> {
1796 let mut enhanced_script = base_script.to_string();
1797 let mut additional_code = String::new();
1798
1799 if let Some(config) = self.build_data_driven_config() {
1801 TerminalReporter::print_progress("Adding data-driven testing support...");
1802 additional_code.push_str(&DataDrivenGenerator::generate_setup(&config));
1803 additional_code.push('\n');
1804 TerminalReporter::print_success("Data-driven testing enabled");
1805 }
1806
1807 if let Some(config) = self.build_invalid_data_config() {
1809 TerminalReporter::print_progress("Adding invalid data testing support...");
1810 additional_code.push_str(&InvalidDataGenerator::generate_invalidation_logic());
1811 additional_code.push('\n');
1812 additional_code
1813 .push_str(&InvalidDataGenerator::generate_should_invalidate(config.error_rate));
1814 additional_code.push('\n');
1815 additional_code
1816 .push_str(&InvalidDataGenerator::generate_type_selection(&config.error_types));
1817 additional_code.push('\n');
1818 TerminalReporter::print_success(&format!(
1819 "Invalid data testing enabled ({}% error rate)",
1820 (self.error_rate.unwrap_or(0.0) * 100.0) as u32
1821 ));
1822 }
1823
1824 let verbatim = self.wafbench_verbatim;
1831 if verbatim && self.security_test {
1832 TerminalReporter::print_warning(
1833 "--security-test is ignored under --wafbench-verbatim: verbatim mode sends your \
1834 traffic cases exactly as written and will not append attack payloads to them. \
1835 Drop --wafbench-verbatim if you want payload injection.",
1836 );
1837 }
1838 let security_config = if verbatim {
1839 None
1840 } else {
1841 self.build_security_config()
1842 };
1843 let wafbench_payloads = if verbatim {
1844 Vec::new()
1845 } else {
1846 self.load_wafbench_payloads()?
1847 };
1848 let security_requested =
1849 !verbatim && (security_config.is_some() || self.wafbench_dir.is_some());
1850
1851 if security_config.is_some() || !wafbench_payloads.is_empty() {
1852 TerminalReporter::print_progress("Adding security testing support...");
1853
1854 let mut payload_list: Vec<SecurityPayload> = Vec::new();
1856
1857 if let Some(ref config) = security_config {
1858 payload_list.extend(SecurityPayloads::get_payloads(config));
1859 }
1860
1861 if !wafbench_payloads.is_empty() {
1863 TerminalReporter::print_progress(&format!(
1864 "Loading {} WAFBench attack patterns...",
1865 wafbench_payloads.len()
1866 ));
1867 payload_list.extend(wafbench_payloads);
1868 }
1869
1870 let target_fields =
1871 security_config.as_ref().map(|c| c.target_fields.clone()).unwrap_or_default();
1872
1873 additional_code.push_str(&SecurityTestGenerator::generate_payload_selection(
1874 &payload_list,
1875 self.wafbench_cycle_all,
1876 ));
1877 additional_code.push('\n');
1878 additional_code
1879 .push_str(&SecurityTestGenerator::generate_apply_payload(&target_fields));
1880 additional_code.push('\n');
1881 additional_code.push_str(&SecurityTestGenerator::generate_security_checks());
1882 additional_code.push('\n');
1883
1884 let mode = if self.wafbench_cycle_all {
1885 "cycle-all"
1886 } else {
1887 "random"
1888 };
1889 TerminalReporter::print_success(&format!(
1890 "Security testing enabled ({} payloads, {} mode)",
1891 payload_list.len(),
1892 mode
1893 ));
1894 } else if security_requested {
1895 TerminalReporter::print_warning(
1899 "Security testing was requested but no payloads were loaded. \
1900 Ensure --wafbench-dir points to valid CRS YAML files or add --security-test.",
1901 );
1902 additional_code
1903 .push_str(&SecurityTestGenerator::generate_payload_selection(&[], false));
1904 additional_code.push('\n');
1905 additional_code.push_str(&SecurityTestGenerator::generate_apply_payload(&[]));
1906 additional_code.push('\n');
1907 }
1908
1909 if let Some(config) = self.build_parallel_config() {
1911 TerminalReporter::print_progress("Adding parallel execution support...");
1912 additional_code.push_str(&ParallelRequestGenerator::generate_batch_helper(&config));
1913 additional_code.push('\n');
1914 TerminalReporter::print_success(&format!(
1915 "Parallel execution enabled (count: {})",
1916 config.count
1917 ));
1918 }
1919
1920 if !additional_code.is_empty() {
1922 if let Some(import_end) = enhanced_script.find("export const options") {
1924 enhanced_script.insert_str(
1925 import_end,
1926 &format!("\n// === Advanced Testing Features ===\n{}\n", additional_code),
1927 );
1928 }
1929 }
1930
1931 Ok(enhanced_script)
1932 }
1933
1934 async fn execute_sequential_specs(&self) -> Result<()> {
1936 TerminalReporter::print_progress("Sequential spec mode: Loading specs individually...");
1937
1938 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
1940
1941 if !self.spec.is_empty() {
1942 let specs = load_specs_from_files(self.spec.clone())
1943 .await
1944 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
1945 all_specs.extend(specs);
1946 }
1947
1948 if let Some(spec_dir) = &self.spec_dir {
1949 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
1950 BenchError::Other(format!("Failed to load specs from directory: {}", e))
1951 })?;
1952 all_specs.extend(dir_specs);
1953 }
1954
1955 if all_specs.is_empty() {
1956 return Err(BenchError::Other(
1957 "No spec files found for sequential execution".to_string(),
1958 ));
1959 }
1960
1961 TerminalReporter::print_success(&format!("Loaded {} spec(s)", all_specs.len()));
1962
1963 let execution_order = if let Some(config_path) = &self.dependency_config {
1965 TerminalReporter::print_progress("Loading dependency configuration...");
1966 let config = SpecDependencyConfig::from_file(config_path)?;
1967
1968 if !config.disable_auto_detect && config.execution_order.is_empty() {
1969 self.detect_and_sort_specs(&all_specs)?
1971 } else {
1972 config.execution_order.iter().flat_map(|g| g.specs.clone()).collect()
1974 }
1975 } else {
1976 self.detect_and_sort_specs(&all_specs)?
1978 };
1979
1980 TerminalReporter::print_success(&format!(
1981 "Execution order: {}",
1982 execution_order
1983 .iter()
1984 .map(|p| p.file_name().unwrap_or_default().to_string_lossy().to_string())
1985 .collect::<Vec<_>>()
1986 .join(" → ")
1987 ));
1988
1989 let mut extracted_values = ExtractedValues::new();
1991 let total_specs = execution_order.len();
1992
1993 for (index, spec_path) in execution_order.iter().enumerate() {
1994 let spec_name = spec_path.file_name().unwrap_or_default().to_string_lossy().to_string();
1995
1996 TerminalReporter::print_progress(&format!(
1997 "[{}/{}] Executing spec: {}",
1998 index + 1,
1999 total_specs,
2000 spec_name
2001 ));
2002
2003 let spec = all_specs
2005 .iter()
2006 .find(|(p, _)| {
2007 p == spec_path
2008 || p.file_name() == spec_path.file_name()
2009 || p.file_name() == Some(spec_path.as_os_str())
2010 })
2011 .map(|(_, s)| s.clone())
2012 .ok_or_else(|| {
2013 BenchError::Other(format!("Spec not found: {}", spec_path.display()))
2014 })?;
2015
2016 let new_values = self.execute_single_spec(&spec, &spec_name, &extracted_values).await?;
2018
2019 extracted_values.merge(&new_values);
2021
2022 TerminalReporter::print_success(&format!(
2023 "[{}/{}] Completed: {} (extracted {} values)",
2024 index + 1,
2025 total_specs,
2026 spec_name,
2027 new_values.values.len()
2028 ));
2029 }
2030
2031 TerminalReporter::print_success(&format!(
2032 "Sequential execution complete: {} specs executed",
2033 total_specs
2034 ));
2035
2036 Ok(())
2037 }
2038
2039 fn detect_and_sort_specs(&self, specs: &[(PathBuf, OpenApiSpec)]) -> Result<Vec<PathBuf>> {
2041 TerminalReporter::print_progress("Auto-detecting spec dependencies...");
2042
2043 let mut detector = DependencyDetector::new();
2044 let dependencies = detector.detect_dependencies(specs);
2045
2046 if dependencies.is_empty() {
2047 TerminalReporter::print_progress("No dependencies detected, using file order");
2048 return Ok(specs.iter().map(|(p, _)| p.clone()).collect());
2049 }
2050
2051 TerminalReporter::print_progress(&format!(
2052 "Detected {} cross-spec dependencies",
2053 dependencies.len()
2054 ));
2055
2056 for dep in &dependencies {
2057 TerminalReporter::print_progress(&format!(
2058 " {} → {} (via field '{}')",
2059 dep.dependency_spec.file_name().unwrap_or_default().to_string_lossy(),
2060 dep.dependent_spec.file_name().unwrap_or_default().to_string_lossy(),
2061 dep.field_name
2062 ));
2063 }
2064
2065 topological_sort(specs, &dependencies)
2066 }
2067
2068 async fn execute_single_spec(
2070 &self,
2071 spec: &OpenApiSpec,
2072 spec_name: &str,
2073 _external_values: &ExtractedValues,
2074 ) -> Result<ExtractedValues> {
2075 let parser = SpecParser::from_spec(spec.clone());
2076
2077 if self.crud_flow {
2079 self.execute_crud_flow_with_extraction(&parser, spec_name).await
2081 } else {
2082 self.execute_standard_spec(&parser, spec_name).await?;
2084 Ok(ExtractedValues::new())
2085 }
2086 }
2087
2088 async fn execute_crud_flow_with_extraction(
2090 &self,
2091 parser: &SpecParser,
2092 spec_name: &str,
2093 ) -> Result<ExtractedValues> {
2094 let operations = parser.get_operations();
2095 let flows = CrudFlowDetector::detect_flows(&operations);
2096
2097 if flows.is_empty() {
2098 TerminalReporter::print_warning(&format!("No CRUD flows detected in {}", spec_name));
2099 return Ok(ExtractedValues::new());
2100 }
2101
2102 TerminalReporter::print_progress(&format!(
2103 " {} CRUD flow(s) in {}",
2104 flows.len(),
2105 spec_name
2106 ));
2107
2108 let mut handlebars = handlebars::Handlebars::new();
2110 handlebars.register_helper(
2112 "json",
2113 Box::new(
2114 |h: &handlebars::Helper,
2115 _: &handlebars::Handlebars,
2116 _: &handlebars::Context,
2117 _: &mut handlebars::RenderContext,
2118 out: &mut dyn handlebars::Output|
2119 -> handlebars::HelperResult {
2120 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2121 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2122 Ok(())
2123 },
2124 ),
2125 );
2126 let template = include_str!("templates/k6_crud_flow.hbs");
2127 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
2128
2129 let custom_headers = self.parse_headers()?;
2130 let config = self.build_crud_flow_config().unwrap_or_default();
2131
2132 let param_overrides = if let Some(params_file) = &self.params_file {
2134 let overrides = ParameterOverrides::from_file(params_file)?;
2135 Some(overrides)
2136 } else {
2137 None
2138 };
2139
2140 let duration_secs = Self::parse_duration(&self.duration)?;
2142 let scenario =
2143 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2144 let stages = scenario.generate_stages(duration_secs, self.vus);
2145
2146 let api_base_path = self.resolve_base_path(parser);
2148
2149 let mut all_headers = custom_headers.clone();
2151 if let Some(auth) = &self.auth {
2152 all_headers.insert("Authorization".to_string(), auth.clone());
2153 }
2154 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2155
2156 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2158
2159 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2160 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2164 serde_json::json!({
2165 "name": sanitized_name.clone(),
2166 "display_name": f.name,
2167 "base_path": f.base_path,
2168 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2169 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2171 let method_raw = if !parts.is_empty() {
2172 parts[0].to_uppercase()
2173 } else {
2174 "GET".to_string()
2175 };
2176 let method = if !parts.is_empty() {
2177 let m = parts[0].to_lowercase();
2178 if m == "delete" { "del".to_string() } else { m }
2180 } else {
2181 "get".to_string()
2182 };
2183 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2184 let path = if let Some(ref bp) = api_base_path {
2186 format!("{}{}", bp, raw_path)
2187 } else {
2188 raw_path.to_string()
2189 };
2190 let is_get_or_head = method == "get" || method == "head";
2191 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2193
2194 let body_value = if has_body {
2196 param_overrides.as_ref()
2197 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2198 .and_then(|oo| oo.body)
2199 .unwrap_or_else(|| serde_json::json!({}))
2200 } else {
2201 serde_json::json!({})
2202 };
2203
2204 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2206
2207 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2209 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2210
2211 serde_json::json!({
2212 "operation": s.operation,
2213 "method": method,
2214 "path": path,
2215 "extract": s.extract,
2216 "use_values": s.use_values,
2217 "use_body": s.use_body,
2218 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2219 "inject_attacks": s.inject_attacks,
2220 "attack_types": s.attack_types,
2221 "description": s.description,
2222 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2223 "is_get_or_head": is_get_or_head,
2224 "has_body": has_body,
2225 "body": processed_body.value,
2226 "body_is_dynamic": body_is_dynamic,
2227 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2228 })
2229 }).collect::<Vec<_>>(),
2230 })
2231 }).collect();
2232
2233 for flow_data in &flows_data {
2235 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2236 for step in steps {
2237 if let Some(placeholders_arr) =
2238 step.get("_placeholders").and_then(|p| p.as_array())
2239 {
2240 for p_str in placeholders_arr {
2241 if let Some(p_name) = p_str.as_str() {
2242 match p_name {
2243 "VU" => {
2244 all_placeholders.insert(DynamicPlaceholder::VU);
2245 }
2246 "Iteration" => {
2247 all_placeholders.insert(DynamicPlaceholder::Iteration);
2248 }
2249 "Timestamp" => {
2250 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2251 }
2252 "UUID" => {
2253 all_placeholders.insert(DynamicPlaceholder::UUID);
2254 }
2255 "Random" => {
2256 all_placeholders.insert(DynamicPlaceholder::Random);
2257 }
2258 "Counter" => {
2259 all_placeholders.insert(DynamicPlaceholder::Counter);
2260 }
2261 "Date" => {
2262 all_placeholders.insert(DynamicPlaceholder::Date);
2263 }
2264 "VuIter" => {
2265 all_placeholders.insert(DynamicPlaceholder::VuIter);
2266 }
2267 _ => {}
2268 }
2269 }
2270 }
2271 }
2272 }
2273 }
2274 }
2275
2276 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2278 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2279
2280 let security_testing_enabled = self.security_testing_enabled();
2282
2283 let data = serde_json::json!({
2284 "base_url": self.target,
2285 "flows": flows_data,
2286 "extract_fields": config.default_extract_fields,
2287 "duration_secs": duration_secs,
2288 "max_vus": self.vus,
2289 "auth_header": self.auth,
2290 "custom_headers": custom_headers,
2291 "skip_tls_verify": self.skip_tls_verify,
2292 "stages": stages.iter().map(|s| serde_json::json!({
2294 "duration": s.duration,
2295 "target": s.target,
2296 })).collect::<Vec<_>>(),
2297 "threshold_percentile": self.threshold_percentile,
2298 "threshold_ms": self.threshold_ms,
2299 "max_error_rate": self.max_error_rate,
2300 "abort_on_error": self.abort_on_error,
2301 "abort_on_error_rate": self.abort_on_error_rate,
2302 "headers": headers_json,
2303 "dynamic_imports": required_imports,
2304 "dynamic_globals": required_globals,
2305 "extracted_values_output_path": output_dir.join("extracted_values.json").to_string_lossy(),
2306 "security_testing_enabled": security_testing_enabled,
2308 "has_custom_headers": !custom_headers.is_empty(),
2309 });
2310
2311 let mut script = handlebars
2312 .render_template(template, &data)
2313 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2314
2315 if security_testing_enabled {
2317 script = self.generate_enhanced_script(&script)?;
2318 }
2319
2320 let script_path =
2322 self.output.join(format!("k6-{}-crud-flow.js", spec_name.replace('.', "_")));
2323
2324 std::fs::create_dir_all(self.output.clone())?;
2325 std::fs::write(&script_path, &script)?;
2326
2327 if !self.generate_only {
2328 let executor = K6Executor::new()?
2329 .with_local_ips(self.source_ips.join(","))
2330 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
2331 std::fs::create_dir_all(&output_dir)?;
2332
2333 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
2334
2335 let extracted = Self::parse_extracted_values(&output_dir)?;
2336 TerminalReporter::print_progress(&format!(
2337 " Extracted {} value(s) from {}",
2338 extracted.values.len(),
2339 spec_name
2340 ));
2341 return Ok(extracted);
2342 }
2343
2344 Ok(ExtractedValues::new())
2345 }
2346
2347 async fn execute_standard_spec(&self, parser: &SpecParser, spec_name: &str) -> Result<()> {
2349 let mut operations = if let Some(filter) = &self.operations {
2350 parser.filter_operations(filter)?
2351 } else {
2352 parser.get_operations()
2353 };
2354
2355 if let Some(exclude) = &self.exclude_operations {
2356 operations = parser.exclude_operations(operations, exclude)?;
2357 }
2358
2359 if operations.is_empty() {
2360 TerminalReporter::print_warning(&format!("No operations found in {}", spec_name));
2361 return Ok(());
2362 }
2363
2364 TerminalReporter::print_progress(&format!(
2365 " {} operations in {}",
2366 operations.len(),
2367 spec_name
2368 ));
2369
2370 let templates: Vec<_> = operations
2372 .iter()
2373 .map(RequestGenerator::generate_template)
2374 .collect::<Result<Vec<_>>>()?;
2375
2376 let custom_headers = self.parse_headers()?;
2378
2379 let base_path = self.resolve_base_path(parser);
2381
2382 let scenario =
2384 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2385
2386 let security_testing_enabled = self.security_testing_enabled();
2387
2388 let k6_config = K6Config {
2389 target_url: self.target.clone(),
2390 base_path,
2391 scenario,
2392 duration_secs: Self::parse_duration(&self.duration)?,
2393 max_vus: self.vus,
2394 threshold_percentile: self.threshold_percentile.clone(),
2395 threshold_ms: self.threshold_ms,
2396 max_error_rate: self.max_error_rate,
2397 auth_header: self.auth.clone(),
2398 custom_headers,
2399 skip_tls_verify: self.skip_tls_verify,
2400 security_testing_enabled,
2401 chunked_request_bodies: self.chunked_request_bodies,
2402 target_rps: self.target_rps,
2403 no_keep_alive: self.no_keep_alive,
2404 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
2406 .into_iter()
2407 .map(|ip| ip.to_string())
2408 .collect(),
2409 geo_source_headers: if self.geo_source_headers.is_empty()
2410 && !self.geo_source_ips.is_empty()
2411 {
2412 crate::conformance::self_test::default_geo_source_headers()
2413 } else {
2414 self.geo_source_headers.clone()
2415 },
2416 };
2417
2418 let generator = K6ScriptGenerator::new(k6_config, templates)
2419 .with_abort_valve(self.abort_on_error, self.abort_on_error_rate);
2420 let mut script = generator.generate()?;
2421
2422 let has_advanced_features = self.data_file.is_some()
2424 || self.error_rate.is_some()
2425 || self.security_test
2426 || self.parallel_create.is_some()
2427 || self.wafbench_dir.is_some();
2428
2429 if has_advanced_features {
2430 script = self.generate_enhanced_script(&script)?;
2431 }
2432
2433 let script_path = self.output.join(format!("k6-{}.js", spec_name.replace('.', "_")));
2435
2436 std::fs::create_dir_all(self.output.clone())?;
2437 std::fs::write(&script_path, &script)?;
2438
2439 if !self.generate_only {
2440 let executor = K6Executor::new()?
2443 .with_local_ips(self.source_ips.join(","))
2444 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
2445 .with_discard_response_bodies(self.discard_response_bodies);
2446 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
2447 std::fs::create_dir_all(&output_dir)?;
2448
2449 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
2450 }
2451
2452 Ok(())
2453 }
2454
2455 async fn execute_crud_flow(&self, parser: &SpecParser) -> Result<()> {
2457 let config = self.build_crud_flow_config().unwrap_or_default();
2459
2460 let flows = if !config.flows.is_empty() {
2462 TerminalReporter::print_progress("Using custom flow configuration...");
2463 config.flows.clone()
2464 } else {
2465 TerminalReporter::print_progress("Detecting CRUD operations...");
2466 let operations = parser.get_operations();
2467 CrudFlowDetector::detect_flows(&operations)
2468 };
2469
2470 if flows.is_empty() {
2471 return Err(BenchError::Other(
2472 "No CRUD flows detected in spec. Ensure spec has POST/GET/PUT/DELETE operations on related paths.".to_string(),
2473 ));
2474 }
2475
2476 if config.flows.is_empty() {
2477 TerminalReporter::print_success(&format!("Detected {} CRUD flow(s)", flows.len()));
2478 } else {
2479 TerminalReporter::print_success(&format!("Loaded {} custom flow(s)", flows.len()));
2480 }
2481
2482 for flow in &flows {
2483 TerminalReporter::print_progress(&format!(
2484 " - {}: {} steps",
2485 flow.name,
2486 flow.steps.len()
2487 ));
2488 }
2489
2490 let mut handlebars = handlebars::Handlebars::new();
2492 handlebars.register_helper(
2494 "json",
2495 Box::new(
2496 |h: &handlebars::Helper,
2497 _: &handlebars::Handlebars,
2498 _: &handlebars::Context,
2499 _: &mut handlebars::RenderContext,
2500 out: &mut dyn handlebars::Output|
2501 -> handlebars::HelperResult {
2502 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2503 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2504 Ok(())
2505 },
2506 ),
2507 );
2508 let template = include_str!("templates/k6_crud_flow.hbs");
2509
2510 let custom_headers = self.parse_headers()?;
2511
2512 let param_overrides = if let Some(params_file) = &self.params_file {
2514 TerminalReporter::print_progress("Loading parameter overrides...");
2515 let overrides = ParameterOverrides::from_file(params_file)?;
2516 TerminalReporter::print_success(&format!(
2517 "Loaded parameter overrides ({} operation-specific, {} defaults)",
2518 overrides.operations.len(),
2519 if overrides.defaults.is_empty() { 0 } else { 1 }
2520 ));
2521 Some(overrides)
2522 } else {
2523 None
2524 };
2525
2526 let duration_secs = Self::parse_duration(&self.duration)?;
2528 let scenario =
2529 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2530 let stages = scenario.generate_stages(duration_secs, self.vus);
2531
2532 let api_base_path = self.resolve_base_path(parser);
2534 if let Some(ref bp) = api_base_path {
2535 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
2536 }
2537
2538 let mut all_headers = custom_headers.clone();
2540 if let Some(auth) = &self.auth {
2541 all_headers.insert("Authorization".to_string(), auth.clone());
2542 }
2543 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2544
2545 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2547
2548 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2549 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2554 serde_json::json!({
2555 "name": sanitized_name.clone(), "display_name": f.name, "base_path": f.base_path,
2558 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2559 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2561 let method_raw = if !parts.is_empty() {
2562 parts[0].to_uppercase()
2563 } else {
2564 "GET".to_string()
2565 };
2566 let method = if !parts.is_empty() {
2567 let m = parts[0].to_lowercase();
2568 if m == "delete" { "del".to_string() } else { m }
2570 } else {
2571 "get".to_string()
2572 };
2573 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2574 let path = if let Some(ref bp) = api_base_path {
2576 format!("{}{}", bp, raw_path)
2577 } else {
2578 raw_path.to_string()
2579 };
2580 let is_get_or_head = method == "get" || method == "head";
2581 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2583
2584 let body_value = if has_body {
2586 param_overrides.as_ref()
2587 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2588 .and_then(|oo| oo.body)
2589 .unwrap_or_else(|| serde_json::json!({}))
2590 } else {
2591 serde_json::json!({})
2592 };
2593
2594 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2596 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2601 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2602
2603 serde_json::json!({
2604 "operation": s.operation,
2605 "method": method,
2606 "path": path,
2607 "extract": s.extract,
2608 "use_values": s.use_values,
2609 "use_body": s.use_body,
2610 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2611 "inject_attacks": s.inject_attacks,
2612 "attack_types": s.attack_types,
2613 "description": s.description,
2614 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2615 "is_get_or_head": is_get_or_head,
2616 "has_body": has_body,
2617 "body": processed_body.value,
2618 "body_is_dynamic": body_is_dynamic,
2619 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2620 })
2621 }).collect::<Vec<_>>(),
2622 })
2623 }).collect();
2624
2625 for flow_data in &flows_data {
2627 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2628 for step in steps {
2629 if let Some(placeholders_arr) =
2630 step.get("_placeholders").and_then(|p| p.as_array())
2631 {
2632 for p_str in placeholders_arr {
2633 if let Some(p_name) = p_str.as_str() {
2634 match p_name {
2636 "VU" => {
2637 all_placeholders.insert(DynamicPlaceholder::VU);
2638 }
2639 "Iteration" => {
2640 all_placeholders.insert(DynamicPlaceholder::Iteration);
2641 }
2642 "Timestamp" => {
2643 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2644 }
2645 "UUID" => {
2646 all_placeholders.insert(DynamicPlaceholder::UUID);
2647 }
2648 "Random" => {
2649 all_placeholders.insert(DynamicPlaceholder::Random);
2650 }
2651 "Counter" => {
2652 all_placeholders.insert(DynamicPlaceholder::Counter);
2653 }
2654 "Date" => {
2655 all_placeholders.insert(DynamicPlaceholder::Date);
2656 }
2657 "VuIter" => {
2658 all_placeholders.insert(DynamicPlaceholder::VuIter);
2659 }
2660 _ => {}
2661 }
2662 }
2663 }
2664 }
2665 }
2666 }
2667 }
2668
2669 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2671 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2672
2673 let invalid_data_config = self.build_invalid_data_config();
2675 let error_injection_enabled = invalid_data_config.is_some();
2676 let error_rate = self.error_rate.unwrap_or(0.0);
2677 let error_types: Vec<String> = invalid_data_config
2678 .as_ref()
2679 .map(|c| c.error_types.iter().map(|t| format!("{:?}", t)).collect())
2680 .unwrap_or_default();
2681
2682 if error_injection_enabled {
2683 TerminalReporter::print_progress(&format!(
2684 "Error injection enabled ({}% rate)",
2685 (error_rate * 100.0) as u32
2686 ));
2687 }
2688
2689 let security_testing_enabled = self.security_testing_enabled();
2691
2692 let data = serde_json::json!({
2693 "base_url": self.target,
2694 "flows": flows_data,
2695 "extract_fields": config.default_extract_fields,
2696 "duration_secs": duration_secs,
2697 "max_vus": self.vus,
2698 "auth_header": self.auth,
2699 "custom_headers": custom_headers,
2700 "skip_tls_verify": self.skip_tls_verify,
2701 "stages": stages.iter().map(|s| serde_json::json!({
2703 "duration": s.duration,
2704 "target": s.target,
2705 })).collect::<Vec<_>>(),
2706 "threshold_percentile": self.threshold_percentile,
2707 "threshold_ms": self.threshold_ms,
2708 "max_error_rate": self.max_error_rate,
2709 "abort_on_error": self.abort_on_error,
2710 "abort_on_error_rate": self.abort_on_error_rate,
2711 "headers": headers_json,
2712 "dynamic_imports": required_imports,
2713 "dynamic_globals": required_globals,
2714 "extracted_values_output_path": self
2715 .output
2716 .join("crud_flow_extracted_values.json")
2717 .to_string_lossy(),
2718 "error_injection_enabled": error_injection_enabled,
2720 "error_rate": error_rate,
2721 "error_types": error_types,
2722 "security_testing_enabled": security_testing_enabled,
2724 "has_custom_headers": !custom_headers.is_empty(),
2725 });
2726
2727 let mut script = handlebars
2728 .render_template(template, &data)
2729 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2730
2731 if security_testing_enabled {
2733 script = self.generate_enhanced_script(&script)?;
2734 }
2735
2736 TerminalReporter::print_progress("Validating CRUD flow script...");
2738 let validation_errors = K6ScriptGenerator::validate_script(&script);
2739 if !validation_errors.is_empty() {
2740 TerminalReporter::print_error("CRUD flow script validation failed");
2741 for error in &validation_errors {
2742 eprintln!(" {}", error);
2743 }
2744 return Err(BenchError::Other(format!(
2745 "CRUD flow script validation failed with {} error(s)",
2746 validation_errors.len()
2747 )));
2748 }
2749
2750 TerminalReporter::print_success("CRUD flow script generated");
2751
2752 let script_path = if let Some(output) = &self.script_output {
2754 output.clone()
2755 } else {
2756 self.output.join("k6-crud-flow-script.js")
2757 };
2758
2759 if let Some(parent) = script_path.parent() {
2760 std::fs::create_dir_all(parent)?;
2761 }
2762 std::fs::write(&script_path, &script)?;
2763 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
2764
2765 if self.generate_only {
2766 println!("\nScript generated successfully. Run it with:");
2767 println!(" k6 run {}", script_path.display());
2768 return Ok(());
2769 }
2770
2771 TerminalReporter::print_progress("Executing CRUD flow test...");
2773 let executor = K6Executor::new()?
2774 .with_local_ips(self.source_ips.join(","))
2775 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
2776 std::fs::create_dir_all(&self.output)?;
2777
2778 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
2779
2780 let duration_secs = Self::parse_duration(&self.duration)?;
2781 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
2782
2783 Ok(())
2784 }
2785
2786 async fn execute_conformance_test(&self) -> Result<()> {
2788 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
2789 use crate::conformance::report::ConformanceReport;
2790 use crate::conformance::spec::ConformanceFeature;
2791
2792 TerminalReporter::print_progress("OpenAPI 3.0.0 Conformance Testing Mode");
2793
2794 TerminalReporter::print_progress(CONFORMANCE_REPLACES_LOAD_ADVISORY);
2795
2796 let categories = self.conformance_categories.as_ref().map(|cats_str| {
2798 cats_str
2799 .split(',')
2800 .filter_map(|s| {
2801 let trimmed = s.trim();
2802 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
2803 Some(canonical.to_string())
2804 } else {
2805 TerminalReporter::print_warning(&format!(
2806 "Unknown conformance category: '{}'. Valid categories: {}",
2807 trimmed,
2808 ConformanceFeature::cli_category_names()
2809 .iter()
2810 .map(|(cli, _)| *cli)
2811 .collect::<Vec<_>>()
2812 .join(", ")
2813 ));
2814 None
2815 }
2816 })
2817 .collect::<Vec<String>>()
2818 });
2819
2820 let custom_headers: Vec<(String, String)> = self
2822 .conformance_headers
2823 .iter()
2824 .filter_map(|h| {
2825 let (name, value) = h.split_once(':')?;
2826 Some((name.trim().to_string(), value.trim().to_string()))
2827 })
2828 .collect();
2829
2830 if !custom_headers.is_empty() {
2831 TerminalReporter::print_progress(&format!(
2832 "Using {} custom header(s) for authentication",
2833 custom_headers.len()
2834 ));
2835 }
2836
2837 if self.conformance_delay_ms > 0 {
2838 TerminalReporter::print_progress(&format!(
2839 "Using {}ms delay between conformance requests",
2840 self.conformance_delay_ms
2841 ));
2842 }
2843
2844 std::fs::create_dir_all(&self.output)?;
2846
2847 let config = ConformanceConfig {
2848 target_url: self.target.clone(),
2849 api_key: self.conformance_api_key.clone(),
2850 basic_auth: self.conformance_basic_auth.clone(),
2851 skip_tls_verify: self.skip_tls_verify,
2852 categories,
2853 base_path: self.base_path.clone(),
2854 custom_headers,
2855 output_dir: Some(self.output.clone()),
2856 all_operations: self.conformance_all_operations,
2857 custom_checks_file: self.conformance_custom.clone(),
2858 request_delay_ms: self.conformance_delay_ms,
2859 custom_filter: self.conformance_custom_filter.clone(),
2860 export_requests: self.export_requests,
2861 validate_requests: self.validate_requests,
2862 };
2863
2864 let mut resolved_base_path: Option<String> = None;
2872 let annotated_ops = if !self.spec.is_empty() {
2873 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
2874 let parser = SpecParser::from_file(&self.spec[0]).await?;
2875 resolved_base_path = self.resolve_base_path(&parser);
2876
2877 let mut operations = if let Some(filter) = &self.operations {
2882 parser.filter_operations(filter)?
2883 } else {
2884 parser.get_operations()
2885 };
2886 if let Some(exclude) = &self.exclude_operations {
2887 let before_count = operations.len();
2888 operations = parser.exclude_operations(operations, exclude)?;
2889 let excluded_count = before_count - operations.len();
2890 if excluded_count > 0 {
2891 TerminalReporter::print_progress(&format!(
2892 "Excluded {} operations matching '{}'",
2893 excluded_count, exclude
2894 ));
2895 }
2896 }
2897
2898 let annotated =
2899 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
2900 &operations,
2901 parser.spec(),
2902 );
2903 TerminalReporter::print_success(&format!(
2904 "Analyzed {} operations, found {} feature annotations",
2905 operations.len(),
2906 annotated.iter().map(|a| a.features.len()).sum::<usize>()
2907 ));
2908 Some(annotated)
2909 } else {
2910 None
2911 };
2912
2913 if self.conformance_self_test {
2920 let Some(ops) = annotated_ops else {
2921 TerminalReporter::print_error(
2922 "--conformance-self-test requires --spec; no operations to test",
2923 );
2924 return Ok(());
2925 };
2926 let cfg = crate::conformance::self_test::SelfTestConfig {
2927 target_url: self.target.clone(),
2928 skip_tls_verify: self.skip_tls_verify,
2929 timeout: std::time::Duration::from_secs(30),
2930 extra_headers: self
2934 .conformance_headers
2935 .iter()
2936 .filter_map(|h| {
2937 let (n, v) = h.split_once(':')?;
2938 Some((n.trim().to_string(), v.trim().to_string()))
2939 })
2940 .collect(),
2941 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
2942 base_path: resolved_base_path.clone(),
2946 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
2950 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
2951 geo_source_headers: if self.geo_source_headers.is_empty() {
2952 crate::conformance::self_test::default_geo_source_headers()
2953 } else {
2954 self.geo_source_headers.clone()
2955 },
2956 capture: if self.conformance_self_test_capture
2960 || self.validate_response_schemas
2961 || self.validate_requests
2962 {
2963 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
2974 } else {
2975 None
2976 },
2977 validate_response_schemas: self.validate_response_schemas,
2978 spec_label: self.spec.first().map(|p| {
2984 p.file_name()
2985 .map(|s| s.to_string_lossy().into_owned())
2986 .unwrap_or_else(|| p.to_string_lossy().into_owned())
2987 }),
2988 network_events: Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))),
2995 current_iteration: 1,
2996 };
2997 let capture_sink = cfg.capture.clone();
2998 let network_events_sink = cfg.network_events.clone();
2999 TerminalReporter::print_progress(&format!(
3000 "Self-test mode: driving {} operations with positive + per-category negative cases",
3001 ops.len()
3002 ));
3003 let target_iterations = self.conformance_self_test_iterations.max(1);
3010 let duration_budget = self
3011 .conformance_self_test_duration
3012 .as_ref()
3013 .map(|s| Self::parse_duration(s))
3014 .transpose()?
3015 .map(std::time::Duration::from_secs);
3016 let start = std::time::Instant::now();
3017 let deadline = duration_budget.map(|d| start + d);
3026 let mut cfg = cfg;
3030 cfg.current_iteration = 1;
3031 let mut report =
3032 crate::conformance::self_test::run_self_test_with_deadline(&ops, &cfg, deadline)
3033 .await
3034 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3035 let mut iter_done: u32 = 1;
3036 loop {
3037 let by_iter = iter_done >= target_iterations;
3038 let by_dur = duration_budget.map(|d| start.elapsed() >= d).unwrap_or(true);
3039 if by_iter && by_dur {
3040 break;
3041 }
3042 cfg.current_iteration = iter_done.saturating_add(1);
3043 let next = crate::conformance::self_test::run_self_test_with_deadline(
3044 &ops, &cfg, deadline,
3045 )
3046 .await
3047 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3048 report.merge_iteration(next);
3049 iter_done = iter_done.saturating_add(1);
3050 }
3051 if iter_done > 1 {
3052 TerminalReporter::print_progress(&format!(
3053 "Self-test repeated {} iteration(s) ({:.1?} elapsed)",
3054 iter_done,
3055 start.elapsed(),
3056 ));
3057 }
3058 let per_endpoint_summary: Vec<
3068 crate::conformance::per_endpoint_summary::PerEndpointSummary,
3069 >;
3070 if let Some(sink) = capture_sink {
3071 if let Ok(guard) = sink.lock() {
3072 let jsonl_path = self.output.join("conformance-self-test-requests.jsonl");
3073 let mut lines = String::with_capacity(guard.len() * 256);
3074 for entry in guard.iter() {
3075 if let Ok(line) = serde_json::to_string(entry) {
3076 lines.push_str(&line);
3077 lines.push('\n');
3078 }
3079 }
3080 let _ = std::fs::write(&jsonl_path, lines);
3081 let html_path = self.output.join("conformance-self-test-requests.html");
3082 let html =
3083 crate::conformance::capture_html::render_capture_html(guard.as_slice());
3084 let _ = std::fs::write(&html_path, html);
3085
3086 per_endpoint_summary =
3090 crate::conformance::per_endpoint_summary::build_summary(guard.as_slice());
3091 let summary_path = self.output.join("conformance-per-endpoint.json");
3092 if let Ok(json) = serde_json::to_string_pretty(&per_endpoint_summary) {
3093 let _ = std::fs::write(&summary_path, json);
3094 TerminalReporter::print_progress(&format!(
3095 "Self-test request/response capture written to {} ({} entries) + {} + {}",
3096 jsonl_path.display(),
3097 guard.len(),
3098 html_path.display(),
3099 summary_path.display(),
3100 ));
3101 } else {
3102 TerminalReporter::print_progress(&format!(
3103 "Self-test request/response capture written to {} ({} entries) + {}",
3104 jsonl_path.display(),
3105 guard.len(),
3106 html_path.display(),
3107 ));
3108 }
3109 } else {
3110 per_endpoint_summary = Vec::new();
3111 }
3112 } else {
3113 per_endpoint_summary = Vec::new();
3114 }
3115 TerminalReporter::print_progress(&report.render_summary());
3116 if let Some(sink) = network_events_sink {
3123 if let Ok(guard) = sink.lock() {
3124 let path = self.output.join("conformance-network-events.json");
3125 if let Ok(json) = serde_json::to_string_pretty(&*guard) {
3126 let _ = std::fs::write(&path, json);
3127 if guard.is_empty() {
3128 TerminalReporter::print_progress(
3129 "No wire-level network failures during self-test (file written empty)",
3130 );
3131 } else {
3132 TerminalReporter::print_warning(&format!(
3133 "Recorded {} wire-level network event(s) to {}",
3134 guard.len(),
3135 path.display()
3136 ));
3137 }
3138 }
3139 }
3140 }
3141 let json_path = self.output.join("conformance-self-test.json");
3145 if let Ok(json) = serde_json::to_string_pretty(&report) {
3146 let _ = std::fs::write(&json_path, json);
3147 TerminalReporter::print_progress(&format!(
3148 "Self-test report written to {}",
3149 json_path.display()
3150 ));
3151 }
3152 let issues = report.definite_issues();
3156 let issues_path = self.output.join("conformance-definite-issues.json");
3157 if let Ok(json) = serde_json::to_string_pretty(&issues) {
3158 if std::fs::write(&issues_path, json).is_ok() && !issues.is_empty() {
3159 TerminalReporter::print_warning(&format!(
3160 "{} definite issue(s) — see {}",
3161 issues.len(),
3162 issues_path.display()
3163 ));
3164 }
3165 }
3166 let owasp_accepted = report.owasp_accepted_probes();
3169 if !owasp_accepted.is_empty() {
3170 let owasp_path = self.output.join("conformance-owasp-accepted.json");
3171 if let Ok(json) = serde_json::to_string_pretty(&owasp_accepted) {
3172 if std::fs::write(&owasp_path, json).is_ok() {
3173 TerminalReporter::print_warning(&format!(
3174 "{} owasp injection probe(s) accepted by the target — see {}",
3175 owasp_accepted.len(),
3176 owasp_path.display()
3177 ));
3178 }
3179 }
3180 }
3181 if let Some(status) = report.detect_target_misconfiguration() {
3190 let hint = match status {
3191 404 => " Likely cause: spec paths don't match deployed routes — check --base-path and the spec's `servers` block.",
3192 401 | 403 => " Likely cause: authentication header is missing or invalid — check --conformance-header.",
3193 _ => "",
3194 };
3195 TerminalReporter::print_warning(&format!(
3196 "Self-test misconfiguration: every positive case returned {status}.{hint} Negative results below are meaningless under this condition."
3197 ));
3198 } else if !report.all_passed() {
3199 TerminalReporter::print_warning(
3200 "Self-test detected gaps — server let through at least one request that should have been a 4xx",
3201 );
3202 } else {
3203 TerminalReporter::print_success(
3204 "Self-test passed — all positive cases accepted and all negative cases rejected",
3205 );
3206 }
3207 let html_path = self.output.join("conformance-report.html");
3214 let audit_path = self.output.join("conformance-spec-audit.json");
3215 let audit_value = std::fs::read_to_string(&audit_path)
3216 .ok()
3217 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok());
3218 let render_opts = crate::conformance::report_html::RenderOptions {
3223 missed_cap: match self.report_missed_cap {
3224 Some(0) => None,
3225 Some(n) => Some(n as usize),
3226 None => Some(200),
3227 },
3228 };
3229 let mut html = crate::conformance::report_html::render_html_with_options(
3230 &report,
3231 audit_value.as_ref(),
3232 &render_opts,
3233 );
3234 let summary_section = crate::conformance::per_endpoint_summary::render_html_section(
3240 &per_endpoint_summary,
3241 );
3242 if !summary_section.is_empty() {
3243 if let Some(idx) = html.rfind("</body>") {
3244 html.insert_str(idx, &summary_section);
3245 } else {
3246 html.push_str(&summary_section);
3247 }
3248 }
3249 if std::fs::write(&html_path, html).is_ok() {
3250 TerminalReporter::print_progress(&format!(
3251 "HTML report written to {}",
3252 html_path.display()
3253 ));
3254 }
3255
3256 if self.validate_requests && !self.spec.is_empty() {
3268 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3269 &self.spec,
3270 &self.output,
3271 self.base_path.as_deref(),
3272 )
3273 .await?;
3274 if n > 0 {
3275 TerminalReporter::print_warning(&format!(
3276 "{} emitted request(s) recorded against the spec — see conformance-request-violations.json",
3277 n
3278 ));
3279 }
3280 }
3281 return Ok(());
3282 }
3283
3284 if self.validate_requests && !self.spec.is_empty() {
3286 TerminalReporter::print_progress("Validating requests against OpenAPI spec...");
3287 let violation_count = crate::conformance::request_validator::run_request_validation(
3288 &self.spec,
3289 self.conformance_custom.as_deref(),
3290 self.base_path.as_deref(),
3291 &self.output,
3292 )
3293 .await?;
3294 if violation_count > 0 {
3295 TerminalReporter::print_warning(&format!(
3296 "{} request validation violation(s) found — see conformance-request-violations.json",
3297 violation_count
3298 ));
3299 } else {
3300 TerminalReporter::print_success("All requests conform to the OpenAPI spec");
3301 }
3302 }
3303
3304 if self.generate_only || self.use_k6 {
3306 let script = if let Some(annotated) = &annotated_ops {
3307 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
3308 config,
3309 annotated.clone(),
3310 );
3311 let op_count = gen.operation_count();
3312 let (script, check_count) = gen.generate()?;
3313 TerminalReporter::print_success(&format!(
3314 "Conformance: {} operations analyzed, {} unique checks generated",
3315 op_count, check_count
3316 ));
3317 script
3318 } else {
3319 let generator = ConformanceGenerator::new(config);
3320 generator.generate()?
3321 };
3322
3323 let script_path = self.output.join("k6-conformance.js");
3324 std::fs::write(&script_path, &script).map_err(|e| {
3325 BenchError::Other(format!("Failed to write conformance script: {}", e))
3326 })?;
3327 TerminalReporter::print_success(&format!(
3328 "Conformance script generated: {}",
3329 script_path.display()
3330 ));
3331
3332 if self.generate_only {
3333 println!("\nScript generated. Run with:");
3334 println!(" k6 run {}", script_path.display());
3335 return Ok(());
3336 }
3337
3338 if !K6Executor::is_k6_installed() {
3340 TerminalReporter::print_error("k6 is not installed");
3341 TerminalReporter::print_warning(
3342 "Install k6 from: https://k6.io/docs/get-started/installation/",
3343 );
3344 return Err(BenchError::K6NotFound);
3345 }
3346
3347 K6Executor::warn_if_pre_v1().await;
3348 TerminalReporter::print_progress("Running conformance tests via k6...");
3349 let executor = K6Executor::new()?
3350 .with_local_ips(self.source_ips.join(","))
3351 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
3352 executor.execute(&script_path, Some(&self.output), self.verbose).await?;
3353
3354 let report_path = self.output.join("conformance-report.json");
3355 if report_path.exists() {
3356 let report = ConformanceReport::from_file(&report_path)?;
3357 report.print_report_with_options(self.conformance_all_operations);
3358 self.save_conformance_report(&report, &report_path)?;
3359 } else {
3360 TerminalReporter::print_warning(
3361 "Conformance report not generated (k6 handleSummary may not have run)",
3362 );
3363 }
3364
3365 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
3377 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3378 &self.spec,
3379 &self.output,
3380 self.base_path.as_deref(),
3381 )
3382 .await?;
3383 if n > 0 {
3384 TerminalReporter::print_warning(&format!(
3385 "{} emitted request(s) violated the spec — see conformance-request-violations.json",
3386 n
3387 ));
3388 }
3389 }
3390
3391 return Ok(());
3392 }
3393
3394 TerminalReporter::print_progress("Running conformance tests (native executor)...");
3396
3397 let mut executor = crate::conformance::executor::NativeConformanceExecutor::new(config)?;
3398
3399 let custom_only = annotated_ops.is_none() && self.conformance_custom.is_some();
3409 executor = if let Some(annotated) = &annotated_ops {
3410 executor.with_spec_driven_checks(annotated)
3411 } else if custom_only {
3412 executor
3413 } else {
3414 executor.with_reference_checks()
3415 };
3416 executor = executor.with_custom_checks()?;
3417
3418 TerminalReporter::print_success(&format!(
3419 "Executing {} conformance checks...",
3420 executor.check_count()
3421 ));
3422
3423 let report = executor.execute().await?;
3424 report.print_report_with_options(self.conformance_all_operations);
3425
3426 let failure_details = report.failure_details();
3428 if !failure_details.is_empty() {
3429 let details_path = self.output.join("conformance-failure-details.json");
3430 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
3431 let _ = std::fs::write(&details_path, json);
3432 TerminalReporter::print_success(&format!(
3433 "Failure details saved to: {}",
3434 details_path.display()
3435 ));
3436 }
3437 }
3438
3439 let report_path = self.output.join("conformance-report.json");
3441 let report_json = serde_json::to_string_pretty(&report.to_json())
3442 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
3443 std::fs::write(&report_path, &report_json)
3444 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
3445 TerminalReporter::print_success(&format!("Report saved to: {}", report_path.display()));
3446
3447 self.save_conformance_report(&report, &report_path)?;
3448
3449 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
3460 let n =
3461 crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3462 &self.spec,
3463 &self.output,
3464 self.base_path.as_deref(),
3465 )
3466 .await?;
3467 if n > 0 {
3468 TerminalReporter::print_warning(&format!(
3469 "{} emitted request(s) violated the spec — see conformance-request-violations.json",
3470 n
3471 ));
3472 }
3473 }
3474
3475 Ok(())
3476 }
3477
3478 fn save_conformance_report(
3480 &self,
3481 report: &crate::conformance::report::ConformanceReport,
3482 report_path: &Path,
3483 ) -> Result<()> {
3484 if self.conformance_report_format == "sarif" {
3485 use crate::conformance::sarif::ConformanceSarifReport;
3486 ConformanceSarifReport::write(report, &self.target, &self.conformance_report)?;
3487 TerminalReporter::print_success(&format!(
3488 "SARIF report saved to: {}",
3489 self.conformance_report.display()
3490 ));
3491 } else if self.conformance_report != *report_path {
3492 std::fs::copy(report_path, &self.conformance_report)?;
3493 TerminalReporter::print_success(&format!(
3494 "Report saved to: {}",
3495 self.conformance_report.display()
3496 ));
3497 }
3498 Ok(())
3499 }
3500
3501 async fn execute_multi_target_self_test(&self, targets_file: &Path) -> Result<()> {
3513 use crate::conformance::self_test::SelfTestConfig;
3514
3515 TerminalReporter::print_progress("Multi-target conformance self-test mode");
3516 let targets = parse_targets_file(targets_file)?;
3517 if targets.is_empty() {
3518 return Err(BenchError::Other("No targets found in file".to_string()));
3519 }
3520 TerminalReporter::print_success(&format!("Loaded {} target(s)", targets.len()));
3521
3522 let annotated_ops = if !self.spec.is_empty() {
3524 let parser = SpecParser::from_file(&self.spec[0]).await?;
3525 let operations = parser.get_operations();
3526 Some(
3527 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
3528 &operations,
3529 parser.spec(),
3530 ),
3531 )
3532 } else {
3533 return Err(BenchError::Other("--conformance-self-test requires --spec".to_string()));
3534 };
3535 let Some(ops) = annotated_ops else {
3536 unreachable!()
3537 };
3538
3539 std::fs::create_dir_all(&self.output)?;
3540 let resolved_base_path = self.base_path.clone();
3541 let target_iterations = self.conformance_self_test_iterations.max(1);
3542 let duration_budget = self
3543 .conformance_self_test_duration
3544 .as_ref()
3545 .map(|s| Self::parse_duration(s))
3546 .transpose()?
3547 .map(std::time::Duration::from_secs);
3548
3549 for (idx, target) in targets.iter().enumerate() {
3550 let target_dir = self.output.join(format!("target_{}", idx));
3551 std::fs::create_dir_all(&target_dir)?;
3552 TerminalReporter::print_progress(&format!(
3553 "[target {}/{}] {}",
3554 idx + 1,
3555 targets.len(),
3556 target.url
3557 ));
3558
3559 let merged_headers: Vec<(String, String)> = self
3560 .conformance_headers
3561 .iter()
3562 .filter_map(|h| {
3563 let (n, v) = h.split_once(':')?;
3564 Some((n.trim().to_string(), v.trim().to_string()))
3565 })
3566 .collect();
3567
3568 let cfg = SelfTestConfig {
3569 target_url: target.url.clone(),
3570 skip_tls_verify: self.skip_tls_verify,
3571 timeout: std::time::Duration::from_secs(30),
3572 extra_headers: merged_headers,
3573 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
3574 base_path: resolved_base_path.clone(),
3575 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
3576 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
3577 geo_source_headers: if self.geo_source_headers.is_empty() {
3578 crate::conformance::self_test::default_geo_source_headers()
3579 } else {
3580 self.geo_source_headers.clone()
3581 },
3582 capture: if self.conformance_self_test_capture
3583 || self.validate_response_schemas
3584 || self.validate_requests
3585 {
3586 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
3590 } else {
3591 None
3592 },
3593 validate_response_schemas: self.validate_response_schemas,
3594 spec_label: self.spec.first().map(|p| {
3595 p.file_name()
3596 .map(|s| s.to_string_lossy().into_owned())
3597 .unwrap_or_else(|| p.to_string_lossy().into_owned())
3598 }),
3599 network_events: Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))),
3600 current_iteration: 1,
3601 };
3602 let capture_sink = cfg.capture.clone();
3603 let network_events_sink = cfg.network_events.clone();
3604
3605 let start = std::time::Instant::now();
3606 let deadline = duration_budget.map(|d| start + d);
3610 let mut cfg = cfg;
3614 cfg.current_iteration = 1;
3615 let mut report =
3616 crate::conformance::self_test::run_self_test_with_deadline(&ops, &cfg, deadline)
3617 .await
3618 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3619 let mut iter_done: u32 = 1;
3620 loop {
3621 let by_iter = iter_done >= target_iterations;
3622 let by_dur = duration_budget.map(|d| start.elapsed() >= d).unwrap_or(true);
3623 if by_iter && by_dur {
3624 break;
3625 }
3626 cfg.current_iteration = iter_done.saturating_add(1);
3627 let next = crate::conformance::self_test::run_self_test_with_deadline(
3628 &ops, &cfg, deadline,
3629 )
3630 .await
3631 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3632 report.merge_iteration(next);
3633 iter_done = iter_done.saturating_add(1);
3634 }
3635 if iter_done > 1 {
3636 TerminalReporter::print_progress(&format!(
3637 " ran {} iteration(s) in {:.1?}",
3638 iter_done,
3639 start.elapsed(),
3640 ));
3641 }
3642
3643 if let Some(sink) = capture_sink {
3645 if let Ok(guard) = sink.lock() {
3646 let jsonl = target_dir.join("conformance-self-test-requests.jsonl");
3647 let mut lines = String::with_capacity(guard.len() * 256);
3648 for entry in guard.iter() {
3649 if let Ok(line) = serde_json::to_string(entry) {
3650 lines.push_str(&line);
3651 lines.push('\n');
3652 }
3653 }
3654 let _ = std::fs::write(&jsonl, lines);
3655 }
3656 }
3657 if let Some(sink) = network_events_sink {
3658 if let Ok(guard) = sink.lock() {
3659 let path = target_dir.join("conformance-network-events.json");
3660 if let Ok(json) = serde_json::to_string_pretty(&*guard) {
3661 let _ = std::fs::write(&path, json);
3662 if !guard.is_empty() {
3663 TerminalReporter::print_warning(&format!(
3664 " recorded {} wire-level network event(s)",
3665 guard.len()
3666 ));
3667 }
3668 }
3669 }
3670 }
3671
3672 let json_path = target_dir.join("conformance-self-test.json");
3673 if let Ok(json) = serde_json::to_string_pretty(&report) {
3674 let _ = std::fs::write(&json_path, json);
3675 }
3676 let issues = report.definite_issues();
3679 if let Ok(json) = serde_json::to_string_pretty(&issues) {
3680 let issues_path = target_dir.join("conformance-definite-issues.json");
3681 if std::fs::write(&issues_path, json).is_ok() && !issues.is_empty() {
3682 TerminalReporter::print_warning(&format!(
3683 " {} definite issue(s) — see {}",
3684 issues.len(),
3685 issues_path.display()
3686 ));
3687 }
3688 }
3689 let owasp_accepted = report.owasp_accepted_probes();
3691 if !owasp_accepted.is_empty() {
3692 if let Ok(json) = serde_json::to_string_pretty(&owasp_accepted) {
3693 let owasp_path = target_dir.join("conformance-owasp-accepted.json");
3694 if std::fs::write(&owasp_path, json).is_ok() {
3695 TerminalReporter::print_warning(&format!(
3696 " {} owasp injection probe(s) accepted by the target — see {}",
3697 owasp_accepted.len(),
3698 owasp_path.display()
3699 ));
3700 }
3701 }
3702 }
3703 TerminalReporter::print_progress(&report.render_summary());
3704
3705 if self.validate_requests {
3714 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3715 &self.spec,
3716 &target_dir,
3717 self.base_path.as_deref(),
3718 )
3719 .await?;
3720 if n > 0 {
3721 TerminalReporter::print_warning(&format!(
3722 " {} emitted request(s) violated the spec — see {}/conformance-request-violations.json",
3723 n,
3724 target_dir.display(),
3725 ));
3726 }
3727 }
3728 }
3729
3730 Ok(())
3731 }
3732
3733 async fn execute_multi_target_conformance(&self, targets_file: &Path) -> Result<()> {
3739 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
3740 use crate::conformance::report::ConformanceReport;
3741 use crate::conformance::spec::ConformanceFeature;
3742
3743 TerminalReporter::print_progress("Multi-target OpenAPI 3.0.0 Conformance Testing Mode");
3744
3745 TerminalReporter::print_progress("Parsing targets file...");
3747 let targets = parse_targets_file(targets_file)?;
3748 let num_targets = targets.len();
3749 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
3750
3751 if targets.is_empty() {
3752 return Err(BenchError::Other("No targets found in file".to_string()));
3753 }
3754
3755 TerminalReporter::print_progress(CONFORMANCE_REPLACES_LOAD_ADVISORY);
3756
3757 let categories = self.conformance_categories.as_ref().map(|cats_str| {
3759 cats_str
3760 .split(',')
3761 .filter_map(|s| {
3762 let trimmed = s.trim();
3763 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
3764 Some(canonical.to_string())
3765 } else {
3766 TerminalReporter::print_warning(&format!(
3767 "Unknown conformance category: '{}'. Valid categories: {}",
3768 trimmed,
3769 ConformanceFeature::cli_category_names()
3770 .iter()
3771 .map(|(cli, _)| *cli)
3772 .collect::<Vec<_>>()
3773 .join(", ")
3774 ));
3775 None
3776 }
3777 })
3778 .collect::<Vec<String>>()
3779 });
3780
3781 let base_custom_headers: Vec<(String, String)> = self
3783 .conformance_headers
3784 .iter()
3785 .filter_map(|h| {
3786 let (name, value) = h.split_once(':')?;
3787 Some((name.trim().to_string(), value.trim().to_string()))
3788 })
3789 .collect();
3790
3791 if !base_custom_headers.is_empty() {
3792 TerminalReporter::print_progress(&format!(
3793 "Using {} base custom header(s) for authentication",
3794 base_custom_headers.len()
3795 ));
3796 }
3797
3798 let annotated_ops = if !self.spec.is_empty() {
3800 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
3801 let parser = SpecParser::from_file(&self.spec[0]).await?;
3802 let operations = parser.get_operations();
3803 let annotated =
3804 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
3805 &operations,
3806 parser.spec(),
3807 );
3808 TerminalReporter::print_success(&format!(
3809 "Analyzed {} operations, found {} feature annotations",
3810 operations.len(),
3811 annotated.iter().map(|a| a.features.len()).sum::<usize>()
3812 ));
3813 Some(annotated)
3814 } else {
3815 None
3816 };
3817
3818 std::fs::create_dir_all(&self.output)?;
3820
3821 struct TargetResult {
3823 url: String,
3824 passed: usize,
3825 failed: usize,
3826 elapsed: std::time::Duration,
3827 report_json: serde_json::Value,
3828 owasp_coverage: Vec<crate::conformance::report::OwaspCoverageEntry>,
3829 }
3830
3831 let mut target_results: Vec<TargetResult> = Vec::with_capacity(num_targets);
3832 let total_start = std::time::Instant::now();
3833
3834 for (idx, target) in targets.iter().enumerate() {
3835 tracing::info!(
3836 "Running conformance tests against target {}/{}: {}",
3837 idx + 1,
3838 num_targets,
3839 target.url
3840 );
3841 TerminalReporter::print_progress(&format!(
3842 "\n--- Target {}/{}: {} ---",
3843 idx + 1,
3844 num_targets,
3845 target.url
3846 ));
3847
3848 let mut merged_headers = base_custom_headers.clone();
3850 if let Some(ref target_headers) = target.headers {
3851 for (name, value) in target_headers {
3852 if let Some(existing) = merged_headers.iter_mut().find(|(n, _)| n == name) {
3854 existing.1 = value.clone();
3855 } else {
3856 merged_headers.push((name.clone(), value.clone()));
3857 }
3858 }
3859 }
3860 if let Some(ref auth) = target.auth {
3862 if let Some(existing) =
3863 merged_headers.iter_mut().find(|(n, _)| n.eq_ignore_ascii_case("Authorization"))
3864 {
3865 existing.1 = auth.clone();
3866 } else {
3867 merged_headers.push(("Authorization".to_string(), auth.clone()));
3868 }
3869 }
3870
3871 let target_dir = self.output.join(format!("target_{}", idx));
3877 std::fs::create_dir_all(&target_dir)?;
3878
3879 let config = ConformanceConfig {
3880 target_url: target.url.clone(),
3881 api_key: self.conformance_api_key.clone(),
3882 basic_auth: self.conformance_basic_auth.clone(),
3883 skip_tls_verify: self.skip_tls_verify,
3884 categories: categories.clone(),
3885 base_path: self.base_path.clone(),
3886 custom_headers: merged_headers,
3887 output_dir: Some(target_dir.clone()),
3888 all_operations: self.conformance_all_operations,
3889 custom_checks_file: self.conformance_custom.clone(),
3890 request_delay_ms: self.conformance_delay_ms,
3891 custom_filter: self.conformance_custom_filter.clone(),
3892 export_requests: self.export_requests,
3893 validate_requests: self.validate_requests,
3894 };
3895
3896 let target_start = std::time::Instant::now();
3897 let report = if self.use_k6 {
3898 if !K6Executor::is_k6_installed() {
3899 TerminalReporter::print_error("k6 is not installed");
3900 TerminalReporter::print_warning(
3901 "Install k6 from: https://k6.io/docs/get-started/installation/",
3902 );
3903 return Err(BenchError::K6NotFound);
3904 }
3905 K6Executor::warn_if_pre_v1().await;
3906
3907 let script = if let Some(ref annotated) = annotated_ops {
3908 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
3909 config.clone(),
3910 annotated.clone(),
3911 );
3912 let (script, _check_count) = gen.generate()?;
3913 script
3914 } else {
3915 let generator = ConformanceGenerator::new(config.clone());
3916 generator.generate()?
3917 };
3918
3919 let script_path = target_dir.join("k6-conformance.js");
3920 std::fs::write(&script_path, &script).map_err(|e| {
3921 BenchError::Other(format!("Failed to write conformance script: {}", e))
3922 })?;
3923 TerminalReporter::print_success(&format!(
3924 "Conformance script generated: {}",
3925 script_path.display()
3926 ));
3927
3928 TerminalReporter::print_progress(&format!(
3929 "Running conformance tests via k6 against {}...",
3930 target.url
3931 ));
3932 let k6 = K6Executor::new()?
3933 .with_local_ips(self.source_ips.join(","))
3934 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
3935 let api_port = 6565u16.saturating_add(idx as u16);
3937 k6.execute_with_port(&script_path, Some(&target_dir), self.verbose, Some(api_port))
3938 .await?;
3939
3940 let report_path = target_dir.join("conformance-report.json");
3941 if report_path.exists() {
3942 ConformanceReport::from_file(&report_path)?
3943 } else {
3944 TerminalReporter::print_warning(&format!(
3945 "Conformance report not generated for target {} (k6 handleSummary may not have run)",
3946 target.url
3947 ));
3948 continue;
3949 }
3950 } else {
3951 let mut executor =
3952 crate::conformance::executor::NativeConformanceExecutor::new(config)?;
3953
3954 let custom_only = annotated_ops.is_none() && self.conformance_custom.is_some();
3957 executor = if let Some(ref annotated) = annotated_ops {
3958 executor.with_spec_driven_checks(annotated)
3959 } else if custom_only {
3960 executor
3961 } else {
3962 executor.with_reference_checks()
3963 };
3964 executor = executor.with_custom_checks()?;
3965
3966 TerminalReporter::print_success(&format!(
3967 "Executing {} conformance checks against {}...",
3968 executor.check_count(),
3969 target.url
3970 ));
3971
3972 executor.execute().await?
3973 };
3974 let target_elapsed = target_start.elapsed();
3975
3976 let report_json = report.to_json();
3977
3978 let passed = report_json["summary"]["passed"].as_u64().unwrap_or(0) as usize;
3980 let failed = report_json["summary"]["failed"].as_u64().unwrap_or(0) as usize;
3981 let total_checks = passed + failed;
3982 let rate = if total_checks == 0 {
3983 0.0
3984 } else {
3985 (passed as f64 / total_checks as f64) * 100.0
3986 };
3987
3988 TerminalReporter::print_success(&format!(
3989 "Target {}: {}/{} passed ({:.1}%) in {:.1}s",
3990 target.url,
3991 passed,
3992 total_checks,
3993 rate,
3994 target_elapsed.as_secs_f64()
3995 ));
3996
3997 let target_report_path = target_dir.join("conformance-report.json");
3999 let report_str = serde_json::to_string_pretty(&report_json)
4000 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
4001 std::fs::write(&target_report_path, &report_str)
4002 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
4003
4004 let failure_details = report.failure_details();
4006 if !failure_details.is_empty() {
4007 let details_path = target_dir.join("conformance-failure-details.json");
4008 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
4009 let _ = std::fs::write(&details_path, json);
4010 }
4011 }
4012
4013 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
4020 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
4021 &self.spec,
4022 &target_dir,
4023 self.base_path.as_deref(),
4024 )
4025 .await?;
4026 if n > 0 {
4027 TerminalReporter::print_warning(&format!(
4028 "Target {}: {} emitted request(s) violated the spec — see {}/conformance-request-violations.json",
4029 target.url,
4030 n,
4031 target_dir.display()
4032 ));
4033 }
4034 }
4035
4036 let owasp_coverage = report.owasp_coverage_data();
4038
4039 target_results.push(TargetResult {
4040 url: target.url.clone(),
4041 passed,
4042 failed,
4043 elapsed: target_elapsed,
4044 report_json,
4045 owasp_coverage,
4046 });
4047 }
4048
4049 let total_elapsed = total_start.elapsed();
4050
4051 println!("\n{}", "=".repeat(80));
4053 println!(" Multi-Target Conformance Summary");
4054 println!("{}", "=".repeat(80));
4055 println!(
4056 " {:<40} {:>8} {:>8} {:>8} {:>8}",
4057 "Target URL", "Passed", "Failed", "Rate", "Time"
4058 );
4059 println!(" {}", "-".repeat(76));
4060
4061 let mut total_passed = 0usize;
4062 let mut total_failed = 0usize;
4063
4064 for result in &target_results {
4065 let total_checks = result.passed + result.failed;
4066 let rate = if total_checks == 0 {
4067 0.0
4068 } else {
4069 (result.passed as f64 / total_checks as f64) * 100.0
4070 };
4071
4072 let display_url = if result.url.len() > 38 {
4074 format!("{}...", &result.url[..35])
4075 } else {
4076 result.url.clone()
4077 };
4078
4079 println!(
4080 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
4081 display_url,
4082 result.passed,
4083 result.failed,
4084 rate,
4085 result.elapsed.as_secs_f64()
4086 );
4087
4088 total_passed += result.passed;
4089 total_failed += result.failed;
4090 }
4091
4092 let grand_total = total_passed + total_failed;
4093 let overall_rate = if grand_total == 0 {
4094 0.0
4095 } else {
4096 (total_passed as f64 / grand_total as f64) * 100.0
4097 };
4098
4099 println!(" {}", "-".repeat(76));
4100 println!(
4101 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
4102 format!("TOTAL ({} targets)", num_targets),
4103 total_passed,
4104 total_failed,
4105 overall_rate,
4106 total_elapsed.as_secs_f64()
4107 );
4108 println!("{}", "=".repeat(80));
4109
4110 for result in &target_results {
4112 println!("\n OWASP API Security Top 10 Coverage for {}:", result.url);
4113 for entry in &result.owasp_coverage {
4114 let status = if !entry.tested {
4115 "-"
4116 } else if entry.all_passed {
4117 "pass"
4118 } else {
4119 "FAIL"
4120 };
4121 let via = if entry.via_categories.is_empty() {
4122 String::new()
4123 } else {
4124 format!(" (via {})", entry.via_categories.join(", "))
4125 };
4126 println!(" {:<12} {:<40} {}{}", entry.id, entry.name, status, via);
4127 }
4128 }
4129
4130 let per_target_summaries: Vec<serde_json::Value> = target_results
4132 .iter()
4133 .enumerate()
4134 .map(|(idx, r)| {
4135 let total_checks = r.passed + r.failed;
4136 let rate = if total_checks == 0 {
4137 0.0
4138 } else {
4139 (r.passed as f64 / total_checks as f64) * 100.0
4140 };
4141 let owasp_json: Vec<serde_json::Value> = r
4142 .owasp_coverage
4143 .iter()
4144 .map(|e| {
4145 serde_json::json!({
4146 "id": e.id,
4147 "name": e.name,
4148 "tested": e.tested,
4149 "all_passed": e.all_passed,
4150 "via_categories": e.via_categories,
4151 })
4152 })
4153 .collect();
4154 serde_json::json!({
4155 "target_url": r.url,
4156 "target_index": idx,
4157 "checks_passed": r.passed,
4158 "checks_failed": r.failed,
4159 "total_checks": total_checks,
4160 "pass_rate": rate,
4161 "elapsed_seconds": r.elapsed.as_secs_f64(),
4162 "report": r.report_json,
4163 "owasp_coverage": owasp_json,
4164 })
4165 })
4166 .collect();
4167
4168 let combined_summary = serde_json::json!({
4169 "total_targets": num_targets,
4170 "total_checks_passed": total_passed,
4171 "total_checks_failed": total_failed,
4172 "overall_pass_rate": overall_rate,
4173 "total_elapsed_seconds": total_elapsed.as_secs_f64(),
4174 "targets": per_target_summaries,
4175 });
4176
4177 let summary_path = self.output.join("multi-target-conformance-summary.json");
4178 let summary_str = serde_json::to_string_pretty(&combined_summary)
4179 .map_err(|e| BenchError::Other(format!("Failed to serialize summary: {}", e)))?;
4180 std::fs::write(&summary_path, &summary_str)
4181 .map_err(|e| BenchError::Other(format!("Failed to write summary: {}", e)))?;
4182 TerminalReporter::print_success(&format!(
4183 "Combined summary saved to: {}",
4184 summary_path.display()
4185 ));
4186
4187 Ok(())
4188 }
4189
4190 async fn execute_owasp_test(&self, parser: &SpecParser) -> Result<()> {
4192 TerminalReporter::print_progress("OWASP API Security Top 10 Testing Mode");
4193
4194 let custom_headers = self.parse_headers()?;
4196
4197 let mut config = OwaspApiConfig::new()
4199 .with_auth_header(&self.owasp_auth_header)
4200 .with_verbose(self.verbose)
4201 .with_insecure(self.skip_tls_verify)
4202 .with_concurrency(self.vus as usize)
4203 .with_iterations(self.owasp_iterations as usize)
4204 .with_base_path(self.base_path.clone())
4205 .with_custom_headers(custom_headers);
4206
4207 if let Some(ref token) = self.owasp_auth_token {
4209 config = config.with_valid_auth_token(token);
4210 }
4211
4212 if let Some(ref cats_str) = self.owasp_categories {
4214 let categories: Vec<OwaspCategory> = cats_str
4215 .split(',')
4216 .filter_map(|s| {
4217 let trimmed = s.trim();
4218 match trimmed.parse::<OwaspCategory>() {
4219 Ok(cat) => Some(cat),
4220 Err(e) => {
4221 TerminalReporter::print_warning(&e);
4222 None
4223 }
4224 }
4225 })
4226 .collect();
4227
4228 if !categories.is_empty() {
4229 config = config.with_categories(categories);
4230 }
4231 }
4232
4233 if let Some(ref admin_paths_file) = self.owasp_admin_paths {
4235 config.admin_paths_file = Some(admin_paths_file.clone());
4236 if let Err(e) = config.load_admin_paths() {
4237 TerminalReporter::print_warning(&format!("Failed to load admin paths file: {}", e));
4238 }
4239 }
4240
4241 if let Some(ref id_fields_str) = self.owasp_id_fields {
4243 let id_fields: Vec<String> = id_fields_str
4244 .split(',')
4245 .map(|s| s.trim().to_string())
4246 .filter(|s| !s.is_empty())
4247 .collect();
4248 if !id_fields.is_empty() {
4249 config = config.with_id_fields(id_fields);
4250 }
4251 }
4252
4253 if let Some(ref report_path) = self.owasp_report {
4255 config = config.with_report_path(report_path);
4256 }
4257 if let Ok(format) = self.owasp_report_format.parse::<ReportFormat>() {
4258 config = config.with_report_format(format);
4259 }
4260
4261 let categories = config.categories_to_test();
4263 TerminalReporter::print_success(&format!(
4264 "Testing {} OWASP categories: {}",
4265 categories.len(),
4266 categories.iter().map(|c| c.cli_name()).collect::<Vec<_>>().join(", ")
4267 ));
4268
4269 if config.valid_auth_token.is_some() {
4270 TerminalReporter::print_progress("Using provided auth token for baseline requests");
4271 }
4272
4273 TerminalReporter::print_progress("Generating OWASP security test script...");
4275 let generator = OwaspApiGenerator::new(config, self.target.clone(), parser);
4276
4277 let script = generator.generate()?;
4279 TerminalReporter::print_success("OWASP security test script generated");
4280
4281 let script_path = if let Some(output) = &self.script_output {
4283 output.clone()
4284 } else {
4285 self.output.join("k6-owasp-security-test.js")
4286 };
4287
4288 if let Some(parent) = script_path.parent() {
4289 std::fs::create_dir_all(parent)?;
4290 }
4291 std::fs::write(&script_path, &script)?;
4292 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
4293
4294 if self.generate_only {
4296 println!("\nOWASP security test script generated. Run it with:");
4297 println!(" k6 run {}", script_path.display());
4298 return Ok(());
4299 }
4300
4301 TerminalReporter::print_progress("Executing OWASP security tests...");
4303 let executor = K6Executor::new()?
4304 .with_local_ips(self.source_ips.join(","))
4305 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
4306 std::fs::create_dir_all(&self.output)?;
4307
4308 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
4309
4310 let duration_secs = Self::parse_duration(&self.duration)?;
4311 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
4312
4313 println!("\nOWASP security test results saved to: {}", self.output.display());
4314
4315 Ok(())
4316 }
4317}
4318
4319#[cfg(test)]
4320mod tests {
4321 use super::*;
4322 use tempfile::tempdir;
4323
4324 #[test]
4325 fn test_parse_duration() {
4326 assert_eq!(BenchCommand::parse_duration("30s").unwrap(), 30);
4327 assert_eq!(BenchCommand::parse_duration("5m").unwrap(), 300);
4328 assert_eq!(BenchCommand::parse_duration("1h").unwrap(), 3600);
4329 assert_eq!(BenchCommand::parse_duration("60").unwrap(), 60);
4330 }
4331
4332 #[test]
4336 fn parse_ip_list_ipv4_range_inclusive() {
4337 let v = parse_ip_list(&["10.0.0.5-10.0.0.27".into()], "source-ip");
4338 assert_eq!(v.len(), 23);
4339 assert_eq!(v.first().unwrap().to_string(), "10.0.0.5");
4340 assert_eq!(v.last().unwrap().to_string(), "10.0.0.27");
4341 }
4342
4343 #[test]
4346 fn parse_ip_list_range_rejects_backwards() {
4347 let v = parse_ip_list(&["10.0.0.10-10.0.0.5".into()], "source-ip");
4348 assert!(v.is_empty(), "backwards range should produce no IPs; got {v:?}");
4349 }
4350
4351 #[test]
4355 fn parse_ip_list_rejects_ipv6_range_syntax() {
4356 let v = parse_ip_list(&["2001:db8::1-2001:db8::5".into()], "geo-source-ip");
4357 assert!(v.is_empty(), "IPv6 range should be rejected; got {v:?}");
4358 }
4359
4360 #[test]
4362 fn parse_ip_list_range_capped_at_256() {
4363 let v = parse_ip_list(&["10.0.0.0-10.0.5.0".into()], "source-ip");
4364 assert_eq!(v.len(), 256);
4365 assert_eq!(v.first().unwrap().to_string(), "10.0.0.0");
4366 }
4367
4368 #[test]
4371 fn parse_ip_list_plain_and_comma() {
4372 let v = parse_ip_list(&["10.0.0.5".into(), "10.0.0.6,10.0.0.7".into()], "source-ip");
4373 assert_eq!(v.len(), 3);
4374 assert_eq!(v[0].to_string(), "10.0.0.5");
4375 assert_eq!(v[2].to_string(), "10.0.0.7");
4376 }
4377
4378 #[test]
4381 fn parse_ip_list_ipv4_cidr_29_expands_to_8() {
4382 let v = parse_ip_list(&["10.0.0.0/29".into()], "source-ip");
4383 assert_eq!(v.len(), 8);
4384 assert_eq!(v[0].to_string(), "10.0.0.0");
4385 assert_eq!(v[7].to_string(), "10.0.0.7");
4386 }
4387
4388 #[test]
4391 fn parse_ip_list_ipv4_cidr_8_capped_at_256() {
4392 let v = parse_ip_list(&["10.0.0.0/8".into()], "source-ip");
4393 assert_eq!(v.len(), 256);
4394 assert_eq!(v[0].to_string(), "10.0.0.0");
4395 assert_eq!(v[255].to_string(), "10.0.0.255");
4396 }
4397
4398 #[test]
4400 fn parse_ip_list_ipv6_cidr_126_expands_to_4() {
4401 let v = parse_ip_list(&["2001:db8::/126".into()], "geo-source-ip");
4402 assert_eq!(v.len(), 4);
4403 assert!(v[0].is_ipv6());
4404 assert_eq!(v[0].to_string(), "2001:db8::");
4405 assert_eq!(v[3].to_string(), "2001:db8::3");
4406 }
4407
4408 #[test]
4410 fn parse_ip_list_mixed_v4_v6_cidr() {
4411 let v = parse_ip_list(&["10.0.0.0/30,2001:db8::1,203.0.113.42".into()], "geo-source-ip");
4412 assert_eq!(v.len(), 6); assert!(v.iter().any(|ip| ip.to_string() == "2001:db8::1"));
4414 assert!(v.iter().any(|ip| ip.to_string() == "203.0.113.42"));
4415 }
4416
4417 #[test]
4420 fn parse_ip_list_skips_malformed() {
4421 let v = parse_ip_list(
4422 &[
4423 "10.0.0.5".into(),
4424 "not-an-ip".into(),
4425 "10.0.0.6".into(),
4426 "/24".into(),
4427 "1.2.3.4/200".into(),
4428 ],
4429 "source-ip",
4430 );
4431 assert_eq!(v.len(), 2);
4432 assert_eq!(v[0].to_string(), "10.0.0.5");
4433 assert_eq!(v[1].to_string(), "10.0.0.6");
4434 }
4435
4436 #[test]
4437 fn test_parse_duration_invalid() {
4438 assert!(BenchCommand::parse_duration("invalid").is_err());
4439 assert!(BenchCommand::parse_duration("30x").is_err());
4440 }
4441
4442 #[test]
4443 fn test_parse_headers() {
4444 let cmd = BenchCommand {
4445 spec: vec![PathBuf::from("test.yaml")],
4446 spec_dir: None,
4447 merge_conflicts: "error".to_string(),
4448 spec_mode: "merge".to_string(),
4449 dependency_config: None,
4450 target: "http://localhost".to_string(),
4451 base_path: None,
4452 duration: "1m".to_string(),
4453 vus: 10,
4454 scenario: "ramp-up".to_string(),
4455 operations: None,
4456 exclude_operations: None,
4457 auth: None,
4458 headers: vec![
4459 "X-API-Key:test123".to_string(),
4460 "X-Client-ID:client456".to_string(),
4461 ],
4462 output: PathBuf::from("output"),
4463 generate_only: false,
4464 script_output: None,
4465 threshold_percentile: "p(95)".to_string(),
4466 threshold_ms: 500,
4467 max_error_rate: 0.05,
4468 abort_on_error: true,
4469 abort_on_error_rate: 0.95,
4470 verbose: false,
4471 skip_tls_verify: false,
4472 chunked_request_bodies: false,
4473 target_rps: None,
4474 no_keep_alive: false,
4475 targets_file: None,
4476 max_concurrency: None,
4477 results_format: "both".to_string(),
4478 params_file: None,
4479 crud_flow: false,
4480 flow_config: None,
4481 extract_fields: None,
4482 parallel_create: None,
4483 data_file: None,
4484 data_distribution: "unique-per-vu".to_string(),
4485 data_mappings: None,
4486 per_uri_control: false,
4487 error_rate: None,
4488 error_types: None,
4489 security_test: false,
4490 security_payloads: None,
4491 security_categories: None,
4492 security_target_fields: None,
4493 wafbench_dir: None,
4494 wafbench_cycle_all: false,
4495 wafbench_verbatim: false,
4496 owasp_api_top10: false,
4497 owasp_categories: None,
4498 owasp_auth_header: "Authorization".to_string(),
4499 owasp_auth_token: None,
4500 owasp_admin_paths: None,
4501 owasp_id_fields: None,
4502 owasp_report: None,
4503 owasp_report_format: "json".to_string(),
4504 owasp_iterations: 1,
4505 conformance: false,
4506 conformance_api_key: None,
4507 conformance_basic_auth: None,
4508 conformance_report: PathBuf::from("conformance-report.json"),
4509 conformance_categories: None,
4510 conformance_report_format: "json".to_string(),
4511 conformance_headers: vec![],
4512 conformance_all_operations: false,
4513 conformance_custom: None,
4514 conformance_delay_ms: 0,
4515 use_k6: false,
4516 conformance_custom_filter: None,
4517 export_requests: false,
4518 validate_requests: false,
4519 conformance_self_test: false,
4520 conformance_self_test_capture: false,
4521 conformance_self_test_iterations: 1,
4522 conformance_self_test_duration: None,
4523 validate_response_schemas: false,
4524 source_ips: Vec::new(),
4525 geo_source_ips: Vec::new(),
4526 geo_source_headers: Vec::new(),
4527 report_missed_cap: None,
4528 discard_response_bodies: false,
4529 dns_policy: None,
4530 };
4531
4532 let headers = cmd.parse_headers().unwrap();
4533 assert_eq!(headers.get("X-API-Key"), Some(&"test123".to_string()));
4534 assert_eq!(headers.get("X-Client-ID"), Some(&"client456".to_string()));
4535 }
4536
4537 #[test]
4538 fn test_parse_header_string_preserves_comma_in_value() {
4539 let inputs = vec![
4542 "Cookie:session=abc; expires=Thu, 01 Jan 2099 00:00:00 GMT".to_string(),
4543 "X-Trace:1".to_string(),
4544 ];
4545 let headers = parse_header_string(&inputs).unwrap();
4546 assert_eq!(
4547 headers.get("Cookie"),
4548 Some(&"session=abc; expires=Thu, 01 Jan 2099 00:00:00 GMT".to_string())
4549 );
4550 assert_eq!(headers.get("X-Trace"), Some(&"1".to_string()));
4551 }
4552
4553 #[test]
4561 fn conformance_advisory_names_every_discarded_flag() {
4562 let msg = CONFORMANCE_REPLACES_LOAD_ADVISORY;
4563 for flag in ["--vus", "--rps", "-d"] {
4564 assert!(
4565 msg.contains(flag),
4566 "conformance advisory must name `{flag}` as ignored; it is discarded on that \
4567 path and silently dropping it is how users end up tuning a knob that does \
4568 nothing (#980). Message was: {msg}"
4569 );
4570 }
4571 assert!(
4572 msg.contains("REPLACES"),
4573 "conformance advisory must say the load run is REPLACED, not merely that some \
4574 flags are ignored — `--conformance` returns before the load path runs, so no \
4575 load traffic is generated at all (#980). Message was: {msg}"
4576 );
4577 }
4578
4579 #[test]
4593 fn multi_target_clone_preserves_fields_parse_headers_reads() {
4594 let src = include_str!("command.rs");
4595
4596 let fn_start = src
4597 .find("async fn execute_multi_target(")
4598 .expect("execute_multi_target should exist");
4599 let block_start = src[fn_start..]
4600 .find("ParallelExecutor::new(")
4601 .map(|i| i + fn_start)
4602 .expect("multi-target path should build a ParallelExecutor");
4603 let block_end = src[block_start..]
4605 .find("\n );")
4606 .map(|i| i + block_start)
4607 .expect("ParallelExecutor::new(..) should be closed");
4608 let block = &src[block_start..block_end];
4609
4610 for field in ["conformance_basic_auth", "conformance_headers"] {
4613 for zeroed in [format!("{field}: None"), format!("{field}: vec![]")] {
4614 assert!(
4615 !block.contains(&zeroed),
4616 "execute_multi_target zeroes `{zeroed}`. parse_headers() folds `{field}` \
4617 into the header map, so zeroing it here strips auth from every \
4618 multi-target run while single-target keeps working (#79 round 64)."
4619 );
4620 }
4621 let passthrough = format!("{field}: self.{field}.clone()");
4622 assert!(
4623 block.contains(&passthrough),
4624 "execute_multi_target must carry `{field}` through as `{passthrough}` so \
4625 parse_headers() can fold it (#79 round 64)."
4626 );
4627 }
4628 }
4629
4630 #[test]
4631 fn test_get_spec_display_name() {
4632 let cmd = BenchCommand {
4633 spec: vec![PathBuf::from("test.yaml")],
4634 spec_dir: None,
4635 merge_conflicts: "error".to_string(),
4636 spec_mode: "merge".to_string(),
4637 dependency_config: None,
4638 target: "http://localhost".to_string(),
4639 base_path: None,
4640 duration: "1m".to_string(),
4641 vus: 10,
4642 scenario: "ramp-up".to_string(),
4643 operations: None,
4644 exclude_operations: None,
4645 auth: None,
4646 headers: Vec::new(),
4647 output: PathBuf::from("output"),
4648 generate_only: false,
4649 script_output: None,
4650 threshold_percentile: "p(95)".to_string(),
4651 threshold_ms: 500,
4652 max_error_rate: 0.05,
4653 abort_on_error: true,
4654 abort_on_error_rate: 0.95,
4655 verbose: false,
4656 skip_tls_verify: false,
4657 chunked_request_bodies: false,
4658 target_rps: None,
4659 no_keep_alive: false,
4660 targets_file: None,
4661 max_concurrency: None,
4662 results_format: "both".to_string(),
4663 params_file: None,
4664 crud_flow: false,
4665 flow_config: None,
4666 extract_fields: None,
4667 parallel_create: None,
4668 data_file: None,
4669 data_distribution: "unique-per-vu".to_string(),
4670 data_mappings: None,
4671 per_uri_control: false,
4672 error_rate: None,
4673 error_types: None,
4674 security_test: false,
4675 security_payloads: None,
4676 security_categories: None,
4677 security_target_fields: None,
4678 wafbench_dir: None,
4679 wafbench_cycle_all: false,
4680 wafbench_verbatim: false,
4681 owasp_api_top10: false,
4682 owasp_categories: None,
4683 owasp_auth_header: "Authorization".to_string(),
4684 owasp_auth_token: None,
4685 owasp_admin_paths: None,
4686 owasp_id_fields: None,
4687 owasp_report: None,
4688 owasp_report_format: "json".to_string(),
4689 owasp_iterations: 1,
4690 conformance: false,
4691 conformance_api_key: None,
4692 conformance_basic_auth: None,
4693 conformance_report: PathBuf::from("conformance-report.json"),
4694 conformance_categories: None,
4695 conformance_report_format: "json".to_string(),
4696 conformance_headers: vec![],
4697 conformance_all_operations: false,
4698 conformance_custom: None,
4699 conformance_delay_ms: 0,
4700 use_k6: false,
4701 conformance_custom_filter: None,
4702 export_requests: false,
4703 validate_requests: false,
4704 conformance_self_test: false,
4705 conformance_self_test_capture: false,
4706 conformance_self_test_iterations: 1,
4707 conformance_self_test_duration: None,
4708 validate_response_schemas: false,
4709 source_ips: Vec::new(),
4710 geo_source_ips: Vec::new(),
4711 geo_source_headers: Vec::new(),
4712 report_missed_cap: None,
4713 discard_response_bodies: false,
4714 dns_policy: None,
4715 };
4716
4717 assert_eq!(cmd.get_spec_display_name(), "test.yaml");
4718
4719 let cmd_multi = BenchCommand {
4721 spec: vec![PathBuf::from("a.yaml"), PathBuf::from("b.yaml")],
4722 spec_dir: None,
4723 merge_conflicts: "error".to_string(),
4724 spec_mode: "merge".to_string(),
4725 dependency_config: None,
4726 target: "http://localhost".to_string(),
4727 base_path: None,
4728 duration: "1m".to_string(),
4729 vus: 10,
4730 scenario: "ramp-up".to_string(),
4731 operations: None,
4732 exclude_operations: None,
4733 auth: None,
4734 headers: Vec::new(),
4735 output: PathBuf::from("output"),
4736 generate_only: false,
4737 script_output: None,
4738 threshold_percentile: "p(95)".to_string(),
4739 threshold_ms: 500,
4740 max_error_rate: 0.05,
4741 abort_on_error: true,
4742 abort_on_error_rate: 0.95,
4743 verbose: false,
4744 skip_tls_verify: false,
4745 chunked_request_bodies: false,
4746 target_rps: None,
4747 no_keep_alive: false,
4748 targets_file: None,
4749 max_concurrency: None,
4750 results_format: "both".to_string(),
4751 params_file: None,
4752 crud_flow: false,
4753 flow_config: None,
4754 extract_fields: None,
4755 parallel_create: None,
4756 data_file: None,
4757 data_distribution: "unique-per-vu".to_string(),
4758 data_mappings: None,
4759 per_uri_control: false,
4760 error_rate: None,
4761 error_types: None,
4762 security_test: false,
4763 security_payloads: None,
4764 security_categories: None,
4765 security_target_fields: None,
4766 wafbench_dir: None,
4767 wafbench_cycle_all: false,
4768 wafbench_verbatim: false,
4769 owasp_api_top10: false,
4770 owasp_categories: None,
4771 owasp_auth_header: "Authorization".to_string(),
4772 owasp_auth_token: None,
4773 owasp_admin_paths: None,
4774 owasp_id_fields: None,
4775 owasp_report: None,
4776 owasp_report_format: "json".to_string(),
4777 owasp_iterations: 1,
4778 conformance: false,
4779 conformance_api_key: None,
4780 conformance_basic_auth: None,
4781 conformance_report: PathBuf::from("conformance-report.json"),
4782 conformance_categories: None,
4783 conformance_report_format: "json".to_string(),
4784 conformance_headers: vec![],
4785 conformance_all_operations: false,
4786 conformance_custom: None,
4787 conformance_delay_ms: 0,
4788 use_k6: false,
4789 conformance_custom_filter: None,
4790 export_requests: false,
4791 validate_requests: false,
4792 conformance_self_test: false,
4793 conformance_self_test_capture: false,
4794 conformance_self_test_iterations: 1,
4795 conformance_self_test_duration: None,
4796 validate_response_schemas: false,
4797 source_ips: Vec::new(),
4798 geo_source_ips: Vec::new(),
4799 geo_source_headers: Vec::new(),
4800 report_missed_cap: None,
4801 discard_response_bodies: false,
4802 dns_policy: None,
4803 };
4804
4805 assert_eq!(cmd_multi.get_spec_display_name(), "2 spec files");
4806 }
4807
4808 #[test]
4809 fn test_parse_extracted_values_from_output_dir() {
4810 let dir = tempdir().unwrap();
4811 let path = dir.path().join("extracted_values.json");
4812 std::fs::write(
4813 &path,
4814 r#"{
4815 "pool_id": "abc123",
4816 "count": 0,
4817 "enabled": false,
4818 "metadata": { "owner": "team-a" }
4819}"#,
4820 )
4821 .unwrap();
4822
4823 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
4824 assert_eq!(extracted.get("pool_id"), Some(&serde_json::json!("abc123")));
4825 assert_eq!(extracted.get("count"), Some(&serde_json::json!(0)));
4826 assert_eq!(extracted.get("enabled"), Some(&serde_json::json!(false)));
4827 assert_eq!(extracted.get("metadata"), Some(&serde_json::json!({"owner": "team-a"})));
4828 }
4829
4830 #[test]
4831 fn test_parse_extracted_values_missing_file() {
4832 let dir = tempdir().unwrap();
4833 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
4834 assert!(extracted.values.is_empty());
4835 }
4836
4837 fn sample_bench_command() -> BenchCommand {
4840 BenchCommand {
4841 spec: vec![PathBuf::from("test.yaml")],
4842 spec_dir: None,
4843 merge_conflicts: "error".to_string(),
4844 spec_mode: "merge".to_string(),
4845 dependency_config: None,
4846 target: "http://localhost".to_string(),
4847 base_path: None,
4848 duration: "1m".to_string(),
4849 vus: 10,
4850 scenario: "ramp-up".to_string(),
4851 operations: None,
4852 exclude_operations: None,
4853 auth: None,
4854 headers: vec![
4855 "X-API-Key:test123".to_string(),
4856 "X-Client-ID:client456".to_string(),
4857 ],
4858 output: PathBuf::from("output"),
4859 generate_only: false,
4860 script_output: None,
4861 threshold_percentile: "p(95)".to_string(),
4862 threshold_ms: 500,
4863 max_error_rate: 0.05,
4864 abort_on_error: true,
4865 abort_on_error_rate: 0.95,
4866 verbose: false,
4867 skip_tls_verify: false,
4868 chunked_request_bodies: false,
4869 target_rps: None,
4870 no_keep_alive: false,
4871 targets_file: None,
4872 max_concurrency: None,
4873 results_format: "both".to_string(),
4874 params_file: None,
4875 crud_flow: false,
4876 flow_config: None,
4877 extract_fields: None,
4878 parallel_create: None,
4879 data_file: None,
4880 data_distribution: "unique-per-vu".to_string(),
4881 data_mappings: None,
4882 per_uri_control: false,
4883 error_rate: None,
4884 error_types: None,
4885 security_test: false,
4886 security_payloads: None,
4887 security_categories: None,
4888 security_target_fields: None,
4889 wafbench_dir: None,
4890 wafbench_cycle_all: false,
4891 wafbench_verbatim: false,
4892 owasp_api_top10: false,
4893 owasp_categories: None,
4894 owasp_auth_header: "Authorization".to_string(),
4895 owasp_auth_token: None,
4896 owasp_admin_paths: None,
4897 owasp_id_fields: None,
4898 owasp_report: None,
4899 owasp_report_format: "json".to_string(),
4900 owasp_iterations: 1,
4901 conformance: false,
4902 conformance_api_key: None,
4903 conformance_basic_auth: None,
4904 conformance_report: PathBuf::from("conformance-report.json"),
4905 conformance_categories: None,
4906 conformance_report_format: "json".to_string(),
4907 conformance_headers: vec![],
4908 conformance_all_operations: false,
4909 conformance_custom: None,
4910 conformance_delay_ms: 0,
4911 use_k6: false,
4912 conformance_custom_filter: None,
4913 export_requests: false,
4914 validate_requests: false,
4915 conformance_self_test: false,
4916 conformance_self_test_capture: false,
4917 conformance_self_test_iterations: 1,
4918 conformance_self_test_duration: None,
4919 validate_response_schemas: false,
4920 source_ips: Vec::new(),
4921 geo_source_ips: Vec::new(),
4922 geo_source_headers: Vec::new(),
4923 report_missed_cap: None,
4924 discard_response_bodies: false,
4925 dns_policy: None,
4926 }
4927 }
4928
4929 #[test]
4937 fn verbatim_disables_security_payload_injection() {
4938 let mut cmd = sample_bench_command();
4939 cmd.wafbench_dir = Some("traffic.yaml".to_string());
4940
4941 assert!(
4942 cmd.security_testing_enabled(),
4943 "--wafbench-dir alone must still enable payload injection"
4944 );
4945
4946 cmd.wafbench_verbatim = true;
4947 assert!(
4948 !cmd.security_testing_enabled(),
4949 "verbatim mode must not inject payloads into requests sent as written"
4950 );
4951
4952 cmd.security_test = true;
4955 assert!(
4956 !cmd.security_testing_enabled(),
4957 "--security-test must not re-enable injection under --wafbench-verbatim"
4958 );
4959 }
4960
4961 #[test]
4966 fn security_testing_enabled_has_a_single_definition() {
4967 let src = include_str!("command.rs");
4968 let parallel = include_str!("parallel_executor.rs");
4969 let a = format!("self.{} || self.{}.is_some()", "security_test", "wafbench_dir");
4971 let b = format!("self.{}.is_some() || self.{}", "wafbench_dir", "security_test");
4972 let inline = src.matches(a.as_str()).count() + src.matches(b.as_str()).count();
4973 assert_eq!(
4974 inline, 1,
4975 "expected the security_testing_enabled() method to be the only place this is \
4976 computed, found {inline} inline copies -- collapse them or the render paths drift"
4977 );
4978
4979 let parallel_inline = format!(
4984 "{}.{} || {}.{}.is_some()",
4985 "base_command", "security_test", "self.base_command", "wafbench_dir"
4986 );
4987 assert!(
4988 !parallel.contains(¶llel_inline),
4989 "ParallelExecutor must not recompute the security flag inline"
4990 );
4991 assert!(
4992 parallel.contains("security_testing_enabled()"),
4993 "ParallelExecutor must call security_testing_enabled() so --wafbench-verbatim \
4994 turns injection off on --targets-file runs too"
4995 );
4996 }
4997
4998 #[test]
5002 fn missing_wafbench_dir_is_not_swallowed() {
5003 let src = include_str!("command.rs");
5004 let swallowed = format!("Failed to {} WAFBench tests", "load");
5006 let impl_line = src
5007 .lines()
5008 .filter(|l| !l.trim_start().starts_with("//"))
5009 .any(|l| l.contains(&swallowed));
5010 assert!(!impl_line, "missing --wafbench-dir must not be downgraded to a warning");
5011 assert!(
5012 src.contains("self.load_wafbench_payloads()?"),
5013 "payload load errors must reach generate_enhanced_script"
5014 );
5015 }
5016
5017 #[test]
5022 fn multi_target_path_honors_verbatim_templates() {
5023 let src = include_str!("parallel_executor.rs");
5024 assert!(
5025 src.contains("load_verbatim_templates"),
5026 "ParallelExecutor must load traffic-file requests under --wafbench-verbatim. \
5027 Requiring a spec and generating templates from its operations is how \
5028 --targets-file ignored the flag and fuzzed spec URLs (#79)."
5029 );
5030 }
5031
5032 #[test]
5034 fn traffic_breakdown_json_multiplies_unique_by_rps() {
5035 let dir = std::env::temp_dir().join(format!(
5036 "mf-traffic-breakdown-{}-{}",
5037 std::process::id(),
5038 std::time::SystemTime::now()
5039 .duration_since(std::time::UNIX_EPOCH)
5040 .unwrap()
5041 .as_nanos()
5042 ));
5043 let _ = std::fs::create_dir_all(&dir);
5044 let mut cmd = sample_bench_command();
5045 cmd.output = dir.clone();
5046 cmd.target_rps = Some(50);
5047 cmd.duration = "1200s".to_string();
5048 let stats = crate::wafbench::WafBenchStats {
5049 per_file: vec![crate::wafbench::TrafficFileSummary {
5050 file: "apisix_cve-2026-44087.yaml".into(),
5051 sent: 5,
5052 attack: 3,
5053 normal: 2,
5054 omitted: 1,
5055 other: 0,
5056 }],
5057 ..Default::default()
5058 };
5059 cmd.emit_traffic_file_breakdown(&stats, "what to expect in proxy logs");
5060 let raw = std::fs::read_to_string(dir.join("traffic-breakdown.json"))
5061 .expect("traffic-breakdown.json");
5062 let v: serde_json::Value = serde_json::from_str(&raw).unwrap();
5063 assert_eq!(v["rps"], 50);
5064 assert_eq!(v["duration_secs"], 1200);
5065 assert_eq!(v["files"][0]["sent"]["unique"], 5);
5066 assert_eq!(v["files"][0]["sent"]["total"], 250);
5067 assert_eq!(v["files"][0]["sent"]["expected_over_duration"], 300000);
5068 assert_eq!(v["files"][0]["attack"]["total"], 150);
5069 assert_eq!(v["files"][0]["normal"]["total"], 100);
5070 assert_eq!(
5071 BenchCommand::format_unique_total(5, Some(50)),
5072 "unique=5 total=250 (5 * 50 RPS)"
5073 );
5074 let _ = std::fs::remove_dir_all(&dir);
5075 }
5076}