1use crate::crud_flow::{CrudFlowConfig, CrudFlowDetector};
4use crate::data_driven::{DataDistribution, DataDrivenConfig, DataDrivenGenerator, DataMapping};
5use crate::dynamic_params::{DynamicParamProcessor, DynamicPlaceholder};
6use crate::error::{BenchError, Result};
7use crate::executor::K6Executor;
8use crate::invalid_data::{InvalidDataConfig, InvalidDataGenerator};
9use crate::k6_gen::{K6Config, K6ScriptGenerator};
10use crate::mock_integration::{
11 MockIntegrationConfig, MockIntegrationGenerator, MockServerDetector,
12};
13use crate::owasp_api::{OwaspApiConfig, OwaspApiGenerator, OwaspCategory, ReportFormat};
14use crate::parallel_executor::{AggregatedResults, ParallelExecutor};
15use crate::parallel_requests::{ParallelConfig, ParallelRequestGenerator};
16use crate::param_overrides::ParameterOverrides;
17use crate::reporter::TerminalReporter;
18use crate::request_gen::RequestGenerator;
19use crate::scenarios::LoadScenario;
20use crate::security_payloads::{
21 SecurityCategory, SecurityPayload, SecurityPayloads, SecurityTestConfig, SecurityTestGenerator,
22};
23use crate::spec_dependencies::{
24 topological_sort, DependencyDetector, ExtractedValues, SpecDependencyConfig,
25};
26use crate::spec_parser::SpecParser;
27use crate::target_parser::parse_targets_file;
28use crate::wafbench::WafBenchLoader;
29use mockforge_openapi::multi_spec::{
30 load_specs_from_directory, load_specs_from_files, merge_specs, ConflictStrategy,
31};
32use mockforge_openapi::spec::OpenApiSpec;
33use std::collections::{HashMap, HashSet};
34use std::path::{Path, PathBuf};
35use std::str::FromStr;
36
37pub fn parse_header_string(inputs: &[String]) -> Result<HashMap<String, String>> {
45 let mut headers = HashMap::new();
46
47 for pair in inputs {
48 let pair = pair.trim();
49 if pair.is_empty() {
50 continue;
51 }
52 let parts: Vec<&str> = pair.splitn(2, ':').collect();
53 if parts.len() != 2 {
54 return Err(BenchError::Other(format!(
55 "Invalid header format: '{}'. Expected 'Key:Value'",
56 pair
57 )));
58 }
59 headers.insert(parts[0].trim().to_string(), parts[1].trim().to_string());
60 }
61
62 Ok(headers)
63}
64
65const CONFORMANCE_REPLACES_LOAD_ADVISORY: &str =
85 "Conformance mode REPLACES the load run: 1 VU, 1 iteration per endpoint. \
86 --vus, --rps and -d are ignored. Run bench a second time without \
87 --conformance if you also want a load test.";
88
89pub struct BenchCommand {
91 pub spec: Vec<PathBuf>,
93 pub spec_dir: Option<PathBuf>,
95 pub merge_conflicts: String,
97 pub spec_mode: String,
99 pub dependency_config: Option<PathBuf>,
101 pub target: String,
102 pub base_path: Option<String>,
105 pub duration: String,
106 pub vus: u32,
107 pub target_rps: Option<u32>,
113 pub no_keep_alive: bool,
118 pub scenario: String,
119 pub operations: Option<String>,
120 pub exclude_operations: Option<String>,
124 pub auth: Option<String>,
125 pub headers: Vec<String>,
128 pub output: PathBuf,
129 pub generate_only: bool,
130 pub script_output: Option<PathBuf>,
131 pub threshold_percentile: String,
132 pub threshold_ms: u64,
133 pub max_error_rate: f64,
134 pub abort_on_error: bool,
139 pub abort_on_error_rate: f64,
143 pub per_op_metrics: Option<bool>,
147 pub verbose: bool,
148 pub skip_tls_verify: bool,
149 pub chunked_request_bodies: bool,
154 pub targets_file: Option<PathBuf>,
156 pub max_concurrency: Option<u32>,
158 pub repeat_until: Option<String>,
163 pub rounds: Option<u32>,
167 pub results_format: String,
169 pub params_file: Option<PathBuf>,
174
175 pub crud_flow: bool,
178 pub flow_config: Option<PathBuf>,
180 pub extract_fields: Option<String>,
182
183 pub parallel_create: Option<u32>,
186
187 pub data_file: Option<PathBuf>,
190 pub data_distribution: String,
192 pub data_mappings: Option<String>,
194 pub per_uri_control: bool,
196
197 pub error_rate: Option<f64>,
200 pub error_types: Option<String>,
202
203 pub security_test: bool,
206 pub security_payloads: Option<PathBuf>,
208 pub security_categories: Option<String>,
210 pub security_target_fields: Option<String>,
212
213 pub wafbench_dir: Option<String>,
216 pub wafbench_cycle_all: bool,
218 pub wafbench_verbatim: bool,
221
222 pub conformance: bool,
225 pub conformance_api_key: Option<String>,
227 pub conformance_basic_auth: Option<String>,
229 pub conformance_report: PathBuf,
231 pub conformance_categories: Option<String>,
233 pub conformance_report_format: String,
235 pub conformance_headers: Vec<String>,
238 pub conformance_all_operations: bool,
241 pub conformance_custom: Option<PathBuf>,
243 pub conformance_delay_ms: u64,
246 pub use_k6: bool,
248 pub conformance_custom_filter: Option<String>,
252 pub export_requests: bool,
255 pub validate_requests: bool,
258 pub conformance_self_test: bool,
265 pub conformance_self_test_capture: bool,
269 pub validate_response_schemas: bool,
275 pub conformance_self_test_iterations: u32,
280 pub conformance_self_test_duration: Option<String>,
285
286 pub source_ips: Vec<String>,
291 pub geo_source_ips: Vec<String>,
295 pub geo_source_headers: Vec<String>,
299
300 pub report_missed_cap: Option<u32>,
307
308 pub discard_response_bodies: bool,
315
316 pub dns_policy: Option<String>,
322
323 pub no_k6_logs: bool,
331
332 pub keep_rounds: Option<u32>,
338
339 pub owasp_api_top10: bool,
342 pub owasp_categories: Option<String>,
344 pub owasp_auth_header: String,
346 pub owasp_auth_token: Option<String>,
348 pub owasp_admin_paths: Option<PathBuf>,
350 pub owasp_id_fields: Option<String>,
352 pub owasp_report: Option<PathBuf>,
354 pub owasp_report_format: String,
356 pub owasp_iterations: u32,
358}
359
360fn parse_ip_list(raw: &[String], flag_name: &str) -> Vec<std::net::IpAddr> {
374 use std::net::IpAddr;
375 const MAX_CIDR_EXPANSION: usize = 256;
376 let mut out = Vec::new();
377 for entry in raw {
378 for piece in entry.split(',') {
379 let s = piece.trim();
380 if s.is_empty() {
381 continue;
382 }
383 if let Some((addr_part, prefix_part)) = s.split_once('/') {
385 let prefix: u32 = match prefix_part.parse() {
386 Ok(p) => p,
387 Err(e) => {
388 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR prefix: {e}");
389 continue;
390 }
391 };
392 let net_addr: IpAddr = match addr_part.parse() {
393 Ok(a) => a,
394 Err(e) => {
395 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR address: {e}");
396 continue;
397 }
398 };
399 expand_cidr(net_addr, prefix, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
400 continue;
401 }
402 if let Some((start_str, end_str)) = s.split_once('-') {
408 let start_s = start_str.trim();
409 let end_s = end_str.trim();
410 if start_s.contains(':') || end_s.contains(':') {
414 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{s}': IPv6 range syntax not supported (use CIDR like 2001:db8::/126 instead)");
415 continue;
416 }
417 let start: IpAddr = match start_s.parse() {
418 Ok(a) => a,
419 Err(e) => {
420 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range start: {e}");
421 continue;
422 }
423 };
424 let end: IpAddr = match end_s.parse() {
425 Ok(a) => a,
426 Err(e) => {
427 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range end: {e}");
428 continue;
429 }
430 };
431 expand_range(start, end, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
432 continue;
433 }
434 match s.parse::<IpAddr>() {
436 Ok(ip) => out.push(ip),
437 Err(e) => {
438 tracing::warn!(target: "mockforge::bench", "ignoring malformed --{flag_name} value '{s}': {e}");
439 }
440 }
441 }
442 }
443 out
444}
445
446fn expand_range(
450 start: std::net::IpAddr,
451 end: std::net::IpAddr,
452 cap: usize,
453 flag_name: &str,
454 raw: &str,
455 out: &mut Vec<std::net::IpAddr>,
456) {
457 use std::net::{IpAddr, Ipv4Addr};
458 let (start_v4, end_v4) = match (start, end) {
459 (IpAddr::V4(a), IpAddr::V4(b)) => (a, b),
460 _ => {
461 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range start/end must both be IPv4");
462 return;
463 }
464 };
465 let start_u32 = u32::from(start_v4);
466 let end_u32 = u32::from(end_v4);
467 if end_u32 < start_u32 {
468 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range end {end_v4} is before start {start_v4}");
469 return;
470 }
471 let total = (end_u32 - start_u32).saturating_add(1) as usize;
472 let take = total.min(cap);
473 if total > cap {
474 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range has {total} addresses, capping at {cap}");
475 }
476 for i in 0..take as u32 {
477 out.push(IpAddr::V4(Ipv4Addr::from(start_u32 + i)));
478 }
479}
480
481fn expand_cidr(
485 net: std::net::IpAddr,
486 prefix: u32,
487 cap: usize,
488 flag_name: &str,
489 raw: &str,
490 out: &mut Vec<std::net::IpAddr>,
491) {
492 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
493 match net {
494 IpAddr::V4(ipv4) => {
495 if prefix > 32 {
496 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv4 prefix must be <= 32");
497 return;
498 }
499 let total: u64 = 1u64 << (32 - prefix);
500 let take = total.min(cap as u64) as u32;
501 if total > cap as u64 {
502 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': CIDR has {total} addresses, capping at {cap}");
503 }
504 let mask: u32 = if prefix == 0 {
505 0
506 } else {
507 !0u32 << (32 - prefix)
508 };
509 let net_u32 = u32::from(ipv4) & mask;
510 for i in 0..take {
511 out.push(IpAddr::V4(Ipv4Addr::from(net_u32.wrapping_add(i))));
512 }
513 }
514 IpAddr::V6(ipv6) => {
515 if prefix > 128 {
516 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv6 prefix must be <= 128");
517 return;
518 }
519 let mask: u128 = if prefix == 0 {
523 0
524 } else {
525 !0u128 << (128 - prefix)
526 };
527 let net_u128 = u128::from(ipv6) & mask;
528 let remaining_bits = 128 - prefix;
529 let total_capped = if remaining_bits >= 64 {
532 cap as u128
533 } else {
534 (1u128 << remaining_bits).min(cap as u128)
535 };
536 if remaining_bits < 128 && (1u128 << remaining_bits) > cap as u128 {
537 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': IPv6 CIDR exceeds {cap} addresses, capping");
538 }
539 for i in 0..total_capped {
540 out.push(IpAddr::V6(Ipv6Addr::from(net_u128.wrapping_add(i))));
541 }
542 }
543 }
544}
545
546impl BenchCommand {
547 pub fn security_testing_enabled(&self) -> bool {
563 if self.wafbench_verbatim {
564 return false;
565 }
566 self.security_test || self.wafbench_dir.is_some()
567 }
568
569 pub async fn load_and_merge_specs(&self) -> Result<OpenApiSpec> {
571 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
572
573 if !self.spec.is_empty() {
575 let specs = load_specs_from_files(self.spec.clone())
576 .await
577 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
578 all_specs.extend(specs);
579 }
580
581 if let Some(spec_dir) = &self.spec_dir {
583 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
584 BenchError::Other(format!("Failed to load specs from directory: {}", e))
585 })?;
586 all_specs.extend(dir_specs);
587 }
588
589 if all_specs.is_empty() {
590 return Err(BenchError::Other(
591 "No spec files provided. Use --spec or --spec-dir.".to_string(),
592 ));
593 }
594
595 if all_specs.len() == 1 {
597 return Ok(all_specs.into_iter().next().expect("checked len() == 1 above").1);
599 }
600
601 let conflict_strategy = match self.merge_conflicts.as_str() {
603 "first" => ConflictStrategy::First,
604 "last" => ConflictStrategy::Last,
605 _ => ConflictStrategy::Error,
606 };
607
608 merge_specs(all_specs, conflict_strategy)
609 .map_err(|e| BenchError::Other(format!("Failed to merge specs: {}", e)))
610 }
611
612 fn get_spec_display_name(&self) -> String {
614 if self.spec.len() == 1 {
615 self.spec[0].to_string_lossy().to_string()
616 } else if !self.spec.is_empty() {
617 format!("{} spec files", self.spec.len())
618 } else if let Some(dir) = &self.spec_dir {
619 format!("specs from {}", dir.display())
620 } else {
621 "no specs".to_string()
622 }
623 }
624
625 fn advise_capacity(&self) {
632 let target_count = self
633 .targets_file
634 .as_ref()
635 .and_then(|p| std::fs::read_to_string(p).ok())
636 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
637 .and_then(|v| v.as_array().map(|a| a.len()))
638 .unwrap_or(1);
639 let vus = self.vus.max(1);
640 let rps_total = self.target_rps.unwrap_or(0) as usize * target_count.max(1);
641 let load_product = target_count * vus as usize;
645 if load_product >= 150 {
646 let est_ram_gb =
647 (vus as usize * 50) / 1024 + (target_count * 10 * 2) / 1024 + target_count / 2;
648 let est_cores = ((vus as usize) / 50).max(2);
649 TerminalReporter::print_warning(&format!(
650 "Capacity advisory: targets={target_count}, VUs={vus}, RPS-total≈{rps_total}. \
651 Single-client estimate: ~{est_cores} CPU cores, ~{est_ram_gb} GB RAM. \
652 If your machine is below that, expect OOM hangs partway through the run. \
653 See https://docs.mockforge.dev/reference/bench-capacity-sizing.html \
654 for the sizing table and sharding guide."
655 ));
656 }
657 }
658
659 pub async fn execute(&self) -> Result<()> {
661 if self.conformance_self_test && self.use_k6 {
668 TerminalReporter::print_warning(
669 "--use-k6 has no effect with --conformance-self-test: the self-test driver runs and returns before k6 is invoked. Drop one or the other depending on whether you want the spec-driven self-test or a k6 bench run.",
670 );
671 }
672
673 self.advise_capacity();
679
680 if let Some(targets_file) = &self.targets_file {
682 if self.conformance && self.conformance_self_test {
691 return self.execute_multi_target_self_test(targets_file).await;
692 }
693 if self.conformance {
694 return self.execute_multi_target_conformance(targets_file).await;
695 }
696 return self.execute_multi_target(targets_file).await;
697 }
698
699 if self.spec_mode == "sequential" && (self.spec.len() > 1 || self.spec_dir.is_some()) {
701 return self.execute_sequential_specs().await;
702 }
703
704 TerminalReporter::print_header(
707 &self.get_spec_display_name(),
708 &self.target,
709 0, &self.scenario,
711 Self::parse_duration(&self.duration)?,
712 );
713
714 if !K6Executor::is_k6_installed() {
716 TerminalReporter::print_error("k6 is not installed");
717 TerminalReporter::print_warning(
718 "Install k6 from: https://k6.io/docs/get-started/installation/",
719 );
720 return Err(BenchError::K6NotFound);
721 }
722 K6Executor::warn_if_pre_v1().await;
723
724 if self.conformance {
726 return self.execute_conformance_test().await;
727 }
728
729 let spec_supplied = !self.spec.is_empty() || self.spec_dir.is_some();
736 let merged_spec = if self.wafbench_verbatim && !spec_supplied {
737 tracing::info!(
738 target: "mockforge::bench",
739 "--wafbench-verbatim without --spec: sending only the traffic file's requests"
740 );
741 OpenApiSpec {
742 spec: Default::default(),
743 file_path: None,
744 raw_document: None,
745 }
746 } else {
747 TerminalReporter::print_progress("Loading OpenAPI specification(s)...");
748 self.load_and_merge_specs().await?
749 };
750 let parser = SpecParser::from_spec(merged_spec);
751 if self.spec.len() > 1 || self.spec_dir.is_some() {
752 TerminalReporter::print_success(&format!(
753 "Loaded and merged {} specification(s)",
754 self.spec.len() + self.spec_dir.as_ref().map(|_| 1).unwrap_or(0)
755 ));
756 } else {
757 TerminalReporter::print_success("Specification loaded");
758 }
759
760 let mock_config = self.build_mock_config().await;
762 if mock_config.is_mock_server {
763 TerminalReporter::print_progress("Mock server integration enabled");
764 }
765
766 if self.crud_flow {
768 return self.execute_crud_flow(&parser).await;
769 }
770
771 if self.owasp_api_top10 {
773 return self.execute_owasp_test(&parser).await;
774 }
775
776 TerminalReporter::print_progress("Extracting API operations...");
778 let mut operations = if let Some(filter) = &self.operations {
779 parser.filter_operations(filter)?
780 } else {
781 parser.get_operations()
782 };
783
784 if let Some(exclude) = &self.exclude_operations {
786 let before_count = operations.len();
787 operations = parser.exclude_operations(operations, exclude)?;
788 let excluded_count = before_count - operations.len();
789 if excluded_count > 0 {
790 TerminalReporter::print_progress(&format!(
791 "Excluded {} operations matching '{}'",
792 excluded_count, exclude
793 ));
794 }
795 }
796
797 if operations.is_empty() && !self.wafbench_verbatim {
803 return Err(BenchError::Other("No operations found in spec".to_string()));
804 }
805
806 TerminalReporter::print_success(&format!("Found {} operations", operations.len()));
807
808 let param_overrides = if let Some(params_file) = &self.params_file {
810 TerminalReporter::print_progress("Loading parameter overrides...");
811 let overrides = ParameterOverrides::from_file(params_file)?;
812 TerminalReporter::print_success(&format!(
813 "Loaded parameter overrides ({} operation-specific, {} defaults)",
814 overrides.operations.len(),
815 if overrides.defaults.is_empty() { 0 } else { 1 }
816 ));
817 Some(overrides)
818 } else {
819 None
820 };
821
822 TerminalReporter::print_progress("Generating request templates...");
824 let templates: Vec<_> = operations
825 .iter()
826 .map(|op| {
827 let op_overrides = param_overrides.as_ref().map(|po| {
828 po.get_for_operation(op.operation_id.as_deref(), &op.method, &op.path)
829 });
830 RequestGenerator::generate_template_with_overrides(op, op_overrides.as_ref())
831 })
832 .collect::<Result<Vec<_>>>()?;
833 TerminalReporter::print_success("Request templates generated");
834
835 let templates = if self.wafbench_verbatim {
841 let verbatim = self.load_verbatim_templates()?;
842 if verbatim.is_empty() {
843 return Err(BenchError::Other(
844 "--wafbench-verbatim was set but no traffic cases were loaded. Check \
845 --wafbench-dir points at a file, directory or glob containing cases with \
846 a `request.uri`."
847 .to_string(),
848 ));
849 }
850 TerminalReporter::print_success(&format!(
851 "Verbatim mode: {} request(s) will be sent exactly as written (spec endpoints not used)",
852 verbatim.len()
853 ));
854 verbatim
855 } else {
856 templates
857 };
858
859 let custom_headers = self.parse_headers()?;
861
862 let force_http1 = crate::request_gen::should_force_k6_http1(
865 self.wafbench_verbatim,
866 &templates,
867 &custom_headers,
868 );
869
870 let base_path = self.resolve_base_path(&parser);
872 if let Some(ref bp) = base_path {
873 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
874 }
875
876 TerminalReporter::print_progress("Generating k6 load test script...");
878 let scenario =
879 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
880
881 let security_testing_enabled = self.security_testing_enabled();
882
883 let num_ops = operations.len() as u32;
901 if let Some(rps) = self.target_rps {
902 let probe =
903 crate::preflight::probe_target_latency(&self.target, 3, self.skip_tls_verify).await;
904
905 let (required_vus, basis) = match probe {
906 Some(p) => (
907 p.required_vus(rps, num_ops),
908 format!("avg {:.1}ms (measured)", p.avg_latency.as_secs_f64() * 1000.0),
909 ),
910 None => {
911 let fallback = (rps as u64)
913 .saturating_mul(num_ops.max(1) as u64)
914 .div_ceil(10)
915 .min(u32::MAX as u64) as u32;
916 (fallback, "~100ms (default — probe failed)".to_string())
917 }
918 };
919
920 if self.vus < required_vus {
921 const VU_RECOMMENDATION_CAP: u32 = 1000;
927 let recommendation = required_vus.max(self.vus + 1);
928 if recommendation > VU_RECOMMENDATION_CAP {
929 TerminalReporter::print_warning(&format!(
930 "Workload is very large: --rps {} × {} ops/iteration × {} \
931 baseline ⇒ ~{} VUs needed end-to-end, far beyond what's \
932 practical to drive. Two ways to fix:\n 1. Reduce \
933 operations per iteration with `--operations 'pattern,…'` \
934 (or `--exclude-operations`) to focus the bench on a \
935 representative subset.\n 2. Drop `--rps` and use \
936 `--vus {}` alone — closed-model load runs as fast as \
937 the VU pool allows, bounded by latency, with no per-\
938 iteration deadline. Expect 1-iteration coverage of ~{} \
939 operations in {}s.",
940 rps,
941 num_ops,
942 basis,
943 recommendation,
944 self.vus.max(5),
945 num_ops,
946 Self::parse_duration(&self.duration).unwrap_or(0),
947 ));
948 } else {
949 TerminalReporter::print_warning(&format!(
950 "--vus {} may be insufficient for --rps {} × {} ops/iteration \
951 (baseline latency {}). k6's constant-arrival-rate counts ITERATIONS \
952 and each runs every operation in the spec — required ≈ rps × ops × \
953 latency_secs VUs. Bump --vus to ~{} if you see \"Insufficient VUs\" \
954 warnings.",
955 self.vus, rps, num_ops, basis, recommendation,
956 ));
957 }
958 } else if probe.is_some() {
959 TerminalReporter::print_progress(&format!(
960 "Pre-flight probe: target latency {}, {} ops/iteration — --vus {} \
961 is sufficient for --rps {}",
962 basis, num_ops, self.vus, rps,
963 ));
964 }
965 }
966
967 let k6_config = K6Config {
968 target_url: self.target.clone(),
969 base_path,
970 scenario,
971 duration_secs: Self::parse_duration(&self.duration)?,
972 max_vus: self.vus,
973 threshold_percentile: self.threshold_percentile.clone(),
974 threshold_ms: self.threshold_ms,
975 max_error_rate: self.max_error_rate,
976 auth_header: self.auth.clone(),
977 custom_headers,
978 skip_tls_verify: self.skip_tls_verify,
979 security_testing_enabled,
980 chunked_request_bodies: self.chunked_request_bodies,
981 target_rps: self.target_rps,
982 no_keep_alive: self.no_keep_alive,
983 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
989 .into_iter()
990 .map(|ip| ip.to_string())
991 .collect(),
992 geo_source_headers: if self.geo_source_headers.is_empty()
993 && !self.geo_source_ips.is_empty()
994 {
995 crate::conformance::self_test::default_geo_source_headers()
996 } else {
997 self.geo_source_headers.clone()
998 },
999 };
1000
1001 let duration_secs = Self::parse_duration(&self.duration)?;
1004 let (per_op_metrics, per_op_warn) = crate::k6_gen::resolve_per_op_metrics(
1005 self.per_op_metrics,
1006 templates.len(),
1007 duration_secs,
1008 );
1009 if let Some(msg) = per_op_warn {
1010 TerminalReporter::print_warning(&msg);
1011 }
1012
1013 let generator = K6ScriptGenerator::new(k6_config, templates)
1014 .with_abort_valve(self.abort_on_error, self.abort_on_error_rate)
1015 .with_force_http1(force_http1)
1016 .with_per_op_metrics(per_op_metrics);
1017 let mut script = generator.generate()?;
1018 TerminalReporter::print_success("k6 script generated");
1019
1020 let has_advanced_features = self.data_file.is_some()
1022 || self.error_rate.is_some()
1023 || self.security_test
1024 || self.parallel_create.is_some()
1025 || self.wafbench_dir.is_some();
1026
1027 if has_advanced_features {
1029 script = self.generate_enhanced_script(&script)?;
1030 }
1031
1032 if mock_config.is_mock_server {
1034 let setup_code = MockIntegrationGenerator::generate_setup(&mock_config);
1035 let teardown_code = MockIntegrationGenerator::generate_teardown(&mock_config);
1036 let helper_code = MockIntegrationGenerator::generate_vu_id_helper();
1037
1038 if let Some(import_end) = script.find("export const options") {
1040 script.insert_str(
1041 import_end,
1042 &format!(
1043 "\n// === Mock Server Integration ===\n{}\n{}\n{}\n",
1044 helper_code, setup_code, teardown_code
1045 ),
1046 );
1047 }
1048 }
1049
1050 TerminalReporter::print_progress("Validating k6 script...");
1052 let validation_errors = K6ScriptGenerator::validate_script(&script);
1053 if !validation_errors.is_empty() {
1054 TerminalReporter::print_error("Script validation failed");
1055 for error in &validation_errors {
1056 eprintln!(" {}", error);
1057 }
1058 return Err(BenchError::Other(format!(
1059 "Generated k6 script has {} validation error(s). Please check the output above.",
1060 validation_errors.len()
1061 )));
1062 }
1063 TerminalReporter::print_success("Script validation passed");
1064
1065 let script_path = if let Some(output) = &self.script_output {
1067 output.clone()
1068 } else {
1069 self.output.join("k6-script.js")
1070 };
1071
1072 if let Some(parent) = script_path.parent() {
1073 std::fs::create_dir_all(parent)?;
1074 }
1075 std::fs::write(&script_path, &script)?;
1076 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
1077
1078 if self.generate_only {
1080 Self::print_k6_run_hint(&script_path, force_http1);
1081 return Ok(());
1082 }
1083
1084 TerminalReporter::print_progress("Executing load test...");
1086 if force_http1 {
1087 TerminalReporter::print_progress(
1088 "Forcing HTTP/1.1 (GODEBUG=http2client=0): Connection headers are hop-by-hop and HTTP/2 rejects them. The header stays on the wire.",
1089 );
1090 }
1091 let executor = K6Executor::new()?
1095 .with_local_ips(self.source_ips.join(","))
1096 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
1097 .with_discard_response_bodies(self.discard_response_bodies)
1098 .with_force_http1(force_http1)
1099 .with_capture_logs(!self.no_k6_logs);
1100
1101 std::fs::create_dir_all(&self.output)?;
1102
1103 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
1104
1105 let duration_secs = Self::parse_duration(&self.duration)?;
1107 TerminalReporter::print_summary_full(
1108 &results,
1109 duration_secs,
1110 self.no_keep_alive,
1111 Some(num_ops),
1112 );
1113
1114 self.reprint_traffic_file_breakdown();
1115 println!("\nResults saved to: {}", self.output.display());
1116
1117 Ok(())
1118 }
1119
1120 async fn execute_multi_target(&self, targets_file: &Path) -> Result<()> {
1122 TerminalReporter::print_progress("Parsing targets file...");
1123 let targets = parse_targets_file(targets_file)?;
1124 let num_targets = targets.len();
1125 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
1126
1127 if targets.is_empty() {
1128 return Err(BenchError::Other("No targets found in file".to_string()));
1129 }
1130
1131 let max_concurrency = self.max_concurrency.map(|n| n as usize);
1135
1136 TerminalReporter::print_header(
1138 &self.get_spec_display_name(),
1139 &format!("{} targets", num_targets),
1140 0,
1141 &self.scenario,
1142 Self::parse_duration(&self.duration)?,
1143 );
1144
1145 let executor = ParallelExecutor::new(
1147 BenchCommand {
1148 spec: self.spec.clone(),
1150 spec_dir: self.spec_dir.clone(),
1151 merge_conflicts: self.merge_conflicts.clone(),
1152 spec_mode: self.spec_mode.clone(),
1153 dependency_config: self.dependency_config.clone(),
1154 target: self.target.clone(), base_path: self.base_path.clone(),
1156 duration: self.duration.clone(),
1157 vus: self.vus,
1158 target_rps: self.target_rps,
1159 no_keep_alive: self.no_keep_alive,
1160 scenario: self.scenario.clone(),
1161 operations: self.operations.clone(),
1162 exclude_operations: self.exclude_operations.clone(),
1163 auth: self.auth.clone(),
1164 headers: self.headers.clone(),
1165 output: self.output.clone(),
1166 generate_only: self.generate_only,
1167 script_output: self.script_output.clone(),
1168 threshold_percentile: self.threshold_percentile.clone(),
1169 threshold_ms: self.threshold_ms,
1170 max_error_rate: self.max_error_rate,
1171 abort_on_error: self.abort_on_error,
1172 abort_on_error_rate: self.abort_on_error_rate,
1173 per_op_metrics: self.per_op_metrics,
1174 verbose: self.verbose,
1175 skip_tls_verify: self.skip_tls_verify,
1176 chunked_request_bodies: self.chunked_request_bodies,
1177 targets_file: None,
1178 max_concurrency: None,
1179 repeat_until: self.repeat_until.clone(),
1180 rounds: self.rounds,
1181 results_format: self.results_format.clone(),
1182 params_file: self.params_file.clone(),
1183 crud_flow: self.crud_flow,
1184 flow_config: self.flow_config.clone(),
1185 extract_fields: self.extract_fields.clone(),
1186 parallel_create: self.parallel_create,
1187 data_file: self.data_file.clone(),
1188 data_distribution: self.data_distribution.clone(),
1189 data_mappings: self.data_mappings.clone(),
1190 per_uri_control: self.per_uri_control,
1191 error_rate: self.error_rate,
1192 error_types: self.error_types.clone(),
1193 security_test: self.security_test,
1194 security_payloads: self.security_payloads.clone(),
1195 security_categories: self.security_categories.clone(),
1196 security_target_fields: self.security_target_fields.clone(),
1197 wafbench_dir: self.wafbench_dir.clone(),
1198 wafbench_cycle_all: self.wafbench_cycle_all,
1199 wafbench_verbatim: self.wafbench_verbatim,
1200 owasp_api_top10: self.owasp_api_top10,
1201 owasp_categories: self.owasp_categories.clone(),
1202 owasp_auth_header: self.owasp_auth_header.clone(),
1203 owasp_auth_token: self.owasp_auth_token.clone(),
1204 owasp_admin_paths: self.owasp_admin_paths.clone(),
1205 owasp_id_fields: self.owasp_id_fields.clone(),
1206 owasp_report: self.owasp_report.clone(),
1207 owasp_report_format: self.owasp_report_format.clone(),
1208 owasp_iterations: self.owasp_iterations,
1209 conformance: false,
1210 conformance_api_key: self.conformance_api_key.clone(),
1226 conformance_basic_auth: self.conformance_basic_auth.clone(),
1227 conformance_report: PathBuf::from("conformance-report.json"),
1228 conformance_categories: None,
1229 conformance_report_format: "json".to_string(),
1230 conformance_headers: self.conformance_headers.clone(),
1234 conformance_all_operations: false,
1235 conformance_custom: None,
1236 conformance_delay_ms: 0,
1237 use_k6: false,
1238 conformance_custom_filter: None,
1239 export_requests: false,
1240 validate_requests: false,
1241 conformance_self_test: false,
1242 conformance_self_test_capture: false,
1243 conformance_self_test_iterations: 1,
1244 conformance_self_test_duration: None,
1245 validate_response_schemas: false,
1246 source_ips: self.source_ips.clone(),
1251 geo_source_ips: self.geo_source_ips.clone(),
1252 geo_source_headers: self.geo_source_headers.clone(),
1253 report_missed_cap: None,
1254 discard_response_bodies: self.discard_response_bodies,
1258 dns_policy: self.dns_policy.clone(),
1261 no_k6_logs: self.no_k6_logs,
1264 keep_rounds: self.keep_rounds,
1265 },
1266 targets,
1267 max_concurrency,
1268 );
1269
1270 let start_time = std::time::Instant::now();
1272 let aggregated_results = executor.execute_all().await?;
1273 let elapsed = start_time.elapsed();
1274
1275 self.report_multi_target_results(&aggregated_results, elapsed)?;
1277
1278 Ok(())
1279 }
1280
1281 fn report_multi_target_results(
1283 &self,
1284 results: &AggregatedResults,
1285 elapsed: std::time::Duration,
1286 ) -> Result<()> {
1287 TerminalReporter::print_multi_target_summary(results);
1289
1290 let total_secs = elapsed.as_secs();
1292 let hours = total_secs / 3600;
1293 let minutes = (total_secs % 3600) / 60;
1294 let seconds = total_secs % 60;
1295 if hours > 0 {
1296 println!("\n Total Elapsed Time: {}h {}m {}s", hours, minutes, seconds);
1297 } else if minutes > 0 {
1298 println!("\n Total Elapsed Time: {}m {}s", minutes, seconds);
1299 } else {
1300 println!("\n Total Elapsed Time: {}s", seconds);
1301 }
1302
1303 if self.results_format == "aggregated" || self.results_format == "both" {
1305 let summary_path = self.output.join("aggregated_summary.json");
1306 let summary_json =
1307 crate::parallel_executor::targets_summary_json(results, elapsed.as_secs());
1308
1309 std::fs::write(&summary_path, serde_json::to_string_pretty(&summary_json)?)?;
1310 TerminalReporter::print_success(&format!(
1311 "Aggregated summary saved to: {}",
1312 summary_path.display()
1313 ));
1314 }
1315
1316 let csv_path = self.output.join("all_targets.csv");
1318 let _ = std::fs::write(&csv_path, crate::parallel_executor::targets_csv(results));
1319
1320 self.reprint_traffic_file_breakdown();
1321 println!("\nResults saved to: {}", self.output.display());
1322 println!(" - Per-target results: {}", self.output.join("target_*").display());
1323 println!(" - All targets CSV: {}", csv_path.display());
1324 if self.results_format == "aggregated" || self.results_format == "both" {
1325 println!(
1326 " - Aggregated summary: {}",
1327 self.output.join("aggregated_summary.json").display()
1328 );
1329 }
1330 if self.repeat_until.is_some() || self.rounds.map(|r| r > 1).unwrap_or(false) {
1333 println!(
1334 " - Per-round stats: {} (+ {})",
1335 self.output.join("campaign.jsonl").display(),
1336 self.output.join("round-summaries").display()
1337 );
1338 }
1339
1340 Ok(())
1341 }
1342
1343 pub fn parse_duration(duration: &str) -> Result<u64> {
1345 let duration = duration.trim();
1346
1347 if let Some(secs) = duration.strip_suffix('s') {
1348 secs.parse::<u64>()
1349 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1350 } else if let Some(mins) = duration.strip_suffix('m') {
1351 mins.parse::<u64>()
1352 .map(|m| m * 60)
1353 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1354 } else if let Some(hours) = duration.strip_suffix('h') {
1355 hours
1356 .parse::<u64>()
1357 .map(|h| h * 3600)
1358 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1359 } else {
1360 duration
1362 .parse::<u64>()
1363 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1364 }
1365 }
1366
1367 fn print_k6_run_hint(script_path: &Path, force_http1: bool) {
1371 println!("\nScript generated successfully. Run it with:");
1372 if force_http1 {
1373 println!(" GODEBUG=http2client=0 k6 run {}", script_path.display());
1374 println!(
1375 " (HTTP/1.1: a Connection header is on the wire; HTTP/2 rejects it. mockforge bench sets this automatically when it invokes k6.)"
1376 );
1377 } else {
1378 println!(" k6 run {}", script_path.display());
1379 }
1380 }
1381
1382 pub(crate) fn load_verbatim_templates(
1389 &self,
1390 ) -> Result<Vec<crate::request_gen::RequestTemplate>> {
1391 let Some(pattern) = self.wafbench_dir.as_ref() else {
1392 return Err(BenchError::Other(
1393 "--wafbench-verbatim requires --wafbench-dir pointing at your traffic file(s)"
1394 .to_string(),
1395 ));
1396 };
1397
1398 let mut loader = WafBenchLoader::new();
1399 loader.load_from_pattern(pattern)?;
1400 self.emit_traffic_file_breakdown(loader.stats(), "what to expect in proxy logs");
1401
1402 Ok(crate::wafbench::traffic_cases_to_templates(loader.test_cases()))
1403 }
1404
1405 pub fn parse_headers(&self) -> Result<HashMap<String, String>> {
1407 let mut headers = parse_header_string(&self.headers)?;
1408
1409 let already_has = |hs: &HashMap<String, String>, name: &str| -> bool {
1420 hs.keys().any(|k| k.eq_ignore_ascii_case(name))
1421 };
1422
1423 if !already_has(&headers, "Authorization") {
1424 if let Some(b) = self.conformance_basic_auth.as_ref().filter(|s| !s.is_empty()) {
1425 use base64::Engine as _;
1426 let encoded = base64::engine::general_purpose::STANDARD.encode(b.as_bytes());
1427 headers.insert("Authorization".to_string(), format!("Basic {}", encoded));
1428 }
1429 }
1430
1431 for line in &self.conformance_headers {
1437 let Some((name, value)) = line.split_once(':') else {
1438 continue;
1439 };
1440 let name = name.trim();
1441 let value = value.trim();
1442 if name.is_empty() || already_has(&headers, name) {
1443 continue;
1444 }
1445 headers.insert(name.to_string(), value.to_string());
1446 }
1447
1448 if !self.conformance && self.conformance_api_key.is_some() {
1454 TerminalReporter::print_warning(
1455 "--conformance-api-key only fires under --conformance. For plain bench use --header 'X-API-Key: ...'.",
1456 );
1457 }
1458
1459 Ok(headers)
1460 }
1461
1462 fn parse_extracted_values(output_dir: &Path) -> Result<ExtractedValues> {
1463 let extracted_path = output_dir.join("extracted_values.json");
1464 if !extracted_path.exists() {
1465 return Ok(ExtractedValues::new());
1466 }
1467
1468 let content = std::fs::read_to_string(&extracted_path)
1469 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1470 let parsed: serde_json::Value = serde_json::from_str(&content)
1471 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1472
1473 let mut extracted = ExtractedValues::new();
1474 if let Some(values) = parsed.as_object() {
1475 for (key, value) in values {
1476 extracted.set(key.clone(), value.clone());
1477 }
1478 }
1479
1480 Ok(extracted)
1481 }
1482
1483 fn resolve_base_path(&self, parser: &SpecParser) -> Option<String> {
1492 if let Some(cli_base_path) = &self.base_path {
1494 if cli_base_path.is_empty() {
1495 return None;
1497 }
1498 return Some(cli_base_path.clone());
1499 }
1500
1501 parser.get_base_path()
1503 }
1504
1505 async fn build_mock_config(&self) -> MockIntegrationConfig {
1507 if MockServerDetector::looks_like_mock_server(&self.target) {
1509 if let Ok(info) = MockServerDetector::detect(&self.target).await {
1511 if info.is_mockforge {
1512 TerminalReporter::print_success(&format!(
1513 "Detected MockForge server (version: {})",
1514 info.version.as_deref().unwrap_or("unknown")
1515 ));
1516 return MockIntegrationConfig::mock_server();
1517 }
1518 }
1519 }
1520 MockIntegrationConfig::real_api()
1521 }
1522
1523 fn build_crud_flow_config(&self) -> Option<CrudFlowConfig> {
1525 if !self.crud_flow {
1526 return None;
1527 }
1528
1529 if let Some(config_path) = &self.flow_config {
1531 match CrudFlowConfig::from_file(config_path) {
1532 Ok(config) => return Some(config),
1533 Err(e) => {
1534 TerminalReporter::print_warning(&format!(
1535 "Failed to load flow config: {}. Using auto-detection.",
1536 e
1537 ));
1538 }
1539 }
1540 }
1541
1542 let extract_fields = self
1544 .extract_fields
1545 .as_ref()
1546 .map(|f| f.split(',').map(|s| s.trim().to_string()).collect())
1547 .unwrap_or_else(|| vec!["id".to_string(), "uuid".to_string()]);
1548
1549 Some(CrudFlowConfig {
1550 flows: Vec::new(), default_extract_fields: extract_fields,
1552 })
1553 }
1554
1555 fn build_data_driven_config(&self) -> Option<DataDrivenConfig> {
1557 let data_file = self.data_file.as_ref()?;
1558
1559 let distribution = DataDistribution::from_str(&self.data_distribution)
1560 .unwrap_or(DataDistribution::UniquePerVu);
1561
1562 let mappings = self
1563 .data_mappings
1564 .as_ref()
1565 .map(|m| DataMapping::parse_mappings(m).unwrap_or_default())
1566 .unwrap_or_default();
1567
1568 Some(DataDrivenConfig {
1569 file_path: data_file.to_string_lossy().to_string(),
1570 distribution,
1571 mappings,
1572 csv_has_header: true,
1573 per_uri_control: self.per_uri_control,
1574 per_uri_columns: crate::data_driven::PerUriColumns::default(),
1575 })
1576 }
1577
1578 fn build_invalid_data_config(&self) -> Option<InvalidDataConfig> {
1580 let error_rate = self.error_rate?;
1581
1582 let error_types = self
1583 .error_types
1584 .as_ref()
1585 .map(|types| InvalidDataConfig::parse_error_types(types).unwrap_or_default())
1586 .unwrap_or_default();
1587
1588 Some(InvalidDataConfig {
1589 error_rate,
1590 error_types,
1591 target_fields: Vec::new(),
1592 })
1593 }
1594
1595 fn build_security_config(&self) -> Option<SecurityTestConfig> {
1597 if !self.security_test {
1598 return None;
1599 }
1600
1601 let categories = self
1602 .security_categories
1603 .as_ref()
1604 .map(|cats| SecurityTestConfig::parse_categories(cats).unwrap_or_default())
1605 .unwrap_or_else(|| {
1606 let mut default = HashSet::new();
1607 default.insert(SecurityCategory::SqlInjection);
1608 default.insert(SecurityCategory::Xss);
1609 default
1610 });
1611
1612 let target_fields = self
1613 .security_target_fields
1614 .as_ref()
1615 .map(|fields| fields.split(',').map(|f| f.trim().to_string()).collect())
1616 .unwrap_or_default();
1617
1618 let custom_payloads_file =
1619 self.security_payloads.as_ref().map(|p| p.to_string_lossy().to_string());
1620
1621 Some(SecurityTestConfig {
1622 enabled: true,
1623 categories,
1624 target_fields,
1625 custom_payloads_file,
1626 include_high_risk: false,
1627 })
1628 }
1629
1630 fn build_parallel_config(&self) -> Option<ParallelConfig> {
1632 let count = self.parallel_create?;
1633
1634 Some(ParallelConfig::new(count))
1635 }
1636
1637 fn format_unique_total(unique: usize, rps: Option<u32>) -> String {
1640 match rps {
1641 Some(r) if r > 0 => {
1642 let projected = unique.saturating_mul(r as usize);
1643 format!(
1644 "unique_cases={unique} projected_per_second={projected} ({unique} * {r} RPS)"
1645 )
1646 }
1647 _ => format!("unique_cases={unique}"),
1648 }
1649 }
1650
1651 fn traffic_bucket_json(
1652 unique: usize,
1653 rps: Option<u32>,
1654 duration_secs: Option<u64>,
1655 ) -> serde_json::Value {
1656 let per_second = rps.filter(|&r| r > 0).map(|r| (unique as u64).saturating_mul(r as u64));
1663 let projected_over_run = match (rps.filter(|&r| r > 0), duration_secs) {
1664 (Some(r), Some(d)) => Some((unique as u64).saturating_mul(r as u64).saturating_mul(d)),
1665 _ => None,
1666 };
1667 serde_json::json!({
1668 "unique_cases": unique,
1669 "projected_per_second": per_second,
1670 "projected_over_run": projected_over_run,
1671 })
1672 }
1673
1674 fn emit_traffic_file_breakdown(&self, stats: &crate::wafbench::WafBenchStats, phase: &str) {
1677 if stats.per_file.is_empty() {
1678 return;
1679 }
1680 let rps = self.target_rps.filter(|&r| r > 0);
1681 TerminalReporter::print_success(&format!("Traffic file breakdown ({phase}):"));
1682 for file in &stats.per_file {
1683 let other = if file.other > 0 {
1684 format!(" other={}", file.other)
1685 } else {
1686 String::new()
1687 };
1688 TerminalReporter::print_progress(&format!(
1689 " {}: sent {} attack(expected 403) {} normal(expected 200) {} omitted={}{other}",
1690 file.file,
1691 Self::format_unique_total(file.sent, rps),
1692 Self::format_unique_total(file.attack, rps),
1693 Self::format_unique_total(file.normal, rps),
1694 file.omitted
1695 ));
1696 }
1697 self.write_traffic_breakdown_json(stats);
1698 }
1699
1700 fn write_traffic_breakdown_json(&self, stats: &crate::wafbench::WafBenchStats) {
1702 if stats.per_file.is_empty() {
1703 return;
1704 }
1705 let rps = self.target_rps.filter(|&r| r > 0);
1706 let duration_secs = Self::parse_duration(&self.duration).ok();
1707 let files: Vec<serde_json::Value> = stats
1708 .per_file
1709 .iter()
1710 .map(|file| {
1711 serde_json::json!({
1712 "file": file.file,
1713 "sent": Self::traffic_bucket_json(file.sent, rps, duration_secs),
1714 "attack": Self::traffic_bucket_json(file.attack, rps, duration_secs),
1715 "normal": Self::traffic_bucket_json(file.normal, rps, duration_secs),
1716 "omitted": file.omitted,
1717 "other": file.other,
1718 })
1719 })
1720 .collect();
1721 let payload = serde_json::json!({
1722 "rps": rps,
1723 "duration_secs": duration_secs,
1724 "note": "Plan, not k6 counters. unique_cases is the YAML case count (not traffic on the wire). projected_per_second is unique_cases * rps. projected_over_run is unique_cases * rps * duration_secs, assuming each k6 iteration sends every unique case. projected_* are null when --rps is unset.",
1725 "files": files,
1726 });
1727 if let Some(parent) = self.output.parent() {
1728 let _ = std::fs::create_dir_all(parent);
1729 }
1730 let _ = std::fs::create_dir_all(&self.output);
1731 let path = self.output.join("traffic-breakdown.json");
1732 if let Ok(body) = serde_json::to_string_pretty(&payload) {
1733 if std::fs::write(&path, body).is_ok() {
1734 TerminalReporter::print_progress(&format!(
1735 "Traffic breakdown written to: {}",
1736 path.display()
1737 ));
1738 }
1739 }
1740 }
1741
1742 fn reprint_traffic_file_breakdown(&self) {
1745 let path = self.output.join("traffic-breakdown.json");
1746 let Ok(raw) = std::fs::read_to_string(&path) else {
1747 return;
1748 };
1749 let Ok(v) = serde_json::from_str::<serde_json::Value>(&raw) else {
1750 return;
1751 };
1752 let Some(files) = v.get("files").and_then(|f| f.as_array()) else {
1753 return;
1754 };
1755 if files.is_empty() {
1756 return;
1757 }
1758 TerminalReporter::print_success("Traffic file breakdown (end of run):");
1759 for file in files {
1760 let name = file.get("file").and_then(|x| x.as_str()).unwrap_or("?");
1761 let bucket = |key: &str| -> String {
1762 let unique = file
1767 .get(key)
1768 .and_then(|b| b.get("unique_cases").or_else(|| b.get("unique")))
1769 .and_then(|u| u.as_u64())
1770 .unwrap_or(0) as usize;
1771 Self::format_unique_total(unique, self.target_rps.filter(|&r| r > 0))
1772 };
1773 let omitted = file.get("omitted").and_then(|o| o.as_u64()).unwrap_or(0);
1774 let other = file.get("other").and_then(|o| o.as_u64()).unwrap_or(0);
1775 let other = if other > 0 {
1776 format!(" other={other}")
1777 } else {
1778 String::new()
1779 };
1780 TerminalReporter::print_progress(&format!(
1781 " {name}: sent {} attack(expected 403) {} normal(expected 200) {} omitted={omitted}{other}",
1782 bucket("sent"),
1783 bucket("attack"),
1784 bucket("normal"),
1785 ));
1786 }
1787 TerminalReporter::print_progress(&format!(" (also in {})", path.display()));
1788 }
1789
1790 fn load_wafbench_payloads(&self) -> Result<Vec<SecurityPayload>> {
1797 let Some(ref wafbench_dir) = self.wafbench_dir else {
1798 return Ok(Vec::new());
1799 };
1800
1801 let mut loader = WafBenchLoader::new();
1802 loader.load_from_pattern(wafbench_dir)?;
1803
1804 let stats = loader.stats();
1805
1806 if stats.files_processed == 0 {
1807 let mut msg = format!(
1808 "No WAFBench YAML files found matching '{wafbench_dir}'. \
1809 --wafbench-dir is a file, a directory or a glob. A missing \
1810 file is an error, not an empty payload pool."
1811 );
1812 if !stats.parse_errors.is_empty() {
1813 msg.push_str(" Parse errors:");
1814 for error in &stats.parse_errors {
1815 msg.push_str(&format!("\n - {error}"));
1816 }
1817 }
1818 return Err(BenchError::Other(msg));
1819 }
1820
1821 TerminalReporter::print_progress(&format!(
1822 "Loaded {} WAFBench files, {} test cases, {} payloads",
1823 stats.files_processed, stats.test_cases_loaded, stats.payloads_extracted
1824 ));
1825 self.emit_traffic_file_breakdown(stats, "what to expect in proxy logs");
1826
1827 for (category, count) in &stats.by_category {
1829 TerminalReporter::print_progress(&format!(" - {}: {} tests", category, count));
1830 }
1831
1832 for error in &stats.parse_errors {
1834 TerminalReporter::print_warning(&format!(" Parse error: {}", error));
1835 }
1836
1837 Ok(loader.to_security_payloads())
1838 }
1839
1840 pub(crate) fn generate_enhanced_script(&self, base_script: &str) -> Result<String> {
1842 let mut enhanced_script = base_script.to_string();
1843 let mut additional_code = String::new();
1844
1845 if let Some(config) = self.build_data_driven_config() {
1847 TerminalReporter::print_progress("Adding data-driven testing support...");
1848 additional_code.push_str(&DataDrivenGenerator::generate_setup(&config));
1849 additional_code.push('\n');
1850 TerminalReporter::print_success("Data-driven testing enabled");
1851 }
1852
1853 if let Some(config) = self.build_invalid_data_config() {
1855 TerminalReporter::print_progress("Adding invalid data testing support...");
1856 additional_code.push_str(&InvalidDataGenerator::generate_invalidation_logic());
1857 additional_code.push('\n');
1858 additional_code
1859 .push_str(&InvalidDataGenerator::generate_should_invalidate(config.error_rate));
1860 additional_code.push('\n');
1861 additional_code
1862 .push_str(&InvalidDataGenerator::generate_type_selection(&config.error_types));
1863 additional_code.push('\n');
1864 TerminalReporter::print_success(&format!(
1865 "Invalid data testing enabled ({}% error rate)",
1866 (self.error_rate.unwrap_or(0.0) * 100.0) as u32
1867 ));
1868 }
1869
1870 let verbatim = self.wafbench_verbatim;
1877 if verbatim && self.security_test {
1878 TerminalReporter::print_warning(
1879 "--security-test is ignored under --wafbench-verbatim: verbatim mode sends your \
1880 traffic cases exactly as written and will not append attack payloads to them. \
1881 Drop --wafbench-verbatim if you want payload injection.",
1882 );
1883 }
1884 let security_config = if verbatim {
1885 None
1886 } else {
1887 self.build_security_config()
1888 };
1889 let wafbench_payloads = if verbatim {
1890 Vec::new()
1891 } else {
1892 self.load_wafbench_payloads()?
1893 };
1894 let security_requested =
1895 !verbatim && (security_config.is_some() || self.wafbench_dir.is_some());
1896
1897 if security_config.is_some() || !wafbench_payloads.is_empty() {
1898 TerminalReporter::print_progress("Adding security testing support...");
1899
1900 let mut payload_list: Vec<SecurityPayload> = Vec::new();
1902
1903 if let Some(ref config) = security_config {
1904 payload_list.extend(SecurityPayloads::get_payloads(config));
1905 }
1906
1907 if !wafbench_payloads.is_empty() {
1909 TerminalReporter::print_progress(&format!(
1910 "Loading {} WAFBench attack patterns...",
1911 wafbench_payloads.len()
1912 ));
1913 payload_list.extend(wafbench_payloads);
1914 }
1915
1916 let target_fields =
1917 security_config.as_ref().map(|c| c.target_fields.clone()).unwrap_or_default();
1918
1919 additional_code.push_str(&SecurityTestGenerator::generate_payload_selection(
1920 &payload_list,
1921 self.wafbench_cycle_all,
1922 ));
1923 additional_code.push('\n');
1924 additional_code
1925 .push_str(&SecurityTestGenerator::generate_apply_payload(&target_fields));
1926 additional_code.push('\n');
1927 additional_code.push_str(&SecurityTestGenerator::generate_security_checks());
1928 additional_code.push('\n');
1929
1930 let mode = if self.wafbench_cycle_all {
1931 "cycle-all"
1932 } else {
1933 "random"
1934 };
1935 TerminalReporter::print_success(&format!(
1936 "Security testing enabled ({} payloads, {} mode)",
1937 payload_list.len(),
1938 mode
1939 ));
1940 } else if security_requested {
1941 TerminalReporter::print_warning(
1945 "Security testing was requested but no payloads were loaded. \
1946 Ensure --wafbench-dir points to valid CRS YAML files or add --security-test.",
1947 );
1948 additional_code
1949 .push_str(&SecurityTestGenerator::generate_payload_selection(&[], false));
1950 additional_code.push('\n');
1951 additional_code.push_str(&SecurityTestGenerator::generate_apply_payload(&[]));
1952 additional_code.push('\n');
1953 }
1954
1955 if let Some(config) = self.build_parallel_config() {
1957 TerminalReporter::print_progress("Adding parallel execution support...");
1958 additional_code.push_str(&ParallelRequestGenerator::generate_batch_helper(&config));
1959 additional_code.push('\n');
1960 TerminalReporter::print_success(&format!(
1961 "Parallel execution enabled (count: {})",
1962 config.count
1963 ));
1964 }
1965
1966 if !additional_code.is_empty() {
1968 if let Some(import_end) = enhanced_script.find("export const options") {
1970 enhanced_script.insert_str(
1971 import_end,
1972 &format!("\n// === Advanced Testing Features ===\n{}\n", additional_code),
1973 );
1974 }
1975 }
1976
1977 Ok(enhanced_script)
1978 }
1979
1980 async fn execute_sequential_specs(&self) -> Result<()> {
1982 TerminalReporter::print_progress("Sequential spec mode: Loading specs individually...");
1983
1984 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
1986
1987 if !self.spec.is_empty() {
1988 let specs = load_specs_from_files(self.spec.clone())
1989 .await
1990 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
1991 all_specs.extend(specs);
1992 }
1993
1994 if let Some(spec_dir) = &self.spec_dir {
1995 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
1996 BenchError::Other(format!("Failed to load specs from directory: {}", e))
1997 })?;
1998 all_specs.extend(dir_specs);
1999 }
2000
2001 if all_specs.is_empty() {
2002 return Err(BenchError::Other(
2003 "No spec files found for sequential execution".to_string(),
2004 ));
2005 }
2006
2007 TerminalReporter::print_success(&format!("Loaded {} spec(s)", all_specs.len()));
2008
2009 let execution_order = if let Some(config_path) = &self.dependency_config {
2011 TerminalReporter::print_progress("Loading dependency configuration...");
2012 let config = SpecDependencyConfig::from_file(config_path)?;
2013
2014 if !config.disable_auto_detect && config.execution_order.is_empty() {
2015 self.detect_and_sort_specs(&all_specs)?
2017 } else {
2018 config.execution_order.iter().flat_map(|g| g.specs.clone()).collect()
2020 }
2021 } else {
2022 self.detect_and_sort_specs(&all_specs)?
2024 };
2025
2026 TerminalReporter::print_success(&format!(
2027 "Execution order: {}",
2028 execution_order
2029 .iter()
2030 .map(|p| p.file_name().unwrap_or_default().to_string_lossy().to_string())
2031 .collect::<Vec<_>>()
2032 .join(" → ")
2033 ));
2034
2035 let mut extracted_values = ExtractedValues::new();
2037 let total_specs = execution_order.len();
2038
2039 for (index, spec_path) in execution_order.iter().enumerate() {
2040 let spec_name = spec_path.file_name().unwrap_or_default().to_string_lossy().to_string();
2041
2042 TerminalReporter::print_progress(&format!(
2043 "[{}/{}] Executing spec: {}",
2044 index + 1,
2045 total_specs,
2046 spec_name
2047 ));
2048
2049 let spec = all_specs
2051 .iter()
2052 .find(|(p, _)| {
2053 p == spec_path
2054 || p.file_name() == spec_path.file_name()
2055 || p.file_name() == Some(spec_path.as_os_str())
2056 })
2057 .map(|(_, s)| s.clone())
2058 .ok_or_else(|| {
2059 BenchError::Other(format!("Spec not found: {}", spec_path.display()))
2060 })?;
2061
2062 let new_values = self.execute_single_spec(&spec, &spec_name, &extracted_values).await?;
2064
2065 extracted_values.merge(&new_values);
2067
2068 TerminalReporter::print_success(&format!(
2069 "[{}/{}] Completed: {} (extracted {} values)",
2070 index + 1,
2071 total_specs,
2072 spec_name,
2073 new_values.values.len()
2074 ));
2075 }
2076
2077 TerminalReporter::print_success(&format!(
2078 "Sequential execution complete: {} specs executed",
2079 total_specs
2080 ));
2081
2082 Ok(())
2083 }
2084
2085 fn detect_and_sort_specs(&self, specs: &[(PathBuf, OpenApiSpec)]) -> Result<Vec<PathBuf>> {
2087 TerminalReporter::print_progress("Auto-detecting spec dependencies...");
2088
2089 let mut detector = DependencyDetector::new();
2090 let dependencies = detector.detect_dependencies(specs);
2091
2092 if dependencies.is_empty() {
2093 TerminalReporter::print_progress("No dependencies detected, using file order");
2094 return Ok(specs.iter().map(|(p, _)| p.clone()).collect());
2095 }
2096
2097 TerminalReporter::print_progress(&format!(
2098 "Detected {} cross-spec dependencies",
2099 dependencies.len()
2100 ));
2101
2102 for dep in &dependencies {
2103 TerminalReporter::print_progress(&format!(
2104 " {} → {} (via field '{}')",
2105 dep.dependency_spec.file_name().unwrap_or_default().to_string_lossy(),
2106 dep.dependent_spec.file_name().unwrap_or_default().to_string_lossy(),
2107 dep.field_name
2108 ));
2109 }
2110
2111 topological_sort(specs, &dependencies)
2112 }
2113
2114 async fn execute_single_spec(
2116 &self,
2117 spec: &OpenApiSpec,
2118 spec_name: &str,
2119 _external_values: &ExtractedValues,
2120 ) -> Result<ExtractedValues> {
2121 let parser = SpecParser::from_spec(spec.clone());
2122
2123 if self.crud_flow {
2125 self.execute_crud_flow_with_extraction(&parser, spec_name).await
2127 } else {
2128 self.execute_standard_spec(&parser, spec_name).await?;
2130 Ok(ExtractedValues::new())
2131 }
2132 }
2133
2134 async fn execute_crud_flow_with_extraction(
2136 &self,
2137 parser: &SpecParser,
2138 spec_name: &str,
2139 ) -> Result<ExtractedValues> {
2140 let operations = parser.get_operations();
2141 let flows = CrudFlowDetector::detect_flows(&operations);
2142
2143 if flows.is_empty() {
2144 TerminalReporter::print_warning(&format!("No CRUD flows detected in {}", spec_name));
2145 return Ok(ExtractedValues::new());
2146 }
2147
2148 TerminalReporter::print_progress(&format!(
2149 " {} CRUD flow(s) in {}",
2150 flows.len(),
2151 spec_name
2152 ));
2153
2154 let mut handlebars = handlebars::Handlebars::new();
2156 handlebars.register_helper(
2158 "json",
2159 Box::new(
2160 |h: &handlebars::Helper,
2161 _: &handlebars::Handlebars,
2162 _: &handlebars::Context,
2163 _: &mut handlebars::RenderContext,
2164 out: &mut dyn handlebars::Output|
2165 -> handlebars::HelperResult {
2166 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2167 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2168 Ok(())
2169 },
2170 ),
2171 );
2172 let template = include_str!("templates/k6_crud_flow.hbs");
2173 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
2174
2175 let custom_headers = self.parse_headers()?;
2176 let config = self.build_crud_flow_config().unwrap_or_default();
2177
2178 let param_overrides = if let Some(params_file) = &self.params_file {
2180 let overrides = ParameterOverrides::from_file(params_file)?;
2181 Some(overrides)
2182 } else {
2183 None
2184 };
2185
2186 let duration_secs = Self::parse_duration(&self.duration)?;
2188 let scenario =
2189 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2190 let stages = scenario.generate_stages(duration_secs, self.vus);
2191
2192 let api_base_path = self.resolve_base_path(parser);
2194
2195 let mut all_headers = custom_headers.clone();
2197 if let Some(auth) = &self.auth {
2198 all_headers.insert("Authorization".to_string(), auth.clone());
2199 }
2200 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2201
2202 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2204
2205 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2206 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2210 serde_json::json!({
2211 "name": sanitized_name.clone(),
2212 "display_name": f.name,
2213 "base_path": f.base_path,
2214 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2215 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2217 let method_raw = if !parts.is_empty() {
2218 parts[0].to_uppercase()
2219 } else {
2220 "GET".to_string()
2221 };
2222 let method = if !parts.is_empty() {
2223 let m = parts[0].to_lowercase();
2224 if m == "delete" { "del".to_string() } else { m }
2226 } else {
2227 "get".to_string()
2228 };
2229 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2230 let path = if let Some(ref bp) = api_base_path {
2232 format!("{}{}", bp, raw_path)
2233 } else {
2234 raw_path.to_string()
2235 };
2236 let is_get_or_head = method == "get" || method == "head";
2237 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2239
2240 let body_value = if has_body {
2242 param_overrides.as_ref()
2243 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2244 .and_then(|oo| oo.body)
2245 .unwrap_or_else(|| serde_json::json!({}))
2246 } else {
2247 serde_json::json!({})
2248 };
2249
2250 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2252
2253 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2255 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2256
2257 serde_json::json!({
2258 "operation": s.operation,
2259 "method": method,
2260 "path": path,
2261 "extract": s.extract,
2262 "use_values": s.use_values,
2263 "use_body": s.use_body,
2264 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2265 "inject_attacks": s.inject_attacks,
2266 "attack_types": s.attack_types,
2267 "description": s.description,
2268 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2269 "is_get_or_head": is_get_or_head,
2270 "has_body": has_body,
2271 "body": processed_body.value,
2272 "body_is_dynamic": body_is_dynamic,
2273 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2274 })
2275 }).collect::<Vec<_>>(),
2276 })
2277 }).collect();
2278
2279 for flow_data in &flows_data {
2281 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2282 for step in steps {
2283 if let Some(placeholders_arr) =
2284 step.get("_placeholders").and_then(|p| p.as_array())
2285 {
2286 for p_str in placeholders_arr {
2287 if let Some(p_name) = p_str.as_str() {
2288 match p_name {
2289 "VU" => {
2290 all_placeholders.insert(DynamicPlaceholder::VU);
2291 }
2292 "Iteration" => {
2293 all_placeholders.insert(DynamicPlaceholder::Iteration);
2294 }
2295 "Timestamp" => {
2296 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2297 }
2298 "UUID" => {
2299 all_placeholders.insert(DynamicPlaceholder::UUID);
2300 }
2301 "Random" => {
2302 all_placeholders.insert(DynamicPlaceholder::Random);
2303 }
2304 "Counter" => {
2305 all_placeholders.insert(DynamicPlaceholder::Counter);
2306 }
2307 "Date" => {
2308 all_placeholders.insert(DynamicPlaceholder::Date);
2309 }
2310 "VuIter" => {
2311 all_placeholders.insert(DynamicPlaceholder::VuIter);
2312 }
2313 _ => {}
2314 }
2315 }
2316 }
2317 }
2318 }
2319 }
2320 }
2321
2322 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2324 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2325
2326 let security_testing_enabled = self.security_testing_enabled();
2328
2329 let data = serde_json::json!({
2330 "base_url": self.target,
2331 "flows": flows_data,
2332 "extract_fields": config.default_extract_fields,
2333 "duration_secs": duration_secs,
2334 "max_vus": self.vus,
2335 "auth_header": self.auth,
2336 "custom_headers": custom_headers,
2337 "skip_tls_verify": self.skip_tls_verify,
2338 "stages": stages.iter().map(|s| serde_json::json!({
2340 "duration": s.duration,
2341 "target": s.target,
2342 })).collect::<Vec<_>>(),
2343 "threshold_percentile": self.threshold_percentile,
2344 "threshold_ms": self.threshold_ms,
2345 "max_error_rate": self.max_error_rate,
2346 "abort_on_error": self.abort_on_error,
2347 "abort_on_error_rate": self.abort_on_error_rate,
2348 "headers": headers_json,
2349 "dynamic_imports": required_imports,
2350 "dynamic_globals": required_globals,
2351 "extracted_values_output_path": output_dir.join("extracted_values.json").to_string_lossy(),
2352 "security_testing_enabled": security_testing_enabled,
2354 "has_custom_headers": !custom_headers.is_empty(),
2355 });
2356
2357 let mut script = handlebars
2358 .render_template(template, &data)
2359 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2360
2361 if security_testing_enabled {
2363 script = self.generate_enhanced_script(&script)?;
2364 }
2365
2366 let script_path =
2368 self.output.join(format!("k6-{}-crud-flow.js", spec_name.replace('.', "_")));
2369
2370 std::fs::create_dir_all(self.output.clone())?;
2371 std::fs::write(&script_path, &script)?;
2372
2373 if !self.generate_only {
2374 let executor = K6Executor::new()?
2375 .with_local_ips(self.source_ips.join(","))
2376 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
2377 .with_capture_logs(!self.no_k6_logs);
2378 std::fs::create_dir_all(&output_dir)?;
2379
2380 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
2381
2382 let extracted = Self::parse_extracted_values(&output_dir)?;
2383 TerminalReporter::print_progress(&format!(
2384 " Extracted {} value(s) from {}",
2385 extracted.values.len(),
2386 spec_name
2387 ));
2388 return Ok(extracted);
2389 }
2390
2391 Ok(ExtractedValues::new())
2392 }
2393
2394 async fn execute_standard_spec(&self, parser: &SpecParser, spec_name: &str) -> Result<()> {
2396 let mut operations = if let Some(filter) = &self.operations {
2397 parser.filter_operations(filter)?
2398 } else {
2399 parser.get_operations()
2400 };
2401
2402 if let Some(exclude) = &self.exclude_operations {
2403 operations = parser.exclude_operations(operations, exclude)?;
2404 }
2405
2406 if operations.is_empty() {
2407 TerminalReporter::print_warning(&format!("No operations found in {}", spec_name));
2408 return Ok(());
2409 }
2410
2411 TerminalReporter::print_progress(&format!(
2412 " {} operations in {}",
2413 operations.len(),
2414 spec_name
2415 ));
2416
2417 let templates: Vec<_> = operations
2419 .iter()
2420 .map(RequestGenerator::generate_template)
2421 .collect::<Result<Vec<_>>>()?;
2422
2423 let custom_headers = self.parse_headers()?;
2425
2426 let base_path = self.resolve_base_path(parser);
2428
2429 let scenario =
2431 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2432
2433 let security_testing_enabled = self.security_testing_enabled();
2434
2435 let force_http1 = crate::request_gen::should_force_k6_http1(
2436 self.wafbench_verbatim,
2437 &templates,
2438 &custom_headers,
2439 );
2440
2441 let duration_secs = Self::parse_duration(&self.duration)?;
2442 let (per_op_metrics, per_op_warn) = crate::k6_gen::resolve_per_op_metrics(
2443 self.per_op_metrics,
2444 templates.len(),
2445 duration_secs,
2446 );
2447 if let Some(msg) = per_op_warn {
2448 TerminalReporter::print_warning(&msg);
2449 }
2450
2451 let k6_config = K6Config {
2452 target_url: self.target.clone(),
2453 base_path,
2454 scenario,
2455 duration_secs,
2456 max_vus: self.vus,
2457 threshold_percentile: self.threshold_percentile.clone(),
2458 threshold_ms: self.threshold_ms,
2459 max_error_rate: self.max_error_rate,
2460 auth_header: self.auth.clone(),
2461 custom_headers,
2462 skip_tls_verify: self.skip_tls_verify,
2463 security_testing_enabled,
2464 chunked_request_bodies: self.chunked_request_bodies,
2465 target_rps: self.target_rps,
2466 no_keep_alive: self.no_keep_alive,
2467 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
2469 .into_iter()
2470 .map(|ip| ip.to_string())
2471 .collect(),
2472 geo_source_headers: if self.geo_source_headers.is_empty()
2473 && !self.geo_source_ips.is_empty()
2474 {
2475 crate::conformance::self_test::default_geo_source_headers()
2476 } else {
2477 self.geo_source_headers.clone()
2478 },
2479 };
2480
2481 let generator = K6ScriptGenerator::new(k6_config, templates)
2482 .with_abort_valve(self.abort_on_error, self.abort_on_error_rate)
2483 .with_force_http1(force_http1)
2484 .with_per_op_metrics(per_op_metrics);
2485 let mut script = generator.generate()?;
2486
2487 let has_advanced_features = self.data_file.is_some()
2489 || self.error_rate.is_some()
2490 || self.security_test
2491 || self.parallel_create.is_some()
2492 || self.wafbench_dir.is_some();
2493
2494 if has_advanced_features {
2495 script = self.generate_enhanced_script(&script)?;
2496 }
2497
2498 let script_path = self.output.join(format!("k6-{}.js", spec_name.replace('.', "_")));
2500
2501 std::fs::create_dir_all(self.output.clone())?;
2502 std::fs::write(&script_path, &script)?;
2503
2504 if !self.generate_only {
2505 let executor = K6Executor::new()?
2508 .with_local_ips(self.source_ips.join(","))
2509 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
2510 .with_discard_response_bodies(self.discard_response_bodies)
2511 .with_force_http1(force_http1)
2512 .with_capture_logs(!self.no_k6_logs);
2513 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
2514 std::fs::create_dir_all(&output_dir)?;
2515
2516 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
2517 }
2518
2519 Ok(())
2520 }
2521
2522 async fn execute_crud_flow(&self, parser: &SpecParser) -> Result<()> {
2524 let config = self.build_crud_flow_config().unwrap_or_default();
2526
2527 let flows = if !config.flows.is_empty() {
2529 TerminalReporter::print_progress("Using custom flow configuration...");
2530 config.flows.clone()
2531 } else {
2532 TerminalReporter::print_progress("Detecting CRUD operations...");
2533 let operations = parser.get_operations();
2534 CrudFlowDetector::detect_flows(&operations)
2535 };
2536
2537 if flows.is_empty() {
2538 return Err(BenchError::Other(
2539 "No CRUD flows detected in spec. Ensure spec has POST/GET/PUT/DELETE operations on related paths.".to_string(),
2540 ));
2541 }
2542
2543 if config.flows.is_empty() {
2544 TerminalReporter::print_success(&format!("Detected {} CRUD flow(s)", flows.len()));
2545 } else {
2546 TerminalReporter::print_success(&format!("Loaded {} custom flow(s)", flows.len()));
2547 }
2548
2549 for flow in &flows {
2550 TerminalReporter::print_progress(&format!(
2551 " - {}: {} steps",
2552 flow.name,
2553 flow.steps.len()
2554 ));
2555 }
2556
2557 let mut handlebars = handlebars::Handlebars::new();
2559 handlebars.register_helper(
2561 "json",
2562 Box::new(
2563 |h: &handlebars::Helper,
2564 _: &handlebars::Handlebars,
2565 _: &handlebars::Context,
2566 _: &mut handlebars::RenderContext,
2567 out: &mut dyn handlebars::Output|
2568 -> handlebars::HelperResult {
2569 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2570 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2571 Ok(())
2572 },
2573 ),
2574 );
2575 let template = include_str!("templates/k6_crud_flow.hbs");
2576
2577 let custom_headers = self.parse_headers()?;
2578
2579 let param_overrides = if let Some(params_file) = &self.params_file {
2581 TerminalReporter::print_progress("Loading parameter overrides...");
2582 let overrides = ParameterOverrides::from_file(params_file)?;
2583 TerminalReporter::print_success(&format!(
2584 "Loaded parameter overrides ({} operation-specific, {} defaults)",
2585 overrides.operations.len(),
2586 if overrides.defaults.is_empty() { 0 } else { 1 }
2587 ));
2588 Some(overrides)
2589 } else {
2590 None
2591 };
2592
2593 let duration_secs = Self::parse_duration(&self.duration)?;
2595 let scenario =
2596 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2597 let stages = scenario.generate_stages(duration_secs, self.vus);
2598
2599 let api_base_path = self.resolve_base_path(parser);
2601 if let Some(ref bp) = api_base_path {
2602 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
2603 }
2604
2605 let mut all_headers = custom_headers.clone();
2607 if let Some(auth) = &self.auth {
2608 all_headers.insert("Authorization".to_string(), auth.clone());
2609 }
2610 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2611
2612 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2614
2615 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2616 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2621 serde_json::json!({
2622 "name": sanitized_name.clone(), "display_name": f.name, "base_path": f.base_path,
2625 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2626 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2628 let method_raw = if !parts.is_empty() {
2629 parts[0].to_uppercase()
2630 } else {
2631 "GET".to_string()
2632 };
2633 let method = if !parts.is_empty() {
2634 let m = parts[0].to_lowercase();
2635 if m == "delete" { "del".to_string() } else { m }
2637 } else {
2638 "get".to_string()
2639 };
2640 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2641 let path = if let Some(ref bp) = api_base_path {
2643 format!("{}{}", bp, raw_path)
2644 } else {
2645 raw_path.to_string()
2646 };
2647 let is_get_or_head = method == "get" || method == "head";
2648 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2650
2651 let body_value = if has_body {
2653 param_overrides.as_ref()
2654 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2655 .and_then(|oo| oo.body)
2656 .unwrap_or_else(|| serde_json::json!({}))
2657 } else {
2658 serde_json::json!({})
2659 };
2660
2661 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2663 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2668 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2669
2670 serde_json::json!({
2671 "operation": s.operation,
2672 "method": method,
2673 "path": path,
2674 "extract": s.extract,
2675 "use_values": s.use_values,
2676 "use_body": s.use_body,
2677 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2678 "inject_attacks": s.inject_attacks,
2679 "attack_types": s.attack_types,
2680 "description": s.description,
2681 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2682 "is_get_or_head": is_get_or_head,
2683 "has_body": has_body,
2684 "body": processed_body.value,
2685 "body_is_dynamic": body_is_dynamic,
2686 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2687 })
2688 }).collect::<Vec<_>>(),
2689 })
2690 }).collect();
2691
2692 for flow_data in &flows_data {
2694 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2695 for step in steps {
2696 if let Some(placeholders_arr) =
2697 step.get("_placeholders").and_then(|p| p.as_array())
2698 {
2699 for p_str in placeholders_arr {
2700 if let Some(p_name) = p_str.as_str() {
2701 match p_name {
2703 "VU" => {
2704 all_placeholders.insert(DynamicPlaceholder::VU);
2705 }
2706 "Iteration" => {
2707 all_placeholders.insert(DynamicPlaceholder::Iteration);
2708 }
2709 "Timestamp" => {
2710 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2711 }
2712 "UUID" => {
2713 all_placeholders.insert(DynamicPlaceholder::UUID);
2714 }
2715 "Random" => {
2716 all_placeholders.insert(DynamicPlaceholder::Random);
2717 }
2718 "Counter" => {
2719 all_placeholders.insert(DynamicPlaceholder::Counter);
2720 }
2721 "Date" => {
2722 all_placeholders.insert(DynamicPlaceholder::Date);
2723 }
2724 "VuIter" => {
2725 all_placeholders.insert(DynamicPlaceholder::VuIter);
2726 }
2727 _ => {}
2728 }
2729 }
2730 }
2731 }
2732 }
2733 }
2734 }
2735
2736 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2738 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2739
2740 let invalid_data_config = self.build_invalid_data_config();
2742 let error_injection_enabled = invalid_data_config.is_some();
2743 let error_rate = self.error_rate.unwrap_or(0.0);
2744 let error_types: Vec<String> = invalid_data_config
2745 .as_ref()
2746 .map(|c| c.error_types.iter().map(|t| format!("{:?}", t)).collect())
2747 .unwrap_or_default();
2748
2749 if error_injection_enabled {
2750 TerminalReporter::print_progress(&format!(
2751 "Error injection enabled ({}% rate)",
2752 (error_rate * 100.0) as u32
2753 ));
2754 }
2755
2756 let security_testing_enabled = self.security_testing_enabled();
2758
2759 let data = serde_json::json!({
2760 "base_url": self.target,
2761 "flows": flows_data,
2762 "extract_fields": config.default_extract_fields,
2763 "duration_secs": duration_secs,
2764 "max_vus": self.vus,
2765 "auth_header": self.auth,
2766 "custom_headers": custom_headers,
2767 "skip_tls_verify": self.skip_tls_verify,
2768 "stages": stages.iter().map(|s| serde_json::json!({
2770 "duration": s.duration,
2771 "target": s.target,
2772 })).collect::<Vec<_>>(),
2773 "threshold_percentile": self.threshold_percentile,
2774 "threshold_ms": self.threshold_ms,
2775 "max_error_rate": self.max_error_rate,
2776 "abort_on_error": self.abort_on_error,
2777 "abort_on_error_rate": self.abort_on_error_rate,
2778 "headers": headers_json,
2779 "dynamic_imports": required_imports,
2780 "dynamic_globals": required_globals,
2781 "extracted_values_output_path": self
2782 .output
2783 .join("crud_flow_extracted_values.json")
2784 .to_string_lossy(),
2785 "error_injection_enabled": error_injection_enabled,
2787 "error_rate": error_rate,
2788 "error_types": error_types,
2789 "security_testing_enabled": security_testing_enabled,
2791 "has_custom_headers": !custom_headers.is_empty(),
2792 });
2793
2794 let mut script = handlebars
2795 .render_template(template, &data)
2796 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2797
2798 if security_testing_enabled {
2800 script = self.generate_enhanced_script(&script)?;
2801 }
2802
2803 TerminalReporter::print_progress("Validating CRUD flow script...");
2805 let validation_errors = K6ScriptGenerator::validate_script(&script);
2806 if !validation_errors.is_empty() {
2807 TerminalReporter::print_error("CRUD flow script validation failed");
2808 for error in &validation_errors {
2809 eprintln!(" {}", error);
2810 }
2811 return Err(BenchError::Other(format!(
2812 "CRUD flow script validation failed with {} error(s)",
2813 validation_errors.len()
2814 )));
2815 }
2816
2817 TerminalReporter::print_success("CRUD flow script generated");
2818
2819 let script_path = if let Some(output) = &self.script_output {
2821 output.clone()
2822 } else {
2823 self.output.join("k6-crud-flow-script.js")
2824 };
2825
2826 if let Some(parent) = script_path.parent() {
2827 std::fs::create_dir_all(parent)?;
2828 }
2829 std::fs::write(&script_path, &script)?;
2830 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
2831
2832 if self.generate_only {
2833 println!("\nScript generated successfully. Run it with:");
2834 println!(" k6 run {}", script_path.display());
2835 return Ok(());
2836 }
2837
2838 TerminalReporter::print_progress("Executing CRUD flow test...");
2840 let executor = K6Executor::new()?
2841 .with_local_ips(self.source_ips.join(","))
2842 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
2843 .with_capture_logs(!self.no_k6_logs);
2844 std::fs::create_dir_all(&self.output)?;
2845
2846 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
2847
2848 let duration_secs = Self::parse_duration(&self.duration)?;
2849 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
2850
2851 Ok(())
2852 }
2853
2854 async fn execute_conformance_test(&self) -> Result<()> {
2856 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
2857 use crate::conformance::report::ConformanceReport;
2858 use crate::conformance::spec::ConformanceFeature;
2859
2860 TerminalReporter::print_progress("OpenAPI 3.0.0 Conformance Testing Mode");
2861
2862 TerminalReporter::print_progress(CONFORMANCE_REPLACES_LOAD_ADVISORY);
2863
2864 let categories = self.conformance_categories.as_ref().map(|cats_str| {
2866 cats_str
2867 .split(',')
2868 .filter_map(|s| {
2869 let trimmed = s.trim();
2870 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
2871 Some(canonical.to_string())
2872 } else {
2873 TerminalReporter::print_warning(&format!(
2874 "Unknown conformance category: '{}'. Valid categories: {}",
2875 trimmed,
2876 ConformanceFeature::cli_category_names()
2877 .iter()
2878 .map(|(cli, _)| *cli)
2879 .collect::<Vec<_>>()
2880 .join(", ")
2881 ));
2882 None
2883 }
2884 })
2885 .collect::<Vec<String>>()
2886 });
2887
2888 let custom_headers: Vec<(String, String)> = self
2890 .conformance_headers
2891 .iter()
2892 .filter_map(|h| {
2893 let (name, value) = h.split_once(':')?;
2894 Some((name.trim().to_string(), value.trim().to_string()))
2895 })
2896 .collect();
2897
2898 if !custom_headers.is_empty() {
2899 TerminalReporter::print_progress(&format!(
2900 "Using {} custom header(s) for authentication",
2901 custom_headers.len()
2902 ));
2903 }
2904
2905 if self.conformance_delay_ms > 0 {
2906 TerminalReporter::print_progress(&format!(
2907 "Using {}ms delay between conformance requests",
2908 self.conformance_delay_ms
2909 ));
2910 }
2911
2912 std::fs::create_dir_all(&self.output)?;
2914
2915 let config = ConformanceConfig {
2916 target_url: self.target.clone(),
2917 api_key: self.conformance_api_key.clone(),
2918 basic_auth: self.conformance_basic_auth.clone(),
2919 skip_tls_verify: self.skip_tls_verify,
2920 categories,
2921 base_path: self.base_path.clone(),
2922 custom_headers,
2923 output_dir: Some(self.output.clone()),
2924 all_operations: self.conformance_all_operations,
2925 custom_checks_file: self.conformance_custom.clone(),
2926 request_delay_ms: self.conformance_delay_ms,
2927 custom_filter: self.conformance_custom_filter.clone(),
2928 export_requests: self.export_requests,
2929 validate_requests: self.validate_requests,
2930 };
2931
2932 let mut resolved_base_path: Option<String> = None;
2940 let annotated_ops = if !self.spec.is_empty() {
2941 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
2942 let parser = SpecParser::from_file(&self.spec[0]).await?;
2943 resolved_base_path = self.resolve_base_path(&parser);
2944
2945 let mut operations = if let Some(filter) = &self.operations {
2950 parser.filter_operations(filter)?
2951 } else {
2952 parser.get_operations()
2953 };
2954 if let Some(exclude) = &self.exclude_operations {
2955 let before_count = operations.len();
2956 operations = parser.exclude_operations(operations, exclude)?;
2957 let excluded_count = before_count - operations.len();
2958 if excluded_count > 0 {
2959 TerminalReporter::print_progress(&format!(
2960 "Excluded {} operations matching '{}'",
2961 excluded_count, exclude
2962 ));
2963 }
2964 }
2965
2966 let annotated =
2967 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
2968 &operations,
2969 parser.spec(),
2970 );
2971 TerminalReporter::print_success(&format!(
2972 "Analyzed {} operations, found {} feature annotations",
2973 operations.len(),
2974 annotated.iter().map(|a| a.features.len()).sum::<usize>()
2975 ));
2976 Some(annotated)
2977 } else {
2978 None
2979 };
2980
2981 if self.conformance_self_test {
2988 let Some(ops) = annotated_ops else {
2989 TerminalReporter::print_error(
2990 "--conformance-self-test requires --spec; no operations to test",
2991 );
2992 return Ok(());
2993 };
2994 let cfg = crate::conformance::self_test::SelfTestConfig {
2995 target_url: self.target.clone(),
2996 skip_tls_verify: self.skip_tls_verify,
2997 timeout: std::time::Duration::from_secs(30),
2998 extra_headers: self
3002 .conformance_headers
3003 .iter()
3004 .filter_map(|h| {
3005 let (n, v) = h.split_once(':')?;
3006 Some((n.trim().to_string(), v.trim().to_string()))
3007 })
3008 .collect(),
3009 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
3010 base_path: resolved_base_path.clone(),
3014 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
3018 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
3019 geo_source_headers: if self.geo_source_headers.is_empty() {
3020 crate::conformance::self_test::default_geo_source_headers()
3021 } else {
3022 self.geo_source_headers.clone()
3023 },
3024 capture: if self.conformance_self_test_capture
3028 || self.validate_response_schemas
3029 || self.validate_requests
3030 {
3031 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
3042 } else {
3043 None
3044 },
3045 validate_response_schemas: self.validate_response_schemas,
3046 spec_label: self.spec.first().map(|p| {
3052 p.file_name()
3053 .map(|s| s.to_string_lossy().into_owned())
3054 .unwrap_or_else(|| p.to_string_lossy().into_owned())
3055 }),
3056 network_events: Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))),
3063 current_iteration: 1,
3064 };
3065 let capture_sink = cfg.capture.clone();
3066 let network_events_sink = cfg.network_events.clone();
3067 TerminalReporter::print_progress(&format!(
3068 "Self-test mode: driving {} operations with positive + per-category negative cases",
3069 ops.len()
3070 ));
3071 let target_iterations = self.conformance_self_test_iterations.max(1);
3078 let duration_budget = self
3079 .conformance_self_test_duration
3080 .as_ref()
3081 .map(|s| Self::parse_duration(s))
3082 .transpose()?
3083 .map(std::time::Duration::from_secs);
3084 let start = std::time::Instant::now();
3085 let deadline = duration_budget.map(|d| start + d);
3094 let mut cfg = cfg;
3098 cfg.current_iteration = 1;
3099 let mut report =
3100 crate::conformance::self_test::run_self_test_with_deadline(&ops, &cfg, deadline)
3101 .await
3102 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3103 let mut iter_done: u32 = 1;
3104 loop {
3105 let by_iter = iter_done >= target_iterations;
3106 let by_dur = duration_budget.map(|d| start.elapsed() >= d).unwrap_or(true);
3107 if by_iter && by_dur {
3108 break;
3109 }
3110 cfg.current_iteration = iter_done.saturating_add(1);
3111 let next = crate::conformance::self_test::run_self_test_with_deadline(
3112 &ops, &cfg, deadline,
3113 )
3114 .await
3115 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3116 report.merge_iteration(next);
3117 iter_done = iter_done.saturating_add(1);
3118 }
3119 if iter_done > 1 {
3120 TerminalReporter::print_progress(&format!(
3121 "Self-test repeated {} iteration(s) ({:.1?} elapsed)",
3122 iter_done,
3123 start.elapsed(),
3124 ));
3125 }
3126 let per_endpoint_summary: Vec<
3136 crate::conformance::per_endpoint_summary::PerEndpointSummary,
3137 >;
3138 if let Some(sink) = capture_sink {
3139 if let Ok(guard) = sink.lock() {
3140 let jsonl_path = self.output.join("conformance-self-test-requests.jsonl");
3141 let mut lines = String::with_capacity(guard.len() * 256);
3142 for entry in guard.iter() {
3143 if let Ok(line) = serde_json::to_string(entry) {
3144 lines.push_str(&line);
3145 lines.push('\n');
3146 }
3147 }
3148 let _ = std::fs::write(&jsonl_path, lines);
3149 let html_path = self.output.join("conformance-self-test-requests.html");
3150 let html =
3151 crate::conformance::capture_html::render_capture_html(guard.as_slice());
3152 let _ = std::fs::write(&html_path, html);
3153
3154 per_endpoint_summary =
3158 crate::conformance::per_endpoint_summary::build_summary(guard.as_slice());
3159 let summary_path = self.output.join("conformance-per-endpoint.json");
3160 if let Ok(json) = serde_json::to_string_pretty(&per_endpoint_summary) {
3161 let _ = std::fs::write(&summary_path, json);
3162 TerminalReporter::print_progress(&format!(
3163 "Self-test request/response capture written to {} ({} entries) + {} + {}",
3164 jsonl_path.display(),
3165 guard.len(),
3166 html_path.display(),
3167 summary_path.display(),
3168 ));
3169 } else {
3170 TerminalReporter::print_progress(&format!(
3171 "Self-test request/response capture written to {} ({} entries) + {}",
3172 jsonl_path.display(),
3173 guard.len(),
3174 html_path.display(),
3175 ));
3176 }
3177 } else {
3178 per_endpoint_summary = Vec::new();
3179 }
3180 } else {
3181 per_endpoint_summary = Vec::new();
3182 }
3183 TerminalReporter::print_progress(&report.render_summary());
3184 if let Some(sink) = network_events_sink {
3191 if let Ok(guard) = sink.lock() {
3192 let path = self.output.join("conformance-network-events.json");
3193 if let Ok(json) = serde_json::to_string_pretty(&*guard) {
3194 let _ = std::fs::write(&path, json);
3195 if guard.is_empty() {
3196 TerminalReporter::print_progress(
3197 "No wire-level network failures during self-test (file written empty)",
3198 );
3199 } else {
3200 TerminalReporter::print_warning(&format!(
3201 "Recorded {} wire-level network event(s) to {}",
3202 guard.len(),
3203 path.display()
3204 ));
3205 }
3206 }
3207 }
3208 }
3209 let json_path = self.output.join("conformance-self-test.json");
3213 if let Ok(json) = serde_json::to_string_pretty(&report) {
3214 let _ = std::fs::write(&json_path, json);
3215 TerminalReporter::print_progress(&format!(
3216 "Self-test report written to {}",
3217 json_path.display()
3218 ));
3219 }
3220 let issues = report.definite_issues();
3224 let issues_path = self.output.join("conformance-definite-issues.json");
3225 if let Ok(json) = serde_json::to_string_pretty(&issues) {
3226 if std::fs::write(&issues_path, json).is_ok() && !issues.is_empty() {
3227 TerminalReporter::print_warning(&format!(
3228 "{} definite issue(s) — see {}",
3229 issues.len(),
3230 issues_path.display()
3231 ));
3232 }
3233 }
3234 let owasp_accepted = report.owasp_accepted_probes();
3237 if !owasp_accepted.is_empty() {
3238 let owasp_path = self.output.join("conformance-owasp-accepted.json");
3239 if let Ok(json) = serde_json::to_string_pretty(&owasp_accepted) {
3240 if std::fs::write(&owasp_path, json).is_ok() {
3241 TerminalReporter::print_warning(&format!(
3242 "{} owasp injection probe(s) accepted by the target — see {}",
3243 owasp_accepted.len(),
3244 owasp_path.display()
3245 ));
3246 }
3247 }
3248 }
3249 if let Some(status) = report.detect_target_misconfiguration() {
3258 let hint = match status {
3259 404 => " Likely cause: spec paths don't match deployed routes — check --base-path and the spec's `servers` block.",
3260 401 | 403 => " Likely cause: authentication header is missing or invalid — check --conformance-header.",
3261 _ => "",
3262 };
3263 TerminalReporter::print_warning(&format!(
3264 "Self-test misconfiguration: every positive case returned {status}.{hint} Negative results below are meaningless under this condition."
3265 ));
3266 } else if !report.all_passed() {
3267 TerminalReporter::print_warning(
3268 "Self-test detected gaps — server let through at least one request that should have been a 4xx",
3269 );
3270 } else {
3271 TerminalReporter::print_success(
3272 "Self-test passed — all positive cases accepted and all negative cases rejected",
3273 );
3274 }
3275 let html_path = self.output.join("conformance-report.html");
3282 let audit_path = self.output.join("conformance-spec-audit.json");
3283 let audit_value = std::fs::read_to_string(&audit_path)
3284 .ok()
3285 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok());
3286 let render_opts = crate::conformance::report_html::RenderOptions {
3291 missed_cap: match self.report_missed_cap {
3292 Some(0) => None,
3293 Some(n) => Some(n as usize),
3294 None => Some(200),
3295 },
3296 };
3297 let mut html = crate::conformance::report_html::render_html_with_options(
3298 &report,
3299 audit_value.as_ref(),
3300 &render_opts,
3301 );
3302 let summary_section = crate::conformance::per_endpoint_summary::render_html_section(
3308 &per_endpoint_summary,
3309 );
3310 if !summary_section.is_empty() {
3311 if let Some(idx) = html.rfind("</body>") {
3312 html.insert_str(idx, &summary_section);
3313 } else {
3314 html.push_str(&summary_section);
3315 }
3316 }
3317 if std::fs::write(&html_path, html).is_ok() {
3318 TerminalReporter::print_progress(&format!(
3319 "HTML report written to {}",
3320 html_path.display()
3321 ));
3322 }
3323
3324 if self.validate_requests && !self.spec.is_empty() {
3336 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3337 &self.spec,
3338 &self.output,
3339 self.base_path.as_deref(),
3340 )
3341 .await?;
3342 if n > 0 {
3343 TerminalReporter::print_warning(&format!(
3344 "{} emitted request(s) recorded against the spec — see conformance-request-violations.json",
3345 n
3346 ));
3347 }
3348 }
3349 return Ok(());
3350 }
3351
3352 if self.validate_requests && !self.spec.is_empty() {
3354 TerminalReporter::print_progress("Validating requests against OpenAPI spec...");
3355 let violation_count = crate::conformance::request_validator::run_request_validation(
3356 &self.spec,
3357 self.conformance_custom.as_deref(),
3358 self.base_path.as_deref(),
3359 &self.output,
3360 )
3361 .await?;
3362 if violation_count > 0 {
3363 TerminalReporter::print_warning(&format!(
3364 "{} request validation violation(s) found — see conformance-request-violations.json",
3365 violation_count
3366 ));
3367 } else {
3368 TerminalReporter::print_success("All requests conform to the OpenAPI spec");
3369 }
3370 }
3371
3372 if self.generate_only || self.use_k6 {
3374 let script = if let Some(annotated) = &annotated_ops {
3375 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
3376 config,
3377 annotated.clone(),
3378 );
3379 let op_count = gen.operation_count();
3380 let (script, check_count) = gen.generate()?;
3381 TerminalReporter::print_success(&format!(
3382 "Conformance: {} operations analyzed, {} unique checks generated",
3383 op_count, check_count
3384 ));
3385 script
3386 } else {
3387 let generator = ConformanceGenerator::new(config);
3388 generator.generate()?
3389 };
3390
3391 let script_path = self.output.join("k6-conformance.js");
3392 std::fs::write(&script_path, &script).map_err(|e| {
3393 BenchError::Other(format!("Failed to write conformance script: {}", e))
3394 })?;
3395 TerminalReporter::print_success(&format!(
3396 "Conformance script generated: {}",
3397 script_path.display()
3398 ));
3399
3400 if self.generate_only {
3401 println!("\nScript generated. Run with:");
3402 println!(" k6 run {}", script_path.display());
3403 return Ok(());
3404 }
3405
3406 if !K6Executor::is_k6_installed() {
3408 TerminalReporter::print_error("k6 is not installed");
3409 TerminalReporter::print_warning(
3410 "Install k6 from: https://k6.io/docs/get-started/installation/",
3411 );
3412 return Err(BenchError::K6NotFound);
3413 }
3414
3415 K6Executor::warn_if_pre_v1().await;
3416 TerminalReporter::print_progress("Running conformance tests via k6...");
3417 let executor = K6Executor::new()?
3418 .with_local_ips(self.source_ips.join(","))
3419 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
3420 .with_capture_logs(!self.no_k6_logs);
3421 executor.execute(&script_path, Some(&self.output), self.verbose).await?;
3422
3423 let report_path = self.output.join("conformance-report.json");
3424 if report_path.exists() {
3425 let report = ConformanceReport::from_file(&report_path)?;
3426 report.print_report_with_options(self.conformance_all_operations);
3427 self.save_conformance_report(&report, &report_path)?;
3428 } else {
3429 TerminalReporter::print_warning(
3430 "Conformance report not generated (k6 handleSummary may not have run)",
3431 );
3432 }
3433
3434 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
3446 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3447 &self.spec,
3448 &self.output,
3449 self.base_path.as_deref(),
3450 )
3451 .await?;
3452 if n > 0 {
3453 TerminalReporter::print_warning(&format!(
3454 "{} emitted request(s) violated the spec — see conformance-request-violations.json",
3455 n
3456 ));
3457 }
3458 }
3459
3460 return Ok(());
3461 }
3462
3463 TerminalReporter::print_progress("Running conformance tests (native executor)...");
3465
3466 let mut executor = crate::conformance::executor::NativeConformanceExecutor::new(config)?;
3467
3468 let custom_only = annotated_ops.is_none() && self.conformance_custom.is_some();
3478 executor = if let Some(annotated) = &annotated_ops {
3479 executor.with_spec_driven_checks(annotated)
3480 } else if custom_only {
3481 executor
3482 } else {
3483 executor.with_reference_checks()
3484 };
3485 executor = executor.with_custom_checks()?;
3486
3487 TerminalReporter::print_success(&format!(
3488 "Executing {} conformance checks...",
3489 executor.check_count()
3490 ));
3491
3492 let report = executor.execute().await?;
3493 report.print_report_with_options(self.conformance_all_operations);
3494
3495 let failure_details = report.failure_details();
3497 if !failure_details.is_empty() {
3498 let details_path = self.output.join("conformance-failure-details.json");
3499 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
3500 let _ = std::fs::write(&details_path, json);
3501 TerminalReporter::print_success(&format!(
3502 "Failure details saved to: {}",
3503 details_path.display()
3504 ));
3505 }
3506 }
3507
3508 let report_path = self.output.join("conformance-report.json");
3510 let report_json = serde_json::to_string_pretty(&report.to_json())
3511 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
3512 std::fs::write(&report_path, &report_json)
3513 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
3514 TerminalReporter::print_success(&format!("Report saved to: {}", report_path.display()));
3515
3516 self.save_conformance_report(&report, &report_path)?;
3517
3518 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
3529 let n =
3530 crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3531 &self.spec,
3532 &self.output,
3533 self.base_path.as_deref(),
3534 )
3535 .await?;
3536 if n > 0 {
3537 TerminalReporter::print_warning(&format!(
3538 "{} emitted request(s) violated the spec — see conformance-request-violations.json",
3539 n
3540 ));
3541 }
3542 }
3543
3544 Ok(())
3545 }
3546
3547 fn save_conformance_report(
3549 &self,
3550 report: &crate::conformance::report::ConformanceReport,
3551 report_path: &Path,
3552 ) -> Result<()> {
3553 if self.conformance_report_format == "sarif" {
3554 use crate::conformance::sarif::ConformanceSarifReport;
3555 ConformanceSarifReport::write(report, &self.target, &self.conformance_report)?;
3556 TerminalReporter::print_success(&format!(
3557 "SARIF report saved to: {}",
3558 self.conformance_report.display()
3559 ));
3560 } else if self.conformance_report != *report_path {
3561 std::fs::copy(report_path, &self.conformance_report)?;
3562 TerminalReporter::print_success(&format!(
3563 "Report saved to: {}",
3564 self.conformance_report.display()
3565 ));
3566 }
3567 Ok(())
3568 }
3569
3570 async fn execute_multi_target_self_test(&self, targets_file: &Path) -> Result<()> {
3582 use crate::conformance::self_test::SelfTestConfig;
3583
3584 TerminalReporter::print_progress("Multi-target conformance self-test mode");
3585 let targets = parse_targets_file(targets_file)?;
3586 if targets.is_empty() {
3587 return Err(BenchError::Other("No targets found in file".to_string()));
3588 }
3589 TerminalReporter::print_success(&format!("Loaded {} target(s)", targets.len()));
3590
3591 let annotated_ops = if !self.spec.is_empty() {
3593 let parser = SpecParser::from_file(&self.spec[0]).await?;
3594 let operations = parser.get_operations();
3595 Some(
3596 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
3597 &operations,
3598 parser.spec(),
3599 ),
3600 )
3601 } else {
3602 return Err(BenchError::Other("--conformance-self-test requires --spec".to_string()));
3603 };
3604 let Some(ops) = annotated_ops else {
3605 unreachable!()
3606 };
3607
3608 std::fs::create_dir_all(&self.output)?;
3609 let resolved_base_path = self.base_path.clone();
3610 let target_iterations = self.conformance_self_test_iterations.max(1);
3611 let duration_budget = self
3612 .conformance_self_test_duration
3613 .as_ref()
3614 .map(|s| Self::parse_duration(s))
3615 .transpose()?
3616 .map(std::time::Duration::from_secs);
3617
3618 for (idx, target) in targets.iter().enumerate() {
3619 let target_dir = self.output.join(format!("target_{}", idx));
3620 std::fs::create_dir_all(&target_dir)?;
3621 TerminalReporter::print_progress(&format!(
3622 "[target {}/{}] {}",
3623 idx + 1,
3624 targets.len(),
3625 target.url
3626 ));
3627
3628 let merged_headers: Vec<(String, String)> = self
3629 .conformance_headers
3630 .iter()
3631 .filter_map(|h| {
3632 let (n, v) = h.split_once(':')?;
3633 Some((n.trim().to_string(), v.trim().to_string()))
3634 })
3635 .collect();
3636
3637 let cfg = SelfTestConfig {
3638 target_url: target.url.clone(),
3639 skip_tls_verify: self.skip_tls_verify,
3640 timeout: std::time::Duration::from_secs(30),
3641 extra_headers: merged_headers,
3642 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
3643 base_path: resolved_base_path.clone(),
3644 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
3645 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
3646 geo_source_headers: if self.geo_source_headers.is_empty() {
3647 crate::conformance::self_test::default_geo_source_headers()
3648 } else {
3649 self.geo_source_headers.clone()
3650 },
3651 capture: if self.conformance_self_test_capture
3652 || self.validate_response_schemas
3653 || self.validate_requests
3654 {
3655 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
3659 } else {
3660 None
3661 },
3662 validate_response_schemas: self.validate_response_schemas,
3663 spec_label: self.spec.first().map(|p| {
3664 p.file_name()
3665 .map(|s| s.to_string_lossy().into_owned())
3666 .unwrap_or_else(|| p.to_string_lossy().into_owned())
3667 }),
3668 network_events: Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))),
3669 current_iteration: 1,
3670 };
3671 let capture_sink = cfg.capture.clone();
3672 let network_events_sink = cfg.network_events.clone();
3673
3674 let start = std::time::Instant::now();
3675 let deadline = duration_budget.map(|d| start + d);
3679 let mut cfg = cfg;
3683 cfg.current_iteration = 1;
3684 let mut report =
3685 crate::conformance::self_test::run_self_test_with_deadline(&ops, &cfg, deadline)
3686 .await
3687 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3688 let mut iter_done: u32 = 1;
3689 loop {
3690 let by_iter = iter_done >= target_iterations;
3691 let by_dur = duration_budget.map(|d| start.elapsed() >= d).unwrap_or(true);
3692 if by_iter && by_dur {
3693 break;
3694 }
3695 cfg.current_iteration = iter_done.saturating_add(1);
3696 let next = crate::conformance::self_test::run_self_test_with_deadline(
3697 &ops, &cfg, deadline,
3698 )
3699 .await
3700 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3701 report.merge_iteration(next);
3702 iter_done = iter_done.saturating_add(1);
3703 }
3704 if iter_done > 1 {
3705 TerminalReporter::print_progress(&format!(
3706 " ran {} iteration(s) in {:.1?}",
3707 iter_done,
3708 start.elapsed(),
3709 ));
3710 }
3711
3712 if let Some(sink) = capture_sink {
3714 if let Ok(guard) = sink.lock() {
3715 let jsonl = target_dir.join("conformance-self-test-requests.jsonl");
3716 let mut lines = String::with_capacity(guard.len() * 256);
3717 for entry in guard.iter() {
3718 if let Ok(line) = serde_json::to_string(entry) {
3719 lines.push_str(&line);
3720 lines.push('\n');
3721 }
3722 }
3723 let _ = std::fs::write(&jsonl, lines);
3724 }
3725 }
3726 if let Some(sink) = network_events_sink {
3727 if let Ok(guard) = sink.lock() {
3728 let path = target_dir.join("conformance-network-events.json");
3729 if let Ok(json) = serde_json::to_string_pretty(&*guard) {
3730 let _ = std::fs::write(&path, json);
3731 if !guard.is_empty() {
3732 TerminalReporter::print_warning(&format!(
3733 " recorded {} wire-level network event(s)",
3734 guard.len()
3735 ));
3736 }
3737 }
3738 }
3739 }
3740
3741 let json_path = target_dir.join("conformance-self-test.json");
3742 if let Ok(json) = serde_json::to_string_pretty(&report) {
3743 let _ = std::fs::write(&json_path, json);
3744 }
3745 let issues = report.definite_issues();
3748 if let Ok(json) = serde_json::to_string_pretty(&issues) {
3749 let issues_path = target_dir.join("conformance-definite-issues.json");
3750 if std::fs::write(&issues_path, json).is_ok() && !issues.is_empty() {
3751 TerminalReporter::print_warning(&format!(
3752 " {} definite issue(s) — see {}",
3753 issues.len(),
3754 issues_path.display()
3755 ));
3756 }
3757 }
3758 let owasp_accepted = report.owasp_accepted_probes();
3760 if !owasp_accepted.is_empty() {
3761 if let Ok(json) = serde_json::to_string_pretty(&owasp_accepted) {
3762 let owasp_path = target_dir.join("conformance-owasp-accepted.json");
3763 if std::fs::write(&owasp_path, json).is_ok() {
3764 TerminalReporter::print_warning(&format!(
3765 " {} owasp injection probe(s) accepted by the target — see {}",
3766 owasp_accepted.len(),
3767 owasp_path.display()
3768 ));
3769 }
3770 }
3771 }
3772 TerminalReporter::print_progress(&report.render_summary());
3773
3774 if self.validate_requests {
3783 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3784 &self.spec,
3785 &target_dir,
3786 self.base_path.as_deref(),
3787 )
3788 .await?;
3789 if n > 0 {
3790 TerminalReporter::print_warning(&format!(
3791 " {} emitted request(s) violated the spec — see {}/conformance-request-violations.json",
3792 n,
3793 target_dir.display(),
3794 ));
3795 }
3796 }
3797 }
3798
3799 Ok(())
3800 }
3801
3802 async fn execute_multi_target_conformance(&self, targets_file: &Path) -> Result<()> {
3808 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
3809 use crate::conformance::report::ConformanceReport;
3810 use crate::conformance::spec::ConformanceFeature;
3811
3812 TerminalReporter::print_progress("Multi-target OpenAPI 3.0.0 Conformance Testing Mode");
3813
3814 TerminalReporter::print_progress("Parsing targets file...");
3816 let targets = parse_targets_file(targets_file)?;
3817 let num_targets = targets.len();
3818 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
3819
3820 if targets.is_empty() {
3821 return Err(BenchError::Other("No targets found in file".to_string()));
3822 }
3823
3824 TerminalReporter::print_progress(CONFORMANCE_REPLACES_LOAD_ADVISORY);
3825
3826 let categories = self.conformance_categories.as_ref().map(|cats_str| {
3828 cats_str
3829 .split(',')
3830 .filter_map(|s| {
3831 let trimmed = s.trim();
3832 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
3833 Some(canonical.to_string())
3834 } else {
3835 TerminalReporter::print_warning(&format!(
3836 "Unknown conformance category: '{}'. Valid categories: {}",
3837 trimmed,
3838 ConformanceFeature::cli_category_names()
3839 .iter()
3840 .map(|(cli, _)| *cli)
3841 .collect::<Vec<_>>()
3842 .join(", ")
3843 ));
3844 None
3845 }
3846 })
3847 .collect::<Vec<String>>()
3848 });
3849
3850 let base_custom_headers: Vec<(String, String)> = self
3852 .conformance_headers
3853 .iter()
3854 .filter_map(|h| {
3855 let (name, value) = h.split_once(':')?;
3856 Some((name.trim().to_string(), value.trim().to_string()))
3857 })
3858 .collect();
3859
3860 if !base_custom_headers.is_empty() {
3861 TerminalReporter::print_progress(&format!(
3862 "Using {} base custom header(s) for authentication",
3863 base_custom_headers.len()
3864 ));
3865 }
3866
3867 let annotated_ops = if !self.spec.is_empty() {
3869 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
3870 let parser = SpecParser::from_file(&self.spec[0]).await?;
3871 let operations = parser.get_operations();
3872 let annotated =
3873 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
3874 &operations,
3875 parser.spec(),
3876 );
3877 TerminalReporter::print_success(&format!(
3878 "Analyzed {} operations, found {} feature annotations",
3879 operations.len(),
3880 annotated.iter().map(|a| a.features.len()).sum::<usize>()
3881 ));
3882 Some(annotated)
3883 } else {
3884 None
3885 };
3886
3887 std::fs::create_dir_all(&self.output)?;
3889
3890 struct TargetResult {
3892 url: String,
3893 passed: usize,
3894 failed: usize,
3895 elapsed: std::time::Duration,
3896 report_json: serde_json::Value,
3897 owasp_coverage: Vec<crate::conformance::report::OwaspCoverageEntry>,
3898 }
3899
3900 let mut target_results: Vec<TargetResult> = Vec::with_capacity(num_targets);
3901 let total_start = std::time::Instant::now();
3902
3903 for (idx, target) in targets.iter().enumerate() {
3904 tracing::info!(
3905 "Running conformance tests against target {}/{}: {}",
3906 idx + 1,
3907 num_targets,
3908 target.url
3909 );
3910 TerminalReporter::print_progress(&format!(
3911 "\n--- Target {}/{}: {} ---",
3912 idx + 1,
3913 num_targets,
3914 target.url
3915 ));
3916
3917 let mut merged_headers = base_custom_headers.clone();
3919 if let Some(ref target_headers) = target.headers {
3920 for (name, value) in target_headers {
3921 if let Some(existing) = merged_headers.iter_mut().find(|(n, _)| n == name) {
3923 existing.1 = value.clone();
3924 } else {
3925 merged_headers.push((name.clone(), value.clone()));
3926 }
3927 }
3928 }
3929 if let Some(ref auth) = target.auth {
3931 if let Some(existing) =
3932 merged_headers.iter_mut().find(|(n, _)| n.eq_ignore_ascii_case("Authorization"))
3933 {
3934 existing.1 = auth.clone();
3935 } else {
3936 merged_headers.push(("Authorization".to_string(), auth.clone()));
3937 }
3938 }
3939
3940 let target_dir = self.output.join(format!("target_{}", idx));
3946 std::fs::create_dir_all(&target_dir)?;
3947
3948 let config = ConformanceConfig {
3949 target_url: target.url.clone(),
3950 api_key: self.conformance_api_key.clone(),
3951 basic_auth: self.conformance_basic_auth.clone(),
3952 skip_tls_verify: self.skip_tls_verify,
3953 categories: categories.clone(),
3954 base_path: self.base_path.clone(),
3955 custom_headers: merged_headers,
3956 output_dir: Some(target_dir.clone()),
3957 all_operations: self.conformance_all_operations,
3958 custom_checks_file: self.conformance_custom.clone(),
3959 request_delay_ms: self.conformance_delay_ms,
3960 custom_filter: self.conformance_custom_filter.clone(),
3961 export_requests: self.export_requests,
3962 validate_requests: self.validate_requests,
3963 };
3964
3965 let target_start = std::time::Instant::now();
3966 let report = if self.use_k6 {
3967 if !K6Executor::is_k6_installed() {
3968 TerminalReporter::print_error("k6 is not installed");
3969 TerminalReporter::print_warning(
3970 "Install k6 from: https://k6.io/docs/get-started/installation/",
3971 );
3972 return Err(BenchError::K6NotFound);
3973 }
3974 K6Executor::warn_if_pre_v1().await;
3975
3976 let script = if let Some(ref annotated) = annotated_ops {
3977 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
3978 config.clone(),
3979 annotated.clone(),
3980 );
3981 let (script, _check_count) = gen.generate()?;
3982 script
3983 } else {
3984 let generator = ConformanceGenerator::new(config.clone());
3985 generator.generate()?
3986 };
3987
3988 let script_path = target_dir.join("k6-conformance.js");
3989 std::fs::write(&script_path, &script).map_err(|e| {
3990 BenchError::Other(format!("Failed to write conformance script: {}", e))
3991 })?;
3992 TerminalReporter::print_success(&format!(
3993 "Conformance script generated: {}",
3994 script_path.display()
3995 ));
3996
3997 TerminalReporter::print_progress(&format!(
3998 "Running conformance tests via k6 against {}...",
3999 target.url
4000 ));
4001 let k6 = K6Executor::new()?
4002 .with_local_ips(self.source_ips.join(","))
4003 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
4004 .with_capture_logs(!self.no_k6_logs);
4005 let api_port = 6565u16.saturating_add(idx as u16);
4007 k6.execute_with_port(&script_path, Some(&target_dir), self.verbose, Some(api_port))
4008 .await?;
4009
4010 let report_path = target_dir.join("conformance-report.json");
4011 if report_path.exists() {
4012 ConformanceReport::from_file(&report_path)?
4013 } else {
4014 TerminalReporter::print_warning(&format!(
4015 "Conformance report not generated for target {} (k6 handleSummary may not have run)",
4016 target.url
4017 ));
4018 continue;
4019 }
4020 } else {
4021 let mut executor =
4022 crate::conformance::executor::NativeConformanceExecutor::new(config)?;
4023
4024 let custom_only = annotated_ops.is_none() && self.conformance_custom.is_some();
4027 executor = if let Some(ref annotated) = annotated_ops {
4028 executor.with_spec_driven_checks(annotated)
4029 } else if custom_only {
4030 executor
4031 } else {
4032 executor.with_reference_checks()
4033 };
4034 executor = executor.with_custom_checks()?;
4035
4036 TerminalReporter::print_success(&format!(
4037 "Executing {} conformance checks against {}...",
4038 executor.check_count(),
4039 target.url
4040 ));
4041
4042 executor.execute().await?
4043 };
4044 let target_elapsed = target_start.elapsed();
4045
4046 let report_json = report.to_json();
4047
4048 let passed = report_json["summary"]["passed"].as_u64().unwrap_or(0) as usize;
4050 let failed = report_json["summary"]["failed"].as_u64().unwrap_or(0) as usize;
4051 let total_checks = passed + failed;
4052 let rate = if total_checks == 0 {
4053 0.0
4054 } else {
4055 (passed as f64 / total_checks as f64) * 100.0
4056 };
4057
4058 TerminalReporter::print_success(&format!(
4059 "Target {}: {}/{} passed ({:.1}%) in {:.1}s",
4060 target.url,
4061 passed,
4062 total_checks,
4063 rate,
4064 target_elapsed.as_secs_f64()
4065 ));
4066
4067 let target_report_path = target_dir.join("conformance-report.json");
4069 let report_str = serde_json::to_string_pretty(&report_json)
4070 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
4071 std::fs::write(&target_report_path, &report_str)
4072 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
4073
4074 let failure_details = report.failure_details();
4076 if !failure_details.is_empty() {
4077 let details_path = target_dir.join("conformance-failure-details.json");
4078 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
4079 let _ = std::fs::write(&details_path, json);
4080 }
4081 }
4082
4083 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
4090 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
4091 &self.spec,
4092 &target_dir,
4093 self.base_path.as_deref(),
4094 )
4095 .await?;
4096 if n > 0 {
4097 TerminalReporter::print_warning(&format!(
4098 "Target {}: {} emitted request(s) violated the spec — see {}/conformance-request-violations.json",
4099 target.url,
4100 n,
4101 target_dir.display()
4102 ));
4103 }
4104 }
4105
4106 let owasp_coverage = report.owasp_coverage_data();
4108
4109 target_results.push(TargetResult {
4110 url: target.url.clone(),
4111 passed,
4112 failed,
4113 elapsed: target_elapsed,
4114 report_json,
4115 owasp_coverage,
4116 });
4117 }
4118
4119 let total_elapsed = total_start.elapsed();
4120
4121 println!("\n{}", "=".repeat(80));
4123 println!(" Multi-Target Conformance Summary");
4124 println!("{}", "=".repeat(80));
4125 println!(
4126 " {:<40} {:>8} {:>8} {:>8} {:>8}",
4127 "Target URL", "Passed", "Failed", "Rate", "Time"
4128 );
4129 println!(" {}", "-".repeat(76));
4130
4131 let mut total_passed = 0usize;
4132 let mut total_failed = 0usize;
4133
4134 for result in &target_results {
4135 let total_checks = result.passed + result.failed;
4136 let rate = if total_checks == 0 {
4137 0.0
4138 } else {
4139 (result.passed as f64 / total_checks as f64) * 100.0
4140 };
4141
4142 let display_url = if result.url.len() > 38 {
4144 format!("{}...", &result.url[..35])
4145 } else {
4146 result.url.clone()
4147 };
4148
4149 println!(
4150 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
4151 display_url,
4152 result.passed,
4153 result.failed,
4154 rate,
4155 result.elapsed.as_secs_f64()
4156 );
4157
4158 total_passed += result.passed;
4159 total_failed += result.failed;
4160 }
4161
4162 let grand_total = total_passed + total_failed;
4163 let overall_rate = if grand_total == 0 {
4164 0.0
4165 } else {
4166 (total_passed as f64 / grand_total as f64) * 100.0
4167 };
4168
4169 println!(" {}", "-".repeat(76));
4170 println!(
4171 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
4172 format!("TOTAL ({} targets)", num_targets),
4173 total_passed,
4174 total_failed,
4175 overall_rate,
4176 total_elapsed.as_secs_f64()
4177 );
4178 println!("{}", "=".repeat(80));
4179
4180 for result in &target_results {
4182 println!("\n OWASP API Security Top 10 Coverage for {}:", result.url);
4183 for entry in &result.owasp_coverage {
4184 let status = if !entry.tested {
4185 "-"
4186 } else if entry.all_passed {
4187 "pass"
4188 } else {
4189 "FAIL"
4190 };
4191 let via = if entry.via_categories.is_empty() {
4192 String::new()
4193 } else {
4194 format!(" (via {})", entry.via_categories.join(", "))
4195 };
4196 println!(" {:<12} {:<40} {}{}", entry.id, entry.name, status, via);
4197 }
4198 }
4199
4200 let per_target_summaries: Vec<serde_json::Value> = target_results
4202 .iter()
4203 .enumerate()
4204 .map(|(idx, r)| {
4205 let total_checks = r.passed + r.failed;
4206 let rate = if total_checks == 0 {
4207 0.0
4208 } else {
4209 (r.passed as f64 / total_checks as f64) * 100.0
4210 };
4211 let owasp_json: Vec<serde_json::Value> = r
4212 .owasp_coverage
4213 .iter()
4214 .map(|e| {
4215 serde_json::json!({
4216 "id": e.id,
4217 "name": e.name,
4218 "tested": e.tested,
4219 "all_passed": e.all_passed,
4220 "via_categories": e.via_categories,
4221 })
4222 })
4223 .collect();
4224 serde_json::json!({
4225 "target_url": r.url,
4226 "target_index": idx,
4227 "checks_passed": r.passed,
4228 "checks_failed": r.failed,
4229 "total_checks": total_checks,
4230 "pass_rate": rate,
4231 "elapsed_seconds": r.elapsed.as_secs_f64(),
4232 "report": r.report_json,
4233 "owasp_coverage": owasp_json,
4234 })
4235 })
4236 .collect();
4237
4238 let combined_summary = serde_json::json!({
4239 "total_targets": num_targets,
4240 "total_checks_passed": total_passed,
4241 "total_checks_failed": total_failed,
4242 "overall_pass_rate": overall_rate,
4243 "total_elapsed_seconds": total_elapsed.as_secs_f64(),
4244 "targets": per_target_summaries,
4245 });
4246
4247 let summary_path = self.output.join("multi-target-conformance-summary.json");
4248 let summary_str = serde_json::to_string_pretty(&combined_summary)
4249 .map_err(|e| BenchError::Other(format!("Failed to serialize summary: {}", e)))?;
4250 std::fs::write(&summary_path, &summary_str)
4251 .map_err(|e| BenchError::Other(format!("Failed to write summary: {}", e)))?;
4252 TerminalReporter::print_success(&format!(
4253 "Combined summary saved to: {}",
4254 summary_path.display()
4255 ));
4256
4257 Ok(())
4258 }
4259
4260 async fn execute_owasp_test(&self, parser: &SpecParser) -> Result<()> {
4262 TerminalReporter::print_progress("OWASP API Security Top 10 Testing Mode");
4263
4264 let custom_headers = self.parse_headers()?;
4266
4267 let mut config = OwaspApiConfig::new()
4269 .with_auth_header(&self.owasp_auth_header)
4270 .with_verbose(self.verbose)
4271 .with_insecure(self.skip_tls_verify)
4272 .with_concurrency(self.vus as usize)
4273 .with_iterations(self.owasp_iterations as usize)
4274 .with_base_path(self.base_path.clone())
4275 .with_custom_headers(custom_headers);
4276
4277 if let Some(ref token) = self.owasp_auth_token {
4279 config = config.with_valid_auth_token(token);
4280 }
4281
4282 if let Some(ref cats_str) = self.owasp_categories {
4284 let categories: Vec<OwaspCategory> = cats_str
4285 .split(',')
4286 .filter_map(|s| {
4287 let trimmed = s.trim();
4288 match trimmed.parse::<OwaspCategory>() {
4289 Ok(cat) => Some(cat),
4290 Err(e) => {
4291 TerminalReporter::print_warning(&e);
4292 None
4293 }
4294 }
4295 })
4296 .collect();
4297
4298 if !categories.is_empty() {
4299 config = config.with_categories(categories);
4300 }
4301 }
4302
4303 if let Some(ref admin_paths_file) = self.owasp_admin_paths {
4305 config.admin_paths_file = Some(admin_paths_file.clone());
4306 if let Err(e) = config.load_admin_paths() {
4307 TerminalReporter::print_warning(&format!("Failed to load admin paths file: {}", e));
4308 }
4309 }
4310
4311 if let Some(ref id_fields_str) = self.owasp_id_fields {
4313 let id_fields: Vec<String> = id_fields_str
4314 .split(',')
4315 .map(|s| s.trim().to_string())
4316 .filter(|s| !s.is_empty())
4317 .collect();
4318 if !id_fields.is_empty() {
4319 config = config.with_id_fields(id_fields);
4320 }
4321 }
4322
4323 if let Some(ref report_path) = self.owasp_report {
4325 config = config.with_report_path(report_path);
4326 }
4327 if let Ok(format) = self.owasp_report_format.parse::<ReportFormat>() {
4328 config = config.with_report_format(format);
4329 }
4330
4331 let categories = config.categories_to_test();
4333 TerminalReporter::print_success(&format!(
4334 "Testing {} OWASP categories: {}",
4335 categories.len(),
4336 categories.iter().map(|c| c.cli_name()).collect::<Vec<_>>().join(", ")
4337 ));
4338
4339 if config.valid_auth_token.is_some() {
4340 TerminalReporter::print_progress("Using provided auth token for baseline requests");
4341 }
4342
4343 TerminalReporter::print_progress("Generating OWASP security test script...");
4345 let generator = OwaspApiGenerator::new(config, self.target.clone(), parser);
4346
4347 let script = generator.generate()?;
4349 TerminalReporter::print_success("OWASP security test script generated");
4350
4351 let script_path = if let Some(output) = &self.script_output {
4353 output.clone()
4354 } else {
4355 self.output.join("k6-owasp-security-test.js")
4356 };
4357
4358 if let Some(parent) = script_path.parent() {
4359 std::fs::create_dir_all(parent)?;
4360 }
4361 std::fs::write(&script_path, &script)?;
4362 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
4363
4364 if self.generate_only {
4366 println!("\nOWASP security test script generated. Run it with:");
4367 println!(" k6 run {}", script_path.display());
4368 return Ok(());
4369 }
4370
4371 TerminalReporter::print_progress("Executing OWASP security tests...");
4373 let executor = K6Executor::new()?
4374 .with_local_ips(self.source_ips.join(","))
4375 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
4376 .with_capture_logs(!self.no_k6_logs);
4377 std::fs::create_dir_all(&self.output)?;
4378
4379 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
4380
4381 let duration_secs = Self::parse_duration(&self.duration)?;
4382 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
4383
4384 println!("\nOWASP security test results saved to: {}", self.output.display());
4385
4386 Ok(())
4387 }
4388}
4389
4390#[cfg(test)]
4391mod tests {
4392 use super::*;
4393 use tempfile::tempdir;
4394
4395 #[test]
4396 fn test_parse_duration() {
4397 assert_eq!(BenchCommand::parse_duration("30s").unwrap(), 30);
4398 assert_eq!(BenchCommand::parse_duration("5m").unwrap(), 300);
4399 assert_eq!(BenchCommand::parse_duration("1h").unwrap(), 3600);
4400 assert_eq!(BenchCommand::parse_duration("60").unwrap(), 60);
4401 }
4402
4403 #[test]
4407 fn parse_ip_list_ipv4_range_inclusive() {
4408 let v = parse_ip_list(&["10.0.0.5-10.0.0.27".into()], "source-ip");
4409 assert_eq!(v.len(), 23);
4410 assert_eq!(v.first().unwrap().to_string(), "10.0.0.5");
4411 assert_eq!(v.last().unwrap().to_string(), "10.0.0.27");
4412 }
4413
4414 #[test]
4417 fn parse_ip_list_range_rejects_backwards() {
4418 let v = parse_ip_list(&["10.0.0.10-10.0.0.5".into()], "source-ip");
4419 assert!(v.is_empty(), "backwards range should produce no IPs; got {v:?}");
4420 }
4421
4422 #[test]
4426 fn parse_ip_list_rejects_ipv6_range_syntax() {
4427 let v = parse_ip_list(&["2001:db8::1-2001:db8::5".into()], "geo-source-ip");
4428 assert!(v.is_empty(), "IPv6 range should be rejected; got {v:?}");
4429 }
4430
4431 #[test]
4433 fn parse_ip_list_range_capped_at_256() {
4434 let v = parse_ip_list(&["10.0.0.0-10.0.5.0".into()], "source-ip");
4435 assert_eq!(v.len(), 256);
4436 assert_eq!(v.first().unwrap().to_string(), "10.0.0.0");
4437 }
4438
4439 #[test]
4442 fn parse_ip_list_plain_and_comma() {
4443 let v = parse_ip_list(&["10.0.0.5".into(), "10.0.0.6,10.0.0.7".into()], "source-ip");
4444 assert_eq!(v.len(), 3);
4445 assert_eq!(v[0].to_string(), "10.0.0.5");
4446 assert_eq!(v[2].to_string(), "10.0.0.7");
4447 }
4448
4449 #[test]
4452 fn parse_ip_list_ipv4_cidr_29_expands_to_8() {
4453 let v = parse_ip_list(&["10.0.0.0/29".into()], "source-ip");
4454 assert_eq!(v.len(), 8);
4455 assert_eq!(v[0].to_string(), "10.0.0.0");
4456 assert_eq!(v[7].to_string(), "10.0.0.7");
4457 }
4458
4459 #[test]
4462 fn parse_ip_list_ipv4_cidr_8_capped_at_256() {
4463 let v = parse_ip_list(&["10.0.0.0/8".into()], "source-ip");
4464 assert_eq!(v.len(), 256);
4465 assert_eq!(v[0].to_string(), "10.0.0.0");
4466 assert_eq!(v[255].to_string(), "10.0.0.255");
4467 }
4468
4469 #[test]
4471 fn parse_ip_list_ipv6_cidr_126_expands_to_4() {
4472 let v = parse_ip_list(&["2001:db8::/126".into()], "geo-source-ip");
4473 assert_eq!(v.len(), 4);
4474 assert!(v[0].is_ipv6());
4475 assert_eq!(v[0].to_string(), "2001:db8::");
4476 assert_eq!(v[3].to_string(), "2001:db8::3");
4477 }
4478
4479 #[test]
4481 fn parse_ip_list_mixed_v4_v6_cidr() {
4482 let v = parse_ip_list(&["10.0.0.0/30,2001:db8::1,203.0.113.42".into()], "geo-source-ip");
4483 assert_eq!(v.len(), 6); assert!(v.iter().any(|ip| ip.to_string() == "2001:db8::1"));
4485 assert!(v.iter().any(|ip| ip.to_string() == "203.0.113.42"));
4486 }
4487
4488 #[test]
4491 fn parse_ip_list_skips_malformed() {
4492 let v = parse_ip_list(
4493 &[
4494 "10.0.0.5".into(),
4495 "not-an-ip".into(),
4496 "10.0.0.6".into(),
4497 "/24".into(),
4498 "1.2.3.4/200".into(),
4499 ],
4500 "source-ip",
4501 );
4502 assert_eq!(v.len(), 2);
4503 assert_eq!(v[0].to_string(), "10.0.0.5");
4504 assert_eq!(v[1].to_string(), "10.0.0.6");
4505 }
4506
4507 #[test]
4508 fn test_parse_duration_invalid() {
4509 assert!(BenchCommand::parse_duration("invalid").is_err());
4510 assert!(BenchCommand::parse_duration("30x").is_err());
4511 }
4512
4513 #[test]
4514 fn test_parse_headers() {
4515 let cmd = BenchCommand {
4516 spec: vec![PathBuf::from("test.yaml")],
4517 spec_dir: None,
4518 merge_conflicts: "error".to_string(),
4519 spec_mode: "merge".to_string(),
4520 dependency_config: None,
4521 target: "http://localhost".to_string(),
4522 base_path: None,
4523 duration: "1m".to_string(),
4524 vus: 10,
4525 scenario: "ramp-up".to_string(),
4526 operations: None,
4527 exclude_operations: None,
4528 auth: None,
4529 headers: vec![
4530 "X-API-Key:test123".to_string(),
4531 "X-Client-ID:client456".to_string(),
4532 ],
4533 output: PathBuf::from("output"),
4534 generate_only: false,
4535 script_output: None,
4536 threshold_percentile: "p(95)".to_string(),
4537 threshold_ms: 500,
4538 max_error_rate: 0.05,
4539 abort_on_error: true,
4540 abort_on_error_rate: 0.95,
4541 per_op_metrics: None,
4542 verbose: false,
4543 skip_tls_verify: false,
4544 chunked_request_bodies: false,
4545 target_rps: None,
4546 no_keep_alive: false,
4547 targets_file: None,
4548 max_concurrency: None,
4549 repeat_until: None,
4550 rounds: None,
4551 results_format: "both".to_string(),
4552 params_file: None,
4553 crud_flow: false,
4554 flow_config: None,
4555 extract_fields: None,
4556 parallel_create: None,
4557 data_file: None,
4558 data_distribution: "unique-per-vu".to_string(),
4559 data_mappings: None,
4560 per_uri_control: false,
4561 error_rate: None,
4562 error_types: None,
4563 security_test: false,
4564 security_payloads: None,
4565 security_categories: None,
4566 security_target_fields: None,
4567 wafbench_dir: None,
4568 wafbench_cycle_all: false,
4569 wafbench_verbatim: false,
4570 owasp_api_top10: false,
4571 owasp_categories: None,
4572 owasp_auth_header: "Authorization".to_string(),
4573 owasp_auth_token: None,
4574 owasp_admin_paths: None,
4575 owasp_id_fields: None,
4576 owasp_report: None,
4577 owasp_report_format: "json".to_string(),
4578 owasp_iterations: 1,
4579 conformance: false,
4580 conformance_api_key: None,
4581 conformance_basic_auth: None,
4582 conformance_report: PathBuf::from("conformance-report.json"),
4583 conformance_categories: None,
4584 conformance_report_format: "json".to_string(),
4585 conformance_headers: vec![],
4586 conformance_all_operations: false,
4587 conformance_custom: None,
4588 conformance_delay_ms: 0,
4589 use_k6: false,
4590 conformance_custom_filter: None,
4591 export_requests: false,
4592 validate_requests: false,
4593 conformance_self_test: false,
4594 conformance_self_test_capture: false,
4595 conformance_self_test_iterations: 1,
4596 conformance_self_test_duration: None,
4597 validate_response_schemas: false,
4598 source_ips: Vec::new(),
4599 geo_source_ips: Vec::new(),
4600 geo_source_headers: Vec::new(),
4601 report_missed_cap: None,
4602 discard_response_bodies: false,
4603 dns_policy: None,
4604 no_k6_logs: false,
4605 keep_rounds: None,
4606 };
4607
4608 let headers = cmd.parse_headers().unwrap();
4609 assert_eq!(headers.get("X-API-Key"), Some(&"test123".to_string()));
4610 assert_eq!(headers.get("X-Client-ID"), Some(&"client456".to_string()));
4611 }
4612
4613 #[test]
4614 fn test_parse_header_string_preserves_comma_in_value() {
4615 let inputs = vec![
4618 "Cookie:session=abc; expires=Thu, 01 Jan 2099 00:00:00 GMT".to_string(),
4619 "X-Trace:1".to_string(),
4620 ];
4621 let headers = parse_header_string(&inputs).unwrap();
4622 assert_eq!(
4623 headers.get("Cookie"),
4624 Some(&"session=abc; expires=Thu, 01 Jan 2099 00:00:00 GMT".to_string())
4625 );
4626 assert_eq!(headers.get("X-Trace"), Some(&"1".to_string()));
4627 }
4628
4629 #[test]
4637 fn conformance_advisory_names_every_discarded_flag() {
4638 let msg = CONFORMANCE_REPLACES_LOAD_ADVISORY;
4639 for flag in ["--vus", "--rps", "-d"] {
4640 assert!(
4641 msg.contains(flag),
4642 "conformance advisory must name `{flag}` as ignored; it is discarded on that \
4643 path and silently dropping it is how users end up tuning a knob that does \
4644 nothing (#980). Message was: {msg}"
4645 );
4646 }
4647 assert!(
4648 msg.contains("REPLACES"),
4649 "conformance advisory must say the load run is REPLACED, not merely that some \
4650 flags are ignored — `--conformance` returns before the load path runs, so no \
4651 load traffic is generated at all (#980). Message was: {msg}"
4652 );
4653 }
4654
4655 #[test]
4669 fn multi_target_clone_preserves_fields_parse_headers_reads() {
4670 let src = include_str!("command.rs");
4671
4672 let fn_start = src
4673 .find("async fn execute_multi_target(")
4674 .expect("execute_multi_target should exist");
4675 let block_start = src[fn_start..]
4676 .find("ParallelExecutor::new(")
4677 .map(|i| i + fn_start)
4678 .expect("multi-target path should build a ParallelExecutor");
4679 let block_end = src[block_start..]
4681 .find("\n );")
4682 .map(|i| i + block_start)
4683 .expect("ParallelExecutor::new(..) should be closed");
4684 let block = &src[block_start..block_end];
4685
4686 for field in ["conformance_basic_auth", "conformance_headers"] {
4689 for zeroed in [format!("{field}: None"), format!("{field}: vec![]")] {
4690 assert!(
4691 !block.contains(&zeroed),
4692 "execute_multi_target zeroes `{zeroed}`. parse_headers() folds `{field}` \
4693 into the header map, so zeroing it here strips auth from every \
4694 multi-target run while single-target keeps working (#79 round 64)."
4695 );
4696 }
4697 let passthrough = format!("{field}: self.{field}.clone()");
4698 assert!(
4699 block.contains(&passthrough),
4700 "execute_multi_target must carry `{field}` through as `{passthrough}` so \
4701 parse_headers() can fold it (#79 round 64)."
4702 );
4703 }
4704 }
4705
4706 #[test]
4707 fn test_get_spec_display_name() {
4708 let cmd = BenchCommand {
4709 spec: vec![PathBuf::from("test.yaml")],
4710 spec_dir: None,
4711 merge_conflicts: "error".to_string(),
4712 spec_mode: "merge".to_string(),
4713 dependency_config: None,
4714 target: "http://localhost".to_string(),
4715 base_path: None,
4716 duration: "1m".to_string(),
4717 vus: 10,
4718 scenario: "ramp-up".to_string(),
4719 operations: None,
4720 exclude_operations: None,
4721 auth: None,
4722 headers: Vec::new(),
4723 output: PathBuf::from("output"),
4724 generate_only: false,
4725 script_output: None,
4726 threshold_percentile: "p(95)".to_string(),
4727 threshold_ms: 500,
4728 max_error_rate: 0.05,
4729 abort_on_error: true,
4730 abort_on_error_rate: 0.95,
4731 per_op_metrics: None,
4732 verbose: false,
4733 skip_tls_verify: false,
4734 chunked_request_bodies: false,
4735 target_rps: None,
4736 no_keep_alive: false,
4737 targets_file: None,
4738 max_concurrency: None,
4739 repeat_until: None,
4740 rounds: None,
4741 results_format: "both".to_string(),
4742 params_file: None,
4743 crud_flow: false,
4744 flow_config: None,
4745 extract_fields: None,
4746 parallel_create: None,
4747 data_file: None,
4748 data_distribution: "unique-per-vu".to_string(),
4749 data_mappings: None,
4750 per_uri_control: false,
4751 error_rate: None,
4752 error_types: None,
4753 security_test: false,
4754 security_payloads: None,
4755 security_categories: None,
4756 security_target_fields: None,
4757 wafbench_dir: None,
4758 wafbench_cycle_all: false,
4759 wafbench_verbatim: false,
4760 owasp_api_top10: false,
4761 owasp_categories: None,
4762 owasp_auth_header: "Authorization".to_string(),
4763 owasp_auth_token: None,
4764 owasp_admin_paths: None,
4765 owasp_id_fields: None,
4766 owasp_report: None,
4767 owasp_report_format: "json".to_string(),
4768 owasp_iterations: 1,
4769 conformance: false,
4770 conformance_api_key: None,
4771 conformance_basic_auth: None,
4772 conformance_report: PathBuf::from("conformance-report.json"),
4773 conformance_categories: None,
4774 conformance_report_format: "json".to_string(),
4775 conformance_headers: vec![],
4776 conformance_all_operations: false,
4777 conformance_custom: None,
4778 conformance_delay_ms: 0,
4779 use_k6: false,
4780 conformance_custom_filter: None,
4781 export_requests: false,
4782 validate_requests: false,
4783 conformance_self_test: false,
4784 conformance_self_test_capture: false,
4785 conformance_self_test_iterations: 1,
4786 conformance_self_test_duration: None,
4787 validate_response_schemas: false,
4788 source_ips: Vec::new(),
4789 geo_source_ips: Vec::new(),
4790 geo_source_headers: Vec::new(),
4791 report_missed_cap: None,
4792 discard_response_bodies: false,
4793 dns_policy: None,
4794 no_k6_logs: false,
4795 keep_rounds: None,
4796 };
4797
4798 assert_eq!(cmd.get_spec_display_name(), "test.yaml");
4799
4800 let cmd_multi = BenchCommand {
4802 spec: vec![PathBuf::from("a.yaml"), PathBuf::from("b.yaml")],
4803 spec_dir: None,
4804 merge_conflicts: "error".to_string(),
4805 spec_mode: "merge".to_string(),
4806 dependency_config: None,
4807 target: "http://localhost".to_string(),
4808 base_path: None,
4809 duration: "1m".to_string(),
4810 vus: 10,
4811 scenario: "ramp-up".to_string(),
4812 operations: None,
4813 exclude_operations: None,
4814 auth: None,
4815 headers: Vec::new(),
4816 output: PathBuf::from("output"),
4817 generate_only: false,
4818 script_output: None,
4819 threshold_percentile: "p(95)".to_string(),
4820 threshold_ms: 500,
4821 max_error_rate: 0.05,
4822 abort_on_error: true,
4823 abort_on_error_rate: 0.95,
4824 per_op_metrics: None,
4825 verbose: false,
4826 skip_tls_verify: false,
4827 chunked_request_bodies: false,
4828 target_rps: None,
4829 no_keep_alive: false,
4830 targets_file: None,
4831 max_concurrency: None,
4832 repeat_until: None,
4833 rounds: None,
4834 results_format: "both".to_string(),
4835 params_file: None,
4836 crud_flow: false,
4837 flow_config: None,
4838 extract_fields: None,
4839 parallel_create: None,
4840 data_file: None,
4841 data_distribution: "unique-per-vu".to_string(),
4842 data_mappings: None,
4843 per_uri_control: false,
4844 error_rate: None,
4845 error_types: None,
4846 security_test: false,
4847 security_payloads: None,
4848 security_categories: None,
4849 security_target_fields: None,
4850 wafbench_dir: None,
4851 wafbench_cycle_all: false,
4852 wafbench_verbatim: false,
4853 owasp_api_top10: false,
4854 owasp_categories: None,
4855 owasp_auth_header: "Authorization".to_string(),
4856 owasp_auth_token: None,
4857 owasp_admin_paths: None,
4858 owasp_id_fields: None,
4859 owasp_report: None,
4860 owasp_report_format: "json".to_string(),
4861 owasp_iterations: 1,
4862 conformance: false,
4863 conformance_api_key: None,
4864 conformance_basic_auth: None,
4865 conformance_report: PathBuf::from("conformance-report.json"),
4866 conformance_categories: None,
4867 conformance_report_format: "json".to_string(),
4868 conformance_headers: vec![],
4869 conformance_all_operations: false,
4870 conformance_custom: None,
4871 conformance_delay_ms: 0,
4872 use_k6: false,
4873 conformance_custom_filter: None,
4874 export_requests: false,
4875 validate_requests: false,
4876 conformance_self_test: false,
4877 conformance_self_test_capture: false,
4878 conformance_self_test_iterations: 1,
4879 conformance_self_test_duration: None,
4880 validate_response_schemas: false,
4881 source_ips: Vec::new(),
4882 geo_source_ips: Vec::new(),
4883 geo_source_headers: Vec::new(),
4884 report_missed_cap: None,
4885 discard_response_bodies: false,
4886 dns_policy: None,
4887 no_k6_logs: false,
4888 keep_rounds: None,
4889 };
4890
4891 assert_eq!(cmd_multi.get_spec_display_name(), "2 spec files");
4892 }
4893
4894 #[test]
4895 fn test_parse_extracted_values_from_output_dir() {
4896 let dir = tempdir().unwrap();
4897 let path = dir.path().join("extracted_values.json");
4898 std::fs::write(
4899 &path,
4900 r#"{
4901 "pool_id": "abc123",
4902 "count": 0,
4903 "enabled": false,
4904 "metadata": { "owner": "team-a" }
4905}"#,
4906 )
4907 .unwrap();
4908
4909 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
4910 assert_eq!(extracted.get("pool_id"), Some(&serde_json::json!("abc123")));
4911 assert_eq!(extracted.get("count"), Some(&serde_json::json!(0)));
4912 assert_eq!(extracted.get("enabled"), Some(&serde_json::json!(false)));
4913 assert_eq!(extracted.get("metadata"), Some(&serde_json::json!({"owner": "team-a"})));
4914 }
4915
4916 #[test]
4917 fn test_parse_extracted_values_missing_file() {
4918 let dir = tempdir().unwrap();
4919 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
4920 assert!(extracted.values.is_empty());
4921 }
4922
4923 fn sample_bench_command() -> BenchCommand {
4926 BenchCommand {
4927 spec: vec![PathBuf::from("test.yaml")],
4928 spec_dir: None,
4929 merge_conflicts: "error".to_string(),
4930 spec_mode: "merge".to_string(),
4931 dependency_config: None,
4932 target: "http://localhost".to_string(),
4933 base_path: None,
4934 duration: "1m".to_string(),
4935 vus: 10,
4936 scenario: "ramp-up".to_string(),
4937 operations: None,
4938 exclude_operations: None,
4939 auth: None,
4940 headers: vec![
4941 "X-API-Key:test123".to_string(),
4942 "X-Client-ID:client456".to_string(),
4943 ],
4944 output: PathBuf::from("output"),
4945 generate_only: false,
4946 script_output: None,
4947 threshold_percentile: "p(95)".to_string(),
4948 threshold_ms: 500,
4949 max_error_rate: 0.05,
4950 abort_on_error: true,
4951 abort_on_error_rate: 0.95,
4952 per_op_metrics: None,
4953 verbose: false,
4954 skip_tls_verify: false,
4955 chunked_request_bodies: false,
4956 target_rps: None,
4957 no_keep_alive: false,
4958 targets_file: None,
4959 max_concurrency: None,
4960 repeat_until: None,
4961 rounds: None,
4962 results_format: "both".to_string(),
4963 params_file: None,
4964 crud_flow: false,
4965 flow_config: None,
4966 extract_fields: None,
4967 parallel_create: None,
4968 data_file: None,
4969 data_distribution: "unique-per-vu".to_string(),
4970 data_mappings: None,
4971 per_uri_control: false,
4972 error_rate: None,
4973 error_types: None,
4974 security_test: false,
4975 security_payloads: None,
4976 security_categories: None,
4977 security_target_fields: None,
4978 wafbench_dir: None,
4979 wafbench_cycle_all: false,
4980 wafbench_verbatim: false,
4981 owasp_api_top10: false,
4982 owasp_categories: None,
4983 owasp_auth_header: "Authorization".to_string(),
4984 owasp_auth_token: None,
4985 owasp_admin_paths: None,
4986 owasp_id_fields: None,
4987 owasp_report: None,
4988 owasp_report_format: "json".to_string(),
4989 owasp_iterations: 1,
4990 conformance: false,
4991 conformance_api_key: None,
4992 conformance_basic_auth: None,
4993 conformance_report: PathBuf::from("conformance-report.json"),
4994 conformance_categories: None,
4995 conformance_report_format: "json".to_string(),
4996 conformance_headers: vec![],
4997 conformance_all_operations: false,
4998 conformance_custom: None,
4999 conformance_delay_ms: 0,
5000 use_k6: false,
5001 conformance_custom_filter: None,
5002 export_requests: false,
5003 validate_requests: false,
5004 conformance_self_test: false,
5005 conformance_self_test_capture: false,
5006 conformance_self_test_iterations: 1,
5007 conformance_self_test_duration: None,
5008 validate_response_schemas: false,
5009 source_ips: Vec::new(),
5010 geo_source_ips: Vec::new(),
5011 geo_source_headers: Vec::new(),
5012 report_missed_cap: None,
5013 discard_response_bodies: false,
5014 dns_policy: None,
5015 no_k6_logs: false,
5016 keep_rounds: None,
5017 }
5018 }
5019
5020 #[test]
5028 fn verbatim_disables_security_payload_injection() {
5029 let mut cmd = sample_bench_command();
5030 cmd.wafbench_dir = Some("traffic.yaml".to_string());
5031
5032 assert!(
5033 cmd.security_testing_enabled(),
5034 "--wafbench-dir alone must still enable payload injection"
5035 );
5036
5037 cmd.wafbench_verbatim = true;
5038 assert!(
5039 !cmd.security_testing_enabled(),
5040 "verbatim mode must not inject payloads into requests sent as written"
5041 );
5042
5043 cmd.security_test = true;
5046 assert!(
5047 !cmd.security_testing_enabled(),
5048 "--security-test must not re-enable injection under --wafbench-verbatim"
5049 );
5050 }
5051
5052 #[test]
5057 fn security_testing_enabled_has_a_single_definition() {
5058 let src = include_str!("command.rs");
5059 let parallel = include_str!("parallel_executor.rs");
5060 let a = format!("self.{} || self.{}.is_some()", "security_test", "wafbench_dir");
5062 let b = format!("self.{}.is_some() || self.{}", "wafbench_dir", "security_test");
5063 let inline = src.matches(a.as_str()).count() + src.matches(b.as_str()).count();
5064 assert_eq!(
5065 inline, 1,
5066 "expected the security_testing_enabled() method to be the only place this is \
5067 computed, found {inline} inline copies -- collapse them or the render paths drift"
5068 );
5069
5070 let parallel_inline = format!(
5075 "{}.{} || {}.{}.is_some()",
5076 "base_command", "security_test", "self.base_command", "wafbench_dir"
5077 );
5078 assert!(
5079 !parallel.contains(¶llel_inline),
5080 "ParallelExecutor must not recompute the security flag inline"
5081 );
5082 assert!(
5083 parallel.contains("security_testing_enabled()"),
5084 "ParallelExecutor must call security_testing_enabled() so --wafbench-verbatim \
5085 turns injection off on --targets-file runs too"
5086 );
5087 }
5088
5089 #[test]
5093 fn missing_wafbench_dir_is_not_swallowed() {
5094 let src = include_str!("command.rs");
5095 let swallowed = format!("Failed to {} WAFBench tests", "load");
5097 let impl_line = src
5098 .lines()
5099 .filter(|l| !l.trim_start().starts_with("//"))
5100 .any(|l| l.contains(&swallowed));
5101 assert!(!impl_line, "missing --wafbench-dir must not be downgraded to a warning");
5102 assert!(
5103 src.contains("self.load_wafbench_payloads()?"),
5104 "payload load errors must reach generate_enhanced_script"
5105 );
5106 }
5107
5108 #[test]
5113 fn multi_target_path_honors_verbatim_templates() {
5114 let src = include_str!("parallel_executor.rs");
5115 assert!(
5116 src.contains("load_verbatim_templates"),
5117 "ParallelExecutor must load traffic-file requests under --wafbench-verbatim. \
5118 Requiring a spec and generating templates from its operations is how \
5119 --targets-file ignored the flag and fuzzed spec URLs (#79)."
5120 );
5121 }
5122
5123 #[test]
5124 fn single_target_k6_spawn_sets_force_http1() {
5125 let src = include_str!("command.rs");
5126 assert!(
5127 src.contains("with_force_http1(force_http1)"),
5128 "single-target k6 spawn must set GODEBUG=http2client=0 for Connection-header WAF cases"
5129 );
5130 assert!(
5131 src.contains("print_k6_run_hint"),
5132 "generate-only must print GODEBUG=http2client=0 when HTTP/1.1 is required"
5133 );
5134 assert!(
5135 src.contains("with_per_op_metrics(per_op_metrics)"),
5136 "single-target k6 generation must apply Round-65 per-op metrics collapse (#79)"
5137 );
5138 assert!(
5139 src.contains("resolve_per_op_metrics"),
5140 "single-target path must resolve auto/forced per-op metrics (#79)"
5141 );
5142 }
5143
5144 #[test]
5146 fn traffic_breakdown_json_multiplies_unique_by_rps() {
5147 let dir = std::env::temp_dir().join(format!(
5148 "mf-traffic-breakdown-{}-{}",
5149 std::process::id(),
5150 std::time::SystemTime::now()
5151 .duration_since(std::time::UNIX_EPOCH)
5152 .unwrap()
5153 .as_nanos()
5154 ));
5155 let _ = std::fs::create_dir_all(&dir);
5156 let mut cmd = sample_bench_command();
5157 cmd.output = dir.clone();
5158 cmd.target_rps = Some(50);
5159 cmd.duration = "1200s".to_string();
5160 let stats = crate::wafbench::WafBenchStats {
5161 per_file: vec![crate::wafbench::TrafficFileSummary {
5162 file: "apisix_cve-2026-44087.yaml".into(),
5163 sent: 5,
5164 attack: 3,
5165 normal: 2,
5166 omitted: 1,
5167 other: 0,
5168 }],
5169 ..Default::default()
5170 };
5171 cmd.emit_traffic_file_breakdown(&stats, "what to expect in proxy logs");
5172 let raw = std::fs::read_to_string(dir.join("traffic-breakdown.json"))
5173 .expect("traffic-breakdown.json");
5174 let v: serde_json::Value = serde_json::from_str(&raw).unwrap();
5175 assert_eq!(v["rps"], 50);
5176 assert_eq!(v["duration_secs"], 1200);
5177 assert_eq!(v["files"][0]["sent"]["unique_cases"], 5);
5178 assert_eq!(v["files"][0]["sent"]["projected_per_second"], 250);
5179 assert_eq!(v["files"][0]["sent"]["projected_over_run"], 300000);
5180 assert_eq!(v["files"][0]["attack"]["projected_per_second"], 150);
5181 assert_eq!(v["files"][0]["normal"]["projected_per_second"], 100);
5182 for gone in [
5183 "unique",
5184 "total",
5185 "expected_requests",
5186 "expected_requests_unit",
5187 ] {
5188 assert!(
5189 v["files"][0]["sent"].get(gone).is_none(),
5190 "{gone} alias must not appear in traffic-breakdown.json"
5191 );
5192 }
5193 assert!(v["note"].as_str().unwrap().contains("Plan, not k6 counters"));
5194 assert!(v["note"].as_str().unwrap().contains("not traffic on the wire"));
5195 assert_eq!(
5196 BenchCommand::format_unique_total(5, Some(50)),
5197 "unique_cases=5 projected_per_second=250 (5 * 50 RPS)"
5198 );
5199 let _ = std::fs::remove_dir_all(&dir);
5200 }
5201
5202 #[test]
5205 fn traffic_breakdown_json_omits_projected_without_rps() {
5206 let dir = std::env::temp_dir().join(format!(
5207 "mf-traffic-breakdown-norps-{}-{}",
5208 std::process::id(),
5209 std::time::SystemTime::now()
5210 .duration_since(std::time::UNIX_EPOCH)
5211 .unwrap()
5212 .as_nanos()
5213 ));
5214 let _ = std::fs::create_dir_all(&dir);
5215 let mut cmd = sample_bench_command();
5216 cmd.output = dir.clone();
5217 cmd.target_rps = None;
5218 cmd.duration = "60s".to_string();
5219 let stats = crate::wafbench::WafBenchStats {
5220 per_file: vec![crate::wafbench::TrafficFileSummary {
5221 file: "apisix_cve-2026-44087.yaml".into(),
5222 sent: 5,
5223 attack: 3,
5224 normal: 2,
5225 omitted: 1,
5226 other: 0,
5227 }],
5228 ..Default::default()
5229 };
5230 cmd.emit_traffic_file_breakdown(&stats, "what to expect in proxy logs");
5231 let raw = std::fs::read_to_string(dir.join("traffic-breakdown.json"))
5232 .expect("traffic-breakdown.json");
5233 let v: serde_json::Value = serde_json::from_str(&raw).unwrap();
5234 assert!(v["rps"].is_null());
5235 assert_eq!(v["duration_secs"], 60);
5236 assert_eq!(v["files"][0]["sent"]["unique_cases"], 5);
5237 assert!(v["files"][0]["sent"]["projected_per_second"].is_null());
5238 assert!(v["files"][0]["sent"]["projected_over_run"].is_null());
5239 for gone in ["unique", "total", "expected_requests"] {
5240 assert!(
5241 v["files"][0]["sent"].get(gone).is_none(),
5242 "{gone} alias must not appear when --rps is unset"
5243 );
5244 }
5245 let _ = std::fs::remove_dir_all(&dir);
5246 }
5247}