1use crate::crud_flow::{CrudFlowConfig, CrudFlowDetector};
4use crate::data_driven::{DataDistribution, DataDrivenConfig, DataDrivenGenerator, DataMapping};
5use crate::dynamic_params::{DynamicParamProcessor, DynamicPlaceholder};
6use crate::error::{BenchError, Result};
7use crate::executor::K6Executor;
8use crate::invalid_data::{InvalidDataConfig, InvalidDataGenerator};
9use crate::k6_gen::{K6Config, K6ScriptGenerator};
10use crate::mock_integration::{
11 MockIntegrationConfig, MockIntegrationGenerator, MockServerDetector,
12};
13use crate::owasp_api::{OwaspApiConfig, OwaspApiGenerator, OwaspCategory, ReportFormat};
14use crate::parallel_executor::{AggregatedResults, ParallelExecutor};
15use crate::parallel_requests::{ParallelConfig, ParallelRequestGenerator};
16use crate::param_overrides::ParameterOverrides;
17use crate::reporter::TerminalReporter;
18use crate::request_gen::RequestGenerator;
19use crate::scenarios::LoadScenario;
20use crate::security_payloads::{
21 SecurityCategory, SecurityPayload, SecurityPayloads, SecurityTestConfig, SecurityTestGenerator,
22};
23use crate::spec_dependencies::{
24 topological_sort, DependencyDetector, ExtractedValues, SpecDependencyConfig,
25};
26use crate::spec_parser::SpecParser;
27use crate::target_parser::parse_targets_file;
28use crate::wafbench::WafBenchLoader;
29use mockforge_openapi::multi_spec::{
30 load_specs_from_directory, load_specs_from_files, merge_specs, ConflictStrategy,
31};
32use mockforge_openapi::spec::OpenApiSpec;
33use std::collections::{HashMap, HashSet};
34use std::path::{Path, PathBuf};
35use std::str::FromStr;
36
37pub fn parse_header_string(inputs: &[String]) -> Result<HashMap<String, String>> {
45 let mut headers = HashMap::new();
46
47 for pair in inputs {
48 let pair = pair.trim();
49 if pair.is_empty() {
50 continue;
51 }
52 let parts: Vec<&str> = pair.splitn(2, ':').collect();
53 if parts.len() != 2 {
54 return Err(BenchError::Other(format!(
55 "Invalid header format: '{}'. Expected 'Key:Value'",
56 pair
57 )));
58 }
59 headers.insert(parts[0].trim().to_string(), parts[1].trim().to_string());
60 }
61
62 Ok(headers)
63}
64
65const CONFORMANCE_REPLACES_LOAD_ADVISORY: &str =
85 "Conformance mode REPLACES the load run: 1 VU, 1 iteration per endpoint. \
86 --vus, --rps and -d are ignored. Run bench a second time without \
87 --conformance if you also want a load test.";
88
89pub struct BenchCommand {
91 pub spec: Vec<PathBuf>,
93 pub spec_dir: Option<PathBuf>,
95 pub merge_conflicts: String,
97 pub spec_mode: String,
99 pub dependency_config: Option<PathBuf>,
101 pub target: String,
102 pub base_path: Option<String>,
105 pub duration: String,
106 pub vus: u32,
107 pub target_rps: Option<u32>,
113 pub no_keep_alive: bool,
118 pub scenario: String,
119 pub operations: Option<String>,
120 pub exclude_operations: Option<String>,
124 pub auth: Option<String>,
125 pub headers: Vec<String>,
128 pub output: PathBuf,
129 pub generate_only: bool,
130 pub script_output: Option<PathBuf>,
131 pub threshold_percentile: String,
132 pub threshold_ms: u64,
133 pub max_error_rate: f64,
134 pub abort_on_error: bool,
139 pub abort_on_error_rate: f64,
143 pub per_op_metrics: Option<bool>,
147 pub verbose: bool,
148 pub skip_tls_verify: bool,
149 pub chunked_request_bodies: bool,
154 pub targets_file: Option<PathBuf>,
156 pub max_concurrency: Option<u32>,
158 pub repeat_until: Option<String>,
163 pub rounds: Option<u32>,
167 pub results_format: String,
169 pub params_file: Option<PathBuf>,
174
175 pub crud_flow: bool,
178 pub flow_config: Option<PathBuf>,
180 pub extract_fields: Option<String>,
182
183 pub parallel_create: Option<u32>,
186
187 pub data_file: Option<PathBuf>,
190 pub data_distribution: String,
192 pub data_mappings: Option<String>,
194 pub per_uri_control: bool,
196
197 pub error_rate: Option<f64>,
200 pub error_types: Option<String>,
202
203 pub security_test: bool,
206 pub security_payloads: Option<PathBuf>,
208 pub security_categories: Option<String>,
210 pub security_target_fields: Option<String>,
212
213 pub wafbench_dir: Option<String>,
216 pub wafbench_cycle_all: bool,
218 pub wafbench_verbatim: bool,
221
222 pub conformance: bool,
225 pub conformance_api_key: Option<String>,
227 pub conformance_basic_auth: Option<String>,
229 pub conformance_report: PathBuf,
231 pub conformance_categories: Option<String>,
233 pub conformance_report_format: String,
235 pub conformance_headers: Vec<String>,
238 pub conformance_all_operations: bool,
241 pub conformance_custom: Option<PathBuf>,
243 pub conformance_delay_ms: u64,
246 pub use_k6: bool,
248 pub conformance_custom_filter: Option<String>,
252 pub export_requests: bool,
255 pub validate_requests: bool,
258 pub conformance_self_test: bool,
265 pub conformance_self_test_capture: bool,
269 pub validate_response_schemas: bool,
275 pub conformance_self_test_iterations: u32,
280 pub conformance_self_test_duration: Option<String>,
285
286 pub source_ips: Vec<String>,
291 pub geo_source_ips: Vec<String>,
295 pub geo_source_headers: Vec<String>,
299
300 pub report_missed_cap: Option<u32>,
307
308 pub discard_response_bodies: bool,
315
316 pub dns_policy: Option<String>,
322
323 pub owasp_api_top10: bool,
326 pub owasp_categories: Option<String>,
328 pub owasp_auth_header: String,
330 pub owasp_auth_token: Option<String>,
332 pub owasp_admin_paths: Option<PathBuf>,
334 pub owasp_id_fields: Option<String>,
336 pub owasp_report: Option<PathBuf>,
338 pub owasp_report_format: String,
340 pub owasp_iterations: u32,
342}
343
344fn parse_ip_list(raw: &[String], flag_name: &str) -> Vec<std::net::IpAddr> {
358 use std::net::IpAddr;
359 const MAX_CIDR_EXPANSION: usize = 256;
360 let mut out = Vec::new();
361 for entry in raw {
362 for piece in entry.split(',') {
363 let s = piece.trim();
364 if s.is_empty() {
365 continue;
366 }
367 if let Some((addr_part, prefix_part)) = s.split_once('/') {
369 let prefix: u32 = match prefix_part.parse() {
370 Ok(p) => p,
371 Err(e) => {
372 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR prefix: {e}");
373 continue;
374 }
375 };
376 let net_addr: IpAddr = match addr_part.parse() {
377 Ok(a) => a,
378 Err(e) => {
379 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad CIDR address: {e}");
380 continue;
381 }
382 };
383 expand_cidr(net_addr, prefix, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
384 continue;
385 }
386 if let Some((start_str, end_str)) = s.split_once('-') {
392 let start_s = start_str.trim();
393 let end_s = end_str.trim();
394 if start_s.contains(':') || end_s.contains(':') {
398 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{s}': IPv6 range syntax not supported (use CIDR like 2001:db8::/126 instead)");
399 continue;
400 }
401 let start: IpAddr = match start_s.parse() {
402 Ok(a) => a,
403 Err(e) => {
404 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range start: {e}");
405 continue;
406 }
407 };
408 let end: IpAddr = match end_s.parse() {
409 Ok(a) => a,
410 Err(e) => {
411 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{s}': bad range end: {e}");
412 continue;
413 }
414 };
415 expand_range(start, end, MAX_CIDR_EXPANSION, flag_name, s, &mut out);
416 continue;
417 }
418 match s.parse::<IpAddr>() {
420 Ok(ip) => out.push(ip),
421 Err(e) => {
422 tracing::warn!(target: "mockforge::bench", "ignoring malformed --{flag_name} value '{s}': {e}");
423 }
424 }
425 }
426 }
427 out
428}
429
430fn expand_range(
434 start: std::net::IpAddr,
435 end: std::net::IpAddr,
436 cap: usize,
437 flag_name: &str,
438 raw: &str,
439 out: &mut Vec<std::net::IpAddr>,
440) {
441 use std::net::{IpAddr, Ipv4Addr};
442 let (start_v4, end_v4) = match (start, end) {
443 (IpAddr::V4(a), IpAddr::V4(b)) => (a, b),
444 _ => {
445 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range start/end must both be IPv4");
446 return;
447 }
448 };
449 let start_u32 = u32::from(start_v4);
450 let end_u32 = u32::from(end_v4);
451 if end_u32 < start_u32 {
452 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range end {end_v4} is before start {start_v4}");
453 return;
454 }
455 let total = (end_u32 - start_u32).saturating_add(1) as usize;
456 let take = total.min(cap);
457 if total > cap {
458 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': range has {total} addresses, capping at {cap}");
459 }
460 for i in 0..take as u32 {
461 out.push(IpAddr::V4(Ipv4Addr::from(start_u32 + i)));
462 }
463}
464
465fn expand_cidr(
469 net: std::net::IpAddr,
470 prefix: u32,
471 cap: usize,
472 flag_name: &str,
473 raw: &str,
474 out: &mut Vec<std::net::IpAddr>,
475) {
476 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
477 match net {
478 IpAddr::V4(ipv4) => {
479 if prefix > 32 {
480 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv4 prefix must be <= 32");
481 return;
482 }
483 let total: u64 = 1u64 << (32 - prefix);
484 let take = total.min(cap as u64) as u32;
485 if total > cap as u64 {
486 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': CIDR has {total} addresses, capping at {cap}");
487 }
488 let mask: u32 = if prefix == 0 {
489 0
490 } else {
491 !0u32 << (32 - prefix)
492 };
493 let net_u32 = u32::from(ipv4) & mask;
494 for i in 0..take {
495 out.push(IpAddr::V4(Ipv4Addr::from(net_u32.wrapping_add(i))));
496 }
497 }
498 IpAddr::V6(ipv6) => {
499 if prefix > 128 {
500 tracing::warn!(target: "mockforge::bench", "ignoring --{flag_name} '{raw}': IPv6 prefix must be <= 128");
501 return;
502 }
503 let mask: u128 = if prefix == 0 {
507 0
508 } else {
509 !0u128 << (128 - prefix)
510 };
511 let net_u128 = u128::from(ipv6) & mask;
512 let remaining_bits = 128 - prefix;
513 let total_capped = if remaining_bits >= 64 {
516 cap as u128
517 } else {
518 (1u128 << remaining_bits).min(cap as u128)
519 };
520 if remaining_bits < 128 && (1u128 << remaining_bits) > cap as u128 {
521 tracing::warn!(target: "mockforge::bench", "--{flag_name} '{raw}': IPv6 CIDR exceeds {cap} addresses, capping");
522 }
523 for i in 0..total_capped {
524 out.push(IpAddr::V6(Ipv6Addr::from(net_u128.wrapping_add(i))));
525 }
526 }
527 }
528}
529
530impl BenchCommand {
531 pub fn security_testing_enabled(&self) -> bool {
547 if self.wafbench_verbatim {
548 return false;
549 }
550 self.security_test || self.wafbench_dir.is_some()
551 }
552
553 pub async fn load_and_merge_specs(&self) -> Result<OpenApiSpec> {
555 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
556
557 if !self.spec.is_empty() {
559 let specs = load_specs_from_files(self.spec.clone())
560 .await
561 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
562 all_specs.extend(specs);
563 }
564
565 if let Some(spec_dir) = &self.spec_dir {
567 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
568 BenchError::Other(format!("Failed to load specs from directory: {}", e))
569 })?;
570 all_specs.extend(dir_specs);
571 }
572
573 if all_specs.is_empty() {
574 return Err(BenchError::Other(
575 "No spec files provided. Use --spec or --spec-dir.".to_string(),
576 ));
577 }
578
579 if all_specs.len() == 1 {
581 return Ok(all_specs.into_iter().next().expect("checked len() == 1 above").1);
583 }
584
585 let conflict_strategy = match self.merge_conflicts.as_str() {
587 "first" => ConflictStrategy::First,
588 "last" => ConflictStrategy::Last,
589 _ => ConflictStrategy::Error,
590 };
591
592 merge_specs(all_specs, conflict_strategy)
593 .map_err(|e| BenchError::Other(format!("Failed to merge specs: {}", e)))
594 }
595
596 fn get_spec_display_name(&self) -> String {
598 if self.spec.len() == 1 {
599 self.spec[0].to_string_lossy().to_string()
600 } else if !self.spec.is_empty() {
601 format!("{} spec files", self.spec.len())
602 } else if let Some(dir) = &self.spec_dir {
603 format!("specs from {}", dir.display())
604 } else {
605 "no specs".to_string()
606 }
607 }
608
609 fn advise_capacity(&self) {
616 let target_count = self
617 .targets_file
618 .as_ref()
619 .and_then(|p| std::fs::read_to_string(p).ok())
620 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
621 .and_then(|v| v.as_array().map(|a| a.len()))
622 .unwrap_or(1);
623 let vus = self.vus.max(1);
624 let rps_total = self.target_rps.unwrap_or(0) as usize * target_count.max(1);
625 let load_product = target_count * vus as usize;
629 if load_product >= 150 {
630 let est_ram_gb =
631 (vus as usize * 50) / 1024 + (target_count * 10 * 2) / 1024 + target_count / 2;
632 let est_cores = ((vus as usize) / 50).max(2);
633 TerminalReporter::print_warning(&format!(
634 "Capacity advisory: targets={target_count}, VUs={vus}, RPS-total≈{rps_total}. \
635 Single-client estimate: ~{est_cores} CPU cores, ~{est_ram_gb} GB RAM. \
636 If your machine is below that, expect OOM hangs partway through the run. \
637 See https://docs.mockforge.dev/reference/bench-capacity-sizing.html \
638 for the sizing table and sharding guide."
639 ));
640 }
641 }
642
643 pub async fn execute(&self) -> Result<()> {
645 if self.conformance_self_test && self.use_k6 {
652 TerminalReporter::print_warning(
653 "--use-k6 has no effect with --conformance-self-test: the self-test driver runs and returns before k6 is invoked. Drop one or the other depending on whether you want the spec-driven self-test or a k6 bench run.",
654 );
655 }
656
657 self.advise_capacity();
663
664 if let Some(targets_file) = &self.targets_file {
666 if self.conformance && self.conformance_self_test {
675 return self.execute_multi_target_self_test(targets_file).await;
676 }
677 if self.conformance {
678 return self.execute_multi_target_conformance(targets_file).await;
679 }
680 return self.execute_multi_target(targets_file).await;
681 }
682
683 if self.spec_mode == "sequential" && (self.spec.len() > 1 || self.spec_dir.is_some()) {
685 return self.execute_sequential_specs().await;
686 }
687
688 TerminalReporter::print_header(
691 &self.get_spec_display_name(),
692 &self.target,
693 0, &self.scenario,
695 Self::parse_duration(&self.duration)?,
696 );
697
698 if !K6Executor::is_k6_installed() {
700 TerminalReporter::print_error("k6 is not installed");
701 TerminalReporter::print_warning(
702 "Install k6 from: https://k6.io/docs/get-started/installation/",
703 );
704 return Err(BenchError::K6NotFound);
705 }
706 K6Executor::warn_if_pre_v1().await;
707
708 if self.conformance {
710 return self.execute_conformance_test().await;
711 }
712
713 let spec_supplied = !self.spec.is_empty() || self.spec_dir.is_some();
720 let merged_spec = if self.wafbench_verbatim && !spec_supplied {
721 tracing::info!(
722 target: "mockforge::bench",
723 "--wafbench-verbatim without --spec: sending only the traffic file's requests"
724 );
725 OpenApiSpec {
726 spec: Default::default(),
727 file_path: None,
728 raw_document: None,
729 }
730 } else {
731 TerminalReporter::print_progress("Loading OpenAPI specification(s)...");
732 self.load_and_merge_specs().await?
733 };
734 let parser = SpecParser::from_spec(merged_spec);
735 if self.spec.len() > 1 || self.spec_dir.is_some() {
736 TerminalReporter::print_success(&format!(
737 "Loaded and merged {} specification(s)",
738 self.spec.len() + self.spec_dir.as_ref().map(|_| 1).unwrap_or(0)
739 ));
740 } else {
741 TerminalReporter::print_success("Specification loaded");
742 }
743
744 let mock_config = self.build_mock_config().await;
746 if mock_config.is_mock_server {
747 TerminalReporter::print_progress("Mock server integration enabled");
748 }
749
750 if self.crud_flow {
752 return self.execute_crud_flow(&parser).await;
753 }
754
755 if self.owasp_api_top10 {
757 return self.execute_owasp_test(&parser).await;
758 }
759
760 TerminalReporter::print_progress("Extracting API operations...");
762 let mut operations = if let Some(filter) = &self.operations {
763 parser.filter_operations(filter)?
764 } else {
765 parser.get_operations()
766 };
767
768 if let Some(exclude) = &self.exclude_operations {
770 let before_count = operations.len();
771 operations = parser.exclude_operations(operations, exclude)?;
772 let excluded_count = before_count - operations.len();
773 if excluded_count > 0 {
774 TerminalReporter::print_progress(&format!(
775 "Excluded {} operations matching '{}'",
776 excluded_count, exclude
777 ));
778 }
779 }
780
781 if operations.is_empty() && !self.wafbench_verbatim {
787 return Err(BenchError::Other("No operations found in spec".to_string()));
788 }
789
790 TerminalReporter::print_success(&format!("Found {} operations", operations.len()));
791
792 let param_overrides = if let Some(params_file) = &self.params_file {
794 TerminalReporter::print_progress("Loading parameter overrides...");
795 let overrides = ParameterOverrides::from_file(params_file)?;
796 TerminalReporter::print_success(&format!(
797 "Loaded parameter overrides ({} operation-specific, {} defaults)",
798 overrides.operations.len(),
799 if overrides.defaults.is_empty() { 0 } else { 1 }
800 ));
801 Some(overrides)
802 } else {
803 None
804 };
805
806 TerminalReporter::print_progress("Generating request templates...");
808 let templates: Vec<_> = operations
809 .iter()
810 .map(|op| {
811 let op_overrides = param_overrides.as_ref().map(|po| {
812 po.get_for_operation(op.operation_id.as_deref(), &op.method, &op.path)
813 });
814 RequestGenerator::generate_template_with_overrides(op, op_overrides.as_ref())
815 })
816 .collect::<Result<Vec<_>>>()?;
817 TerminalReporter::print_success("Request templates generated");
818
819 let templates = if self.wafbench_verbatim {
825 let verbatim = self.load_verbatim_templates()?;
826 if verbatim.is_empty() {
827 return Err(BenchError::Other(
828 "--wafbench-verbatim was set but no traffic cases were loaded. Check \
829 --wafbench-dir points at a file, directory or glob containing cases with \
830 a `request.uri`."
831 .to_string(),
832 ));
833 }
834 TerminalReporter::print_success(&format!(
835 "Verbatim mode: {} request(s) will be sent exactly as written (spec endpoints not used)",
836 verbatim.len()
837 ));
838 verbatim
839 } else {
840 templates
841 };
842
843 let custom_headers = self.parse_headers()?;
845
846 let force_http1 = crate::request_gen::should_force_k6_http1(
849 self.wafbench_verbatim,
850 &templates,
851 &custom_headers,
852 );
853
854 let base_path = self.resolve_base_path(&parser);
856 if let Some(ref bp) = base_path {
857 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
858 }
859
860 TerminalReporter::print_progress("Generating k6 load test script...");
862 let scenario =
863 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
864
865 let security_testing_enabled = self.security_testing_enabled();
866
867 let num_ops = operations.len() as u32;
885 if let Some(rps) = self.target_rps {
886 let probe =
887 crate::preflight::probe_target_latency(&self.target, 3, self.skip_tls_verify).await;
888
889 let (required_vus, basis) = match probe {
890 Some(p) => (
891 p.required_vus(rps, num_ops),
892 format!("avg {:.1}ms (measured)", p.avg_latency.as_secs_f64() * 1000.0),
893 ),
894 None => {
895 let fallback = (rps as u64)
897 .saturating_mul(num_ops.max(1) as u64)
898 .div_ceil(10)
899 .min(u32::MAX as u64) as u32;
900 (fallback, "~100ms (default — probe failed)".to_string())
901 }
902 };
903
904 if self.vus < required_vus {
905 const VU_RECOMMENDATION_CAP: u32 = 1000;
911 let recommendation = required_vus.max(self.vus + 1);
912 if recommendation > VU_RECOMMENDATION_CAP {
913 TerminalReporter::print_warning(&format!(
914 "Workload is very large: --rps {} × {} ops/iteration × {} \
915 baseline ⇒ ~{} VUs needed end-to-end, far beyond what's \
916 practical to drive. Two ways to fix:\n 1. Reduce \
917 operations per iteration with `--operations 'pattern,…'` \
918 (or `--exclude-operations`) to focus the bench on a \
919 representative subset.\n 2. Drop `--rps` and use \
920 `--vus {}` alone — closed-model load runs as fast as \
921 the VU pool allows, bounded by latency, with no per-\
922 iteration deadline. Expect 1-iteration coverage of ~{} \
923 operations in {}s.",
924 rps,
925 num_ops,
926 basis,
927 recommendation,
928 self.vus.max(5),
929 num_ops,
930 Self::parse_duration(&self.duration).unwrap_or(0),
931 ));
932 } else {
933 TerminalReporter::print_warning(&format!(
934 "--vus {} may be insufficient for --rps {} × {} ops/iteration \
935 (baseline latency {}). k6's constant-arrival-rate counts ITERATIONS \
936 and each runs every operation in the spec — required ≈ rps × ops × \
937 latency_secs VUs. Bump --vus to ~{} if you see \"Insufficient VUs\" \
938 warnings.",
939 self.vus, rps, num_ops, basis, recommendation,
940 ));
941 }
942 } else if probe.is_some() {
943 TerminalReporter::print_progress(&format!(
944 "Pre-flight probe: target latency {}, {} ops/iteration — --vus {} \
945 is sufficient for --rps {}",
946 basis, num_ops, self.vus, rps,
947 ));
948 }
949 }
950
951 let k6_config = K6Config {
952 target_url: self.target.clone(),
953 base_path,
954 scenario,
955 duration_secs: Self::parse_duration(&self.duration)?,
956 max_vus: self.vus,
957 threshold_percentile: self.threshold_percentile.clone(),
958 threshold_ms: self.threshold_ms,
959 max_error_rate: self.max_error_rate,
960 auth_header: self.auth.clone(),
961 custom_headers,
962 skip_tls_verify: self.skip_tls_verify,
963 security_testing_enabled,
964 chunked_request_bodies: self.chunked_request_bodies,
965 target_rps: self.target_rps,
966 no_keep_alive: self.no_keep_alive,
967 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
973 .into_iter()
974 .map(|ip| ip.to_string())
975 .collect(),
976 geo_source_headers: if self.geo_source_headers.is_empty()
977 && !self.geo_source_ips.is_empty()
978 {
979 crate::conformance::self_test::default_geo_source_headers()
980 } else {
981 self.geo_source_headers.clone()
982 },
983 };
984
985 let duration_secs = Self::parse_duration(&self.duration)?;
988 let (per_op_metrics, per_op_warn) = crate::k6_gen::resolve_per_op_metrics(
989 self.per_op_metrics,
990 templates.len(),
991 duration_secs,
992 );
993 if let Some(msg) = per_op_warn {
994 TerminalReporter::print_warning(&msg);
995 }
996
997 let generator = K6ScriptGenerator::new(k6_config, templates)
998 .with_abort_valve(self.abort_on_error, self.abort_on_error_rate)
999 .with_force_http1(force_http1)
1000 .with_per_op_metrics(per_op_metrics);
1001 let mut script = generator.generate()?;
1002 TerminalReporter::print_success("k6 script generated");
1003
1004 let has_advanced_features = self.data_file.is_some()
1006 || self.error_rate.is_some()
1007 || self.security_test
1008 || self.parallel_create.is_some()
1009 || self.wafbench_dir.is_some();
1010
1011 if has_advanced_features {
1013 script = self.generate_enhanced_script(&script)?;
1014 }
1015
1016 if mock_config.is_mock_server {
1018 let setup_code = MockIntegrationGenerator::generate_setup(&mock_config);
1019 let teardown_code = MockIntegrationGenerator::generate_teardown(&mock_config);
1020 let helper_code = MockIntegrationGenerator::generate_vu_id_helper();
1021
1022 if let Some(import_end) = script.find("export const options") {
1024 script.insert_str(
1025 import_end,
1026 &format!(
1027 "\n// === Mock Server Integration ===\n{}\n{}\n{}\n",
1028 helper_code, setup_code, teardown_code
1029 ),
1030 );
1031 }
1032 }
1033
1034 TerminalReporter::print_progress("Validating k6 script...");
1036 let validation_errors = K6ScriptGenerator::validate_script(&script);
1037 if !validation_errors.is_empty() {
1038 TerminalReporter::print_error("Script validation failed");
1039 for error in &validation_errors {
1040 eprintln!(" {}", error);
1041 }
1042 return Err(BenchError::Other(format!(
1043 "Generated k6 script has {} validation error(s). Please check the output above.",
1044 validation_errors.len()
1045 )));
1046 }
1047 TerminalReporter::print_success("Script validation passed");
1048
1049 let script_path = if let Some(output) = &self.script_output {
1051 output.clone()
1052 } else {
1053 self.output.join("k6-script.js")
1054 };
1055
1056 if let Some(parent) = script_path.parent() {
1057 std::fs::create_dir_all(parent)?;
1058 }
1059 std::fs::write(&script_path, &script)?;
1060 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
1061
1062 if self.generate_only {
1064 Self::print_k6_run_hint(&script_path, force_http1);
1065 return Ok(());
1066 }
1067
1068 TerminalReporter::print_progress("Executing load test...");
1070 if force_http1 {
1071 TerminalReporter::print_progress(
1072 "Forcing HTTP/1.1 (GODEBUG=http2client=0): Connection headers are hop-by-hop and HTTP/2 rejects them. The header stays on the wire.",
1073 );
1074 }
1075 let executor = K6Executor::new()?
1079 .with_local_ips(self.source_ips.join(","))
1080 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
1081 .with_discard_response_bodies(self.discard_response_bodies)
1082 .with_force_http1(force_http1);
1083
1084 std::fs::create_dir_all(&self.output)?;
1085
1086 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
1087
1088 let duration_secs = Self::parse_duration(&self.duration)?;
1090 TerminalReporter::print_summary_full(
1091 &results,
1092 duration_secs,
1093 self.no_keep_alive,
1094 Some(num_ops),
1095 );
1096
1097 self.reprint_traffic_file_breakdown();
1098 println!("\nResults saved to: {}", self.output.display());
1099
1100 Ok(())
1101 }
1102
1103 async fn execute_multi_target(&self, targets_file: &Path) -> Result<()> {
1105 TerminalReporter::print_progress("Parsing targets file...");
1106 let targets = parse_targets_file(targets_file)?;
1107 let num_targets = targets.len();
1108 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
1109
1110 if targets.is_empty() {
1111 return Err(BenchError::Other("No targets found in file".to_string()));
1112 }
1113
1114 let max_concurrency = self.max_concurrency.map(|n| n as usize);
1118
1119 TerminalReporter::print_header(
1121 &self.get_spec_display_name(),
1122 &format!("{} targets", num_targets),
1123 0,
1124 &self.scenario,
1125 Self::parse_duration(&self.duration)?,
1126 );
1127
1128 let executor = ParallelExecutor::new(
1130 BenchCommand {
1131 spec: self.spec.clone(),
1133 spec_dir: self.spec_dir.clone(),
1134 merge_conflicts: self.merge_conflicts.clone(),
1135 spec_mode: self.spec_mode.clone(),
1136 dependency_config: self.dependency_config.clone(),
1137 target: self.target.clone(), base_path: self.base_path.clone(),
1139 duration: self.duration.clone(),
1140 vus: self.vus,
1141 target_rps: self.target_rps,
1142 no_keep_alive: self.no_keep_alive,
1143 scenario: self.scenario.clone(),
1144 operations: self.operations.clone(),
1145 exclude_operations: self.exclude_operations.clone(),
1146 auth: self.auth.clone(),
1147 headers: self.headers.clone(),
1148 output: self.output.clone(),
1149 generate_only: self.generate_only,
1150 script_output: self.script_output.clone(),
1151 threshold_percentile: self.threshold_percentile.clone(),
1152 threshold_ms: self.threshold_ms,
1153 max_error_rate: self.max_error_rate,
1154 abort_on_error: self.abort_on_error,
1155 abort_on_error_rate: self.abort_on_error_rate,
1156 per_op_metrics: self.per_op_metrics,
1157 verbose: self.verbose,
1158 skip_tls_verify: self.skip_tls_verify,
1159 chunked_request_bodies: self.chunked_request_bodies,
1160 targets_file: None,
1161 max_concurrency: None,
1162 repeat_until: self.repeat_until.clone(),
1163 rounds: self.rounds,
1164 results_format: self.results_format.clone(),
1165 params_file: self.params_file.clone(),
1166 crud_flow: self.crud_flow,
1167 flow_config: self.flow_config.clone(),
1168 extract_fields: self.extract_fields.clone(),
1169 parallel_create: self.parallel_create,
1170 data_file: self.data_file.clone(),
1171 data_distribution: self.data_distribution.clone(),
1172 data_mappings: self.data_mappings.clone(),
1173 per_uri_control: self.per_uri_control,
1174 error_rate: self.error_rate,
1175 error_types: self.error_types.clone(),
1176 security_test: self.security_test,
1177 security_payloads: self.security_payloads.clone(),
1178 security_categories: self.security_categories.clone(),
1179 security_target_fields: self.security_target_fields.clone(),
1180 wafbench_dir: self.wafbench_dir.clone(),
1181 wafbench_cycle_all: self.wafbench_cycle_all,
1182 wafbench_verbatim: self.wafbench_verbatim,
1183 owasp_api_top10: self.owasp_api_top10,
1184 owasp_categories: self.owasp_categories.clone(),
1185 owasp_auth_header: self.owasp_auth_header.clone(),
1186 owasp_auth_token: self.owasp_auth_token.clone(),
1187 owasp_admin_paths: self.owasp_admin_paths.clone(),
1188 owasp_id_fields: self.owasp_id_fields.clone(),
1189 owasp_report: self.owasp_report.clone(),
1190 owasp_report_format: self.owasp_report_format.clone(),
1191 owasp_iterations: self.owasp_iterations,
1192 conformance: false,
1193 conformance_api_key: self.conformance_api_key.clone(),
1209 conformance_basic_auth: self.conformance_basic_auth.clone(),
1210 conformance_report: PathBuf::from("conformance-report.json"),
1211 conformance_categories: None,
1212 conformance_report_format: "json".to_string(),
1213 conformance_headers: self.conformance_headers.clone(),
1217 conformance_all_operations: false,
1218 conformance_custom: None,
1219 conformance_delay_ms: 0,
1220 use_k6: false,
1221 conformance_custom_filter: None,
1222 export_requests: false,
1223 validate_requests: false,
1224 conformance_self_test: false,
1225 conformance_self_test_capture: false,
1226 conformance_self_test_iterations: 1,
1227 conformance_self_test_duration: None,
1228 validate_response_schemas: false,
1229 source_ips: self.source_ips.clone(),
1234 geo_source_ips: self.geo_source_ips.clone(),
1235 geo_source_headers: self.geo_source_headers.clone(),
1236 report_missed_cap: None,
1237 discard_response_bodies: self.discard_response_bodies,
1241 dns_policy: self.dns_policy.clone(),
1244 },
1245 targets,
1246 max_concurrency,
1247 );
1248
1249 let start_time = std::time::Instant::now();
1251 let aggregated_results = executor.execute_all().await?;
1252 let elapsed = start_time.elapsed();
1253
1254 self.report_multi_target_results(&aggregated_results, elapsed)?;
1256
1257 Ok(())
1258 }
1259
1260 fn report_multi_target_results(
1262 &self,
1263 results: &AggregatedResults,
1264 elapsed: std::time::Duration,
1265 ) -> Result<()> {
1266 TerminalReporter::print_multi_target_summary(results);
1268
1269 let total_secs = elapsed.as_secs();
1271 let hours = total_secs / 3600;
1272 let minutes = (total_secs % 3600) / 60;
1273 let seconds = total_secs % 60;
1274 if hours > 0 {
1275 println!("\n Total Elapsed Time: {}h {}m {}s", hours, minutes, seconds);
1276 } else if minutes > 0 {
1277 println!("\n Total Elapsed Time: {}m {}s", minutes, seconds);
1278 } else {
1279 println!("\n Total Elapsed Time: {}s", seconds);
1280 }
1281
1282 if self.results_format == "aggregated" || self.results_format == "both" {
1284 let summary_path = self.output.join("aggregated_summary.json");
1285 let summary_json = serde_json::json!({
1286 "total_elapsed_seconds": elapsed.as_secs(),
1287 "total_targets": results.total_targets,
1288 "successful_targets": results.successful_targets,
1289 "failed_targets": results.failed_targets,
1290 "aggregated_metrics": {
1291 "total_requests": results.aggregated_metrics.total_requests,
1292 "total_failed_requests": results.aggregated_metrics.total_failed_requests,
1293 "avg_duration_ms": results.aggregated_metrics.avg_duration_ms,
1294 "p95_duration_ms": results.aggregated_metrics.p95_duration_ms,
1295 "p99_duration_ms": results.aggregated_metrics.p99_duration_ms,
1296 "error_rate": results.aggregated_metrics.error_rate,
1297 "total_rps": results.aggregated_metrics.total_rps,
1298 "avg_rps": results.aggregated_metrics.avg_rps,
1299 "total_vus_max": results.aggregated_metrics.total_vus_max,
1300 },
1301 "target_results": results.target_results.iter().map(|r| {
1302 serde_json::json!({
1303 "target_url": r.target_url,
1304 "target_index": r.target_index,
1305 "success": r.success,
1306 "error": r.error,
1307 "total_requests": r.results.total_requests,
1308 "failed_requests": r.results.failed_requests,
1309 "avg_duration_ms": r.results.avg_duration_ms,
1310 "min_duration_ms": r.results.min_duration_ms,
1311 "med_duration_ms": r.results.med_duration_ms,
1312 "p90_duration_ms": r.results.p90_duration_ms,
1313 "p95_duration_ms": r.results.p95_duration_ms,
1314 "p99_duration_ms": r.results.p99_duration_ms,
1315 "max_duration_ms": r.results.max_duration_ms,
1316 "rps": r.results.rps,
1317 "vus_max": r.results.vus_max,
1318 "output_dir": r.output_dir.to_string_lossy(),
1319 })
1320 }).collect::<Vec<_>>(),
1321 });
1322
1323 std::fs::write(&summary_path, serde_json::to_string_pretty(&summary_json)?)?;
1324 TerminalReporter::print_success(&format!(
1325 "Aggregated summary saved to: {}",
1326 summary_path.display()
1327 ));
1328 }
1329
1330 let csv_path = self.output.join("all_targets.csv");
1332 let mut csv = String::from(
1333 "target_url,success,requests,failed,rps,vus,min_ms,avg_ms,med_ms,p90_ms,p95_ms,p99_ms,max_ms,error\n",
1334 );
1335 for r in &results.target_results {
1336 csv.push_str(&format!(
1337 "{},{},{},{},{:.1},{},{:.1},{:.1},{:.1},{:.1},{:.1},{:.1},{:.1},{}\n",
1338 r.target_url,
1339 r.success,
1340 r.results.total_requests,
1341 r.results.failed_requests,
1342 r.results.rps,
1343 r.results.vus_max,
1344 r.results.min_duration_ms,
1345 r.results.avg_duration_ms,
1346 r.results.med_duration_ms,
1347 r.results.p90_duration_ms,
1348 r.results.p95_duration_ms,
1349 r.results.p99_duration_ms,
1350 r.results.max_duration_ms,
1351 r.error.as_deref().unwrap_or(""),
1352 ));
1353 }
1354 let _ = std::fs::write(&csv_path, &csv);
1355
1356 self.reprint_traffic_file_breakdown();
1357 println!("\nResults saved to: {}", self.output.display());
1358 println!(" - Per-target results: {}", self.output.join("target_*").display());
1359 println!(" - All targets CSV: {}", csv_path.display());
1360 if self.results_format == "aggregated" || self.results_format == "both" {
1361 println!(
1362 " - Aggregated summary: {}",
1363 self.output.join("aggregated_summary.json").display()
1364 );
1365 }
1366
1367 Ok(())
1368 }
1369
1370 pub fn parse_duration(duration: &str) -> Result<u64> {
1372 let duration = duration.trim();
1373
1374 if let Some(secs) = duration.strip_suffix('s') {
1375 secs.parse::<u64>()
1376 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1377 } else if let Some(mins) = duration.strip_suffix('m') {
1378 mins.parse::<u64>()
1379 .map(|m| m * 60)
1380 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1381 } else if let Some(hours) = duration.strip_suffix('h') {
1382 hours
1383 .parse::<u64>()
1384 .map(|h| h * 3600)
1385 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1386 } else {
1387 duration
1389 .parse::<u64>()
1390 .map_err(|_| BenchError::Other(format!("Invalid duration: {}", duration)))
1391 }
1392 }
1393
1394 fn print_k6_run_hint(script_path: &Path, force_http1: bool) {
1398 println!("\nScript generated successfully. Run it with:");
1399 if force_http1 {
1400 println!(" GODEBUG=http2client=0 k6 run {}", script_path.display());
1401 println!(
1402 " (HTTP/1.1: a Connection header is on the wire; HTTP/2 rejects it. mockforge bench sets this automatically when it invokes k6.)"
1403 );
1404 } else {
1405 println!(" k6 run {}", script_path.display());
1406 }
1407 }
1408
1409 pub(crate) fn load_verbatim_templates(
1416 &self,
1417 ) -> Result<Vec<crate::request_gen::RequestTemplate>> {
1418 let Some(pattern) = self.wafbench_dir.as_ref() else {
1419 return Err(BenchError::Other(
1420 "--wafbench-verbatim requires --wafbench-dir pointing at your traffic file(s)"
1421 .to_string(),
1422 ));
1423 };
1424
1425 let mut loader = WafBenchLoader::new();
1426 loader.load_from_pattern(pattern)?;
1427 self.emit_traffic_file_breakdown(loader.stats(), "what to expect in proxy logs");
1428
1429 Ok(crate::wafbench::traffic_cases_to_templates(loader.test_cases()))
1430 }
1431
1432 pub fn parse_headers(&self) -> Result<HashMap<String, String>> {
1434 let mut headers = parse_header_string(&self.headers)?;
1435
1436 let already_has = |hs: &HashMap<String, String>, name: &str| -> bool {
1447 hs.keys().any(|k| k.eq_ignore_ascii_case(name))
1448 };
1449
1450 if !already_has(&headers, "Authorization") {
1451 if let Some(b) = self.conformance_basic_auth.as_ref().filter(|s| !s.is_empty()) {
1452 use base64::Engine as _;
1453 let encoded = base64::engine::general_purpose::STANDARD.encode(b.as_bytes());
1454 headers.insert("Authorization".to_string(), format!("Basic {}", encoded));
1455 }
1456 }
1457
1458 for line in &self.conformance_headers {
1464 let Some((name, value)) = line.split_once(':') else {
1465 continue;
1466 };
1467 let name = name.trim();
1468 let value = value.trim();
1469 if name.is_empty() || already_has(&headers, name) {
1470 continue;
1471 }
1472 headers.insert(name.to_string(), value.to_string());
1473 }
1474
1475 if !self.conformance && self.conformance_api_key.is_some() {
1481 TerminalReporter::print_warning(
1482 "--conformance-api-key only fires under --conformance. For plain bench use --header 'X-API-Key: ...'.",
1483 );
1484 }
1485
1486 Ok(headers)
1487 }
1488
1489 fn parse_extracted_values(output_dir: &Path) -> Result<ExtractedValues> {
1490 let extracted_path = output_dir.join("extracted_values.json");
1491 if !extracted_path.exists() {
1492 return Ok(ExtractedValues::new());
1493 }
1494
1495 let content = std::fs::read_to_string(&extracted_path)
1496 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1497 let parsed: serde_json::Value = serde_json::from_str(&content)
1498 .map_err(|e| BenchError::ResultsParseError(e.to_string()))?;
1499
1500 let mut extracted = ExtractedValues::new();
1501 if let Some(values) = parsed.as_object() {
1502 for (key, value) in values {
1503 extracted.set(key.clone(), value.clone());
1504 }
1505 }
1506
1507 Ok(extracted)
1508 }
1509
1510 fn resolve_base_path(&self, parser: &SpecParser) -> Option<String> {
1519 if let Some(cli_base_path) = &self.base_path {
1521 if cli_base_path.is_empty() {
1522 return None;
1524 }
1525 return Some(cli_base_path.clone());
1526 }
1527
1528 parser.get_base_path()
1530 }
1531
1532 async fn build_mock_config(&self) -> MockIntegrationConfig {
1534 if MockServerDetector::looks_like_mock_server(&self.target) {
1536 if let Ok(info) = MockServerDetector::detect(&self.target).await {
1538 if info.is_mockforge {
1539 TerminalReporter::print_success(&format!(
1540 "Detected MockForge server (version: {})",
1541 info.version.as_deref().unwrap_or("unknown")
1542 ));
1543 return MockIntegrationConfig::mock_server();
1544 }
1545 }
1546 }
1547 MockIntegrationConfig::real_api()
1548 }
1549
1550 fn build_crud_flow_config(&self) -> Option<CrudFlowConfig> {
1552 if !self.crud_flow {
1553 return None;
1554 }
1555
1556 if let Some(config_path) = &self.flow_config {
1558 match CrudFlowConfig::from_file(config_path) {
1559 Ok(config) => return Some(config),
1560 Err(e) => {
1561 TerminalReporter::print_warning(&format!(
1562 "Failed to load flow config: {}. Using auto-detection.",
1563 e
1564 ));
1565 }
1566 }
1567 }
1568
1569 let extract_fields = self
1571 .extract_fields
1572 .as_ref()
1573 .map(|f| f.split(',').map(|s| s.trim().to_string()).collect())
1574 .unwrap_or_else(|| vec!["id".to_string(), "uuid".to_string()]);
1575
1576 Some(CrudFlowConfig {
1577 flows: Vec::new(), default_extract_fields: extract_fields,
1579 })
1580 }
1581
1582 fn build_data_driven_config(&self) -> Option<DataDrivenConfig> {
1584 let data_file = self.data_file.as_ref()?;
1585
1586 let distribution = DataDistribution::from_str(&self.data_distribution)
1587 .unwrap_or(DataDistribution::UniquePerVu);
1588
1589 let mappings = self
1590 .data_mappings
1591 .as_ref()
1592 .map(|m| DataMapping::parse_mappings(m).unwrap_or_default())
1593 .unwrap_or_default();
1594
1595 Some(DataDrivenConfig {
1596 file_path: data_file.to_string_lossy().to_string(),
1597 distribution,
1598 mappings,
1599 csv_has_header: true,
1600 per_uri_control: self.per_uri_control,
1601 per_uri_columns: crate::data_driven::PerUriColumns::default(),
1602 })
1603 }
1604
1605 fn build_invalid_data_config(&self) -> Option<InvalidDataConfig> {
1607 let error_rate = self.error_rate?;
1608
1609 let error_types = self
1610 .error_types
1611 .as_ref()
1612 .map(|types| InvalidDataConfig::parse_error_types(types).unwrap_or_default())
1613 .unwrap_or_default();
1614
1615 Some(InvalidDataConfig {
1616 error_rate,
1617 error_types,
1618 target_fields: Vec::new(),
1619 })
1620 }
1621
1622 fn build_security_config(&self) -> Option<SecurityTestConfig> {
1624 if !self.security_test {
1625 return None;
1626 }
1627
1628 let categories = self
1629 .security_categories
1630 .as_ref()
1631 .map(|cats| SecurityTestConfig::parse_categories(cats).unwrap_or_default())
1632 .unwrap_or_else(|| {
1633 let mut default = HashSet::new();
1634 default.insert(SecurityCategory::SqlInjection);
1635 default.insert(SecurityCategory::Xss);
1636 default
1637 });
1638
1639 let target_fields = self
1640 .security_target_fields
1641 .as_ref()
1642 .map(|fields| fields.split(',').map(|f| f.trim().to_string()).collect())
1643 .unwrap_or_default();
1644
1645 let custom_payloads_file =
1646 self.security_payloads.as_ref().map(|p| p.to_string_lossy().to_string());
1647
1648 Some(SecurityTestConfig {
1649 enabled: true,
1650 categories,
1651 target_fields,
1652 custom_payloads_file,
1653 include_high_risk: false,
1654 })
1655 }
1656
1657 fn build_parallel_config(&self) -> Option<ParallelConfig> {
1659 let count = self.parallel_create?;
1660
1661 Some(ParallelConfig::new(count))
1662 }
1663
1664 fn format_unique_total(unique: usize, rps: Option<u32>) -> String {
1667 match rps {
1668 Some(r) if r > 0 => {
1669 let projected = unique.saturating_mul(r as usize);
1670 format!(
1671 "unique_cases={unique} projected_per_second={projected} ({unique} * {r} RPS)"
1672 )
1673 }
1674 _ => format!("unique_cases={unique}"),
1675 }
1676 }
1677
1678 fn traffic_bucket_json(
1679 unique: usize,
1680 rps: Option<u32>,
1681 duration_secs: Option<u64>,
1682 ) -> serde_json::Value {
1683 let per_second = rps.filter(|&r| r > 0).map(|r| (unique as u64).saturating_mul(r as u64));
1690 let projected_over_run = match (rps.filter(|&r| r > 0), duration_secs) {
1691 (Some(r), Some(d)) => Some((unique as u64).saturating_mul(r as u64).saturating_mul(d)),
1692 _ => None,
1693 };
1694 serde_json::json!({
1695 "unique_cases": unique,
1696 "projected_per_second": per_second,
1697 "projected_over_run": projected_over_run,
1698 })
1699 }
1700
1701 fn emit_traffic_file_breakdown(&self, stats: &crate::wafbench::WafBenchStats, phase: &str) {
1704 if stats.per_file.is_empty() {
1705 return;
1706 }
1707 let rps = self.target_rps.filter(|&r| r > 0);
1708 TerminalReporter::print_success(&format!("Traffic file breakdown ({phase}):"));
1709 for file in &stats.per_file {
1710 let other = if file.other > 0 {
1711 format!(" other={}", file.other)
1712 } else {
1713 String::new()
1714 };
1715 TerminalReporter::print_progress(&format!(
1716 " {}: sent {} attack(expected 403) {} normal(expected 200) {} omitted={}{other}",
1717 file.file,
1718 Self::format_unique_total(file.sent, rps),
1719 Self::format_unique_total(file.attack, rps),
1720 Self::format_unique_total(file.normal, rps),
1721 file.omitted
1722 ));
1723 }
1724 self.write_traffic_breakdown_json(stats);
1725 }
1726
1727 fn write_traffic_breakdown_json(&self, stats: &crate::wafbench::WafBenchStats) {
1729 if stats.per_file.is_empty() {
1730 return;
1731 }
1732 let rps = self.target_rps.filter(|&r| r > 0);
1733 let duration_secs = Self::parse_duration(&self.duration).ok();
1734 let files: Vec<serde_json::Value> = stats
1735 .per_file
1736 .iter()
1737 .map(|file| {
1738 serde_json::json!({
1739 "file": file.file,
1740 "sent": Self::traffic_bucket_json(file.sent, rps, duration_secs),
1741 "attack": Self::traffic_bucket_json(file.attack, rps, duration_secs),
1742 "normal": Self::traffic_bucket_json(file.normal, rps, duration_secs),
1743 "omitted": file.omitted,
1744 "other": file.other,
1745 })
1746 })
1747 .collect();
1748 let payload = serde_json::json!({
1749 "rps": rps,
1750 "duration_secs": duration_secs,
1751 "note": "Plan, not k6 counters. unique_cases is the YAML case count (not traffic on the wire). projected_per_second is unique_cases * rps. projected_over_run is unique_cases * rps * duration_secs, assuming each k6 iteration sends every unique case. projected_* are null when --rps is unset.",
1752 "files": files,
1753 });
1754 if let Some(parent) = self.output.parent() {
1755 let _ = std::fs::create_dir_all(parent);
1756 }
1757 let _ = std::fs::create_dir_all(&self.output);
1758 let path = self.output.join("traffic-breakdown.json");
1759 if let Ok(body) = serde_json::to_string_pretty(&payload) {
1760 if std::fs::write(&path, body).is_ok() {
1761 TerminalReporter::print_progress(&format!(
1762 "Traffic breakdown written to: {}",
1763 path.display()
1764 ));
1765 }
1766 }
1767 }
1768
1769 fn reprint_traffic_file_breakdown(&self) {
1772 let path = self.output.join("traffic-breakdown.json");
1773 let Ok(raw) = std::fs::read_to_string(&path) else {
1774 return;
1775 };
1776 let Ok(v) = serde_json::from_str::<serde_json::Value>(&raw) else {
1777 return;
1778 };
1779 let Some(files) = v.get("files").and_then(|f| f.as_array()) else {
1780 return;
1781 };
1782 if files.is_empty() {
1783 return;
1784 }
1785 TerminalReporter::print_success("Traffic file breakdown (end of run):");
1786 for file in files {
1787 let name = file.get("file").and_then(|x| x.as_str()).unwrap_or("?");
1788 let bucket = |key: &str| -> String {
1789 let unique = file
1794 .get(key)
1795 .and_then(|b| b.get("unique_cases").or_else(|| b.get("unique")))
1796 .and_then(|u| u.as_u64())
1797 .unwrap_or(0) as usize;
1798 Self::format_unique_total(unique, self.target_rps.filter(|&r| r > 0))
1799 };
1800 let omitted = file.get("omitted").and_then(|o| o.as_u64()).unwrap_or(0);
1801 let other = file.get("other").and_then(|o| o.as_u64()).unwrap_or(0);
1802 let other = if other > 0 {
1803 format!(" other={other}")
1804 } else {
1805 String::new()
1806 };
1807 TerminalReporter::print_progress(&format!(
1808 " {name}: sent {} attack(expected 403) {} normal(expected 200) {} omitted={omitted}{other}",
1809 bucket("sent"),
1810 bucket("attack"),
1811 bucket("normal"),
1812 ));
1813 }
1814 TerminalReporter::print_progress(&format!(" (also in {})", path.display()));
1815 }
1816
1817 fn load_wafbench_payloads(&self) -> Result<Vec<SecurityPayload>> {
1824 let Some(ref wafbench_dir) = self.wafbench_dir else {
1825 return Ok(Vec::new());
1826 };
1827
1828 let mut loader = WafBenchLoader::new();
1829 loader.load_from_pattern(wafbench_dir)?;
1830
1831 let stats = loader.stats();
1832
1833 if stats.files_processed == 0 {
1834 let mut msg = format!(
1835 "No WAFBench YAML files found matching '{wafbench_dir}'. \
1836 --wafbench-dir is a file, a directory or a glob. A missing \
1837 file is an error, not an empty payload pool."
1838 );
1839 if !stats.parse_errors.is_empty() {
1840 msg.push_str(" Parse errors:");
1841 for error in &stats.parse_errors {
1842 msg.push_str(&format!("\n - {error}"));
1843 }
1844 }
1845 return Err(BenchError::Other(msg));
1846 }
1847
1848 TerminalReporter::print_progress(&format!(
1849 "Loaded {} WAFBench files, {} test cases, {} payloads",
1850 stats.files_processed, stats.test_cases_loaded, stats.payloads_extracted
1851 ));
1852 self.emit_traffic_file_breakdown(stats, "what to expect in proxy logs");
1853
1854 for (category, count) in &stats.by_category {
1856 TerminalReporter::print_progress(&format!(" - {}: {} tests", category, count));
1857 }
1858
1859 for error in &stats.parse_errors {
1861 TerminalReporter::print_warning(&format!(" Parse error: {}", error));
1862 }
1863
1864 Ok(loader.to_security_payloads())
1865 }
1866
1867 pub(crate) fn generate_enhanced_script(&self, base_script: &str) -> Result<String> {
1869 let mut enhanced_script = base_script.to_string();
1870 let mut additional_code = String::new();
1871
1872 if let Some(config) = self.build_data_driven_config() {
1874 TerminalReporter::print_progress("Adding data-driven testing support...");
1875 additional_code.push_str(&DataDrivenGenerator::generate_setup(&config));
1876 additional_code.push('\n');
1877 TerminalReporter::print_success("Data-driven testing enabled");
1878 }
1879
1880 if let Some(config) = self.build_invalid_data_config() {
1882 TerminalReporter::print_progress("Adding invalid data testing support...");
1883 additional_code.push_str(&InvalidDataGenerator::generate_invalidation_logic());
1884 additional_code.push('\n');
1885 additional_code
1886 .push_str(&InvalidDataGenerator::generate_should_invalidate(config.error_rate));
1887 additional_code.push('\n');
1888 additional_code
1889 .push_str(&InvalidDataGenerator::generate_type_selection(&config.error_types));
1890 additional_code.push('\n');
1891 TerminalReporter::print_success(&format!(
1892 "Invalid data testing enabled ({}% error rate)",
1893 (self.error_rate.unwrap_or(0.0) * 100.0) as u32
1894 ));
1895 }
1896
1897 let verbatim = self.wafbench_verbatim;
1904 if verbatim && self.security_test {
1905 TerminalReporter::print_warning(
1906 "--security-test is ignored under --wafbench-verbatim: verbatim mode sends your \
1907 traffic cases exactly as written and will not append attack payloads to them. \
1908 Drop --wafbench-verbatim if you want payload injection.",
1909 );
1910 }
1911 let security_config = if verbatim {
1912 None
1913 } else {
1914 self.build_security_config()
1915 };
1916 let wafbench_payloads = if verbatim {
1917 Vec::new()
1918 } else {
1919 self.load_wafbench_payloads()?
1920 };
1921 let security_requested =
1922 !verbatim && (security_config.is_some() || self.wafbench_dir.is_some());
1923
1924 if security_config.is_some() || !wafbench_payloads.is_empty() {
1925 TerminalReporter::print_progress("Adding security testing support...");
1926
1927 let mut payload_list: Vec<SecurityPayload> = Vec::new();
1929
1930 if let Some(ref config) = security_config {
1931 payload_list.extend(SecurityPayloads::get_payloads(config));
1932 }
1933
1934 if !wafbench_payloads.is_empty() {
1936 TerminalReporter::print_progress(&format!(
1937 "Loading {} WAFBench attack patterns...",
1938 wafbench_payloads.len()
1939 ));
1940 payload_list.extend(wafbench_payloads);
1941 }
1942
1943 let target_fields =
1944 security_config.as_ref().map(|c| c.target_fields.clone()).unwrap_or_default();
1945
1946 additional_code.push_str(&SecurityTestGenerator::generate_payload_selection(
1947 &payload_list,
1948 self.wafbench_cycle_all,
1949 ));
1950 additional_code.push('\n');
1951 additional_code
1952 .push_str(&SecurityTestGenerator::generate_apply_payload(&target_fields));
1953 additional_code.push('\n');
1954 additional_code.push_str(&SecurityTestGenerator::generate_security_checks());
1955 additional_code.push('\n');
1956
1957 let mode = if self.wafbench_cycle_all {
1958 "cycle-all"
1959 } else {
1960 "random"
1961 };
1962 TerminalReporter::print_success(&format!(
1963 "Security testing enabled ({} payloads, {} mode)",
1964 payload_list.len(),
1965 mode
1966 ));
1967 } else if security_requested {
1968 TerminalReporter::print_warning(
1972 "Security testing was requested but no payloads were loaded. \
1973 Ensure --wafbench-dir points to valid CRS YAML files or add --security-test.",
1974 );
1975 additional_code
1976 .push_str(&SecurityTestGenerator::generate_payload_selection(&[], false));
1977 additional_code.push('\n');
1978 additional_code.push_str(&SecurityTestGenerator::generate_apply_payload(&[]));
1979 additional_code.push('\n');
1980 }
1981
1982 if let Some(config) = self.build_parallel_config() {
1984 TerminalReporter::print_progress("Adding parallel execution support...");
1985 additional_code.push_str(&ParallelRequestGenerator::generate_batch_helper(&config));
1986 additional_code.push('\n');
1987 TerminalReporter::print_success(&format!(
1988 "Parallel execution enabled (count: {})",
1989 config.count
1990 ));
1991 }
1992
1993 if !additional_code.is_empty() {
1995 if let Some(import_end) = enhanced_script.find("export const options") {
1997 enhanced_script.insert_str(
1998 import_end,
1999 &format!("\n// === Advanced Testing Features ===\n{}\n", additional_code),
2000 );
2001 }
2002 }
2003
2004 Ok(enhanced_script)
2005 }
2006
2007 async fn execute_sequential_specs(&self) -> Result<()> {
2009 TerminalReporter::print_progress("Sequential spec mode: Loading specs individually...");
2010
2011 let mut all_specs: Vec<(PathBuf, OpenApiSpec)> = Vec::new();
2013
2014 if !self.spec.is_empty() {
2015 let specs = load_specs_from_files(self.spec.clone())
2016 .await
2017 .map_err(|e| BenchError::Other(format!("Failed to load spec files: {}", e)))?;
2018 all_specs.extend(specs);
2019 }
2020
2021 if let Some(spec_dir) = &self.spec_dir {
2022 let dir_specs = load_specs_from_directory(spec_dir).await.map_err(|e| {
2023 BenchError::Other(format!("Failed to load specs from directory: {}", e))
2024 })?;
2025 all_specs.extend(dir_specs);
2026 }
2027
2028 if all_specs.is_empty() {
2029 return Err(BenchError::Other(
2030 "No spec files found for sequential execution".to_string(),
2031 ));
2032 }
2033
2034 TerminalReporter::print_success(&format!("Loaded {} spec(s)", all_specs.len()));
2035
2036 let execution_order = if let Some(config_path) = &self.dependency_config {
2038 TerminalReporter::print_progress("Loading dependency configuration...");
2039 let config = SpecDependencyConfig::from_file(config_path)?;
2040
2041 if !config.disable_auto_detect && config.execution_order.is_empty() {
2042 self.detect_and_sort_specs(&all_specs)?
2044 } else {
2045 config.execution_order.iter().flat_map(|g| g.specs.clone()).collect()
2047 }
2048 } else {
2049 self.detect_and_sort_specs(&all_specs)?
2051 };
2052
2053 TerminalReporter::print_success(&format!(
2054 "Execution order: {}",
2055 execution_order
2056 .iter()
2057 .map(|p| p.file_name().unwrap_or_default().to_string_lossy().to_string())
2058 .collect::<Vec<_>>()
2059 .join(" → ")
2060 ));
2061
2062 let mut extracted_values = ExtractedValues::new();
2064 let total_specs = execution_order.len();
2065
2066 for (index, spec_path) in execution_order.iter().enumerate() {
2067 let spec_name = spec_path.file_name().unwrap_or_default().to_string_lossy().to_string();
2068
2069 TerminalReporter::print_progress(&format!(
2070 "[{}/{}] Executing spec: {}",
2071 index + 1,
2072 total_specs,
2073 spec_name
2074 ));
2075
2076 let spec = all_specs
2078 .iter()
2079 .find(|(p, _)| {
2080 p == spec_path
2081 || p.file_name() == spec_path.file_name()
2082 || p.file_name() == Some(spec_path.as_os_str())
2083 })
2084 .map(|(_, s)| s.clone())
2085 .ok_or_else(|| {
2086 BenchError::Other(format!("Spec not found: {}", spec_path.display()))
2087 })?;
2088
2089 let new_values = self.execute_single_spec(&spec, &spec_name, &extracted_values).await?;
2091
2092 extracted_values.merge(&new_values);
2094
2095 TerminalReporter::print_success(&format!(
2096 "[{}/{}] Completed: {} (extracted {} values)",
2097 index + 1,
2098 total_specs,
2099 spec_name,
2100 new_values.values.len()
2101 ));
2102 }
2103
2104 TerminalReporter::print_success(&format!(
2105 "Sequential execution complete: {} specs executed",
2106 total_specs
2107 ));
2108
2109 Ok(())
2110 }
2111
2112 fn detect_and_sort_specs(&self, specs: &[(PathBuf, OpenApiSpec)]) -> Result<Vec<PathBuf>> {
2114 TerminalReporter::print_progress("Auto-detecting spec dependencies...");
2115
2116 let mut detector = DependencyDetector::new();
2117 let dependencies = detector.detect_dependencies(specs);
2118
2119 if dependencies.is_empty() {
2120 TerminalReporter::print_progress("No dependencies detected, using file order");
2121 return Ok(specs.iter().map(|(p, _)| p.clone()).collect());
2122 }
2123
2124 TerminalReporter::print_progress(&format!(
2125 "Detected {} cross-spec dependencies",
2126 dependencies.len()
2127 ));
2128
2129 for dep in &dependencies {
2130 TerminalReporter::print_progress(&format!(
2131 " {} → {} (via field '{}')",
2132 dep.dependency_spec.file_name().unwrap_or_default().to_string_lossy(),
2133 dep.dependent_spec.file_name().unwrap_or_default().to_string_lossy(),
2134 dep.field_name
2135 ));
2136 }
2137
2138 topological_sort(specs, &dependencies)
2139 }
2140
2141 async fn execute_single_spec(
2143 &self,
2144 spec: &OpenApiSpec,
2145 spec_name: &str,
2146 _external_values: &ExtractedValues,
2147 ) -> Result<ExtractedValues> {
2148 let parser = SpecParser::from_spec(spec.clone());
2149
2150 if self.crud_flow {
2152 self.execute_crud_flow_with_extraction(&parser, spec_name).await
2154 } else {
2155 self.execute_standard_spec(&parser, spec_name).await?;
2157 Ok(ExtractedValues::new())
2158 }
2159 }
2160
2161 async fn execute_crud_flow_with_extraction(
2163 &self,
2164 parser: &SpecParser,
2165 spec_name: &str,
2166 ) -> Result<ExtractedValues> {
2167 let operations = parser.get_operations();
2168 let flows = CrudFlowDetector::detect_flows(&operations);
2169
2170 if flows.is_empty() {
2171 TerminalReporter::print_warning(&format!("No CRUD flows detected in {}", spec_name));
2172 return Ok(ExtractedValues::new());
2173 }
2174
2175 TerminalReporter::print_progress(&format!(
2176 " {} CRUD flow(s) in {}",
2177 flows.len(),
2178 spec_name
2179 ));
2180
2181 let mut handlebars = handlebars::Handlebars::new();
2183 handlebars.register_helper(
2185 "json",
2186 Box::new(
2187 |h: &handlebars::Helper,
2188 _: &handlebars::Handlebars,
2189 _: &handlebars::Context,
2190 _: &mut handlebars::RenderContext,
2191 out: &mut dyn handlebars::Output|
2192 -> handlebars::HelperResult {
2193 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2194 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2195 Ok(())
2196 },
2197 ),
2198 );
2199 let template = include_str!("templates/k6_crud_flow.hbs");
2200 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
2201
2202 let custom_headers = self.parse_headers()?;
2203 let config = self.build_crud_flow_config().unwrap_or_default();
2204
2205 let param_overrides = if let Some(params_file) = &self.params_file {
2207 let overrides = ParameterOverrides::from_file(params_file)?;
2208 Some(overrides)
2209 } else {
2210 None
2211 };
2212
2213 let duration_secs = Self::parse_duration(&self.duration)?;
2215 let scenario =
2216 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2217 let stages = scenario.generate_stages(duration_secs, self.vus);
2218
2219 let api_base_path = self.resolve_base_path(parser);
2221
2222 let mut all_headers = custom_headers.clone();
2224 if let Some(auth) = &self.auth {
2225 all_headers.insert("Authorization".to_string(), auth.clone());
2226 }
2227 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2228
2229 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2231
2232 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2233 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2237 serde_json::json!({
2238 "name": sanitized_name.clone(),
2239 "display_name": f.name,
2240 "base_path": f.base_path,
2241 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2242 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2244 let method_raw = if !parts.is_empty() {
2245 parts[0].to_uppercase()
2246 } else {
2247 "GET".to_string()
2248 };
2249 let method = if !parts.is_empty() {
2250 let m = parts[0].to_lowercase();
2251 if m == "delete" { "del".to_string() } else { m }
2253 } else {
2254 "get".to_string()
2255 };
2256 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2257 let path = if let Some(ref bp) = api_base_path {
2259 format!("{}{}", bp, raw_path)
2260 } else {
2261 raw_path.to_string()
2262 };
2263 let is_get_or_head = method == "get" || method == "head";
2264 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2266
2267 let body_value = if has_body {
2269 param_overrides.as_ref()
2270 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2271 .and_then(|oo| oo.body)
2272 .unwrap_or_else(|| serde_json::json!({}))
2273 } else {
2274 serde_json::json!({})
2275 };
2276
2277 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2279
2280 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2282 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2283
2284 serde_json::json!({
2285 "operation": s.operation,
2286 "method": method,
2287 "path": path,
2288 "extract": s.extract,
2289 "use_values": s.use_values,
2290 "use_body": s.use_body,
2291 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2292 "inject_attacks": s.inject_attacks,
2293 "attack_types": s.attack_types,
2294 "description": s.description,
2295 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2296 "is_get_or_head": is_get_or_head,
2297 "has_body": has_body,
2298 "body": processed_body.value,
2299 "body_is_dynamic": body_is_dynamic,
2300 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2301 })
2302 }).collect::<Vec<_>>(),
2303 })
2304 }).collect();
2305
2306 for flow_data in &flows_data {
2308 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2309 for step in steps {
2310 if let Some(placeholders_arr) =
2311 step.get("_placeholders").and_then(|p| p.as_array())
2312 {
2313 for p_str in placeholders_arr {
2314 if let Some(p_name) = p_str.as_str() {
2315 match p_name {
2316 "VU" => {
2317 all_placeholders.insert(DynamicPlaceholder::VU);
2318 }
2319 "Iteration" => {
2320 all_placeholders.insert(DynamicPlaceholder::Iteration);
2321 }
2322 "Timestamp" => {
2323 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2324 }
2325 "UUID" => {
2326 all_placeholders.insert(DynamicPlaceholder::UUID);
2327 }
2328 "Random" => {
2329 all_placeholders.insert(DynamicPlaceholder::Random);
2330 }
2331 "Counter" => {
2332 all_placeholders.insert(DynamicPlaceholder::Counter);
2333 }
2334 "Date" => {
2335 all_placeholders.insert(DynamicPlaceholder::Date);
2336 }
2337 "VuIter" => {
2338 all_placeholders.insert(DynamicPlaceholder::VuIter);
2339 }
2340 _ => {}
2341 }
2342 }
2343 }
2344 }
2345 }
2346 }
2347 }
2348
2349 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2351 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2352
2353 let security_testing_enabled = self.security_testing_enabled();
2355
2356 let data = serde_json::json!({
2357 "base_url": self.target,
2358 "flows": flows_data,
2359 "extract_fields": config.default_extract_fields,
2360 "duration_secs": duration_secs,
2361 "max_vus": self.vus,
2362 "auth_header": self.auth,
2363 "custom_headers": custom_headers,
2364 "skip_tls_verify": self.skip_tls_verify,
2365 "stages": stages.iter().map(|s| serde_json::json!({
2367 "duration": s.duration,
2368 "target": s.target,
2369 })).collect::<Vec<_>>(),
2370 "threshold_percentile": self.threshold_percentile,
2371 "threshold_ms": self.threshold_ms,
2372 "max_error_rate": self.max_error_rate,
2373 "abort_on_error": self.abort_on_error,
2374 "abort_on_error_rate": self.abort_on_error_rate,
2375 "headers": headers_json,
2376 "dynamic_imports": required_imports,
2377 "dynamic_globals": required_globals,
2378 "extracted_values_output_path": output_dir.join("extracted_values.json").to_string_lossy(),
2379 "security_testing_enabled": security_testing_enabled,
2381 "has_custom_headers": !custom_headers.is_empty(),
2382 });
2383
2384 let mut script = handlebars
2385 .render_template(template, &data)
2386 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2387
2388 if security_testing_enabled {
2390 script = self.generate_enhanced_script(&script)?;
2391 }
2392
2393 let script_path =
2395 self.output.join(format!("k6-{}-crud-flow.js", spec_name.replace('.', "_")));
2396
2397 std::fs::create_dir_all(self.output.clone())?;
2398 std::fs::write(&script_path, &script)?;
2399
2400 if !self.generate_only {
2401 let executor = K6Executor::new()?
2402 .with_local_ips(self.source_ips.join(","))
2403 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
2404 std::fs::create_dir_all(&output_dir)?;
2405
2406 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
2407
2408 let extracted = Self::parse_extracted_values(&output_dir)?;
2409 TerminalReporter::print_progress(&format!(
2410 " Extracted {} value(s) from {}",
2411 extracted.values.len(),
2412 spec_name
2413 ));
2414 return Ok(extracted);
2415 }
2416
2417 Ok(ExtractedValues::new())
2418 }
2419
2420 async fn execute_standard_spec(&self, parser: &SpecParser, spec_name: &str) -> Result<()> {
2422 let mut operations = if let Some(filter) = &self.operations {
2423 parser.filter_operations(filter)?
2424 } else {
2425 parser.get_operations()
2426 };
2427
2428 if let Some(exclude) = &self.exclude_operations {
2429 operations = parser.exclude_operations(operations, exclude)?;
2430 }
2431
2432 if operations.is_empty() {
2433 TerminalReporter::print_warning(&format!("No operations found in {}", spec_name));
2434 return Ok(());
2435 }
2436
2437 TerminalReporter::print_progress(&format!(
2438 " {} operations in {}",
2439 operations.len(),
2440 spec_name
2441 ));
2442
2443 let templates: Vec<_> = operations
2445 .iter()
2446 .map(RequestGenerator::generate_template)
2447 .collect::<Result<Vec<_>>>()?;
2448
2449 let custom_headers = self.parse_headers()?;
2451
2452 let base_path = self.resolve_base_path(parser);
2454
2455 let scenario =
2457 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2458
2459 let security_testing_enabled = self.security_testing_enabled();
2460
2461 let force_http1 = crate::request_gen::should_force_k6_http1(
2462 self.wafbench_verbatim,
2463 &templates,
2464 &custom_headers,
2465 );
2466
2467 let duration_secs = Self::parse_duration(&self.duration)?;
2468 let (per_op_metrics, per_op_warn) = crate::k6_gen::resolve_per_op_metrics(
2469 self.per_op_metrics,
2470 templates.len(),
2471 duration_secs,
2472 );
2473 if let Some(msg) = per_op_warn {
2474 TerminalReporter::print_warning(&msg);
2475 }
2476
2477 let k6_config = K6Config {
2478 target_url: self.target.clone(),
2479 base_path,
2480 scenario,
2481 duration_secs,
2482 max_vus: self.vus,
2483 threshold_percentile: self.threshold_percentile.clone(),
2484 threshold_ms: self.threshold_ms,
2485 max_error_rate: self.max_error_rate,
2486 auth_header: self.auth.clone(),
2487 custom_headers,
2488 skip_tls_verify: self.skip_tls_verify,
2489 security_testing_enabled,
2490 chunked_request_bodies: self.chunked_request_bodies,
2491 target_rps: self.target_rps,
2492 no_keep_alive: self.no_keep_alive,
2493 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip")
2495 .into_iter()
2496 .map(|ip| ip.to_string())
2497 .collect(),
2498 geo_source_headers: if self.geo_source_headers.is_empty()
2499 && !self.geo_source_ips.is_empty()
2500 {
2501 crate::conformance::self_test::default_geo_source_headers()
2502 } else {
2503 self.geo_source_headers.clone()
2504 },
2505 };
2506
2507 let generator = K6ScriptGenerator::new(k6_config, templates)
2508 .with_abort_valve(self.abort_on_error, self.abort_on_error_rate)
2509 .with_force_http1(force_http1)
2510 .with_per_op_metrics(per_op_metrics);
2511 let mut script = generator.generate()?;
2512
2513 let has_advanced_features = self.data_file.is_some()
2515 || self.error_rate.is_some()
2516 || self.security_test
2517 || self.parallel_create.is_some()
2518 || self.wafbench_dir.is_some();
2519
2520 if has_advanced_features {
2521 script = self.generate_enhanced_script(&script)?;
2522 }
2523
2524 let script_path = self.output.join(format!("k6-{}.js", spec_name.replace('.', "_")));
2526
2527 std::fs::create_dir_all(self.output.clone())?;
2528 std::fs::write(&script_path, &script)?;
2529
2530 if !self.generate_only {
2531 let executor = K6Executor::new()?
2534 .with_local_ips(self.source_ips.join(","))
2535 .with_dns_policy(self.dns_policy.clone().unwrap_or_default())
2536 .with_discard_response_bodies(self.discard_response_bodies)
2537 .with_force_http1(force_http1);
2538 let output_dir = self.output.join(format!("{}_results", spec_name.replace('.', "_")));
2539 std::fs::create_dir_all(&output_dir)?;
2540
2541 executor.execute(&script_path, Some(&output_dir), self.verbose).await?;
2542 }
2543
2544 Ok(())
2545 }
2546
2547 async fn execute_crud_flow(&self, parser: &SpecParser) -> Result<()> {
2549 let config = self.build_crud_flow_config().unwrap_or_default();
2551
2552 let flows = if !config.flows.is_empty() {
2554 TerminalReporter::print_progress("Using custom flow configuration...");
2555 config.flows.clone()
2556 } else {
2557 TerminalReporter::print_progress("Detecting CRUD operations...");
2558 let operations = parser.get_operations();
2559 CrudFlowDetector::detect_flows(&operations)
2560 };
2561
2562 if flows.is_empty() {
2563 return Err(BenchError::Other(
2564 "No CRUD flows detected in spec. Ensure spec has POST/GET/PUT/DELETE operations on related paths.".to_string(),
2565 ));
2566 }
2567
2568 if config.flows.is_empty() {
2569 TerminalReporter::print_success(&format!("Detected {} CRUD flow(s)", flows.len()));
2570 } else {
2571 TerminalReporter::print_success(&format!("Loaded {} custom flow(s)", flows.len()));
2572 }
2573
2574 for flow in &flows {
2575 TerminalReporter::print_progress(&format!(
2576 " - {}: {} steps",
2577 flow.name,
2578 flow.steps.len()
2579 ));
2580 }
2581
2582 let mut handlebars = handlebars::Handlebars::new();
2584 handlebars.register_helper(
2586 "json",
2587 Box::new(
2588 |h: &handlebars::Helper,
2589 _: &handlebars::Handlebars,
2590 _: &handlebars::Context,
2591 _: &mut handlebars::RenderContext,
2592 out: &mut dyn handlebars::Output|
2593 -> handlebars::HelperResult {
2594 let param = h.param(0).map(|v| v.value()).unwrap_or(&serde_json::Value::Null);
2595 out.write(&serde_json::to_string(param).unwrap_or_else(|_| "[]".to_string()))?;
2596 Ok(())
2597 },
2598 ),
2599 );
2600 let template = include_str!("templates/k6_crud_flow.hbs");
2601
2602 let custom_headers = self.parse_headers()?;
2603
2604 let param_overrides = if let Some(params_file) = &self.params_file {
2606 TerminalReporter::print_progress("Loading parameter overrides...");
2607 let overrides = ParameterOverrides::from_file(params_file)?;
2608 TerminalReporter::print_success(&format!(
2609 "Loaded parameter overrides ({} operation-specific, {} defaults)",
2610 overrides.operations.len(),
2611 if overrides.defaults.is_empty() { 0 } else { 1 }
2612 ));
2613 Some(overrides)
2614 } else {
2615 None
2616 };
2617
2618 let duration_secs = Self::parse_duration(&self.duration)?;
2620 let scenario =
2621 LoadScenario::from_str(&self.scenario).map_err(BenchError::InvalidScenario)?;
2622 let stages = scenario.generate_stages(duration_secs, self.vus);
2623
2624 let api_base_path = self.resolve_base_path(parser);
2626 if let Some(ref bp) = api_base_path {
2627 TerminalReporter::print_progress(&format!("Using base path: {}", bp));
2628 }
2629
2630 let mut all_headers = custom_headers.clone();
2632 if let Some(auth) = &self.auth {
2633 all_headers.insert("Authorization".to_string(), auth.clone());
2634 }
2635 let headers_json = serde_json::to_string(&all_headers).unwrap_or_else(|_| "{}".to_string());
2636
2637 let mut all_placeholders: HashSet<DynamicPlaceholder> = HashSet::new();
2639
2640 let flows_data: Vec<serde_json::Value> = flows.iter().map(|f| {
2641 let sanitized_name = K6ScriptGenerator::sanitize_k6_metric_name(&f.name);
2646 serde_json::json!({
2647 "name": sanitized_name.clone(), "display_name": f.name, "base_path": f.base_path,
2650 "steps": f.steps.iter().enumerate().map(|(idx, s)| {
2651 let parts: Vec<&str> = s.operation.splitn(2, ' ').collect();
2653 let method_raw = if !parts.is_empty() {
2654 parts[0].to_uppercase()
2655 } else {
2656 "GET".to_string()
2657 };
2658 let method = if !parts.is_empty() {
2659 let m = parts[0].to_lowercase();
2660 if m == "delete" { "del".to_string() } else { m }
2662 } else {
2663 "get".to_string()
2664 };
2665 let raw_path = if parts.len() >= 2 { parts[1] } else { "/" };
2666 let path = if let Some(ref bp) = api_base_path {
2668 format!("{}{}", bp, raw_path)
2669 } else {
2670 raw_path.to_string()
2671 };
2672 let is_get_or_head = method == "get" || method == "head";
2673 let has_body = matches!(method.as_str(), "post" | "put" | "patch");
2675
2676 let body_value = if has_body {
2678 param_overrides.as_ref()
2679 .map(|po| po.get_for_operation(None, &method_raw, raw_path))
2680 .and_then(|oo| oo.body)
2681 .unwrap_or_else(|| serde_json::json!({}))
2682 } else {
2683 serde_json::json!({})
2684 };
2685
2686 let processed_body = DynamicParamProcessor::process_json_body(&body_value);
2688 let body_has_extracted_placeholders = processed_body.value.contains("${extracted.");
2693 let body_is_dynamic = processed_body.is_dynamic || body_has_extracted_placeholders;
2694
2695 serde_json::json!({
2696 "operation": s.operation,
2697 "method": method,
2698 "path": path,
2699 "extract": s.extract,
2700 "use_values": s.use_values,
2701 "use_body": s.use_body,
2702 "merge_body": if s.merge_body.is_empty() { None } else { Some(&s.merge_body) },
2703 "inject_attacks": s.inject_attacks,
2704 "attack_types": s.attack_types,
2705 "description": s.description,
2706 "display_name": s.description.clone().unwrap_or_else(|| format!("Step {}", idx)),
2707 "is_get_or_head": is_get_or_head,
2708 "has_body": has_body,
2709 "body": processed_body.value,
2710 "body_is_dynamic": body_is_dynamic,
2711 "_placeholders": processed_body.placeholders.iter().map(|p| format!("{:?}", p)).collect::<Vec<_>>(),
2712 })
2713 }).collect::<Vec<_>>(),
2714 })
2715 }).collect();
2716
2717 for flow_data in &flows_data {
2719 if let Some(steps) = flow_data.get("steps").and_then(|s| s.as_array()) {
2720 for step in steps {
2721 if let Some(placeholders_arr) =
2722 step.get("_placeholders").and_then(|p| p.as_array())
2723 {
2724 for p_str in placeholders_arr {
2725 if let Some(p_name) = p_str.as_str() {
2726 match p_name {
2728 "VU" => {
2729 all_placeholders.insert(DynamicPlaceholder::VU);
2730 }
2731 "Iteration" => {
2732 all_placeholders.insert(DynamicPlaceholder::Iteration);
2733 }
2734 "Timestamp" => {
2735 all_placeholders.insert(DynamicPlaceholder::Timestamp);
2736 }
2737 "UUID" => {
2738 all_placeholders.insert(DynamicPlaceholder::UUID);
2739 }
2740 "Random" => {
2741 all_placeholders.insert(DynamicPlaceholder::Random);
2742 }
2743 "Counter" => {
2744 all_placeholders.insert(DynamicPlaceholder::Counter);
2745 }
2746 "Date" => {
2747 all_placeholders.insert(DynamicPlaceholder::Date);
2748 }
2749 "VuIter" => {
2750 all_placeholders.insert(DynamicPlaceholder::VuIter);
2751 }
2752 _ => {}
2753 }
2754 }
2755 }
2756 }
2757 }
2758 }
2759 }
2760
2761 let required_imports = DynamicParamProcessor::get_required_imports(&all_placeholders);
2763 let required_globals = DynamicParamProcessor::get_required_globals(&all_placeholders);
2764
2765 let invalid_data_config = self.build_invalid_data_config();
2767 let error_injection_enabled = invalid_data_config.is_some();
2768 let error_rate = self.error_rate.unwrap_or(0.0);
2769 let error_types: Vec<String> = invalid_data_config
2770 .as_ref()
2771 .map(|c| c.error_types.iter().map(|t| format!("{:?}", t)).collect())
2772 .unwrap_or_default();
2773
2774 if error_injection_enabled {
2775 TerminalReporter::print_progress(&format!(
2776 "Error injection enabled ({}% rate)",
2777 (error_rate * 100.0) as u32
2778 ));
2779 }
2780
2781 let security_testing_enabled = self.security_testing_enabled();
2783
2784 let data = serde_json::json!({
2785 "base_url": self.target,
2786 "flows": flows_data,
2787 "extract_fields": config.default_extract_fields,
2788 "duration_secs": duration_secs,
2789 "max_vus": self.vus,
2790 "auth_header": self.auth,
2791 "custom_headers": custom_headers,
2792 "skip_tls_verify": self.skip_tls_verify,
2793 "stages": stages.iter().map(|s| serde_json::json!({
2795 "duration": s.duration,
2796 "target": s.target,
2797 })).collect::<Vec<_>>(),
2798 "threshold_percentile": self.threshold_percentile,
2799 "threshold_ms": self.threshold_ms,
2800 "max_error_rate": self.max_error_rate,
2801 "abort_on_error": self.abort_on_error,
2802 "abort_on_error_rate": self.abort_on_error_rate,
2803 "headers": headers_json,
2804 "dynamic_imports": required_imports,
2805 "dynamic_globals": required_globals,
2806 "extracted_values_output_path": self
2807 .output
2808 .join("crud_flow_extracted_values.json")
2809 .to_string_lossy(),
2810 "error_injection_enabled": error_injection_enabled,
2812 "error_rate": error_rate,
2813 "error_types": error_types,
2814 "security_testing_enabled": security_testing_enabled,
2816 "has_custom_headers": !custom_headers.is_empty(),
2817 });
2818
2819 let mut script = handlebars
2820 .render_template(template, &data)
2821 .map_err(|e| BenchError::ScriptGenerationFailed(e.to_string()))?;
2822
2823 if security_testing_enabled {
2825 script = self.generate_enhanced_script(&script)?;
2826 }
2827
2828 TerminalReporter::print_progress("Validating CRUD flow script...");
2830 let validation_errors = K6ScriptGenerator::validate_script(&script);
2831 if !validation_errors.is_empty() {
2832 TerminalReporter::print_error("CRUD flow script validation failed");
2833 for error in &validation_errors {
2834 eprintln!(" {}", error);
2835 }
2836 return Err(BenchError::Other(format!(
2837 "CRUD flow script validation failed with {} error(s)",
2838 validation_errors.len()
2839 )));
2840 }
2841
2842 TerminalReporter::print_success("CRUD flow script generated");
2843
2844 let script_path = if let Some(output) = &self.script_output {
2846 output.clone()
2847 } else {
2848 self.output.join("k6-crud-flow-script.js")
2849 };
2850
2851 if let Some(parent) = script_path.parent() {
2852 std::fs::create_dir_all(parent)?;
2853 }
2854 std::fs::write(&script_path, &script)?;
2855 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
2856
2857 if self.generate_only {
2858 println!("\nScript generated successfully. Run it with:");
2859 println!(" k6 run {}", script_path.display());
2860 return Ok(());
2861 }
2862
2863 TerminalReporter::print_progress("Executing CRUD flow test...");
2865 let executor = K6Executor::new()?
2866 .with_local_ips(self.source_ips.join(","))
2867 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
2868 std::fs::create_dir_all(&self.output)?;
2869
2870 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
2871
2872 let duration_secs = Self::parse_duration(&self.duration)?;
2873 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
2874
2875 Ok(())
2876 }
2877
2878 async fn execute_conformance_test(&self) -> Result<()> {
2880 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
2881 use crate::conformance::report::ConformanceReport;
2882 use crate::conformance::spec::ConformanceFeature;
2883
2884 TerminalReporter::print_progress("OpenAPI 3.0.0 Conformance Testing Mode");
2885
2886 TerminalReporter::print_progress(CONFORMANCE_REPLACES_LOAD_ADVISORY);
2887
2888 let categories = self.conformance_categories.as_ref().map(|cats_str| {
2890 cats_str
2891 .split(',')
2892 .filter_map(|s| {
2893 let trimmed = s.trim();
2894 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
2895 Some(canonical.to_string())
2896 } else {
2897 TerminalReporter::print_warning(&format!(
2898 "Unknown conformance category: '{}'. Valid categories: {}",
2899 trimmed,
2900 ConformanceFeature::cli_category_names()
2901 .iter()
2902 .map(|(cli, _)| *cli)
2903 .collect::<Vec<_>>()
2904 .join(", ")
2905 ));
2906 None
2907 }
2908 })
2909 .collect::<Vec<String>>()
2910 });
2911
2912 let custom_headers: Vec<(String, String)> = self
2914 .conformance_headers
2915 .iter()
2916 .filter_map(|h| {
2917 let (name, value) = h.split_once(':')?;
2918 Some((name.trim().to_string(), value.trim().to_string()))
2919 })
2920 .collect();
2921
2922 if !custom_headers.is_empty() {
2923 TerminalReporter::print_progress(&format!(
2924 "Using {} custom header(s) for authentication",
2925 custom_headers.len()
2926 ));
2927 }
2928
2929 if self.conformance_delay_ms > 0 {
2930 TerminalReporter::print_progress(&format!(
2931 "Using {}ms delay between conformance requests",
2932 self.conformance_delay_ms
2933 ));
2934 }
2935
2936 std::fs::create_dir_all(&self.output)?;
2938
2939 let config = ConformanceConfig {
2940 target_url: self.target.clone(),
2941 api_key: self.conformance_api_key.clone(),
2942 basic_auth: self.conformance_basic_auth.clone(),
2943 skip_tls_verify: self.skip_tls_verify,
2944 categories,
2945 base_path: self.base_path.clone(),
2946 custom_headers,
2947 output_dir: Some(self.output.clone()),
2948 all_operations: self.conformance_all_operations,
2949 custom_checks_file: self.conformance_custom.clone(),
2950 request_delay_ms: self.conformance_delay_ms,
2951 custom_filter: self.conformance_custom_filter.clone(),
2952 export_requests: self.export_requests,
2953 validate_requests: self.validate_requests,
2954 };
2955
2956 let mut resolved_base_path: Option<String> = None;
2964 let annotated_ops = if !self.spec.is_empty() {
2965 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
2966 let parser = SpecParser::from_file(&self.spec[0]).await?;
2967 resolved_base_path = self.resolve_base_path(&parser);
2968
2969 let mut operations = if let Some(filter) = &self.operations {
2974 parser.filter_operations(filter)?
2975 } else {
2976 parser.get_operations()
2977 };
2978 if let Some(exclude) = &self.exclude_operations {
2979 let before_count = operations.len();
2980 operations = parser.exclude_operations(operations, exclude)?;
2981 let excluded_count = before_count - operations.len();
2982 if excluded_count > 0 {
2983 TerminalReporter::print_progress(&format!(
2984 "Excluded {} operations matching '{}'",
2985 excluded_count, exclude
2986 ));
2987 }
2988 }
2989
2990 let annotated =
2991 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
2992 &operations,
2993 parser.spec(),
2994 );
2995 TerminalReporter::print_success(&format!(
2996 "Analyzed {} operations, found {} feature annotations",
2997 operations.len(),
2998 annotated.iter().map(|a| a.features.len()).sum::<usize>()
2999 ));
3000 Some(annotated)
3001 } else {
3002 None
3003 };
3004
3005 if self.conformance_self_test {
3012 let Some(ops) = annotated_ops else {
3013 TerminalReporter::print_error(
3014 "--conformance-self-test requires --spec; no operations to test",
3015 );
3016 return Ok(());
3017 };
3018 let cfg = crate::conformance::self_test::SelfTestConfig {
3019 target_url: self.target.clone(),
3020 skip_tls_verify: self.skip_tls_verify,
3021 timeout: std::time::Duration::from_secs(30),
3022 extra_headers: self
3026 .conformance_headers
3027 .iter()
3028 .filter_map(|h| {
3029 let (n, v) = h.split_once(':')?;
3030 Some((n.trim().to_string(), v.trim().to_string()))
3031 })
3032 .collect(),
3033 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
3034 base_path: resolved_base_path.clone(),
3038 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
3042 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
3043 geo_source_headers: if self.geo_source_headers.is_empty() {
3044 crate::conformance::self_test::default_geo_source_headers()
3045 } else {
3046 self.geo_source_headers.clone()
3047 },
3048 capture: if self.conformance_self_test_capture
3052 || self.validate_response_schemas
3053 || self.validate_requests
3054 {
3055 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
3066 } else {
3067 None
3068 },
3069 validate_response_schemas: self.validate_response_schemas,
3070 spec_label: self.spec.first().map(|p| {
3076 p.file_name()
3077 .map(|s| s.to_string_lossy().into_owned())
3078 .unwrap_or_else(|| p.to_string_lossy().into_owned())
3079 }),
3080 network_events: Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))),
3087 current_iteration: 1,
3088 };
3089 let capture_sink = cfg.capture.clone();
3090 let network_events_sink = cfg.network_events.clone();
3091 TerminalReporter::print_progress(&format!(
3092 "Self-test mode: driving {} operations with positive + per-category negative cases",
3093 ops.len()
3094 ));
3095 let target_iterations = self.conformance_self_test_iterations.max(1);
3102 let duration_budget = self
3103 .conformance_self_test_duration
3104 .as_ref()
3105 .map(|s| Self::parse_duration(s))
3106 .transpose()?
3107 .map(std::time::Duration::from_secs);
3108 let start = std::time::Instant::now();
3109 let deadline = duration_budget.map(|d| start + d);
3118 let mut cfg = cfg;
3122 cfg.current_iteration = 1;
3123 let mut report =
3124 crate::conformance::self_test::run_self_test_with_deadline(&ops, &cfg, deadline)
3125 .await
3126 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3127 let mut iter_done: u32 = 1;
3128 loop {
3129 let by_iter = iter_done >= target_iterations;
3130 let by_dur = duration_budget.map(|d| start.elapsed() >= d).unwrap_or(true);
3131 if by_iter && by_dur {
3132 break;
3133 }
3134 cfg.current_iteration = iter_done.saturating_add(1);
3135 let next = crate::conformance::self_test::run_self_test_with_deadline(
3136 &ops, &cfg, deadline,
3137 )
3138 .await
3139 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3140 report.merge_iteration(next);
3141 iter_done = iter_done.saturating_add(1);
3142 }
3143 if iter_done > 1 {
3144 TerminalReporter::print_progress(&format!(
3145 "Self-test repeated {} iteration(s) ({:.1?} elapsed)",
3146 iter_done,
3147 start.elapsed(),
3148 ));
3149 }
3150 let per_endpoint_summary: Vec<
3160 crate::conformance::per_endpoint_summary::PerEndpointSummary,
3161 >;
3162 if let Some(sink) = capture_sink {
3163 if let Ok(guard) = sink.lock() {
3164 let jsonl_path = self.output.join("conformance-self-test-requests.jsonl");
3165 let mut lines = String::with_capacity(guard.len() * 256);
3166 for entry in guard.iter() {
3167 if let Ok(line) = serde_json::to_string(entry) {
3168 lines.push_str(&line);
3169 lines.push('\n');
3170 }
3171 }
3172 let _ = std::fs::write(&jsonl_path, lines);
3173 let html_path = self.output.join("conformance-self-test-requests.html");
3174 let html =
3175 crate::conformance::capture_html::render_capture_html(guard.as_slice());
3176 let _ = std::fs::write(&html_path, html);
3177
3178 per_endpoint_summary =
3182 crate::conformance::per_endpoint_summary::build_summary(guard.as_slice());
3183 let summary_path = self.output.join("conformance-per-endpoint.json");
3184 if let Ok(json) = serde_json::to_string_pretty(&per_endpoint_summary) {
3185 let _ = std::fs::write(&summary_path, json);
3186 TerminalReporter::print_progress(&format!(
3187 "Self-test request/response capture written to {} ({} entries) + {} + {}",
3188 jsonl_path.display(),
3189 guard.len(),
3190 html_path.display(),
3191 summary_path.display(),
3192 ));
3193 } else {
3194 TerminalReporter::print_progress(&format!(
3195 "Self-test request/response capture written to {} ({} entries) + {}",
3196 jsonl_path.display(),
3197 guard.len(),
3198 html_path.display(),
3199 ));
3200 }
3201 } else {
3202 per_endpoint_summary = Vec::new();
3203 }
3204 } else {
3205 per_endpoint_summary = Vec::new();
3206 }
3207 TerminalReporter::print_progress(&report.render_summary());
3208 if let Some(sink) = network_events_sink {
3215 if let Ok(guard) = sink.lock() {
3216 let path = self.output.join("conformance-network-events.json");
3217 if let Ok(json) = serde_json::to_string_pretty(&*guard) {
3218 let _ = std::fs::write(&path, json);
3219 if guard.is_empty() {
3220 TerminalReporter::print_progress(
3221 "No wire-level network failures during self-test (file written empty)",
3222 );
3223 } else {
3224 TerminalReporter::print_warning(&format!(
3225 "Recorded {} wire-level network event(s) to {}",
3226 guard.len(),
3227 path.display()
3228 ));
3229 }
3230 }
3231 }
3232 }
3233 let json_path = self.output.join("conformance-self-test.json");
3237 if let Ok(json) = serde_json::to_string_pretty(&report) {
3238 let _ = std::fs::write(&json_path, json);
3239 TerminalReporter::print_progress(&format!(
3240 "Self-test report written to {}",
3241 json_path.display()
3242 ));
3243 }
3244 let issues = report.definite_issues();
3248 let issues_path = self.output.join("conformance-definite-issues.json");
3249 if let Ok(json) = serde_json::to_string_pretty(&issues) {
3250 if std::fs::write(&issues_path, json).is_ok() && !issues.is_empty() {
3251 TerminalReporter::print_warning(&format!(
3252 "{} definite issue(s) — see {}",
3253 issues.len(),
3254 issues_path.display()
3255 ));
3256 }
3257 }
3258 let owasp_accepted = report.owasp_accepted_probes();
3261 if !owasp_accepted.is_empty() {
3262 let owasp_path = self.output.join("conformance-owasp-accepted.json");
3263 if let Ok(json) = serde_json::to_string_pretty(&owasp_accepted) {
3264 if std::fs::write(&owasp_path, json).is_ok() {
3265 TerminalReporter::print_warning(&format!(
3266 "{} owasp injection probe(s) accepted by the target — see {}",
3267 owasp_accepted.len(),
3268 owasp_path.display()
3269 ));
3270 }
3271 }
3272 }
3273 if let Some(status) = report.detect_target_misconfiguration() {
3282 let hint = match status {
3283 404 => " Likely cause: spec paths don't match deployed routes — check --base-path and the spec's `servers` block.",
3284 401 | 403 => " Likely cause: authentication header is missing or invalid — check --conformance-header.",
3285 _ => "",
3286 };
3287 TerminalReporter::print_warning(&format!(
3288 "Self-test misconfiguration: every positive case returned {status}.{hint} Negative results below are meaningless under this condition."
3289 ));
3290 } else if !report.all_passed() {
3291 TerminalReporter::print_warning(
3292 "Self-test detected gaps — server let through at least one request that should have been a 4xx",
3293 );
3294 } else {
3295 TerminalReporter::print_success(
3296 "Self-test passed — all positive cases accepted and all negative cases rejected",
3297 );
3298 }
3299 let html_path = self.output.join("conformance-report.html");
3306 let audit_path = self.output.join("conformance-spec-audit.json");
3307 let audit_value = std::fs::read_to_string(&audit_path)
3308 .ok()
3309 .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok());
3310 let render_opts = crate::conformance::report_html::RenderOptions {
3315 missed_cap: match self.report_missed_cap {
3316 Some(0) => None,
3317 Some(n) => Some(n as usize),
3318 None => Some(200),
3319 },
3320 };
3321 let mut html = crate::conformance::report_html::render_html_with_options(
3322 &report,
3323 audit_value.as_ref(),
3324 &render_opts,
3325 );
3326 let summary_section = crate::conformance::per_endpoint_summary::render_html_section(
3332 &per_endpoint_summary,
3333 );
3334 if !summary_section.is_empty() {
3335 if let Some(idx) = html.rfind("</body>") {
3336 html.insert_str(idx, &summary_section);
3337 } else {
3338 html.push_str(&summary_section);
3339 }
3340 }
3341 if std::fs::write(&html_path, html).is_ok() {
3342 TerminalReporter::print_progress(&format!(
3343 "HTML report written to {}",
3344 html_path.display()
3345 ));
3346 }
3347
3348 if self.validate_requests && !self.spec.is_empty() {
3360 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3361 &self.spec,
3362 &self.output,
3363 self.base_path.as_deref(),
3364 )
3365 .await?;
3366 if n > 0 {
3367 TerminalReporter::print_warning(&format!(
3368 "{} emitted request(s) recorded against the spec — see conformance-request-violations.json",
3369 n
3370 ));
3371 }
3372 }
3373 return Ok(());
3374 }
3375
3376 if self.validate_requests && !self.spec.is_empty() {
3378 TerminalReporter::print_progress("Validating requests against OpenAPI spec...");
3379 let violation_count = crate::conformance::request_validator::run_request_validation(
3380 &self.spec,
3381 self.conformance_custom.as_deref(),
3382 self.base_path.as_deref(),
3383 &self.output,
3384 )
3385 .await?;
3386 if violation_count > 0 {
3387 TerminalReporter::print_warning(&format!(
3388 "{} request validation violation(s) found — see conformance-request-violations.json",
3389 violation_count
3390 ));
3391 } else {
3392 TerminalReporter::print_success("All requests conform to the OpenAPI spec");
3393 }
3394 }
3395
3396 if self.generate_only || self.use_k6 {
3398 let script = if let Some(annotated) = &annotated_ops {
3399 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
3400 config,
3401 annotated.clone(),
3402 );
3403 let op_count = gen.operation_count();
3404 let (script, check_count) = gen.generate()?;
3405 TerminalReporter::print_success(&format!(
3406 "Conformance: {} operations analyzed, {} unique checks generated",
3407 op_count, check_count
3408 ));
3409 script
3410 } else {
3411 let generator = ConformanceGenerator::new(config);
3412 generator.generate()?
3413 };
3414
3415 let script_path = self.output.join("k6-conformance.js");
3416 std::fs::write(&script_path, &script).map_err(|e| {
3417 BenchError::Other(format!("Failed to write conformance script: {}", e))
3418 })?;
3419 TerminalReporter::print_success(&format!(
3420 "Conformance script generated: {}",
3421 script_path.display()
3422 ));
3423
3424 if self.generate_only {
3425 println!("\nScript generated. Run with:");
3426 println!(" k6 run {}", script_path.display());
3427 return Ok(());
3428 }
3429
3430 if !K6Executor::is_k6_installed() {
3432 TerminalReporter::print_error("k6 is not installed");
3433 TerminalReporter::print_warning(
3434 "Install k6 from: https://k6.io/docs/get-started/installation/",
3435 );
3436 return Err(BenchError::K6NotFound);
3437 }
3438
3439 K6Executor::warn_if_pre_v1().await;
3440 TerminalReporter::print_progress("Running conformance tests via k6...");
3441 let executor = K6Executor::new()?
3442 .with_local_ips(self.source_ips.join(","))
3443 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
3444 executor.execute(&script_path, Some(&self.output), self.verbose).await?;
3445
3446 let report_path = self.output.join("conformance-report.json");
3447 if report_path.exists() {
3448 let report = ConformanceReport::from_file(&report_path)?;
3449 report.print_report_with_options(self.conformance_all_operations);
3450 self.save_conformance_report(&report, &report_path)?;
3451 } else {
3452 TerminalReporter::print_warning(
3453 "Conformance report not generated (k6 handleSummary may not have run)",
3454 );
3455 }
3456
3457 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
3469 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3470 &self.spec,
3471 &self.output,
3472 self.base_path.as_deref(),
3473 )
3474 .await?;
3475 if n > 0 {
3476 TerminalReporter::print_warning(&format!(
3477 "{} emitted request(s) violated the spec — see conformance-request-violations.json",
3478 n
3479 ));
3480 }
3481 }
3482
3483 return Ok(());
3484 }
3485
3486 TerminalReporter::print_progress("Running conformance tests (native executor)...");
3488
3489 let mut executor = crate::conformance::executor::NativeConformanceExecutor::new(config)?;
3490
3491 let custom_only = annotated_ops.is_none() && self.conformance_custom.is_some();
3501 executor = if let Some(annotated) = &annotated_ops {
3502 executor.with_spec_driven_checks(annotated)
3503 } else if custom_only {
3504 executor
3505 } else {
3506 executor.with_reference_checks()
3507 };
3508 executor = executor.with_custom_checks()?;
3509
3510 TerminalReporter::print_success(&format!(
3511 "Executing {} conformance checks...",
3512 executor.check_count()
3513 ));
3514
3515 let report = executor.execute().await?;
3516 report.print_report_with_options(self.conformance_all_operations);
3517
3518 let failure_details = report.failure_details();
3520 if !failure_details.is_empty() {
3521 let details_path = self.output.join("conformance-failure-details.json");
3522 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
3523 let _ = std::fs::write(&details_path, json);
3524 TerminalReporter::print_success(&format!(
3525 "Failure details saved to: {}",
3526 details_path.display()
3527 ));
3528 }
3529 }
3530
3531 let report_path = self.output.join("conformance-report.json");
3533 let report_json = serde_json::to_string_pretty(&report.to_json())
3534 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
3535 std::fs::write(&report_path, &report_json)
3536 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
3537 TerminalReporter::print_success(&format!("Report saved to: {}", report_path.display()));
3538
3539 self.save_conformance_report(&report, &report_path)?;
3540
3541 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
3552 let n =
3553 crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3554 &self.spec,
3555 &self.output,
3556 self.base_path.as_deref(),
3557 )
3558 .await?;
3559 if n > 0 {
3560 TerminalReporter::print_warning(&format!(
3561 "{} emitted request(s) violated the spec — see conformance-request-violations.json",
3562 n
3563 ));
3564 }
3565 }
3566
3567 Ok(())
3568 }
3569
3570 fn save_conformance_report(
3572 &self,
3573 report: &crate::conformance::report::ConformanceReport,
3574 report_path: &Path,
3575 ) -> Result<()> {
3576 if self.conformance_report_format == "sarif" {
3577 use crate::conformance::sarif::ConformanceSarifReport;
3578 ConformanceSarifReport::write(report, &self.target, &self.conformance_report)?;
3579 TerminalReporter::print_success(&format!(
3580 "SARIF report saved to: {}",
3581 self.conformance_report.display()
3582 ));
3583 } else if self.conformance_report != *report_path {
3584 std::fs::copy(report_path, &self.conformance_report)?;
3585 TerminalReporter::print_success(&format!(
3586 "Report saved to: {}",
3587 self.conformance_report.display()
3588 ));
3589 }
3590 Ok(())
3591 }
3592
3593 async fn execute_multi_target_self_test(&self, targets_file: &Path) -> Result<()> {
3605 use crate::conformance::self_test::SelfTestConfig;
3606
3607 TerminalReporter::print_progress("Multi-target conformance self-test mode");
3608 let targets = parse_targets_file(targets_file)?;
3609 if targets.is_empty() {
3610 return Err(BenchError::Other("No targets found in file".to_string()));
3611 }
3612 TerminalReporter::print_success(&format!("Loaded {} target(s)", targets.len()));
3613
3614 let annotated_ops = if !self.spec.is_empty() {
3616 let parser = SpecParser::from_file(&self.spec[0]).await?;
3617 let operations = parser.get_operations();
3618 Some(
3619 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
3620 &operations,
3621 parser.spec(),
3622 ),
3623 )
3624 } else {
3625 return Err(BenchError::Other("--conformance-self-test requires --spec".to_string()));
3626 };
3627 let Some(ops) = annotated_ops else {
3628 unreachable!()
3629 };
3630
3631 std::fs::create_dir_all(&self.output)?;
3632 let resolved_base_path = self.base_path.clone();
3633 let target_iterations = self.conformance_self_test_iterations.max(1);
3634 let duration_budget = self
3635 .conformance_self_test_duration
3636 .as_ref()
3637 .map(|s| Self::parse_duration(s))
3638 .transpose()?
3639 .map(std::time::Duration::from_secs);
3640
3641 for (idx, target) in targets.iter().enumerate() {
3642 let target_dir = self.output.join(format!("target_{}", idx));
3643 std::fs::create_dir_all(&target_dir)?;
3644 TerminalReporter::print_progress(&format!(
3645 "[target {}/{}] {}",
3646 idx + 1,
3647 targets.len(),
3648 target.url
3649 ));
3650
3651 let merged_headers: Vec<(String, String)> = self
3652 .conformance_headers
3653 .iter()
3654 .filter_map(|h| {
3655 let (n, v) = h.split_once(':')?;
3656 Some((n.trim().to_string(), v.trim().to_string()))
3657 })
3658 .collect();
3659
3660 let cfg = SelfTestConfig {
3661 target_url: target.url.clone(),
3662 skip_tls_verify: self.skip_tls_verify,
3663 timeout: std::time::Duration::from_secs(30),
3664 extra_headers: merged_headers,
3665 delay_between_requests: std::time::Duration::from_millis(self.conformance_delay_ms),
3666 base_path: resolved_base_path.clone(),
3667 source_ips: parse_ip_list(&self.source_ips, "source-ip"),
3668 geo_source_ips: parse_ip_list(&self.geo_source_ips, "geo-source-ip"),
3669 geo_source_headers: if self.geo_source_headers.is_empty() {
3670 crate::conformance::self_test::default_geo_source_headers()
3671 } else {
3672 self.geo_source_headers.clone()
3673 },
3674 capture: if self.conformance_self_test_capture
3675 || self.validate_response_schemas
3676 || self.validate_requests
3677 {
3678 Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new())))
3682 } else {
3683 None
3684 },
3685 validate_response_schemas: self.validate_response_schemas,
3686 spec_label: self.spec.first().map(|p| {
3687 p.file_name()
3688 .map(|s| s.to_string_lossy().into_owned())
3689 .unwrap_or_else(|| p.to_string_lossy().into_owned())
3690 }),
3691 network_events: Some(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))),
3692 current_iteration: 1,
3693 };
3694 let capture_sink = cfg.capture.clone();
3695 let network_events_sink = cfg.network_events.clone();
3696
3697 let start = std::time::Instant::now();
3698 let deadline = duration_budget.map(|d| start + d);
3702 let mut cfg = cfg;
3706 cfg.current_iteration = 1;
3707 let mut report =
3708 crate::conformance::self_test::run_self_test_with_deadline(&ops, &cfg, deadline)
3709 .await
3710 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3711 let mut iter_done: u32 = 1;
3712 loop {
3713 let by_iter = iter_done >= target_iterations;
3714 let by_dur = duration_budget.map(|d| start.elapsed() >= d).unwrap_or(true);
3715 if by_iter && by_dur {
3716 break;
3717 }
3718 cfg.current_iteration = iter_done.saturating_add(1);
3719 let next = crate::conformance::self_test::run_self_test_with_deadline(
3720 &ops, &cfg, deadline,
3721 )
3722 .await
3723 .map_err(|e| BenchError::Other(format!("self-test client error: {e}")))?;
3724 report.merge_iteration(next);
3725 iter_done = iter_done.saturating_add(1);
3726 }
3727 if iter_done > 1 {
3728 TerminalReporter::print_progress(&format!(
3729 " ran {} iteration(s) in {:.1?}",
3730 iter_done,
3731 start.elapsed(),
3732 ));
3733 }
3734
3735 if let Some(sink) = capture_sink {
3737 if let Ok(guard) = sink.lock() {
3738 let jsonl = target_dir.join("conformance-self-test-requests.jsonl");
3739 let mut lines = String::with_capacity(guard.len() * 256);
3740 for entry in guard.iter() {
3741 if let Ok(line) = serde_json::to_string(entry) {
3742 lines.push_str(&line);
3743 lines.push('\n');
3744 }
3745 }
3746 let _ = std::fs::write(&jsonl, lines);
3747 }
3748 }
3749 if let Some(sink) = network_events_sink {
3750 if let Ok(guard) = sink.lock() {
3751 let path = target_dir.join("conformance-network-events.json");
3752 if let Ok(json) = serde_json::to_string_pretty(&*guard) {
3753 let _ = std::fs::write(&path, json);
3754 if !guard.is_empty() {
3755 TerminalReporter::print_warning(&format!(
3756 " recorded {} wire-level network event(s)",
3757 guard.len()
3758 ));
3759 }
3760 }
3761 }
3762 }
3763
3764 let json_path = target_dir.join("conformance-self-test.json");
3765 if let Ok(json) = serde_json::to_string_pretty(&report) {
3766 let _ = std::fs::write(&json_path, json);
3767 }
3768 let issues = report.definite_issues();
3771 if let Ok(json) = serde_json::to_string_pretty(&issues) {
3772 let issues_path = target_dir.join("conformance-definite-issues.json");
3773 if std::fs::write(&issues_path, json).is_ok() && !issues.is_empty() {
3774 TerminalReporter::print_warning(&format!(
3775 " {} definite issue(s) — see {}",
3776 issues.len(),
3777 issues_path.display()
3778 ));
3779 }
3780 }
3781 let owasp_accepted = report.owasp_accepted_probes();
3783 if !owasp_accepted.is_empty() {
3784 if let Ok(json) = serde_json::to_string_pretty(&owasp_accepted) {
3785 let owasp_path = target_dir.join("conformance-owasp-accepted.json");
3786 if std::fs::write(&owasp_path, json).is_ok() {
3787 TerminalReporter::print_warning(&format!(
3788 " {} owasp injection probe(s) accepted by the target — see {}",
3789 owasp_accepted.len(),
3790 owasp_path.display()
3791 ));
3792 }
3793 }
3794 }
3795 TerminalReporter::print_progress(&report.render_summary());
3796
3797 if self.validate_requests {
3806 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
3807 &self.spec,
3808 &target_dir,
3809 self.base_path.as_deref(),
3810 )
3811 .await?;
3812 if n > 0 {
3813 TerminalReporter::print_warning(&format!(
3814 " {} emitted request(s) violated the spec — see {}/conformance-request-violations.json",
3815 n,
3816 target_dir.display(),
3817 ));
3818 }
3819 }
3820 }
3821
3822 Ok(())
3823 }
3824
3825 async fn execute_multi_target_conformance(&self, targets_file: &Path) -> Result<()> {
3831 use crate::conformance::generator::{ConformanceConfig, ConformanceGenerator};
3832 use crate::conformance::report::ConformanceReport;
3833 use crate::conformance::spec::ConformanceFeature;
3834
3835 TerminalReporter::print_progress("Multi-target OpenAPI 3.0.0 Conformance Testing Mode");
3836
3837 TerminalReporter::print_progress("Parsing targets file...");
3839 let targets = parse_targets_file(targets_file)?;
3840 let num_targets = targets.len();
3841 TerminalReporter::print_success(&format!("Loaded {} targets", num_targets));
3842
3843 if targets.is_empty() {
3844 return Err(BenchError::Other("No targets found in file".to_string()));
3845 }
3846
3847 TerminalReporter::print_progress(CONFORMANCE_REPLACES_LOAD_ADVISORY);
3848
3849 let categories = self.conformance_categories.as_ref().map(|cats_str| {
3851 cats_str
3852 .split(',')
3853 .filter_map(|s| {
3854 let trimmed = s.trim();
3855 if let Some(canonical) = ConformanceFeature::category_from_cli_name(trimmed) {
3856 Some(canonical.to_string())
3857 } else {
3858 TerminalReporter::print_warning(&format!(
3859 "Unknown conformance category: '{}'. Valid categories: {}",
3860 trimmed,
3861 ConformanceFeature::cli_category_names()
3862 .iter()
3863 .map(|(cli, _)| *cli)
3864 .collect::<Vec<_>>()
3865 .join(", ")
3866 ));
3867 None
3868 }
3869 })
3870 .collect::<Vec<String>>()
3871 });
3872
3873 let base_custom_headers: Vec<(String, String)> = self
3875 .conformance_headers
3876 .iter()
3877 .filter_map(|h| {
3878 let (name, value) = h.split_once(':')?;
3879 Some((name.trim().to_string(), value.trim().to_string()))
3880 })
3881 .collect();
3882
3883 if !base_custom_headers.is_empty() {
3884 TerminalReporter::print_progress(&format!(
3885 "Using {} base custom header(s) for authentication",
3886 base_custom_headers.len()
3887 ));
3888 }
3889
3890 let annotated_ops = if !self.spec.is_empty() {
3892 TerminalReporter::print_progress("Spec-driven conformance mode: analyzing spec...");
3893 let parser = SpecParser::from_file(&self.spec[0]).await?;
3894 let operations = parser.get_operations();
3895 let annotated =
3896 crate::conformance::spec_driven::SpecDrivenConformanceGenerator::annotate_operations(
3897 &operations,
3898 parser.spec(),
3899 );
3900 TerminalReporter::print_success(&format!(
3901 "Analyzed {} operations, found {} feature annotations",
3902 operations.len(),
3903 annotated.iter().map(|a| a.features.len()).sum::<usize>()
3904 ));
3905 Some(annotated)
3906 } else {
3907 None
3908 };
3909
3910 std::fs::create_dir_all(&self.output)?;
3912
3913 struct TargetResult {
3915 url: String,
3916 passed: usize,
3917 failed: usize,
3918 elapsed: std::time::Duration,
3919 report_json: serde_json::Value,
3920 owasp_coverage: Vec<crate::conformance::report::OwaspCoverageEntry>,
3921 }
3922
3923 let mut target_results: Vec<TargetResult> = Vec::with_capacity(num_targets);
3924 let total_start = std::time::Instant::now();
3925
3926 for (idx, target) in targets.iter().enumerate() {
3927 tracing::info!(
3928 "Running conformance tests against target {}/{}: {}",
3929 idx + 1,
3930 num_targets,
3931 target.url
3932 );
3933 TerminalReporter::print_progress(&format!(
3934 "\n--- Target {}/{}: {} ---",
3935 idx + 1,
3936 num_targets,
3937 target.url
3938 ));
3939
3940 let mut merged_headers = base_custom_headers.clone();
3942 if let Some(ref target_headers) = target.headers {
3943 for (name, value) in target_headers {
3944 if let Some(existing) = merged_headers.iter_mut().find(|(n, _)| n == name) {
3946 existing.1 = value.clone();
3947 } else {
3948 merged_headers.push((name.clone(), value.clone()));
3949 }
3950 }
3951 }
3952 if let Some(ref auth) = target.auth {
3954 if let Some(existing) =
3955 merged_headers.iter_mut().find(|(n, _)| n.eq_ignore_ascii_case("Authorization"))
3956 {
3957 existing.1 = auth.clone();
3958 } else {
3959 merged_headers.push(("Authorization".to_string(), auth.clone()));
3960 }
3961 }
3962
3963 let target_dir = self.output.join(format!("target_{}", idx));
3969 std::fs::create_dir_all(&target_dir)?;
3970
3971 let config = ConformanceConfig {
3972 target_url: target.url.clone(),
3973 api_key: self.conformance_api_key.clone(),
3974 basic_auth: self.conformance_basic_auth.clone(),
3975 skip_tls_verify: self.skip_tls_verify,
3976 categories: categories.clone(),
3977 base_path: self.base_path.clone(),
3978 custom_headers: merged_headers,
3979 output_dir: Some(target_dir.clone()),
3980 all_operations: self.conformance_all_operations,
3981 custom_checks_file: self.conformance_custom.clone(),
3982 request_delay_ms: self.conformance_delay_ms,
3983 custom_filter: self.conformance_custom_filter.clone(),
3984 export_requests: self.export_requests,
3985 validate_requests: self.validate_requests,
3986 };
3987
3988 let target_start = std::time::Instant::now();
3989 let report = if self.use_k6 {
3990 if !K6Executor::is_k6_installed() {
3991 TerminalReporter::print_error("k6 is not installed");
3992 TerminalReporter::print_warning(
3993 "Install k6 from: https://k6.io/docs/get-started/installation/",
3994 );
3995 return Err(BenchError::K6NotFound);
3996 }
3997 K6Executor::warn_if_pre_v1().await;
3998
3999 let script = if let Some(ref annotated) = annotated_ops {
4000 let gen = crate::conformance::spec_driven::SpecDrivenConformanceGenerator::new(
4001 config.clone(),
4002 annotated.clone(),
4003 );
4004 let (script, _check_count) = gen.generate()?;
4005 script
4006 } else {
4007 let generator = ConformanceGenerator::new(config.clone());
4008 generator.generate()?
4009 };
4010
4011 let script_path = target_dir.join("k6-conformance.js");
4012 std::fs::write(&script_path, &script).map_err(|e| {
4013 BenchError::Other(format!("Failed to write conformance script: {}", e))
4014 })?;
4015 TerminalReporter::print_success(&format!(
4016 "Conformance script generated: {}",
4017 script_path.display()
4018 ));
4019
4020 TerminalReporter::print_progress(&format!(
4021 "Running conformance tests via k6 against {}...",
4022 target.url
4023 ));
4024 let k6 = K6Executor::new()?
4025 .with_local_ips(self.source_ips.join(","))
4026 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
4027 let api_port = 6565u16.saturating_add(idx as u16);
4029 k6.execute_with_port(&script_path, Some(&target_dir), self.verbose, Some(api_port))
4030 .await?;
4031
4032 let report_path = target_dir.join("conformance-report.json");
4033 if report_path.exists() {
4034 ConformanceReport::from_file(&report_path)?
4035 } else {
4036 TerminalReporter::print_warning(&format!(
4037 "Conformance report not generated for target {} (k6 handleSummary may not have run)",
4038 target.url
4039 ));
4040 continue;
4041 }
4042 } else {
4043 let mut executor =
4044 crate::conformance::executor::NativeConformanceExecutor::new(config)?;
4045
4046 let custom_only = annotated_ops.is_none() && self.conformance_custom.is_some();
4049 executor = if let Some(ref annotated) = annotated_ops {
4050 executor.with_spec_driven_checks(annotated)
4051 } else if custom_only {
4052 executor
4053 } else {
4054 executor.with_reference_checks()
4055 };
4056 executor = executor.with_custom_checks()?;
4057
4058 TerminalReporter::print_success(&format!(
4059 "Executing {} conformance checks against {}...",
4060 executor.check_count(),
4061 target.url
4062 ));
4063
4064 executor.execute().await?
4065 };
4066 let target_elapsed = target_start.elapsed();
4067
4068 let report_json = report.to_json();
4069
4070 let passed = report_json["summary"]["passed"].as_u64().unwrap_or(0) as usize;
4072 let failed = report_json["summary"]["failed"].as_u64().unwrap_or(0) as usize;
4073 let total_checks = passed + failed;
4074 let rate = if total_checks == 0 {
4075 0.0
4076 } else {
4077 (passed as f64 / total_checks as f64) * 100.0
4078 };
4079
4080 TerminalReporter::print_success(&format!(
4081 "Target {}: {}/{} passed ({:.1}%) in {:.1}s",
4082 target.url,
4083 passed,
4084 total_checks,
4085 rate,
4086 target_elapsed.as_secs_f64()
4087 ));
4088
4089 let target_report_path = target_dir.join("conformance-report.json");
4091 let report_str = serde_json::to_string_pretty(&report_json)
4092 .map_err(|e| BenchError::Other(format!("Failed to serialize report: {}", e)))?;
4093 std::fs::write(&target_report_path, &report_str)
4094 .map_err(|e| BenchError::Other(format!("Failed to write report: {}", e)))?;
4095
4096 let failure_details = report.failure_details();
4098 if !failure_details.is_empty() {
4099 let details_path = target_dir.join("conformance-failure-details.json");
4100 if let Ok(json) = serde_json::to_string_pretty(&failure_details) {
4101 let _ = std::fs::write(&details_path, json);
4102 }
4103 }
4104
4105 if self.validate_requests && self.export_requests && !self.spec.is_empty() {
4112 let n = crate::conformance::request_validator::validate_emitted_requests_with_base_path(
4113 &self.spec,
4114 &target_dir,
4115 self.base_path.as_deref(),
4116 )
4117 .await?;
4118 if n > 0 {
4119 TerminalReporter::print_warning(&format!(
4120 "Target {}: {} emitted request(s) violated the spec — see {}/conformance-request-violations.json",
4121 target.url,
4122 n,
4123 target_dir.display()
4124 ));
4125 }
4126 }
4127
4128 let owasp_coverage = report.owasp_coverage_data();
4130
4131 target_results.push(TargetResult {
4132 url: target.url.clone(),
4133 passed,
4134 failed,
4135 elapsed: target_elapsed,
4136 report_json,
4137 owasp_coverage,
4138 });
4139 }
4140
4141 let total_elapsed = total_start.elapsed();
4142
4143 println!("\n{}", "=".repeat(80));
4145 println!(" Multi-Target Conformance Summary");
4146 println!("{}", "=".repeat(80));
4147 println!(
4148 " {:<40} {:>8} {:>8} {:>8} {:>8}",
4149 "Target URL", "Passed", "Failed", "Rate", "Time"
4150 );
4151 println!(" {}", "-".repeat(76));
4152
4153 let mut total_passed = 0usize;
4154 let mut total_failed = 0usize;
4155
4156 for result in &target_results {
4157 let total_checks = result.passed + result.failed;
4158 let rate = if total_checks == 0 {
4159 0.0
4160 } else {
4161 (result.passed as f64 / total_checks as f64) * 100.0
4162 };
4163
4164 let display_url = if result.url.len() > 38 {
4166 format!("{}...", &result.url[..35])
4167 } else {
4168 result.url.clone()
4169 };
4170
4171 println!(
4172 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
4173 display_url,
4174 result.passed,
4175 result.failed,
4176 rate,
4177 result.elapsed.as_secs_f64()
4178 );
4179
4180 total_passed += result.passed;
4181 total_failed += result.failed;
4182 }
4183
4184 let grand_total = total_passed + total_failed;
4185 let overall_rate = if grand_total == 0 {
4186 0.0
4187 } else {
4188 (total_passed as f64 / grand_total as f64) * 100.0
4189 };
4190
4191 println!(" {}", "-".repeat(76));
4192 println!(
4193 " {:<40} {:>8} {:>8} {:>7.1}% {:>6.1}s",
4194 format!("TOTAL ({} targets)", num_targets),
4195 total_passed,
4196 total_failed,
4197 overall_rate,
4198 total_elapsed.as_secs_f64()
4199 );
4200 println!("{}", "=".repeat(80));
4201
4202 for result in &target_results {
4204 println!("\n OWASP API Security Top 10 Coverage for {}:", result.url);
4205 for entry in &result.owasp_coverage {
4206 let status = if !entry.tested {
4207 "-"
4208 } else if entry.all_passed {
4209 "pass"
4210 } else {
4211 "FAIL"
4212 };
4213 let via = if entry.via_categories.is_empty() {
4214 String::new()
4215 } else {
4216 format!(" (via {})", entry.via_categories.join(", "))
4217 };
4218 println!(" {:<12} {:<40} {}{}", entry.id, entry.name, status, via);
4219 }
4220 }
4221
4222 let per_target_summaries: Vec<serde_json::Value> = target_results
4224 .iter()
4225 .enumerate()
4226 .map(|(idx, r)| {
4227 let total_checks = r.passed + r.failed;
4228 let rate = if total_checks == 0 {
4229 0.0
4230 } else {
4231 (r.passed as f64 / total_checks as f64) * 100.0
4232 };
4233 let owasp_json: Vec<serde_json::Value> = r
4234 .owasp_coverage
4235 .iter()
4236 .map(|e| {
4237 serde_json::json!({
4238 "id": e.id,
4239 "name": e.name,
4240 "tested": e.tested,
4241 "all_passed": e.all_passed,
4242 "via_categories": e.via_categories,
4243 })
4244 })
4245 .collect();
4246 serde_json::json!({
4247 "target_url": r.url,
4248 "target_index": idx,
4249 "checks_passed": r.passed,
4250 "checks_failed": r.failed,
4251 "total_checks": total_checks,
4252 "pass_rate": rate,
4253 "elapsed_seconds": r.elapsed.as_secs_f64(),
4254 "report": r.report_json,
4255 "owasp_coverage": owasp_json,
4256 })
4257 })
4258 .collect();
4259
4260 let combined_summary = serde_json::json!({
4261 "total_targets": num_targets,
4262 "total_checks_passed": total_passed,
4263 "total_checks_failed": total_failed,
4264 "overall_pass_rate": overall_rate,
4265 "total_elapsed_seconds": total_elapsed.as_secs_f64(),
4266 "targets": per_target_summaries,
4267 });
4268
4269 let summary_path = self.output.join("multi-target-conformance-summary.json");
4270 let summary_str = serde_json::to_string_pretty(&combined_summary)
4271 .map_err(|e| BenchError::Other(format!("Failed to serialize summary: {}", e)))?;
4272 std::fs::write(&summary_path, &summary_str)
4273 .map_err(|e| BenchError::Other(format!("Failed to write summary: {}", e)))?;
4274 TerminalReporter::print_success(&format!(
4275 "Combined summary saved to: {}",
4276 summary_path.display()
4277 ));
4278
4279 Ok(())
4280 }
4281
4282 async fn execute_owasp_test(&self, parser: &SpecParser) -> Result<()> {
4284 TerminalReporter::print_progress("OWASP API Security Top 10 Testing Mode");
4285
4286 let custom_headers = self.parse_headers()?;
4288
4289 let mut config = OwaspApiConfig::new()
4291 .with_auth_header(&self.owasp_auth_header)
4292 .with_verbose(self.verbose)
4293 .with_insecure(self.skip_tls_verify)
4294 .with_concurrency(self.vus as usize)
4295 .with_iterations(self.owasp_iterations as usize)
4296 .with_base_path(self.base_path.clone())
4297 .with_custom_headers(custom_headers);
4298
4299 if let Some(ref token) = self.owasp_auth_token {
4301 config = config.with_valid_auth_token(token);
4302 }
4303
4304 if let Some(ref cats_str) = self.owasp_categories {
4306 let categories: Vec<OwaspCategory> = cats_str
4307 .split(',')
4308 .filter_map(|s| {
4309 let trimmed = s.trim();
4310 match trimmed.parse::<OwaspCategory>() {
4311 Ok(cat) => Some(cat),
4312 Err(e) => {
4313 TerminalReporter::print_warning(&e);
4314 None
4315 }
4316 }
4317 })
4318 .collect();
4319
4320 if !categories.is_empty() {
4321 config = config.with_categories(categories);
4322 }
4323 }
4324
4325 if let Some(ref admin_paths_file) = self.owasp_admin_paths {
4327 config.admin_paths_file = Some(admin_paths_file.clone());
4328 if let Err(e) = config.load_admin_paths() {
4329 TerminalReporter::print_warning(&format!("Failed to load admin paths file: {}", e));
4330 }
4331 }
4332
4333 if let Some(ref id_fields_str) = self.owasp_id_fields {
4335 let id_fields: Vec<String> = id_fields_str
4336 .split(',')
4337 .map(|s| s.trim().to_string())
4338 .filter(|s| !s.is_empty())
4339 .collect();
4340 if !id_fields.is_empty() {
4341 config = config.with_id_fields(id_fields);
4342 }
4343 }
4344
4345 if let Some(ref report_path) = self.owasp_report {
4347 config = config.with_report_path(report_path);
4348 }
4349 if let Ok(format) = self.owasp_report_format.parse::<ReportFormat>() {
4350 config = config.with_report_format(format);
4351 }
4352
4353 let categories = config.categories_to_test();
4355 TerminalReporter::print_success(&format!(
4356 "Testing {} OWASP categories: {}",
4357 categories.len(),
4358 categories.iter().map(|c| c.cli_name()).collect::<Vec<_>>().join(", ")
4359 ));
4360
4361 if config.valid_auth_token.is_some() {
4362 TerminalReporter::print_progress("Using provided auth token for baseline requests");
4363 }
4364
4365 TerminalReporter::print_progress("Generating OWASP security test script...");
4367 let generator = OwaspApiGenerator::new(config, self.target.clone(), parser);
4368
4369 let script = generator.generate()?;
4371 TerminalReporter::print_success("OWASP security test script generated");
4372
4373 let script_path = if let Some(output) = &self.script_output {
4375 output.clone()
4376 } else {
4377 self.output.join("k6-owasp-security-test.js")
4378 };
4379
4380 if let Some(parent) = script_path.parent() {
4381 std::fs::create_dir_all(parent)?;
4382 }
4383 std::fs::write(&script_path, &script)?;
4384 TerminalReporter::print_success(&format!("Script written to: {}", script_path.display()));
4385
4386 if self.generate_only {
4388 println!("\nOWASP security test script generated. Run it with:");
4389 println!(" k6 run {}", script_path.display());
4390 return Ok(());
4391 }
4392
4393 TerminalReporter::print_progress("Executing OWASP security tests...");
4395 let executor = K6Executor::new()?
4396 .with_local_ips(self.source_ips.join(","))
4397 .with_dns_policy(self.dns_policy.clone().unwrap_or_default());
4398 std::fs::create_dir_all(&self.output)?;
4399
4400 let results = executor.execute(&script_path, Some(&self.output), self.verbose).await?;
4401
4402 let duration_secs = Self::parse_duration(&self.duration)?;
4403 TerminalReporter::print_summary_with_mode(&results, duration_secs, self.no_keep_alive);
4404
4405 println!("\nOWASP security test results saved to: {}", self.output.display());
4406
4407 Ok(())
4408 }
4409}
4410
4411#[cfg(test)]
4412mod tests {
4413 use super::*;
4414 use tempfile::tempdir;
4415
4416 #[test]
4417 fn test_parse_duration() {
4418 assert_eq!(BenchCommand::parse_duration("30s").unwrap(), 30);
4419 assert_eq!(BenchCommand::parse_duration("5m").unwrap(), 300);
4420 assert_eq!(BenchCommand::parse_duration("1h").unwrap(), 3600);
4421 assert_eq!(BenchCommand::parse_duration("60").unwrap(), 60);
4422 }
4423
4424 #[test]
4428 fn parse_ip_list_ipv4_range_inclusive() {
4429 let v = parse_ip_list(&["10.0.0.5-10.0.0.27".into()], "source-ip");
4430 assert_eq!(v.len(), 23);
4431 assert_eq!(v.first().unwrap().to_string(), "10.0.0.5");
4432 assert_eq!(v.last().unwrap().to_string(), "10.0.0.27");
4433 }
4434
4435 #[test]
4438 fn parse_ip_list_range_rejects_backwards() {
4439 let v = parse_ip_list(&["10.0.0.10-10.0.0.5".into()], "source-ip");
4440 assert!(v.is_empty(), "backwards range should produce no IPs; got {v:?}");
4441 }
4442
4443 #[test]
4447 fn parse_ip_list_rejects_ipv6_range_syntax() {
4448 let v = parse_ip_list(&["2001:db8::1-2001:db8::5".into()], "geo-source-ip");
4449 assert!(v.is_empty(), "IPv6 range should be rejected; got {v:?}");
4450 }
4451
4452 #[test]
4454 fn parse_ip_list_range_capped_at_256() {
4455 let v = parse_ip_list(&["10.0.0.0-10.0.5.0".into()], "source-ip");
4456 assert_eq!(v.len(), 256);
4457 assert_eq!(v.first().unwrap().to_string(), "10.0.0.0");
4458 }
4459
4460 #[test]
4463 fn parse_ip_list_plain_and_comma() {
4464 let v = parse_ip_list(&["10.0.0.5".into(), "10.0.0.6,10.0.0.7".into()], "source-ip");
4465 assert_eq!(v.len(), 3);
4466 assert_eq!(v[0].to_string(), "10.0.0.5");
4467 assert_eq!(v[2].to_string(), "10.0.0.7");
4468 }
4469
4470 #[test]
4473 fn parse_ip_list_ipv4_cidr_29_expands_to_8() {
4474 let v = parse_ip_list(&["10.0.0.0/29".into()], "source-ip");
4475 assert_eq!(v.len(), 8);
4476 assert_eq!(v[0].to_string(), "10.0.0.0");
4477 assert_eq!(v[7].to_string(), "10.0.0.7");
4478 }
4479
4480 #[test]
4483 fn parse_ip_list_ipv4_cidr_8_capped_at_256() {
4484 let v = parse_ip_list(&["10.0.0.0/8".into()], "source-ip");
4485 assert_eq!(v.len(), 256);
4486 assert_eq!(v[0].to_string(), "10.0.0.0");
4487 assert_eq!(v[255].to_string(), "10.0.0.255");
4488 }
4489
4490 #[test]
4492 fn parse_ip_list_ipv6_cidr_126_expands_to_4() {
4493 let v = parse_ip_list(&["2001:db8::/126".into()], "geo-source-ip");
4494 assert_eq!(v.len(), 4);
4495 assert!(v[0].is_ipv6());
4496 assert_eq!(v[0].to_string(), "2001:db8::");
4497 assert_eq!(v[3].to_string(), "2001:db8::3");
4498 }
4499
4500 #[test]
4502 fn parse_ip_list_mixed_v4_v6_cidr() {
4503 let v = parse_ip_list(&["10.0.0.0/30,2001:db8::1,203.0.113.42".into()], "geo-source-ip");
4504 assert_eq!(v.len(), 6); assert!(v.iter().any(|ip| ip.to_string() == "2001:db8::1"));
4506 assert!(v.iter().any(|ip| ip.to_string() == "203.0.113.42"));
4507 }
4508
4509 #[test]
4512 fn parse_ip_list_skips_malformed() {
4513 let v = parse_ip_list(
4514 &[
4515 "10.0.0.5".into(),
4516 "not-an-ip".into(),
4517 "10.0.0.6".into(),
4518 "/24".into(),
4519 "1.2.3.4/200".into(),
4520 ],
4521 "source-ip",
4522 );
4523 assert_eq!(v.len(), 2);
4524 assert_eq!(v[0].to_string(), "10.0.0.5");
4525 assert_eq!(v[1].to_string(), "10.0.0.6");
4526 }
4527
4528 #[test]
4529 fn test_parse_duration_invalid() {
4530 assert!(BenchCommand::parse_duration("invalid").is_err());
4531 assert!(BenchCommand::parse_duration("30x").is_err());
4532 }
4533
4534 #[test]
4535 fn test_parse_headers() {
4536 let cmd = BenchCommand {
4537 spec: vec![PathBuf::from("test.yaml")],
4538 spec_dir: None,
4539 merge_conflicts: "error".to_string(),
4540 spec_mode: "merge".to_string(),
4541 dependency_config: None,
4542 target: "http://localhost".to_string(),
4543 base_path: None,
4544 duration: "1m".to_string(),
4545 vus: 10,
4546 scenario: "ramp-up".to_string(),
4547 operations: None,
4548 exclude_operations: None,
4549 auth: None,
4550 headers: vec![
4551 "X-API-Key:test123".to_string(),
4552 "X-Client-ID:client456".to_string(),
4553 ],
4554 output: PathBuf::from("output"),
4555 generate_only: false,
4556 script_output: None,
4557 threshold_percentile: "p(95)".to_string(),
4558 threshold_ms: 500,
4559 max_error_rate: 0.05,
4560 abort_on_error: true,
4561 abort_on_error_rate: 0.95,
4562 per_op_metrics: None,
4563 verbose: false,
4564 skip_tls_verify: false,
4565 chunked_request_bodies: false,
4566 target_rps: None,
4567 no_keep_alive: false,
4568 targets_file: None,
4569 max_concurrency: None,
4570 repeat_until: None,
4571 rounds: None,
4572 results_format: "both".to_string(),
4573 params_file: None,
4574 crud_flow: false,
4575 flow_config: None,
4576 extract_fields: None,
4577 parallel_create: None,
4578 data_file: None,
4579 data_distribution: "unique-per-vu".to_string(),
4580 data_mappings: None,
4581 per_uri_control: false,
4582 error_rate: None,
4583 error_types: None,
4584 security_test: false,
4585 security_payloads: None,
4586 security_categories: None,
4587 security_target_fields: None,
4588 wafbench_dir: None,
4589 wafbench_cycle_all: false,
4590 wafbench_verbatim: false,
4591 owasp_api_top10: false,
4592 owasp_categories: None,
4593 owasp_auth_header: "Authorization".to_string(),
4594 owasp_auth_token: None,
4595 owasp_admin_paths: None,
4596 owasp_id_fields: None,
4597 owasp_report: None,
4598 owasp_report_format: "json".to_string(),
4599 owasp_iterations: 1,
4600 conformance: false,
4601 conformance_api_key: None,
4602 conformance_basic_auth: None,
4603 conformance_report: PathBuf::from("conformance-report.json"),
4604 conformance_categories: None,
4605 conformance_report_format: "json".to_string(),
4606 conformance_headers: vec![],
4607 conformance_all_operations: false,
4608 conformance_custom: None,
4609 conformance_delay_ms: 0,
4610 use_k6: false,
4611 conformance_custom_filter: None,
4612 export_requests: false,
4613 validate_requests: false,
4614 conformance_self_test: false,
4615 conformance_self_test_capture: false,
4616 conformance_self_test_iterations: 1,
4617 conformance_self_test_duration: None,
4618 validate_response_schemas: false,
4619 source_ips: Vec::new(),
4620 geo_source_ips: Vec::new(),
4621 geo_source_headers: Vec::new(),
4622 report_missed_cap: None,
4623 discard_response_bodies: false,
4624 dns_policy: None,
4625 };
4626
4627 let headers = cmd.parse_headers().unwrap();
4628 assert_eq!(headers.get("X-API-Key"), Some(&"test123".to_string()));
4629 assert_eq!(headers.get("X-Client-ID"), Some(&"client456".to_string()));
4630 }
4631
4632 #[test]
4633 fn test_parse_header_string_preserves_comma_in_value() {
4634 let inputs = vec![
4637 "Cookie:session=abc; expires=Thu, 01 Jan 2099 00:00:00 GMT".to_string(),
4638 "X-Trace:1".to_string(),
4639 ];
4640 let headers = parse_header_string(&inputs).unwrap();
4641 assert_eq!(
4642 headers.get("Cookie"),
4643 Some(&"session=abc; expires=Thu, 01 Jan 2099 00:00:00 GMT".to_string())
4644 );
4645 assert_eq!(headers.get("X-Trace"), Some(&"1".to_string()));
4646 }
4647
4648 #[test]
4656 fn conformance_advisory_names_every_discarded_flag() {
4657 let msg = CONFORMANCE_REPLACES_LOAD_ADVISORY;
4658 for flag in ["--vus", "--rps", "-d"] {
4659 assert!(
4660 msg.contains(flag),
4661 "conformance advisory must name `{flag}` as ignored; it is discarded on that \
4662 path and silently dropping it is how users end up tuning a knob that does \
4663 nothing (#980). Message was: {msg}"
4664 );
4665 }
4666 assert!(
4667 msg.contains("REPLACES"),
4668 "conformance advisory must say the load run is REPLACED, not merely that some \
4669 flags are ignored — `--conformance` returns before the load path runs, so no \
4670 load traffic is generated at all (#980). Message was: {msg}"
4671 );
4672 }
4673
4674 #[test]
4688 fn multi_target_clone_preserves_fields_parse_headers_reads() {
4689 let src = include_str!("command.rs");
4690
4691 let fn_start = src
4692 .find("async fn execute_multi_target(")
4693 .expect("execute_multi_target should exist");
4694 let block_start = src[fn_start..]
4695 .find("ParallelExecutor::new(")
4696 .map(|i| i + fn_start)
4697 .expect("multi-target path should build a ParallelExecutor");
4698 let block_end = src[block_start..]
4700 .find("\n );")
4701 .map(|i| i + block_start)
4702 .expect("ParallelExecutor::new(..) should be closed");
4703 let block = &src[block_start..block_end];
4704
4705 for field in ["conformance_basic_auth", "conformance_headers"] {
4708 for zeroed in [format!("{field}: None"), format!("{field}: vec![]")] {
4709 assert!(
4710 !block.contains(&zeroed),
4711 "execute_multi_target zeroes `{zeroed}`. parse_headers() folds `{field}` \
4712 into the header map, so zeroing it here strips auth from every \
4713 multi-target run while single-target keeps working (#79 round 64)."
4714 );
4715 }
4716 let passthrough = format!("{field}: self.{field}.clone()");
4717 assert!(
4718 block.contains(&passthrough),
4719 "execute_multi_target must carry `{field}` through as `{passthrough}` so \
4720 parse_headers() can fold it (#79 round 64)."
4721 );
4722 }
4723 }
4724
4725 #[test]
4726 fn test_get_spec_display_name() {
4727 let cmd = BenchCommand {
4728 spec: vec![PathBuf::from("test.yaml")],
4729 spec_dir: None,
4730 merge_conflicts: "error".to_string(),
4731 spec_mode: "merge".to_string(),
4732 dependency_config: None,
4733 target: "http://localhost".to_string(),
4734 base_path: None,
4735 duration: "1m".to_string(),
4736 vus: 10,
4737 scenario: "ramp-up".to_string(),
4738 operations: None,
4739 exclude_operations: None,
4740 auth: None,
4741 headers: Vec::new(),
4742 output: PathBuf::from("output"),
4743 generate_only: false,
4744 script_output: None,
4745 threshold_percentile: "p(95)".to_string(),
4746 threshold_ms: 500,
4747 max_error_rate: 0.05,
4748 abort_on_error: true,
4749 abort_on_error_rate: 0.95,
4750 per_op_metrics: None,
4751 verbose: false,
4752 skip_tls_verify: false,
4753 chunked_request_bodies: false,
4754 target_rps: None,
4755 no_keep_alive: false,
4756 targets_file: None,
4757 max_concurrency: None,
4758 repeat_until: None,
4759 rounds: None,
4760 results_format: "both".to_string(),
4761 params_file: None,
4762 crud_flow: false,
4763 flow_config: None,
4764 extract_fields: None,
4765 parallel_create: None,
4766 data_file: None,
4767 data_distribution: "unique-per-vu".to_string(),
4768 data_mappings: None,
4769 per_uri_control: false,
4770 error_rate: None,
4771 error_types: None,
4772 security_test: false,
4773 security_payloads: None,
4774 security_categories: None,
4775 security_target_fields: None,
4776 wafbench_dir: None,
4777 wafbench_cycle_all: false,
4778 wafbench_verbatim: false,
4779 owasp_api_top10: false,
4780 owasp_categories: None,
4781 owasp_auth_header: "Authorization".to_string(),
4782 owasp_auth_token: None,
4783 owasp_admin_paths: None,
4784 owasp_id_fields: None,
4785 owasp_report: None,
4786 owasp_report_format: "json".to_string(),
4787 owasp_iterations: 1,
4788 conformance: false,
4789 conformance_api_key: None,
4790 conformance_basic_auth: None,
4791 conformance_report: PathBuf::from("conformance-report.json"),
4792 conformance_categories: None,
4793 conformance_report_format: "json".to_string(),
4794 conformance_headers: vec![],
4795 conformance_all_operations: false,
4796 conformance_custom: None,
4797 conformance_delay_ms: 0,
4798 use_k6: false,
4799 conformance_custom_filter: None,
4800 export_requests: false,
4801 validate_requests: false,
4802 conformance_self_test: false,
4803 conformance_self_test_capture: false,
4804 conformance_self_test_iterations: 1,
4805 conformance_self_test_duration: None,
4806 validate_response_schemas: false,
4807 source_ips: Vec::new(),
4808 geo_source_ips: Vec::new(),
4809 geo_source_headers: Vec::new(),
4810 report_missed_cap: None,
4811 discard_response_bodies: false,
4812 dns_policy: None,
4813 };
4814
4815 assert_eq!(cmd.get_spec_display_name(), "test.yaml");
4816
4817 let cmd_multi = BenchCommand {
4819 spec: vec![PathBuf::from("a.yaml"), PathBuf::from("b.yaml")],
4820 spec_dir: None,
4821 merge_conflicts: "error".to_string(),
4822 spec_mode: "merge".to_string(),
4823 dependency_config: None,
4824 target: "http://localhost".to_string(),
4825 base_path: None,
4826 duration: "1m".to_string(),
4827 vus: 10,
4828 scenario: "ramp-up".to_string(),
4829 operations: None,
4830 exclude_operations: None,
4831 auth: None,
4832 headers: Vec::new(),
4833 output: PathBuf::from("output"),
4834 generate_only: false,
4835 script_output: None,
4836 threshold_percentile: "p(95)".to_string(),
4837 threshold_ms: 500,
4838 max_error_rate: 0.05,
4839 abort_on_error: true,
4840 abort_on_error_rate: 0.95,
4841 per_op_metrics: None,
4842 verbose: false,
4843 skip_tls_verify: false,
4844 chunked_request_bodies: false,
4845 target_rps: None,
4846 no_keep_alive: false,
4847 targets_file: None,
4848 max_concurrency: None,
4849 repeat_until: None,
4850 rounds: None,
4851 results_format: "both".to_string(),
4852 params_file: None,
4853 crud_flow: false,
4854 flow_config: None,
4855 extract_fields: None,
4856 parallel_create: None,
4857 data_file: None,
4858 data_distribution: "unique-per-vu".to_string(),
4859 data_mappings: None,
4860 per_uri_control: false,
4861 error_rate: None,
4862 error_types: None,
4863 security_test: false,
4864 security_payloads: None,
4865 security_categories: None,
4866 security_target_fields: None,
4867 wafbench_dir: None,
4868 wafbench_cycle_all: false,
4869 wafbench_verbatim: false,
4870 owasp_api_top10: false,
4871 owasp_categories: None,
4872 owasp_auth_header: "Authorization".to_string(),
4873 owasp_auth_token: None,
4874 owasp_admin_paths: None,
4875 owasp_id_fields: None,
4876 owasp_report: None,
4877 owasp_report_format: "json".to_string(),
4878 owasp_iterations: 1,
4879 conformance: false,
4880 conformance_api_key: None,
4881 conformance_basic_auth: None,
4882 conformance_report: PathBuf::from("conformance-report.json"),
4883 conformance_categories: None,
4884 conformance_report_format: "json".to_string(),
4885 conformance_headers: vec![],
4886 conformance_all_operations: false,
4887 conformance_custom: None,
4888 conformance_delay_ms: 0,
4889 use_k6: false,
4890 conformance_custom_filter: None,
4891 export_requests: false,
4892 validate_requests: false,
4893 conformance_self_test: false,
4894 conformance_self_test_capture: false,
4895 conformance_self_test_iterations: 1,
4896 conformance_self_test_duration: None,
4897 validate_response_schemas: false,
4898 source_ips: Vec::new(),
4899 geo_source_ips: Vec::new(),
4900 geo_source_headers: Vec::new(),
4901 report_missed_cap: None,
4902 discard_response_bodies: false,
4903 dns_policy: None,
4904 };
4905
4906 assert_eq!(cmd_multi.get_spec_display_name(), "2 spec files");
4907 }
4908
4909 #[test]
4910 fn test_parse_extracted_values_from_output_dir() {
4911 let dir = tempdir().unwrap();
4912 let path = dir.path().join("extracted_values.json");
4913 std::fs::write(
4914 &path,
4915 r#"{
4916 "pool_id": "abc123",
4917 "count": 0,
4918 "enabled": false,
4919 "metadata": { "owner": "team-a" }
4920}"#,
4921 )
4922 .unwrap();
4923
4924 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
4925 assert_eq!(extracted.get("pool_id"), Some(&serde_json::json!("abc123")));
4926 assert_eq!(extracted.get("count"), Some(&serde_json::json!(0)));
4927 assert_eq!(extracted.get("enabled"), Some(&serde_json::json!(false)));
4928 assert_eq!(extracted.get("metadata"), Some(&serde_json::json!({"owner": "team-a"})));
4929 }
4930
4931 #[test]
4932 fn test_parse_extracted_values_missing_file() {
4933 let dir = tempdir().unwrap();
4934 let extracted = BenchCommand::parse_extracted_values(dir.path()).unwrap();
4935 assert!(extracted.values.is_empty());
4936 }
4937
4938 fn sample_bench_command() -> BenchCommand {
4941 BenchCommand {
4942 spec: vec![PathBuf::from("test.yaml")],
4943 spec_dir: None,
4944 merge_conflicts: "error".to_string(),
4945 spec_mode: "merge".to_string(),
4946 dependency_config: None,
4947 target: "http://localhost".to_string(),
4948 base_path: None,
4949 duration: "1m".to_string(),
4950 vus: 10,
4951 scenario: "ramp-up".to_string(),
4952 operations: None,
4953 exclude_operations: None,
4954 auth: None,
4955 headers: vec![
4956 "X-API-Key:test123".to_string(),
4957 "X-Client-ID:client456".to_string(),
4958 ],
4959 output: PathBuf::from("output"),
4960 generate_only: false,
4961 script_output: None,
4962 threshold_percentile: "p(95)".to_string(),
4963 threshold_ms: 500,
4964 max_error_rate: 0.05,
4965 abort_on_error: true,
4966 abort_on_error_rate: 0.95,
4967 per_op_metrics: None,
4968 verbose: false,
4969 skip_tls_verify: false,
4970 chunked_request_bodies: false,
4971 target_rps: None,
4972 no_keep_alive: false,
4973 targets_file: None,
4974 max_concurrency: None,
4975 repeat_until: None,
4976 rounds: None,
4977 results_format: "both".to_string(),
4978 params_file: None,
4979 crud_flow: false,
4980 flow_config: None,
4981 extract_fields: None,
4982 parallel_create: None,
4983 data_file: None,
4984 data_distribution: "unique-per-vu".to_string(),
4985 data_mappings: None,
4986 per_uri_control: false,
4987 error_rate: None,
4988 error_types: None,
4989 security_test: false,
4990 security_payloads: None,
4991 security_categories: None,
4992 security_target_fields: None,
4993 wafbench_dir: None,
4994 wafbench_cycle_all: false,
4995 wafbench_verbatim: false,
4996 owasp_api_top10: false,
4997 owasp_categories: None,
4998 owasp_auth_header: "Authorization".to_string(),
4999 owasp_auth_token: None,
5000 owasp_admin_paths: None,
5001 owasp_id_fields: None,
5002 owasp_report: None,
5003 owasp_report_format: "json".to_string(),
5004 owasp_iterations: 1,
5005 conformance: false,
5006 conformance_api_key: None,
5007 conformance_basic_auth: None,
5008 conformance_report: PathBuf::from("conformance-report.json"),
5009 conformance_categories: None,
5010 conformance_report_format: "json".to_string(),
5011 conformance_headers: vec![],
5012 conformance_all_operations: false,
5013 conformance_custom: None,
5014 conformance_delay_ms: 0,
5015 use_k6: false,
5016 conformance_custom_filter: None,
5017 export_requests: false,
5018 validate_requests: false,
5019 conformance_self_test: false,
5020 conformance_self_test_capture: false,
5021 conformance_self_test_iterations: 1,
5022 conformance_self_test_duration: None,
5023 validate_response_schemas: false,
5024 source_ips: Vec::new(),
5025 geo_source_ips: Vec::new(),
5026 geo_source_headers: Vec::new(),
5027 report_missed_cap: None,
5028 discard_response_bodies: false,
5029 dns_policy: None,
5030 }
5031 }
5032
5033 #[test]
5041 fn verbatim_disables_security_payload_injection() {
5042 let mut cmd = sample_bench_command();
5043 cmd.wafbench_dir = Some("traffic.yaml".to_string());
5044
5045 assert!(
5046 cmd.security_testing_enabled(),
5047 "--wafbench-dir alone must still enable payload injection"
5048 );
5049
5050 cmd.wafbench_verbatim = true;
5051 assert!(
5052 !cmd.security_testing_enabled(),
5053 "verbatim mode must not inject payloads into requests sent as written"
5054 );
5055
5056 cmd.security_test = true;
5059 assert!(
5060 !cmd.security_testing_enabled(),
5061 "--security-test must not re-enable injection under --wafbench-verbatim"
5062 );
5063 }
5064
5065 #[test]
5070 fn security_testing_enabled_has_a_single_definition() {
5071 let src = include_str!("command.rs");
5072 let parallel = include_str!("parallel_executor.rs");
5073 let a = format!("self.{} || self.{}.is_some()", "security_test", "wafbench_dir");
5075 let b = format!("self.{}.is_some() || self.{}", "wafbench_dir", "security_test");
5076 let inline = src.matches(a.as_str()).count() + src.matches(b.as_str()).count();
5077 assert_eq!(
5078 inline, 1,
5079 "expected the security_testing_enabled() method to be the only place this is \
5080 computed, found {inline} inline copies -- collapse them or the render paths drift"
5081 );
5082
5083 let parallel_inline = format!(
5088 "{}.{} || {}.{}.is_some()",
5089 "base_command", "security_test", "self.base_command", "wafbench_dir"
5090 );
5091 assert!(
5092 !parallel.contains(¶llel_inline),
5093 "ParallelExecutor must not recompute the security flag inline"
5094 );
5095 assert!(
5096 parallel.contains("security_testing_enabled()"),
5097 "ParallelExecutor must call security_testing_enabled() so --wafbench-verbatim \
5098 turns injection off on --targets-file runs too"
5099 );
5100 }
5101
5102 #[test]
5106 fn missing_wafbench_dir_is_not_swallowed() {
5107 let src = include_str!("command.rs");
5108 let swallowed = format!("Failed to {} WAFBench tests", "load");
5110 let impl_line = src
5111 .lines()
5112 .filter(|l| !l.trim_start().starts_with("//"))
5113 .any(|l| l.contains(&swallowed));
5114 assert!(!impl_line, "missing --wafbench-dir must not be downgraded to a warning");
5115 assert!(
5116 src.contains("self.load_wafbench_payloads()?"),
5117 "payload load errors must reach generate_enhanced_script"
5118 );
5119 }
5120
5121 #[test]
5126 fn multi_target_path_honors_verbatim_templates() {
5127 let src = include_str!("parallel_executor.rs");
5128 assert!(
5129 src.contains("load_verbatim_templates"),
5130 "ParallelExecutor must load traffic-file requests under --wafbench-verbatim. \
5131 Requiring a spec and generating templates from its operations is how \
5132 --targets-file ignored the flag and fuzzed spec URLs (#79)."
5133 );
5134 }
5135
5136 #[test]
5137 fn single_target_k6_spawn_sets_force_http1() {
5138 let src = include_str!("command.rs");
5139 assert!(
5140 src.contains("with_force_http1(force_http1)"),
5141 "single-target k6 spawn must set GODEBUG=http2client=0 for Connection-header WAF cases"
5142 );
5143 assert!(
5144 src.contains("print_k6_run_hint"),
5145 "generate-only must print GODEBUG=http2client=0 when HTTP/1.1 is required"
5146 );
5147 assert!(
5148 src.contains("with_per_op_metrics(per_op_metrics)"),
5149 "single-target k6 generation must apply Round-65 per-op metrics collapse (#79)"
5150 );
5151 assert!(
5152 src.contains("resolve_per_op_metrics"),
5153 "single-target path must resolve auto/forced per-op metrics (#79)"
5154 );
5155 }
5156
5157 #[test]
5159 fn traffic_breakdown_json_multiplies_unique_by_rps() {
5160 let dir = std::env::temp_dir().join(format!(
5161 "mf-traffic-breakdown-{}-{}",
5162 std::process::id(),
5163 std::time::SystemTime::now()
5164 .duration_since(std::time::UNIX_EPOCH)
5165 .unwrap()
5166 .as_nanos()
5167 ));
5168 let _ = std::fs::create_dir_all(&dir);
5169 let mut cmd = sample_bench_command();
5170 cmd.output = dir.clone();
5171 cmd.target_rps = Some(50);
5172 cmd.duration = "1200s".to_string();
5173 let stats = crate::wafbench::WafBenchStats {
5174 per_file: vec![crate::wafbench::TrafficFileSummary {
5175 file: "apisix_cve-2026-44087.yaml".into(),
5176 sent: 5,
5177 attack: 3,
5178 normal: 2,
5179 omitted: 1,
5180 other: 0,
5181 }],
5182 ..Default::default()
5183 };
5184 cmd.emit_traffic_file_breakdown(&stats, "what to expect in proxy logs");
5185 let raw = std::fs::read_to_string(dir.join("traffic-breakdown.json"))
5186 .expect("traffic-breakdown.json");
5187 let v: serde_json::Value = serde_json::from_str(&raw).unwrap();
5188 assert_eq!(v["rps"], 50);
5189 assert_eq!(v["duration_secs"], 1200);
5190 assert_eq!(v["files"][0]["sent"]["unique_cases"], 5);
5191 assert_eq!(v["files"][0]["sent"]["projected_per_second"], 250);
5192 assert_eq!(v["files"][0]["sent"]["projected_over_run"], 300000);
5193 assert_eq!(v["files"][0]["attack"]["projected_per_second"], 150);
5194 assert_eq!(v["files"][0]["normal"]["projected_per_second"], 100);
5195 for gone in [
5196 "unique",
5197 "total",
5198 "expected_requests",
5199 "expected_requests_unit",
5200 ] {
5201 assert!(
5202 v["files"][0]["sent"].get(gone).is_none(),
5203 "{gone} alias must not appear in traffic-breakdown.json"
5204 );
5205 }
5206 assert!(v["note"].as_str().unwrap().contains("Plan, not k6 counters"));
5207 assert!(v["note"].as_str().unwrap().contains("not traffic on the wire"));
5208 assert_eq!(
5209 BenchCommand::format_unique_total(5, Some(50)),
5210 "unique_cases=5 projected_per_second=250 (5 * 50 RPS)"
5211 );
5212 let _ = std::fs::remove_dir_all(&dir);
5213 }
5214
5215 #[test]
5218 fn traffic_breakdown_json_omits_projected_without_rps() {
5219 let dir = std::env::temp_dir().join(format!(
5220 "mf-traffic-breakdown-norps-{}-{}",
5221 std::process::id(),
5222 std::time::SystemTime::now()
5223 .duration_since(std::time::UNIX_EPOCH)
5224 .unwrap()
5225 .as_nanos()
5226 ));
5227 let _ = std::fs::create_dir_all(&dir);
5228 let mut cmd = sample_bench_command();
5229 cmd.output = dir.clone();
5230 cmd.target_rps = None;
5231 cmd.duration = "60s".to_string();
5232 let stats = crate::wafbench::WafBenchStats {
5233 per_file: vec![crate::wafbench::TrafficFileSummary {
5234 file: "apisix_cve-2026-44087.yaml".into(),
5235 sent: 5,
5236 attack: 3,
5237 normal: 2,
5238 omitted: 1,
5239 other: 0,
5240 }],
5241 ..Default::default()
5242 };
5243 cmd.emit_traffic_file_breakdown(&stats, "what to expect in proxy logs");
5244 let raw = std::fs::read_to_string(dir.join("traffic-breakdown.json"))
5245 .expect("traffic-breakdown.json");
5246 let v: serde_json::Value = serde_json::from_str(&raw).unwrap();
5247 assert!(v["rps"].is_null());
5248 assert_eq!(v["duration_secs"], 60);
5249 assert_eq!(v["files"][0]["sent"]["unique_cases"], 5);
5250 assert!(v["files"][0]["sent"]["projected_per_second"].is_null());
5251 assert!(v["files"][0]["sent"]["projected_over_run"].is_null());
5252 for gone in ["unique", "total", "expected_requests"] {
5253 assert!(
5254 v["files"][0]["sent"].get(gone).is_none(),
5255 "{gone} alias must not appear when --rps is unset"
5256 );
5257 }
5258 let _ = std::fs::remove_dir_all(&dir);
5259 }
5260}