1use crate::analyzer::BehaviorInferrer;
2use crate::error::Result;
3use crate::types::{
4 Assertion, CliAnalysis, CliOption, NoArgsBehavior, OptionType, TestCase, TestCategory,
5 TestPriority,
6};
7use rayon::prelude::*;
8
9pub struct TestGenerator {
11 analysis: CliAnalysis,
13
14 categories: Vec<TestCategory>,
16}
17
18impl TestGenerator {
19 pub fn new(analysis: CliAnalysis, categories: Vec<TestCategory>) -> Self {
21 Self {
22 analysis,
23 categories,
24 }
25 }
26
27 pub fn generate(&self) -> Result<Vec<TestCase>> {
29 log::info!("Generating tests for {} categories", self.categories.len());
30
31 let mut all_tests = Vec::new();
32
33 for category in &self.categories {
34 let tests = match category {
35 TestCategory::Basic => self.generate_basic_tests()?,
36 TestCategory::Help => self.generate_help_tests()?,
37 TestCategory::Security => self.generate_security_tests()?,
38 TestCategory::Path => self.generate_path_tests()?,
39 TestCategory::InputValidation => self.generate_input_validation_tests()?,
40 TestCategory::DestructiveOps => self.generate_destructive_ops_tests()?,
41 TestCategory::DirectoryTraversal => self.generate_directory_traversal_tests()?,
42 TestCategory::Performance => self.generate_performance_tests()?,
43 TestCategory::MultiShell => self.generate_multi_shell_tests()?,
44 };
45
46 log::info!("Generated {} tests for {:?}", tests.len(), category);
47 all_tests.extend(tests);
48 }
49
50 log::info!("Total tests generated: {}", all_tests.len());
51 Ok(all_tests)
52 }
53
54 pub fn generate_parallel(&self) -> Result<Vec<TestCase>> {
56 log::info!(
57 "Generating tests in parallel for {} categories",
58 self.categories.len()
59 );
60
61 let results: Result<Vec<Vec<TestCase>>> = self
62 .categories
63 .par_iter()
64 .map(|category| match category {
65 TestCategory::Basic => self.generate_basic_tests(),
66 TestCategory::Help => self.generate_help_tests(),
67 TestCategory::Security => self.generate_security_tests(),
68 TestCategory::Path => self.generate_path_tests(),
69 TestCategory::InputValidation => self.generate_input_validation_tests(),
70 TestCategory::DestructiveOps => self.generate_destructive_ops_tests(),
71 TestCategory::DirectoryTraversal => self.generate_directory_traversal_tests(),
72 TestCategory::Performance => self.generate_performance_tests(),
73 TestCategory::MultiShell => self.generate_multi_shell_tests(),
74 })
75 .collect();
76
77 let all_tests: Vec<TestCase> = results?.into_iter().flatten().collect();
78
79 log::info!("Total tests generated (parallel): {}", all_tests.len());
80 Ok(all_tests)
81 }
82
83 fn generate_basic_tests(&self) -> Result<Vec<TestCase>> {
85 let mut tests = Vec::new();
86
87 tests.push(
89 TestCase::new(
90 "basic-001".to_string(),
91 "Display help with --help flag".to_string(),
92 TestCategory::Basic,
93 "\"$CLI_BINARY\" --help".to_string(),
94 )
95 .with_exit_code(0)
96 .with_assertion(Assertion::OutputContains("Usage:".to_string()))
97 .with_tag("help".to_string()),
98 );
99
100 tests.push(
102 TestCase::new(
103 "basic-002".to_string(),
104 "Display help with -h flag".to_string(),
105 TestCategory::Basic,
106 "\"$CLI_BINARY\" -h".to_string(),
107 )
108 .with_exit_code(0)
109 .with_assertion(Assertion::OutputContains("Usage:".to_string()))
110 .with_tag("help".to_string()),
111 );
112
113 if self.analysis.version.is_some() {
115 tests.push(
116 TestCase::new(
117 "basic-003".to_string(),
118 "Display version with --version flag".to_string(),
119 TestCategory::Basic,
120 "\"$CLI_BINARY\" --version".to_string(),
121 )
122 .with_exit_code(0)
123 .with_tag("version".to_string()),
124 );
125 }
126
127 tests.push(
130 TestCase::new(
131 "basic-004".to_string(),
132 "Reject invalid option".to_string(),
133 TestCategory::Basic,
134 "\"$CLI_BINARY\" --invalid-option-xyz".to_string(),
135 )
136 .with_exit_code(2) .with_assertion(Assertion::OutputContains("error".to_string()))
138 .with_tag("error-handling".to_string()),
139 );
140
141 let inferrer = BehaviorInferrer::new();
143 let no_args_behavior = inferrer.infer_no_args_behavior(&self.analysis);
144
145 match no_args_behavior {
146 NoArgsBehavior::ShowHelp => {
147 tests.push(
148 TestCase::new(
149 "basic-005".to_string(),
150 "Show help when invoked without arguments".to_string(),
151 TestCategory::Basic,
152 "\"$CLI_BINARY\"".to_string(),
153 )
154 .with_exit_code(0)
155 .with_priority(TestPriority::Important)
158 .with_tag("no-args".to_string())
159 .with_tag("show-help".to_string()),
160 );
161 }
162
163 NoArgsBehavior::RequireSubcommand => {
164 tests.push(
165 TestCase::new(
166 "basic-005".to_string(),
167 "Require subcommand when invoked without arguments".to_string(),
168 TestCategory::Basic,
169 "\"$CLI_BINARY\"".to_string(),
170 )
171 .expect_nonzero_exit() .with_priority(TestPriority::Important)
175 .with_tag("no-args".to_string())
176 .with_tag("require-subcommand".to_string()),
177 );
178 }
179
180 NoArgsBehavior::Interactive => {
181 tests.push(
182 TestCase::new(
183 "basic-005".to_string(),
184 "Enter interactive mode when invoked without arguments".to_string(),
185 TestCategory::Basic,
186 "echo '' | \"$CLI_BINARY\"".to_string(), )
188 .with_exit_code(0)
189 .with_priority(TestPriority::Important)
190 .with_tag("no-args".to_string())
191 .with_tag("interactive".to_string()),
192 );
193 }
194 }
195
196 Ok(tests)
197 }
198
199 fn generate_help_tests(&self) -> Result<Vec<TestCase>> {
201 let mut tests = Vec::new();
202
203 for (idx, subcommand) in self.analysis.subcommands.iter().enumerate() {
205 tests.push(
206 TestCase::new(
207 format!("help-{:03}", idx + 1),
208 format!("Display help for subcommand '{}'", subcommand.name),
209 TestCategory::Help,
210 format!("\"$CLI_BINARY\" {} --help", subcommand.name),
211 )
212 .with_exit_code(0)
213 .with_assertion(Assertion::OutputContains("Usage:".to_string()))
214 .with_tag("subcommand-help".to_string())
215 .with_tag(subcommand.name.clone()),
216 );
217 }
218
219 Ok(tests)
220 }
221
222 fn generate_security_tests(&self) -> Result<Vec<TestCase>> {
243 let mut tests = Vec::new();
244
245 let string_option = self
247 .analysis
248 .global_options
249 .iter()
250 .find(|opt| {
251 matches!(opt.option_type, OptionType::String | OptionType::Path)
252 && opt.long.is_some()
253 })
254 .and_then(|opt| opt.long.as_ref())
255 .unwrap_or(&"--invalid-option".to_string())
256 .clone();
257
258 tests.push(
261 TestCase::new(
262 "security-001".to_string(),
263 "Reject command injection in option value".to_string(),
264 TestCategory::Security,
265 format!("\"$CLI_BINARY\" {} 'test; rm -rf /'", string_option),
266 )
267 .expect_nonzero_exit() .with_priority(TestPriority::SecurityCheck)
269 .with_tag("injection".to_string())
270 .with_tag("critical".to_string()),
271 );
272
273 tests.push(
276 TestCase::new(
277 "security-002".to_string(),
278 "Reject null byte in option value".to_string(),
279 TestCategory::Security,
280 format!(r#""$CLI_BINARY" {} $'/tmp/test\x00malicious'"#, string_option),
281 )
282 .expect_nonzero_exit() .with_priority(TestPriority::SecurityCheck)
284 .with_tag("injection".to_string())
285 .with_tag("critical".to_string()),
286 );
287
288 tests.push(
291 TestCase::new(
292 "security-003".to_string(),
293 "Reject path traversal attempt".to_string(),
294 TestCategory::Security,
295 format!("\"$CLI_BINARY\" {} ../../../etc/passwd", string_option),
296 )
297 .expect_nonzero_exit() .with_priority(TestPriority::SecurityCheck)
299 .with_tag("path-traversal".to_string())
300 .with_tag("critical".to_string()),
301 );
302
303 let long_input = "A".repeat(10000);
306 tests.push(
307 TestCase::new(
308 "security-004".to_string(),
309 "Handle extremely long input".to_string(),
310 TestCategory::Security,
311 format!("\"$CLI_BINARY\" {} '{}'", string_option, long_input),
312 )
313 .with_priority(TestPriority::Important) .with_tag("buffer-overflow".to_string())
317 .with_tag("informational".to_string()),
318 );
319
320 Ok(tests)
321 }
322
323 fn generate_path_tests(&self) -> Result<Vec<TestCase>> {
325 let mut tests = Vec::new();
326
327 let path_options: Vec<&CliOption> = self
329 .analysis
330 .global_options
331 .iter()
332 .filter(|opt| matches!(opt.option_type, OptionType::Path))
333 .collect();
334
335 if path_options.is_empty() {
336 log::debug!("No path options found, generating generic path tests");
337 return Ok(tests);
338 }
339
340 for (idx, option) in path_options.iter().enumerate() {
341 let flag = option.long.as_ref().or(option.short.as_ref()).unwrap();
342
343 tests.push(
345 TestCase::new(
346 format!("path-{:03}-spaces", idx + 1),
347 format!("Handle path with spaces for {}", flag),
348 TestCategory::Path,
349 format!("\"$CLI_BINARY\" {} '/tmp/test dir/file.txt'", flag),
350 )
351 .with_tag("spaces".to_string()),
352 );
353
354 tests.push(
356 TestCase::new(
357 format!("path-{:03}-unicode", idx + 1),
358 format!("Handle Unicode in path for {}", flag),
359 TestCategory::Path,
360 format!("\"$CLI_BINARY\" {} '/tmp/ใในใ/file.txt'", flag),
361 )
362 .with_tag("unicode".to_string()),
363 );
364
365 tests.push(
367 TestCase::new(
368 format!("path-{:03}-symlink", idx + 1),
369 format!("Handle symbolic links for {}", flag),
370 TestCategory::Path,
371 format!("\"$CLI_BINARY\" {} '/tmp/test-symlink'", flag),
372 )
373 .with_tag("symlink".to_string()),
374 );
375 }
376
377 Ok(tests)
378 }
379
380 fn generate_input_validation_tests(&self) -> Result<Vec<TestCase>> {
382 let mut tests = Vec::new();
383
384 let numeric_options: Vec<&CliOption> = self
386 .analysis
387 .global_options
388 .iter()
389 .filter(|opt| matches!(opt.option_type, OptionType::Numeric { .. }))
390 .collect();
391
392 for (idx, option) in numeric_options.iter().enumerate() {
393 let flag = option.long.as_ref().or(option.short.as_ref()).unwrap();
394
395 tests.push(
397 TestCase::new(
398 format!("input-{:03}-valid", idx + 1),
399 format!("Accept valid numeric value for {}", flag),
400 TestCategory::InputValidation,
401 format!("\"$CLI_BINARY\" {} 10", flag),
402 )
403 .with_tag("numeric".to_string()),
404 );
405
406 tests.push(
408 TestCase::new(
409 format!("input-{:03}-invalid", idx + 1),
410 format!("Reject non-numeric value for {}", flag),
411 TestCategory::InputValidation,
412 format!("\"$CLI_BINARY\" {} 'not-a-number'", flag),
413 )
414 .with_exit_code(1)
415 .with_tag("numeric".to_string())
416 .with_tag("validation".to_string()),
417 );
418
419 if let OptionType::Numeric {
421 min: Some(min_val), ..
422 } = &option.option_type
423 {
424 if *min_val >= 0 {
425 tests.push(
426 TestCase::new(
427 format!("input-{:03}-negative", idx + 1),
428 format!("Reject negative value for {}", flag),
429 TestCategory::InputValidation,
430 format!("\"$CLI_BINARY\" {} -1", flag),
431 )
432 .with_exit_code(1)
433 .with_tag("numeric".to_string())
434 .with_tag("validation".to_string()),
435 );
436 }
437 }
438 }
439
440 let enum_options: Vec<&CliOption> = self
442 .analysis
443 .global_options
444 .iter()
445 .filter(|opt| matches!(opt.option_type, OptionType::Enum { .. }))
446 .collect();
447
448 for (idx, option) in enum_options.iter().enumerate() {
449 let flag = option.long.as_ref().or(option.short.as_ref()).unwrap();
450
451 if let OptionType::Enum { values } = &option.option_type {
452 if let Some(first_value) = values.first() {
453 tests.push(
455 TestCase::new(
456 format!("enum-{:03}-valid", idx + 1),
457 format!("Accept valid enum value for {}", flag),
458 TestCategory::InputValidation,
459 format!("\"$CLI_BINARY\" {} {}", flag, first_value),
460 )
461 .with_tag("enum".to_string()),
462 );
463 }
464
465 tests.push(
467 TestCase::new(
468 format!("enum-{:03}-invalid", idx + 1),
469 format!("Reject invalid enum value for {}", flag),
470 TestCategory::InputValidation,
471 format!("\"$CLI_BINARY\" {} 'invalid-value-xyz'", flag),
472 )
473 .with_exit_code(1)
474 .with_tag("enum".to_string())
475 .with_tag("validation".to_string()),
476 );
477 }
478 }
479
480 Ok(tests)
481 }
482
483 fn generate_destructive_ops_tests(&self) -> Result<Vec<TestCase>> {
485 let mut tests = Vec::new();
486
487 let destructive_keywords = ["delete", "remove", "clean", "destroy", "purge", "drop"];
489
490 for subcommand in &self.analysis.subcommands {
491 let is_destructive = destructive_keywords
492 .iter()
493 .any(|keyword| subcommand.name.to_lowercase().contains(keyword));
494
495 if is_destructive {
496 let dummy_args = subcommand
498 .required_args
499 .iter()
500 .map(|arg| {
501 match arg.to_lowercase().as_str() {
503 "id" | "name" => "test-id",
504 "file" | "path" => "/tmp/test-file",
505 "dir" | "directory" => "/tmp/test-dir",
506 _ => "test-value",
507 }
508 })
509 .collect::<Vec<_>>()
510 .join(" ");
511
512 let args_part = if dummy_args.is_empty() {
513 String::new()
514 } else {
515 format!(" {}", dummy_args)
516 };
517
518 tests.push(
520 TestCase::new(
521 format!("destructive-{}-001", subcommand.name),
522 format!("Subcommand '{}' requires confirmation", subcommand.name),
523 TestCategory::DestructiveOps,
524 format!("echo 'n' | \"$CLI_BINARY\" {}{}", subcommand.name, args_part),
525 )
526 .with_assertion(Assertion::OutputContains("confirm".to_string()))
527 .with_tag("confirmation".to_string())
528 .with_tag(subcommand.name.clone()),
529 );
530
531 let has_yes_flag = subcommand.options.iter().any(|opt| {
533 opt.long
534 .as_ref()
535 .is_some_and(|l| l.contains("yes") || l.contains("force"))
536 });
537
538 if has_yes_flag {
539 tests.push(
540 TestCase::new(
541 format!("destructive-{}-002", subcommand.name),
542 format!("Subcommand '{}' accepts --yes flag", subcommand.name),
543 TestCategory::DestructiveOps,
544 format!("\"$CLI_BINARY\" {}{} --yes", subcommand.name, args_part),
545 )
546 .with_tag("force".to_string())
547 .with_tag(subcommand.name.clone()),
548 );
549 }
550 }
551 }
552
553 if tests.is_empty() {
555 log::debug!("No destructive subcommands detected");
556 }
557
558 Ok(tests)
559 }
560
561 fn generate_directory_traversal_tests(&self) -> Result<Vec<TestCase>> {
563 let mut tests = Vec::new();
564
565 tests.push(
567 TestCase::new(
568 "dir-traversal-001".to_string(),
569 "Handle directory with 1000 files".to_string(),
570 TestCategory::DirectoryTraversal,
571 "\"$CLI_BINARY\" /tmp/test-large-dir".to_string(),
572 )
573 .with_tag("performance".to_string())
574 .with_tag("large-dir".to_string()),
575 );
576
577 tests.push(
579 TestCase::new(
580 "dir-traversal-002".to_string(),
581 "Handle deeply nested directory (50 levels)".to_string(),
582 TestCategory::DirectoryTraversal,
583 "\"$CLI_BINARY\" /tmp/test-deep-dir".to_string(),
584 )
585 .with_tag("performance".to_string())
586 .with_tag("deep-nesting".to_string()),
587 );
588
589 tests.push(
591 TestCase::new(
592 "dir-traversal-003".to_string(),
593 "Detect and handle symlink loops".to_string(),
594 TestCategory::DirectoryTraversal,
595 "\"$CLI_BINARY\" /tmp/test-symlink-loop".to_string(),
596 )
597 .with_tag("symlink".to_string())
598 .with_tag("loop-detection".to_string()),
599 );
600
601 Ok(tests)
602 }
603
604 fn generate_performance_tests(&self) -> Result<Vec<TestCase>> {
606 let mut tests = Vec::new();
607
608 tests.push(
610 TestCase::new(
611 "perf-001".to_string(),
612 "Startup time for --help < 100ms".to_string(),
613 TestCategory::Performance,
614 "\"$CLI_BINARY\" --help".to_string(),
615 )
616 .with_exit_code(0)
617 .with_tag("startup".to_string())
618 .with_tag("benchmark".to_string()),
619 );
620
621 tests.push(
623 TestCase::new(
624 "perf-002".to_string(),
625 "Memory usage stays within reasonable limits".to_string(),
626 TestCategory::Performance,
627 "\"$CLI_BINARY\" --help".to_string(),
628 )
629 .with_exit_code(0)
630 .with_tag("memory".to_string())
631 .with_tag("benchmark".to_string()),
632 );
633
634 Ok(tests)
635 }
636
637 fn generate_multi_shell_tests(&self) -> Result<Vec<TestCase>> {
639 let mut tests = Vec::new();
640
641 for shell in &["bash", "zsh", "sh"] {
643 tests.push(
644 TestCase::new(
645 format!("multi-shell-{}", shell),
646 format!("Run --help in {}", shell),
647 TestCategory::MultiShell,
648 format!("{} -c '\"$CLI_BINARY\" --help'", shell),
649 )
650 .with_exit_code(0)
651 .with_tag(shell.to_string()),
652 );
653 }
654
655 Ok(tests)
656 }
657}
658
659#[cfg(test)]
660mod tests {
661 use super::*;
662 use crate::types::Subcommand;
663 use std::path::PathBuf;
664
665 fn create_test_analysis() -> CliAnalysis {
666 let mut analysis = CliAnalysis::new(
667 PathBuf::from("/usr/bin/test-cli"),
668 "test-cli".to_string(),
669 "Test CLI help output".to_string(),
670 );
671
672 analysis.version = Some("1.0.0".to_string());
673
674 analysis.global_options.push(CliOption {
676 short: Some("-t".to_string()),
677 long: Some("--timeout".to_string()),
678 description: Some("Timeout in seconds".to_string()),
679 option_type: OptionType::Numeric {
680 min: Some(0),
681 max: Some(3600),
682 },
683 required: false,
684 default_value: Some("30".to_string()),
685 });
686
687 analysis.global_options.push(CliOption {
689 short: Some("-f".to_string()),
690 long: Some("--file".to_string()),
691 description: Some("Input file".to_string()),
692 option_type: OptionType::Path,
693 required: false,
694 default_value: None,
695 });
696
697 analysis.global_options.push(CliOption {
699 short: None,
700 long: Some("--format".to_string()),
701 description: Some("Output format".to_string()),
702 option_type: OptionType::Enum {
703 values: vec!["json".to_string(), "yaml".to_string(), "text".to_string()],
704 },
705 required: false,
706 default_value: Some("text".to_string()),
707 });
708
709 analysis.subcommands.push(Subcommand {
711 name: "delete".to_string(),
712 description: Some("Delete resources".to_string()),
713 options: vec![CliOption {
714 short: None,
715 long: Some("--yes".to_string()),
716 description: Some("Skip confirmation".to_string()),
717 option_type: OptionType::Flag,
718 required: false,
719 default_value: None,
720 }],
721 required_args: vec![],
722 subcommands: vec![],
723 depth: 0,
724 });
725
726 analysis
727 }
728
729 #[test]
730 fn test_generator_creation() {
731 let analysis = create_test_analysis();
732 let categories = vec![TestCategory::Basic];
733 let generator = TestGenerator::new(analysis, categories);
734
735 assert_eq!(generator.categories.len(), 1);
736 }
737
738 #[test]
739 fn test_generate_basic_tests() {
740 let analysis = create_test_analysis();
741 let generator = TestGenerator::new(analysis, vec![]);
742
743 let tests = generator.generate_basic_tests().unwrap();
744
745 assert!(!tests.is_empty());
746 assert!(tests.iter().any(|t| t.command.contains("--help")));
747 assert!(tests.iter().any(|t| t.command.contains("--version")));
748 }
749
750 #[test]
751 fn test_generate_security_tests() {
752 let analysis = create_test_analysis();
753 let generator = TestGenerator::new(analysis, vec![]);
754
755 let tests = generator.generate_security_tests().unwrap();
756
757 assert!(!tests.is_empty());
758 assert!(tests
759 .iter()
760 .any(|t| t.tags.contains(&"injection".to_string())));
761 }
762
763 #[test]
764 fn test_generate_input_validation_tests() {
765 let analysis = create_test_analysis();
766 let generator = TestGenerator::new(analysis, vec![]);
767
768 let tests = generator.generate_input_validation_tests().unwrap();
769
770 assert!(!tests.is_empty());
771 assert!(tests
773 .iter()
774 .any(|t| t.tags.contains(&"numeric".to_string())));
775 assert!(tests.iter().any(|t| t.tags.contains(&"enum".to_string())));
776 }
777
778 #[test]
779 fn test_generate_destructive_ops_tests() {
780 let analysis = create_test_analysis();
781 let generator = TestGenerator::new(analysis, vec![]);
782
783 let tests = generator.generate_destructive_ops_tests().unwrap();
784
785 assert!(!tests.is_empty());
786 assert!(tests.iter().any(|t| t.command.contains("delete")));
787 }
788
789 #[test]
790 fn test_generate_all_categories() {
791 let analysis = create_test_analysis();
792 let categories = vec![
793 TestCategory::Basic,
794 TestCategory::Security,
795 TestCategory::InputValidation,
796 ];
797 let generator = TestGenerator::new(analysis, categories);
798
799 let tests = generator.generate().unwrap();
800
801 assert!(!tests.is_empty());
802 assert!(tests.iter().any(|t| t.category == TestCategory::Basic));
803 assert!(tests.iter().any(|t| t.category == TestCategory::Security));
804 assert!(tests
805 .iter()
806 .any(|t| t.category == TestCategory::InputValidation));
807 }
808
809 #[test]
810 fn test_generate_parallel() {
811 let analysis = create_test_analysis();
812 let categories = vec![
813 TestCategory::Basic,
814 TestCategory::Security,
815 TestCategory::Performance,
816 ];
817 let generator = TestGenerator::new(analysis, categories);
818
819 let tests = generator.generate_parallel().unwrap();
820
821 assert!(!tests.is_empty());
822 }
823}