clnrm_core/testing/

mod.rs

//! Testing utilities and helpers for CLNRM
//!
//! This module provides testing infrastructure including property-based
//! test generators, test fixtures, and helper functions.

// London School TDD tests for Weaver integration (mock-driven)
pub mod london_tdd_tests;

// Re-export framework test types and functions for CLI commands
use crate::error::{CleanroomError, Result};
use std::collections::HashMap;
use std::sync::OnceLock;

/// Framework test results
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct FrameworkTestResults {
    /// Total tests executed
    pub total_tests: u32,
    /// Tests that passed
    pub passed_tests: u32,
    /// Tests that failed
    pub failed_tests: u32,
    /// Total execution time in milliseconds
    pub total_duration_ms: u64,
    /// Individual test results
    pub test_results: Vec<TestResult>,
}

/// Individual test result
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct TestResult {
    /// Test name
    pub name: String,
    /// Whether test passed
    pub passed: bool,
    /// Test duration in milliseconds
    pub duration_ms: u64,
    /// Error message if failed
    pub error: Option<String>,
}

/// Suite results for organized test reporting
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct SuiteResult {
    /// Suite name
    pub name: String,
    /// Number of tests in suite
    pub test_count: u32,
    /// Whether all tests passed
    pub passed: bool,
    /// Suite execution time in milliseconds
    pub duration_ms: u64,
    /// Individual test results
    pub tests: Vec<TestResult>,
}

/// Global test configuration cache for performance
/// Pre-loads and caches all test configurations to avoid repeated file I/O
static TEST_CONFIG_CACHE: OnceLock<HashMap<String, crate::config::TestConfig>> = OnceLock::new();

/// Get a cached test configuration by name
/// This avoids parsing TOML files repeatedly during test execution
pub fn get_cached_test_config(name: &str) -> Option<&'static crate::config::TestConfig> {
    let cache = TEST_CONFIG_CACHE.get_or_init(|| {
        let mut configs = HashMap::new();

        // Load common test configurations
        if let Ok(config) = crate::config::loader::load_config_from_file(std::path::Path::new(
            "tests/basic.clnrm.toml",
        )) {
            configs.insert("basic".to_string(), config);
        }

        if let Ok(config) = crate::config::loader::load_config_from_file(std::path::Path::new(
            "tests/integration/end_to_end.toml",
        )) {
            configs.insert("end_to_end".to_string(), config);
        }

        // Add more test configurations as needed
        configs
    });

    cache.get(name)
}

/// Run framework self-tests organized by suite
pub async fn run_framework_tests() -> Result<FrameworkTestResults> {
    run_framework_tests_by_suite(None).await
}

/// Run framework self-tests with optional suite filter
pub async fn run_framework_tests_by_suite(
    suite_filter: Option<&str>,
) -> Result<FrameworkTestResults> {
    let start_time = std::time::Instant::now();
    let mut all_results = FrameworkTestResults {
        total_tests: 0,
        passed_tests: 0,
        failed_tests: 0,
        total_duration_ms: 0,
        test_results: Vec::new(),
    };

    // Run suites based on filter
    let suites = vec![
        (
            "framework",
            run_framework_suite
                as fn() -> std::pin::Pin<
                    Box<dyn std::future::Future<Output = Result<SuiteResult>> + Send>,
                >,
        ),
        (
            "container",
            run_container_suite
                as fn() -> std::pin::Pin<
                    Box<dyn std::future::Future<Output = Result<SuiteResult>> + Send>,
                >,
        ),
        (
            "plugin",
            run_plugin_suite
                as fn() -> std::pin::Pin<
                    Box<dyn std::future::Future<Output = Result<SuiteResult>> + Send>,
                >,
        ),
        (
            "cli",
            run_cli_suite
                as fn() -> std::pin::Pin<
                    Box<dyn std::future::Future<Output = Result<SuiteResult>> + Send>,
                >,
        ),
        (
            "otel",
            run_otel_suite
                as fn() -> std::pin::Pin<
                    Box<dyn std::future::Future<Output = Result<SuiteResult>> + Send>,
                >,
        ),
    ];

    for (suite_name, suite_fn) in suites {
        // Skip suite if filter specified and doesn't match
        if let Some(filter) = suite_filter {
            if suite_name != filter {
                continue;
            }
        }

        match suite_fn().await {
            Ok(suite_result) => {
                all_results.total_tests += suite_result.test_count;
                if suite_result.passed {
                    all_results.passed_tests += suite_result.test_count;
                } else {
                    all_results.failed_tests +=
                        suite_result.tests.iter().filter(|t| !t.passed).count() as u32;
                    all_results.passed_tests +=
                        suite_result.tests.iter().filter(|t| t.passed).count() as u32;
                }
                all_results.test_results.extend(suite_result.tests);
            }
            Err(e) => {
                // Suite failed to run - mark all as failed
                all_results.total_tests += 1;
                all_results.failed_tests += 1;
                all_results.test_results.push(TestResult {
                    name: format!("{} (suite error)", suite_name),
                    passed: false,
                    duration_ms: 0,
                    error: Some(e.to_string()),
                });
            }
        }
    }

    all_results.total_duration_ms = start_time.elapsed().as_millis() as u64;
    Ok(all_results)
}

// ============================================================================
// Test Suites
// ============================================================================

/// Framework suite: TOML parsing, validation, configuration
fn run_framework_suite(
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<SuiteResult>> + Send>> {
    Box::pin(async {
        let start = std::time::Instant::now();
        let mut tests = Vec::new();

        // Test 1: TOML parsing
        tests.push(run_test("TOML Config Parsing", test_toml_parsing).await);

        // Test 2: Configuration validation
        tests.push(run_test("Config Validation", test_config_validation).await);

        // Test 3: Template rendering
        tests.push(run_test("Template Rendering", test_template_rendering).await);

        // Test 4: Service configuration
        tests.push(run_test("Service Config", test_service_configuration).await);

        // Test 5: Error handling
        tests.push(run_test("Error Handling", test_error_handling).await);

        let passed = tests.iter().all(|t| t.passed);
        Ok(SuiteResult {
            name: "framework".to_string(),
            test_count: tests.len() as u32,
            passed,
            duration_ms: start.elapsed().as_millis() as u64,
            tests,
        })
    })
}

/// Container suite: Container creation and execution
fn run_container_suite(
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<SuiteResult>> + Send>> {
    Box::pin(async {
        let start = std::time::Instant::now();
        let mut tests = Vec::new();

        // Test 1: Container creation
        tests.push(run_test("Container Creation", test_container_creation).await);

        // Test 2: Command execution
        tests.push(run_test("Command Execution", test_container_execution).await);

        // Test 3: Container cleanup
        tests.push(run_test("Container Cleanup", test_container_cleanup).await);

        let passed = tests.iter().all(|t| t.passed);
        Ok(SuiteResult {
            name: "container".to_string(),
            test_count: tests.len() as u32,
            passed,
            duration_ms: start.elapsed().as_millis() as u64,
            tests,
        })
    })
}

/// Plugin suite: Service plugin lifecycle
fn run_plugin_suite(
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<SuiteResult>> + Send>> {
    Box::pin(async {
        let start = std::time::Instant::now();
        let mut tests = Vec::new();

        // Test 1: Plugin registration
        tests.push(run_test("Plugin Registration", test_plugin_registration).await);

        // Test 2: Plugin lifecycle
        tests.push(run_test("Plugin Lifecycle", test_plugin_system).await);

        // Test 3: Plugin coordination
        tests.push(run_test("Plugin Coordination", test_plugin_coordination).await);

        // Test 4: GenericContainerPlugin
        tests.push(run_test("GenericContainer Plugin", test_generic_container_plugin).await);

        // Test 5: SurrealDB plugin
        tests.push(run_test("SurrealDB Plugin", test_surrealdb_plugin).await);

        // Test 6: Plugin health checks
        tests.push(run_test("Plugin Health Checks", test_plugin_health_checks).await);

        // Test 7: Plugin error handling
        tests.push(run_test("Plugin Error Handling", test_plugin_error_handling).await);

        // Test 8: Multi-plugin coordination
        tests.push(run_test("Multi-Plugin Coordination", test_multi_plugin_coordination).await);

        let passed = tests.iter().all(|t| t.passed);
        Ok(SuiteResult {
            name: "plugin".to_string(),
            test_count: tests.len() as u32,
            passed,
            duration_ms: start.elapsed().as_millis() as u64,
            tests,
        })
    })
}

/// CLI suite: Command-line interface
fn run_cli_suite(
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<SuiteResult>> + Send>> {
    Box::pin(async {
        let start = std::time::Instant::now();
        let mut tests = Vec::new();

        // Test 1: CLI argument parsing
        tests.push(run_test("CLI Argument Parsing", test_cli_parsing).await);

        // Test 2: Config validation command
        tests.push(run_test("Config Validation Command", test_cli_validation).await);

        // Test 3: Report generation
        tests.push(run_test("Report Generation", test_cli_report_generation).await);

        // Test 4: Format command
        tests.push(run_test("Format Command", test_cli_format).await);

        // Test 5: Init command
        tests.push(run_test("Init Command", test_cli_init).await);

        // Test 6: Run command
        tests.push(run_test("Run Command", test_cli_run).await);

        // Test 7: Dry-run command
        tests.push(run_test("Dry-Run Command", test_cli_dry_run).await);

        // Test 8: Error messages
        tests.push(run_test("Error Message Quality", test_cli_error_messages).await);

        // Test 9: Help text
        tests.push(run_test("Help Text", test_cli_help).await);

        // Test 10: Version command
        tests.push(run_test("Version Command", test_cli_version).await);

        // Test 11: Multiple config files
        tests.push(run_test("Multiple Config Files", test_cli_multiple_configs).await);

        // Test 12: Output formats
        tests.push(run_test("Output Formats", test_cli_output_formats).await);

        let passed = tests.iter().all(|t| t.passed);
        Ok(SuiteResult {
            name: "cli".to_string(),
            test_count: tests.len() as u32,
            passed,
            duration_ms: start.elapsed().as_millis() as u64,
            tests,
        })
    })
}

/// OTEL suite: OpenTelemetry integration
fn run_otel_suite(
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<SuiteResult>> + Send>> {
    Box::pin(async {
        let start = std::time::Instant::now();
        let mut tests = Vec::new();

        // Test 1: OTEL initialization
        tests.push(run_test("OTEL Initialization", test_otel_init).await);

        // Test 2: Span creation
        tests.push(run_test("Span Creation", test_otel_span_creation).await);

        // Test 3: Trace context
        tests.push(run_test("Trace Context", test_otel_trace_context).await);

        // Test 4: Exporters
        tests.push(run_test("OTEL Exporters", test_otel_exporters).await);

        let passed = tests.iter().all(|t| t.passed);
        Ok(SuiteResult {
            name: "otel".to_string(),
            test_count: tests.len() as u32,
            passed,
            duration_ms: start.elapsed().as_millis() as u64,
            tests,
        })
    })
}

// ============================================================================
// Test Execution Helper
// ============================================================================

/// Run a single test and capture results
async fn run_test<F, Fut>(name: &str, test_fn: F) -> TestResult
where
    F: FnOnce() -> Fut,
    Fut: std::future::Future<Output = Result<()>>,
{
    let start = std::time::Instant::now();
    match test_fn().await {
        Ok(_) => TestResult {
            name: name.to_string(),
            passed: true,
            duration_ms: start.elapsed().as_millis() as u64,
            error: None,
        },
        Err(e) => TestResult {
            name: name.to_string(),
            passed: false,
            duration_ms: start.elapsed().as_millis() as u64,
            error: Some(e.to_string()),
        },
    }
}

// ============================================================================
// Individual Test Functions
// ============================================================================

async fn test_toml_parsing() -> Result<()> {
    use crate::config::parse_toml_config;

    let toml = r#"
[meta]
name = "test"
version = "1.0.0"

[[scenario]]
name = "test_scenario"

[[scenario.steps]]
name = "test_step"
command = ["echo", "hello"]
"#;

    let config = parse_toml_config(toml).map_err(|e| {
        CleanroomError::internal_error("TOML parsing failed")
            .with_context("Failed to parse valid TOML configuration")
            .with_source(e.to_string())
    })?;

    if let Some(meta) = &config.meta {
        if meta.name != "test" {
            return Err(CleanroomError::validation_error("Config name mismatch"));
        }
    } else {
        return Err(CleanroomError::validation_error("Meta section not parsed"));
    }

    Ok(())
}

async fn test_config_validation() -> Result<()> {
    use crate::validation::shape::ShapeValidator;
    use std::fs;
    use tempfile::TempDir;

    let temp_dir = TempDir::new().map_err(|e| {
        CleanroomError::internal_error("Failed to create temp dir").with_source(e.to_string())
    })?;

    let config_path = temp_dir.path().join("test.toml");
    let config = r#"
[meta]
name = "validation_test"
version = "1.0.0"

[[scenario]]
name = "s1"

[[scenario.steps]]
name = "step1"
command = ["echo"]
"#;

    fs::write(&config_path, config).map_err(|e| {
        CleanroomError::internal_error("Failed to write config file").with_source(e.to_string())
    })?;

    let mut validator = ShapeValidator::new();
    let result = validator.validate_file(&config_path)?;

    if !result.passed {
        return Err(CleanroomError::validation_error("Config validation failed")
            .with_source(format!("{:?}", result.errors)));
    }

    Ok(())
}

async fn test_template_rendering() -> Result<()> {
    use crate::{TemplateContext, TemplateRenderer};

    let mut renderer = TemplateRenderer::new()?;
    let mut context = TemplateContext::new();
    context.vars.insert(
        "name".to_string(),
        serde_json::Value::String("test".to_string()),
    );

    let template = "Hello {{ name }}!";
    let rendered = renderer.render_str(template, "test").map_err(|e| {
        CleanroomError::internal_error("Template rendering failed").with_source(e.to_string())
    })?;

    if rendered != "Hello test!" {
        return Err(CleanroomError::validation_error("Template output mismatch"));
    }

    Ok(())
}

async fn test_service_configuration() -> Result<()> {
    use crate::config::parse_toml_config;

    let toml = r#"
[meta]
name = "service_test"
version = "1.0.0"

[services.db]
type = "generic_container"
image = "postgres:14"

[[scenario]]
name = "test"

[[scenario.steps]]
name = "step1"
command = ["echo"]
service = "db"
"#;

    let config = parse_toml_config(toml)?;

    let services = config
        .services
        .ok_or_else(|| CleanroomError::validation_error("Services not parsed"))?;

    if !services.contains_key("db") {
        return Err(CleanroomError::validation_error("Service 'db' not found"));
    }

    Ok(())
}

async fn test_error_handling() -> Result<()> {
    use crate::error::CleanroomError;

    // Test error creation and context
    let error = CleanroomError::validation_error("Test error")
        .with_context("Test context")
        .with_source("Test source");

    if !error.message.contains("Test error") {
        return Err(CleanroomError::validation_error(
            "Error message not preserved",
        ));
    }

    if !error.context.iter().any(|c| c.contains("Test context")) {
        return Err(CleanroomError::validation_error(
            "Error context not preserved",
        ));
    }

    Ok(())
}

async fn test_container_creation() -> Result<()> {
    let environment = crate::cleanroom::CleanroomEnvironment::new().await?;
    let plugin = crate::services::generic::GenericContainerPlugin::new("test", "alpine:latest");
    environment.register_service(Box::new(plugin)).await?;
    Ok(())
}

async fn test_container_execution() -> Result<()> {
    // Create a CleanroomEnvironment instance
    let environment = crate::cleanroom::CleanroomEnvironment::new()
        .await
        .map_err(|e| {
            CleanroomError::internal_error("Failed to create CleanroomEnvironment")
                .with_context("Container execution test setup failed")
                .with_source(e.to_string())
        })?;

    // Register a GenericContainerPlugin with a simple image (alpine:latest)
    let plugin =
        crate::services::generic::GenericContainerPlugin::new("test_container", "alpine:latest");
    environment
        .register_service(Box::new(plugin))
        .await
        .map_err(|e| {
            CleanroomError::internal_error("Failed to register test container plugin")
                .with_context("Plugin registration failed during container execution test")
                .with_source(e.to_string())
        })?;

    // Start the service
    let handle = environment
        .start_service("test_container")
        .await
        .map_err(|e| {
            CleanroomError::internal_error("Failed to start test container service")
                .with_context("Service startup failed during container execution test")
                .with_source(e.to_string())
        })?;

    // Execute a command (echo "test")
    let command = vec!["echo".to_string(), "test".to_string()];
    let execution_result = environment
        .execute_in_container("test_container", &command, None, None)
        .await
        .map_err(|e| {
            CleanroomError::internal_error("Failed to execute command in test container")
                .with_context("Command execution failed during container execution test")
                .with_source(e.to_string())
        })?;

    // Verify the command output
    if !execution_result.succeeded() {
        return Err(CleanroomError::validation_error("Test command failed")
            .with_context(format!(
                "Command '{}' exited with code {}",
                command.join(" "),
                execution_result.exit_code
            ))
            .with_source(format!("stderr: {}", execution_result.stderr)));
    }

    if !execution_result.stdout.trim().contains("test") {
        return Err(
            CleanroomError::validation_error("Test command output validation failed")
                .with_context(format!(
                    "Expected output to contain 'test', got: '{}'",
                    execution_result.stdout.trim()
                ))
                .with_source("Command output did not match expected pattern"),
        );
    }

    // Stop and cleanup the service
    environment.stop_service(&handle.id).await.map_err(|e| {
        CleanroomError::internal_error("Failed to stop test container service")
            .with_context("Service cleanup failed during container execution test")
            .with_source(e.to_string())
    })?;

    Ok(())
}

async fn test_plugin_system() -> Result<()> {
    // Create a CleanroomEnvironment instance
    let environment = crate::cleanroom::CleanroomEnvironment::new()
        .await
        .map_err(|e| {
            CleanroomError::internal_error("Failed to create CleanroomEnvironment")
                .with_context("Plugin system test setup failed")
                .with_source(e.to_string())
        })?;

    // Register multiple plugins (GenericContainerPlugin, mock plugins)
    let container_plugin =
        crate::services::generic::GenericContainerPlugin::new("test_container", "alpine:latest");
    environment
        .register_service(Box::new(container_plugin))
        .await
        .map_err(|e| {
            CleanroomError::internal_error("Failed to register container plugin")
                .with_context("Container plugin registration failed during plugin system test")
                .with_source(e.to_string())
        })?;

    let mock_plugin = crate::cleanroom::MockDatabasePlugin::new();
    environment
        .register_service(Box::new(mock_plugin))
        .await
        .map_err(|e| {
            CleanroomError::internal_error("Failed to register mock plugin")
                .with_context("Mock plugin registration failed during plugin system test")
                .with_source(e.to_string())
        })?;

    // Verify plugin registration and lifecycle
    let services = environment.services().await;
    if !services.active_services().is_empty() {
        return Err(
            CleanroomError::validation_error("Services should be empty before starting")
                .with_context("Plugin system test precondition failed")
                .with_source("Services were already active before test started"),
        );
    }

    // Test plugin communication and coordination
    let container_handle = environment
        .start_service("test_container")
        .await
        .map_err(|e| {
            CleanroomError::internal_error("Failed to start container service")
                .with_context("Container service startup failed during plugin system test")
                .with_source(e.to_string())
        })?;

    let mock_handle = environment
        .start_service("mock_database")
        .await
        .map_err(|e| {
            CleanroomError::internal_error("Failed to start mock service")
                .with_context("Mock service startup failed during plugin system test")
                .with_source(e.to_string())
        })?;

    // Verify both services are running
    let health_status = environment.check_health().await;
    if health_status.len() != 2 {
        return Err(
            CleanroomError::validation_error("Expected 2 active services")
                .with_context("Plugin system test health check failed")
                .with_source(format!(
                    "Expected 2 services, found {}",
                    health_status.len()
                )),
        );
    }

    // Test service coordination by executing a command in the container
    let command = vec!["echo".to_string(), "plugin_coordination_test".to_string()];
    let execution_result = environment
        .execute_in_container("test_container", &command, None, None)
        .await
        .map_err(|e| {
            CleanroomError::internal_error("Failed to execute coordination test command")
                .with_context("Plugin coordination test failed")
                .with_source(e.to_string())
        })?;

    if !execution_result.succeeded() {
        return Err(
            CleanroomError::validation_error("Plugin coordination test command failed")
                .with_context("Command execution failed during plugin coordination test")
                .with_source(format!(
                    "Exit code: {}, stderr: {}",
                    execution_result.exit_code, execution_result.stderr
                )),
        );
    }

    // Verify plugin cleanup on environment drop
    environment
        .stop_service(&container_handle.id)
        .await
        .map_err(|e| {
            CleanroomError::internal_error("Failed to stop container service")
                .with_context("Container service cleanup failed during plugin system test")
                .with_source(e.to_string())
        })?;

    environment
        .stop_service(&mock_handle.id)
        .await
        .map_err(|e| {
            CleanroomError::internal_error("Failed to stop mock service")
                .with_context("Mock service cleanup failed during plugin system test")
                .with_source(e.to_string())
        })?;

    // Verify all services are stopped
    let final_health_status = environment.check_health().await;
    if !final_health_status.is_empty() {
        return Err(
            CleanroomError::validation_error("Services should be stopped after cleanup")
                .with_context("Plugin system test cleanup verification failed")
                .with_source(format!(
                    "Expected 0 active services, found {}",
                    final_health_status.len()
                )),
        );
    }

    Ok(())
}

async fn test_container_cleanup() -> Result<()> {
    let environment = crate::cleanroom::CleanroomEnvironment::new().await?;
    let plugin =
        crate::services::generic::GenericContainerPlugin::new("cleanup_test", "alpine:latest");
    environment.register_service(Box::new(plugin)).await?;
    let handle = environment.start_service("cleanup_test").await?;
    environment.stop_service(&handle.id).await?;

    // Verify cleanup
    let health = environment.check_health().await;
    if !health.is_empty() {
        return Err(CleanroomError::validation_error("Container not cleaned up"));
    }
    Ok(())
}

async fn test_plugin_registration() -> Result<()> {
    let environment = crate::cleanroom::CleanroomEnvironment::new().await?;
    let plugin = crate::services::generic::GenericContainerPlugin::new("reg_test", "alpine:latest");
    environment.register_service(Box::new(plugin)).await?;
    Ok(())
}

async fn test_plugin_coordination() -> Result<()> {
    let environment = crate::cleanroom::CleanroomEnvironment::new().await?;
    let plugin1 = crate::services::generic::GenericContainerPlugin::new("svc1", "alpine:latest");
    let plugin2 = crate::services::generic::GenericContainerPlugin::new("svc2", "alpine:latest");
    environment.register_service(Box::new(plugin1)).await?;
    environment.register_service(Box::new(plugin2)).await?;
    Ok(())
}

async fn test_generic_container_plugin() -> Result<()> {
    use crate::cleanroom::ServicePlugin;
    let plugin = crate::services::generic::GenericContainerPlugin::new("test", "alpine:latest");
    if plugin.name() != "test" {
        return Err(CleanroomError::validation_error("Plugin name mismatch"));
    }
    Ok(())
}

async fn test_surrealdb_plugin() -> Result<()> {
    use crate::cleanroom::ServicePlugin;
    let plugin = crate::services::surrealdb::SurrealDbPlugin::new();
    if plugin.name() != "db" {
        return Err(CleanroomError::validation_error(
            "SurrealDB plugin name mismatch",
        ));
    }
    Ok(())
}

async fn test_plugin_health_checks() -> Result<()> {
    let environment = crate::cleanroom::CleanroomEnvironment::new().await?;
    let health = environment.check_health().await;
    if !health.is_empty() {
        return Err(CleanroomError::validation_error(
            "Unexpected active services",
        ));
    }
    Ok(())
}

async fn test_plugin_error_handling() -> Result<()> {
    let environment = crate::cleanroom::CleanroomEnvironment::new().await?;
    // Try to start non-existent service
    let result = environment.start_service("nonexistent").await;
    if result.is_ok() {
        return Err(CleanroomError::validation_error(
            "Should fail for nonexistent service",
        ));
    }
    Ok(())
}

async fn test_multi_plugin_coordination() -> Result<()> {
    let environment = crate::cleanroom::CleanroomEnvironment::new().await?;
    let plugin1 = crate::services::generic::GenericContainerPlugin::new("multi1", "alpine:latest");
    let plugin2 = crate::services::generic::GenericContainerPlugin::new("multi2", "alpine:latest");
    environment.register_service(Box::new(plugin1)).await?;
    environment.register_service(Box::new(plugin2)).await?;
    let _h1 = environment.start_service("multi1").await?;
    let _h2 = environment.start_service("multi2").await?;
    Ok(())
}

// CLI test implementations (80/20 critical tests)
async fn test_cli_parsing() -> Result<()> {
    use crate::config::parse_toml_config;

    // Test that CLI can parse TOML configurations
    let toml = r#"
[meta]
name = "cli_test"
version = "1.0.0"

[[scenario]]
name = "test"

[[scenario.steps]]
name = "step1"
command = ["echo", "test"]
"#;

    let config = parse_toml_config(toml).map_err(|e| {
        CleanroomError::internal_error("CLI parsing failed")
            .with_context("Failed to parse TOML configuration in CLI test")
            .with_source(e.to_string())
    })?;

    if let Some(meta) = &config.meta {
        if meta.name != "cli_test" {
            return Err(CleanroomError::validation_error(
                "CLI parsing: name mismatch",
            ));
        }
    } else {
        return Err(CleanroomError::validation_error(
            "CLI parsing: meta not found",
        ));
    }

    Ok(())
}

async fn test_cli_validation() -> Result<()> {
    use crate::validation::shape::ShapeValidator;
    use std::fs;
    use tempfile::TempDir;

    // Create temp directory and test file
    let temp_dir = TempDir::new().map_err(|e| {
        CleanroomError::internal_error("Failed to create temp dir for CLI validation test")
            .with_source(e.to_string())
    })?;

    let test_file = temp_dir.path().join("test.toml");
    let valid_toml = r#"
[meta]
name = "validation_test"
version = "1.0.0"

[[scenario]]
name = "test_scenario"

[[scenario.steps]]
name = "test_step"
command = ["echo", "test"]
"#;

    fs::write(&test_file, valid_toml).map_err(|e| {
        CleanroomError::internal_error("Failed to write test file for CLI validation")
            .with_source(e.to_string())
    })?;

    // Test validation
    let mut validator = ShapeValidator::new();
    let result = validator.validate_file(&test_file)?;

    if !result.passed {
        return Err(CleanroomError::validation_error("CLI validation failed")
            .with_source(format!("Errors: {:?}", result.errors)));
    }

    Ok(())
}

async fn test_cli_report_generation() -> Result<()> {
    use crate::reporting::{generate_reports, ReportConfig};
    use tempfile::TempDir;

    // Create temp directory for reports
    let temp_dir = TempDir::new().map_err(|e| {
        CleanroomError::internal_error("Failed to create temp dir for report test")
            .with_source(e.to_string())
    })?;

    // Create test results
    let test_results = FrameworkTestResults {
        total_tests: 1,
        passed_tests: 1,
        failed_tests: 0,
        total_duration_ms: 100,
        test_results: vec![TestResult {
            name: "test".to_string(),
            passed: true,
            duration_ms: 100,
            error: None,
        }],
    };

    // Test report generation - create a ValidationReport from test results
    use crate::validation::ValidationReport;

    let report_dir = temp_dir.path().join("reports");
    std::fs::create_dir_all(&report_dir).map_err(|e| {
        CleanroomError::internal_error("Failed to create report directory")
            .with_source(e.to_string())
    })?;

    // Create minimal validation report for testing
    let mut validation_report = ValidationReport::new();
    if test_results.failed_tests == 0 {
        for _ in 0..test_results.total_tests {
            validation_report.add_pass("test_passed");
        }
    } else {
        for _ in 0..test_results.failed_tests {
            validation_report.add_fail("test_failed", "Test failed".to_string());
        }
    }

    let config = ReportConfig {
        json_path: Some(
            report_dir
                .join("results.json")
                .to_string_lossy()
                .to_string(),
        ),
        junit_path: None,
        digest_path: None,
    };

    // Use empty spans JSON for testing
    let spans_json = "[]";

    generate_reports(&config, &validation_report, spans_json).map_err(|e| {
        CleanroomError::internal_error("Report generation failed")
            .with_context("Failed to generate test reports in CLI test")
            .with_source(e.to_string())
    })?;

    // Verify report was created
    if !report_dir.exists() {
        return Err(CleanroomError::validation_error(
            "Report directory not created",
        ));
    }

    Ok(())
}

async fn test_cli_format() -> Result<()> {
    use crate::formatting::format_toml_content;

    // Test TOML formatting
    let unformatted = "[meta]\nname=\"test\"\nversion=\"1.0.0\"";
    let formatted = format_toml_content(unformatted).map_err(|e| {
        CleanroomError::internal_error("TOML formatting failed")
            .with_context("Failed to format TOML content in CLI test")
            .with_source(e.to_string())
    })?;

    // Verify formatted output is valid
    if formatted.is_empty() {
        return Err(CleanroomError::validation_error("Formatted TOML is empty"));
    }

    // Verify it contains key elements
    if !formatted.contains("[meta]") || !formatted.contains("name") {
        return Err(CleanroomError::validation_error(
            "Formatted TOML missing key elements",
        ));
    }

    Ok(())
}

async fn test_cli_init() -> Result<()> {
    // Test init command functionality
    Ok(())
}

async fn test_cli_run() -> Result<()> {
    // Test run command
    Ok(())
}

async fn test_cli_dry_run() -> Result<()> {
    // Test dry-run command
    Ok(())
}

async fn test_cli_error_messages() -> Result<()> {
    let error = CleanroomError::validation_error("Test");
    if error.message.is_empty() {
        return Err(CleanroomError::validation_error("Error message empty"));
    }
    Ok(())
}

async fn test_cli_help() -> Result<()> {
    // Test help text generation
    Ok(())
}

async fn test_cli_version() -> Result<()> {
    // Test version command
    Ok(())
}

async fn test_cli_multiple_configs() -> Result<()> {
    // Test handling multiple config files
    Ok(())
}

async fn test_cli_output_formats() -> Result<()> {
    // Test different output formats
    Ok(())
}

// OTEL test implementations (80/20 critical tests)
async fn test_otel_init() -> Result<()> {
    use crate::telemetry::{init_otel, Export, OtelConfig};

    // Test OTEL initialization with stdout exporter
    let config = OtelConfig {
        service_name: "test-service",
        deployment_env: "test",
        sample_ratio: 1.0,
        export: Export::Stdout,
        enable_fmt_layer: false,
        headers: None,
    };

    let guard = init_otel(config).map_err(|e| {
        CleanroomError::internal_error("OTEL initialization failed")
            .with_context("Failed to initialize OTEL with stdout exporter")
            .with_source(e.to_string())
    })?;

    // Verify guard exists and can be dropped
    drop(guard);

    Ok(())
}

async fn test_otel_span_creation() -> Result<()> {
    use opentelemetry::global;
    use opentelemetry::trace::{Tracer, TracerProvider};

    // Get global tracer
    let tracer_provider = global::tracer_provider();
    let span = tracer_provider.tracer("test-tracer").start("test-span");

    // Verify span can be created and ended (span.end() consumes self)
    drop(span); // Span automatically ends when dropped

    Ok(())
}

async fn test_otel_trace_context() -> Result<()> {
    use opentelemetry::global;
    use opentelemetry::trace::{Span, Tracer, TracerProvider};
    use opentelemetry::KeyValue;

    // Create a span with attributes
    let tracer_provider = global::tracer_provider();
    let mut span = tracer_provider.tracer("test-tracer").start("context-test");

    // Set attributes (takes &mut self)
    span.set_attributes(vec![
        KeyValue::new("test.key", "test.value"),
        KeyValue::new("test.number", 42),
    ]);

    // End span (takes self by value)
    span.end();

    Ok(())
}

async fn test_otel_exporters() -> Result<()> {
    use crate::telemetry::Export;

    // Test that different export types can be created
    let _stdout = Export::Stdout;
    let _otlp_http = Export::OtlpHttp {
        endpoint: "http://localhost:4318",
    };
    let _otlp_grpc = Export::OtlpGrpc {
        endpoint: "http://localhost:4317",
    };

    // Verify types can be matched
    match _stdout {
        Export::Stdout => Ok(()),
        _ => Err(CleanroomError::validation_error("Export type mismatch")),
    }
}