clnrm_core/

cleanroom.rs

//! Cleanroom Environment - Framework Self-Testing Implementation
//!
//! Core cleanroom environment that tests itself through the "eat your own dog food"
//! principle. Every feature of this framework is validated by using the framework
//! to test its own functionality.

use crate::backend::{Backend, Cmd, TestcontainerBackend};
use crate::error::{CleanroomError, Result};
#[cfg(feature = "otel-traces")]
use opentelemetry::global;
#[cfg(feature = "otel-traces")]
use opentelemetry::trace::{Span, Tracer, TracerProvider};
#[cfg(feature = "otel-traces")]
use opentelemetry::KeyValue;
use std::any::Any;
use std::collections::HashMap;
use std::sync::Arc;
use testcontainers::runners::AsyncRunner;
use testcontainers_modules::surrealdb::{SurrealDb, SURREALDB_PORT};
use tokio::sync::RwLock;
use uuid::Uuid;

/// Plugin-based service registry (no hardcoded postgres/redis)
pub trait ServicePlugin: Send + Sync + std::fmt::Debug {
    /// Get service name
    fn name(&self) -> &str;

    /// Start the service
    fn start(&self) -> Result<ServiceHandle>;

    /// Stop the service
    fn stop(&self, handle: ServiceHandle) -> Result<()>;

    /// Check service health
    fn health_check(&self, handle: &ServiceHandle) -> HealthStatus;
}

/// Service handle for managing service instances
#[derive(Debug, Clone)]
pub struct ServiceHandle {
    /// Unique service instance ID
    pub id: String,
    /// Service name
    pub service_name: String,
    /// Service metadata
    pub metadata: HashMap<String, String>,
}

/// Service health status
#[derive(Debug, Clone, PartialEq)]
pub enum HealthStatus {
    /// Service is healthy and running
    Healthy,
    /// Service is unhealthy or not responding
    Unhealthy,
    /// Service status is unknown
    Unknown,
}

/// Plugin-based service registry
#[derive(Debug, Default)]
pub struct ServiceRegistry {
    /// Registered service plugins
    plugins: HashMap<String, Box<dyn ServicePlugin>>,
    /// Active service instances
    active_services: HashMap<String, ServiceHandle>,
}

impl ServiceRegistry {
    /// Create a new service registry
    pub fn new() -> Self {
        Self::default()
    }

    /// Initialize default plugins
    pub fn with_default_plugins(mut self) -> Self {
        use crate::services::{
            generic::GenericContainerPlugin, ollama::OllamaPlugin, tgi::TgiPlugin, vllm::VllmPlugin,
        };

        // Register core plugins
        let generic_plugin = Box::new(GenericContainerPlugin::new(
            "generic_container",
            "alpine:latest",
        ));
        self.register_plugin(generic_plugin);

        // Register AI/LLM proxy plugins for automated rollout testing
        let ollama_config = crate::services::ollama::OllamaConfig {
            endpoint: "http://localhost:11434".to_string(),
            default_model: "qwen3-coder:30b".to_string(),
            timeout_seconds: 60,
        };
        let ollama_plugin = Box::new(OllamaPlugin::new("ollama", ollama_config));
        self.register_plugin(ollama_plugin);

        let vllm_config = crate::services::vllm::VllmConfig {
            endpoint: "http://localhost:8000".to_string(),
            model: "microsoft/DialoGPT-medium".to_string(),
            max_num_seqs: Some(100),
            max_model_len: Some(2048),
            tensor_parallel_size: Some(1),
            gpu_memory_utilization: Some(0.9),
            enable_prefix_caching: Some(true),
            timeout_seconds: 60,
        };
        let vllm_plugin = Box::new(VllmPlugin::new("vllm", vllm_config));
        self.register_plugin(vllm_plugin);

        let tgi_config = crate::services::tgi::TgiConfig {
            endpoint: "http://localhost:8080".to_string(),
            model_id: "microsoft/DialoGPT-medium".to_string(),
            max_total_tokens: Some(2048),
            max_input_length: Some(1024),
            max_batch_prefill_tokens: Some(4096),
            max_concurrent_requests: Some(32),
            max_batch_total_tokens: Some(8192),
            timeout_seconds: 60,
        };
        let tgi_plugin = Box::new(TgiPlugin::new("tgi", tgi_config));
        self.register_plugin(tgi_plugin);

        self
    }

    /// Register a service plugin
    pub fn register_plugin(&mut self, plugin: Box<dyn ServicePlugin>) {
        let name = plugin.name().to_string();
        self.plugins.insert(name, plugin);
    }

    /// Start a service by name
    pub async fn start_service(&mut self, service_name: &str) -> Result<ServiceHandle> {
        let plugin = self.plugins.get(service_name).ok_or_else(|| {
            CleanroomError::internal_error(format!("Service plugin '{}' not found", service_name))
        })?;

        let handle = plugin.start()?;
        self.active_services
            .insert(handle.id.clone(), handle.clone());

        Ok(handle)
    }

    /// Stop a service by handle ID
    pub async fn stop_service(&mut self, handle_id: &str) -> Result<()> {
        if let Some(handle) = self.active_services.remove(handle_id) {
            let plugin = self.plugins.get(&handle.service_name).ok_or_else(|| {
                CleanroomError::internal_error(format!(
                    "Service plugin '{}' not found for handle '{}'",
                    handle.service_name, handle_id
                ))
            })?;

            plugin.stop(handle)?;
        }

        Ok(())
    }

    /// Check health of all services
    pub async fn check_all_health(&self) -> HashMap<String, HealthStatus> {
        let mut health_status = HashMap::new();

        for (handle_id, handle) in &self.active_services {
            if let Some(plugin) = self.plugins.get(&handle.service_name) {
                health_status.insert(handle_id.clone(), plugin.health_check(handle));
            } else {
                health_status.insert(handle_id.clone(), HealthStatus::Unknown);
            }
        }

        health_status
    }

    /// Get all active service handles
    pub fn active_services(&self) -> &HashMap<String, ServiceHandle> {
        &self.active_services
    }

    /// Check if service is running
    pub fn is_service_running(&self, service_name: &str) -> bool {
        self.active_services
            .values()
            .any(|handle| handle.service_name == service_name)
    }

    /// Get service logs
    pub async fn get_service_logs(&self, service_id: &str, lines: usize) -> Result<Vec<String>> {
        let handle = self.active_services.get(service_id).ok_or_else(|| {
            CleanroomError::internal_error(format!("Service with ID '{}' not found", service_id))
        })?;

        let _plugin = self.plugins.get(&handle.service_name).ok_or_else(|| {
            CleanroomError::internal_error(format!(
                "Service plugin '{}' not found",
                handle.service_name
            ))
        })?;

        // For now, return mock logs since actual log retrieval depends on the service implementation
        // In a real implementation, this would call plugin.get_logs(handle, lines)
        let mock_logs = vec![
            format!(
                "[{}] Service '{}' started",
                chrono::Utc::now().format("%Y-%m-%d %H:%M:%S"),
                handle.service_name
            ),
            format!(
                "[{}] Service '{}' is running",
                chrono::Utc::now().format("%Y-%m-%d %H:%M:%S"),
                handle.service_name
            ),
        ];

        // Return only the requested number of lines
        Ok(mock_logs.into_iter().take(lines).collect())
    }
}

/// Simple metrics for quick access
#[derive(Debug, Clone)]
pub struct SimpleMetrics {
    /// Session ID
    pub session_id: Uuid,
    /// Start time
    pub start_time: std::time::Instant,
    /// Tests executed
    pub tests_executed: u32,
    /// Tests passed
    pub tests_passed: u32,
    /// Tests failed
    pub tests_failed: u32,
    /// Total duration
    pub total_duration_ms: u64,
    /// Active containers
    pub active_containers: u32,
    /// Active services
    pub active_services: u32,
    /// Containers created in this session
    pub containers_created: u32,
    /// Containers reused in this session
    pub containers_reused: u32,
}

impl SimpleMetrics {
    pub fn new() -> Self {
        Self {
            session_id: Uuid::new_v4(),
            start_time: std::time::Instant::now(),
            tests_executed: 0,
            tests_passed: 0,
            tests_failed: 0,
            total_duration_ms: 0,
            active_containers: 0,
            active_services: 0,
            containers_created: 0,
            containers_reused: 0,
        }
    }
}

impl Default for SimpleMetrics {
    fn default() -> Self {
        Self::new()
    }
}

/// Execution result for container command execution
#[derive(Debug, Clone)]
pub struct ExecutionResult {
    /// Exit code of the executed command
    pub exit_code: i32,
    /// Standard output from the command
    pub stdout: String,
    /// Standard error from the command
    pub stderr: String,
    /// Duration of command execution
    pub duration: std::time::Duration,
    /// Command that was executed
    pub command: Vec<String>,
    /// Container name where command was executed
    pub container_name: String,
}

impl ExecutionResult {
    /// Check if command output matches a regex pattern
    pub fn matches_regex(&self, pattern: &str) -> Result<bool> {
        use regex::Regex;
        let regex = Regex::new(pattern).map_err(|e| {
            CleanroomError::validation_error(format!("Invalid regex pattern '{}': {}", pattern, e))
        })?;
        Ok(regex.is_match(&self.stdout))
    }

    /// Check if command output does NOT match a regex pattern
    pub fn does_not_match_regex(&self, pattern: &str) -> Result<bool> {
        Ok(!self.matches_regex(pattern)?)
    }

    /// Check if command succeeded (exit code 0)
    pub fn succeeded(&self) -> bool {
        self.exit_code == 0
    }

    /// Check if command failed (non-zero exit code)
    pub fn failed(&self) -> bool {
        !self.succeeded()
    }
}

/// Simple environment wrapper around existing infrastructure
#[allow(dead_code)]
#[derive(Debug)]
pub struct CleanroomEnvironment {
    /// Session ID
    session_id: Uuid,
    /// Backend for container execution
    backend: Arc<dyn Backend>,
    /// Plugin-based service registry
    services: Arc<RwLock<ServiceRegistry>>,
    /// Simple metrics for quick access
    metrics: Arc<RwLock<SimpleMetrics>>,
    /// Container registry for reuse - stores actual container instances
    container_registry: Arc<RwLock<HashMap<String, Box<dyn Any + Send + Sync>>>>,
    /// OpenTelemetry meter for metrics
    #[cfg(feature = "otel-metrics")]
    meter: opentelemetry::metrics::Meter,
    /// Telemetry configuration and state
    telemetry: Arc<RwLock<TelemetryState>>,
}

impl Default for CleanroomEnvironment {
    fn default() -> Self {
        // Use a simple synchronous approach for Default
        // This is acceptable since Default is typically used in tests
        let backend = Arc::new(
            TestcontainerBackend::new("alpine:latest").unwrap_or_else(|_| {
                panic!("Failed to create default backend - check Docker availability")
            }),
        );

        Self {
            session_id: Uuid::new_v4(),
            backend,
            services: Arc::new(RwLock::new(ServiceRegistry::new())),
            metrics: Arc::new(RwLock::new(SimpleMetrics::new())),
            container_registry: Arc::new(RwLock::new(HashMap::new())),
            #[cfg(feature = "otel-metrics")]
            meter: opentelemetry::global::meter("clnrm"),
            telemetry: Arc::new(RwLock::new(TelemetryState {
                tracing_enabled: false,
                metrics_enabled: false,
                traces: Vec::new(),
            })),
        }
    }
}

/// Telemetry state for the cleanroom environment
#[derive(Debug)]
pub struct TelemetryState {
    /// Whether tracing is enabled
    pub tracing_enabled: bool,
    /// Whether metrics are enabled
    pub metrics_enabled: bool,
    /// Collected traces (for testing/debugging)
    pub traces: Vec<String>,
}

impl TelemetryState {
    /// Enable tracing
    pub fn enable_tracing(&mut self) {
        self.tracing_enabled = true;
    }

    /// Enable metrics collection
    pub fn enable_metrics(&mut self) {
        self.metrics_enabled = true;
    }

    /// Get collected traces
    pub fn get_traces(&self) -> Vec<String> {
        self.traces.clone()
    }
}

impl CleanroomEnvironment {
    /// Create a new cleanroom environment
    pub async fn new() -> Result<Self> {
        Ok(Self {
            session_id: Uuid::new_v4(),
            #[cfg(feature = "otel-metrics")]
            meter: {
                let meter_provider = global::meter_provider();
                meter_provider.meter("clnrm-cleanroom")
            },
            backend: Arc::new(TestcontainerBackend::new("alpine:latest")?),
            services: Arc::new(RwLock::new(ServiceRegistry::new().with_default_plugins())),
            metrics: Arc::new(RwLock::new(SimpleMetrics::default())),
            container_registry: Arc::new(RwLock::new(HashMap::new())),
            telemetry: Arc::new(RwLock::new(TelemetryState {
                tracing_enabled: false,
                metrics_enabled: false,
                traces: Vec::new(),
            })),
        })
    }

    /// Execute a test with OTel tracing
    pub async fn execute_test<F, T>(&self, _test_name: &str, test_fn: F) -> Result<T>
    where
        F: FnOnce() -> Result<T>,
    {
        #[cfg(feature = "otel-traces")]
        let tracer_provider = global::tracer_provider();
        #[cfg(feature = "otel-traces")]
        let mut span = tracer_provider
            .tracer("clnrm-cleanroom")
            .start(format!("test.{}", _test_name));
        #[cfg(feature = "otel-traces")]
        span.set_attributes(vec![
            KeyValue::new("test.name", _test_name.to_string()),
            KeyValue::new("session.id", self.session_id.to_string()),
        ]);

        let start_time = std::time::Instant::now();

        // Update metrics
        {
            let mut metrics = self.metrics.write().await;
            metrics.tests_executed += 1;
        }

        let result = test_fn();

        let duration = start_time.elapsed();

        // Record OTel metrics
        let success = result.is_ok();
        if success {
            let mut metrics = self.metrics.write().await;
            metrics.tests_passed += 1;
        } else {
            let mut metrics = self.metrics.write().await;
            metrics.tests_failed += 1;
        }

        let mut metrics = self.metrics.write().await;
        metrics.total_duration_ms += duration.as_millis() as u64;

        #[cfg(feature = "otel-metrics")]
        {
            // OTel metrics
            let attributes = vec![
                KeyValue::new("test.name", _test_name.to_string()),
                KeyValue::new("session.id", self.session_id.to_string()),
            ];

            let counter = self
                .meter
                .u64_counter("test.executions")
                .with_description("Number of test executions")
                .build();
            counter.add(1, &attributes);

            let histogram = self
                .meter
                .f64_histogram("test.duration")
                .with_description("Test execution duration")
                .build();
            histogram.record(duration.as_secs_f64(), &attributes);
        }

        #[cfg(feature = "otel-traces")]
        if !success {
            span.set_status(opentelemetry::trace::Status::error("Test failed"));
        }

        #[cfg(feature = "otel-traces")]
        span.end();

        result
    }

    /// Get current metrics
    pub async fn get_metrics(&self) -> Result<SimpleMetrics> {
        Ok(self.metrics.read().await.clone())
    }

    /// Enable tracing for this environment
    pub async fn enable_tracing(&self) -> Result<()> {
        #[cfg(feature = "otel-traces")]
        {
            let mut telemetry = self.telemetry.write().await;
            telemetry.enable_tracing();
        }
        Ok(())
    }

    /// Enable metrics collection for this environment
    pub async fn enable_metrics(&self) -> Result<()> {
        #[cfg(feature = "otel-traces")]
        {
            let mut telemetry = self.telemetry.write().await;
            telemetry.enable_metrics();
        }
        Ok(())
    }

    /// Get traces from this environment
    pub async fn get_traces(&self) -> Result<Vec<String>> {
        #[cfg(feature = "otel-traces")]
        {
            let telemetry = self.telemetry.read().await;
            Ok(telemetry.get_traces())
        }
        #[cfg(not(feature = "otel-traces"))]
        {
            Ok(Vec::new())
        }
    }

    /// Get container reuse statistics
    pub async fn get_container_reuse_stats(&self) -> (u32, u32) {
        let metrics = self.metrics.read().await;
        (metrics.containers_created, metrics.containers_reused)
    }

    /// Check if a container with the given name has been created in this session
    pub async fn has_container(&self, name: &str) -> bool {
        let registry = self.container_registry.read().await;
        registry.contains_key(name)
    }

    /// Register a service plugin
    pub async fn register_service(&self, plugin: Box<dyn ServicePlugin>) -> Result<()> {
        let mut services = self.services.write().await;
        services.register_plugin(plugin);
        Ok(())
    }

    /// Start a service by name
    pub async fn start_service(&self, service_name: &str) -> Result<ServiceHandle> {
        let mut services = self.services.write().await;
        services.start_service(service_name).await
    }

    /// Stop a service by handle ID
    pub async fn stop_service(&self, handle_id: &str) -> Result<()> {
        let mut services = self.services.write().await;
        services.stop_service(handle_id).await
    }

    /// Get service registry (read-only access)
    pub async fn services(&self) -> tokio::sync::RwLockReadGuard<'_, ServiceRegistry> {
        self.services.read().await
    }

    /// Register a container for reuse
    pub async fn register_container<T: Send + Sync + 'static>(
        &self,
        name: String,
        container: T,
    ) -> Result<()> {
        let mut registry = self.container_registry.write().await;
        registry.insert(name, Box::new(container));
        Ok(())
    }

    /// Get or create container with reuse pattern
    ///
    /// This method implements true container reuse by storing and returning
    /// the actual container instances, providing the promised 10-50x performance improvement.
    pub async fn get_or_create_container<F, T>(&self, name: &str, factory: F) -> Result<T>
    where
        F: FnOnce() -> Result<T>,
        T: Send + Sync + Clone + 'static,
    {
        // Check if we've already created a container with this name in this session
        let existing_container = {
            let registry = self.container_registry.read().await;
            if let Some(existing_container) = registry.get(name) {
                // Try to downcast to the requested type
                existing_container
                    .downcast_ref::<T>()
                    .map(|typed_container| typed_container.clone())
            } else {
                None
            }
        };

        if let Some(container) = existing_container {
            // Update metrics to track actual reuse
            {
                let mut metrics = self.metrics.write().await;
                metrics.containers_reused += 1;
            }

            return Ok(container);
        }

        // First time creating this container
        let container = factory()?;

        // Register the actual container for future reuse
        let mut registry = self.container_registry.write().await;
        registry.insert(name.to_string(), Box::new(container.clone()));

        // Update metrics
        {
            let mut metrics = self.metrics.write().await;
            metrics.containers_created += 1;
        }

        Ok(container)
    }

    /// Check health of all services
    pub async fn check_health(&self) -> HashMap<String, HealthStatus> {
        self.services.read().await.check_all_health().await
    }

    /// Get service logs
    pub async fn get_service_logs(&self, service_id: &str, lines: usize) -> Result<Vec<String>> {
        let services = self.services.read().await;
        services.get_service_logs(service_id, lines).await
    }

    /// Get session ID
    pub fn session_id(&self) -> Uuid {
        self.session_id
    }

    /// Get backend
    pub fn backend(&self) -> &dyn Backend {
        self.backend.as_ref() as &dyn Backend
    }

    /// Execute a command in a container with proper error handling and observability
    /// Core Team Compliance: Async for I/O operations, proper error handling, no unwrap/expect
    ///
    /// This method creates a fresh container for each command execution, which is appropriate
    /// for testing scenarios where isolation is more important than performance.
    pub async fn execute_in_container(
        &self,
        container_name: &str,
        command: &[String],
    ) -> Result<ExecutionResult> {
        #[cfg(feature = "otel-traces")]
        let tracer_provider = global::tracer_provider();
        #[cfg(feature = "otel-traces")]
        let mut span = tracer_provider
            .tracer("clnrm-cleanroom")
            .start(format!("container.exec.{}", container_name));
        #[cfg(feature = "otel-traces")]
        span.set_attributes(vec![
            KeyValue::new("container.name", container_name.to_string()),
            KeyValue::new("command", command.join(" ")),
            KeyValue::new("session.id", self.session_id.to_string()),
        ]);

        let start_time = std::time::Instant::now();

        // Execute command using backend - this creates a fresh container for each command
        // This provides maximum isolation and is appropriate for testing scenarios
        let cmd = Cmd::new("sh")
            .arg("-c")
            .arg(command.join(" "))
            .env("CONTAINER_NAME", container_name);

        // Use spawn_blocking to avoid runtime conflicts with testcontainers
        // Clone the backend to move it into the blocking task
        let backend = self.backend.clone();
        let execution_result = tokio::task::spawn_blocking(move || backend.run_cmd(cmd))
            .await
            .map_err(|e| {
                #[cfg(feature = "otel-traces")]
                {
                    span.set_status(opentelemetry::trace::Status::error("Task join failed"));
                    span.end();
                }
                CleanroomError::internal_error("Failed to execute command in blocking task")
                    .with_context("Command execution task failed")
                    .with_source(e.to_string())
            })?
            .map_err(|e| {
                #[cfg(feature = "otel-traces")]
                {
                    span.set_status(opentelemetry::trace::Status::error(
                        "Command execution failed",
                    ));
                    span.end();
                }
                CleanroomError::container_error("Failed to execute command in container")
                    .with_context(format!(
                        "Container: {}, Command: {}",
                        container_name,
                        command.join(" ")
                    ))
                    .with_source(e.to_string())
            })?;

        let duration = start_time.elapsed();

        // Record metrics
        #[cfg(feature = "otel-metrics")]
        {
            let histogram = self
                .meter
                .f64_histogram("container.command.duration")
                .with_description("Container command execution duration")
                .build();
            histogram.record(
                duration.as_secs_f64(),
                &[
                    KeyValue::new("container.name", container_name.to_string()),
                    KeyValue::new("command", command.join(" ")),
                ],
            );
        }

        #[cfg(feature = "otel-traces")]
        span.set_attributes(vec![
            KeyValue::new(
                "execution.exit_code",
                execution_result.exit_code.to_string(),
            ),
            KeyValue::new("execution.duration_ms", duration.as_millis().to_string()),
        ]);

        #[cfg(feature = "otel-traces")]
        if execution_result.exit_code != 0 {
            span.set_status(opentelemetry::trace::Status::error("Command failed"));
        }

        #[cfg(feature = "otel-traces")]
        span.end();

        Ok(ExecutionResult {
            exit_code: execution_result.exit_code,
            stdout: execution_result.stdout,
            stderr: execution_result.stderr,
            duration,
            command: command.to_vec(),
            container_name: container_name.to_string(),
        })
    }
}

// Default implementation removed to avoid panic in production code
// Use CleanroomEnvironment::new() instead for proper error handling

/// Example custom service plugin implementation
///
/// This demonstrates how to create custom services without hardcoded dependencies
#[derive(Debug)]
pub struct MockDatabasePlugin {
    name: String,
    container_id: Arc<RwLock<Option<String>>>,
}

impl Default for MockDatabasePlugin {
    fn default() -> Self {
        Self::new()
    }
}

impl MockDatabasePlugin {
    pub fn new() -> Self {
        Self {
            name: "mock_database".to_string(),
            container_id: Arc::new(RwLock::new(None)),
        }
    }
}

impl ServicePlugin for MockDatabasePlugin {
    fn name(&self) -> &str {
        &self.name
    }

    fn start(&self) -> Result<ServiceHandle> {
        // For testing, create a simple mock handle without actual container
        // In production, this would use proper async container startup

        // Build metadata with mock connection details
        let mut metadata = HashMap::new();
        metadata.insert("host".to_string(), "127.0.0.1".to_string());
        metadata.insert("port".to_string(), "8000".to_string());
        metadata.insert("username".to_string(), "root".to_string());
        metadata.insert("password".to_string(), "root".to_string());

        Ok(ServiceHandle {
            id: Uuid::new_v4().to_string(),
            service_name: "mock_database".to_string(),
            metadata,
        })
    }

    fn stop(&self, _handle: ServiceHandle) -> Result<()> {
        // For testing, just return success without actual container cleanup
        // In production, this would properly stop the container
        Ok(())
    }

    fn health_check(&self, handle: &ServiceHandle) -> HealthStatus {
        // Quick check if we have port information
        if handle.metadata.contains_key("port") {
            HealthStatus::Healthy
        } else {
            HealthStatus::Unknown
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_cleanroom_creation() {
        let result = CleanroomEnvironment::new().await;
        assert!(result.is_ok()); // Should succeed with default implementation
    }

    #[test]
    fn test_service_plugin_dyn_compatibility() {
        // This test verifies that ServicePlugin is dyn compatible
        let plugin: Arc<dyn ServicePlugin> = Arc::new(MockDatabasePlugin::new());

        // Test that we can call methods on the trait object
        assert_eq!(plugin.name(), "mock_database");

        // Test that we can store multiple plugins in a collection
        let mut plugins: Vec<Arc<dyn ServicePlugin>> = Vec::new();
        plugins.push(plugin);

        // Test that we can iterate over them
        for plugin in &plugins {
            assert_eq!(plugin.name(), "mock_database");
        }

        println!("✅ ServicePlugin trait is dyn compatible!");
    }

    #[tokio::test]
    async fn test_cleanroom_session_id() -> Result<()> {
        let env = CleanroomEnvironment::new().await?;
        assert!(!env.session_id().is_nil());
        Ok(())
    }

    #[tokio::test]
    async fn test_cleanroom_execute_test() -> Result<()> {
        let env = CleanroomEnvironment::new().await?;
        let result = env
            .execute_test("test", || Ok::<i32, CleanroomError>(42))
            .await?;
        assert_eq!(result, 42);
        Ok(())
    }

    #[tokio::test]
    async fn test_service_registry() -> Result<()> {
        let env = CleanroomEnvironment::new().await?;
        let services = env.services().await;
        assert!(services.active_services().is_empty());
        Ok(())
    }

    #[tokio::test]
    async fn test_service_plugin_registration() -> Result<()> {
        let env = CleanroomEnvironment::new().await?;
        let plugin = Box::new(MockDatabasePlugin::new());
        env.register_service(plugin).await?;
        Ok(())
    }

    #[tokio::test]
    async fn test_service_start_stop() -> Result<()> {
        let env = CleanroomEnvironment::new().await?;
        let plugin = Box::new(MockDatabasePlugin::new());
        env.register_service(plugin).await?;

        let handle = env.start_service("mock_database").await?;
        assert_eq!(handle.service_name, "mock_database");

        env.stop_service(&handle.id).await?;
        Ok(())
    }

    #[tokio::test]
    async fn test_register_container() {
        let env = CleanroomEnvironment::default();
        let result = env
            .register_container("test-container".to_string(), "container-123".to_string())
            .await;
        assert!(result.is_ok());

        // Verify container was registered
        assert!(env.has_container("test-container").await);
    }

    #[tokio::test]
    async fn test_get_or_create_container() -> Result<()> {
        let env = CleanroomEnvironment::new().await?;

        // First call should create and register container
        let result1 = env
            .get_or_create_container("test-container", || {
                Ok::<String, CleanroomError>("container-instance".to_string())
            })
            .await?;
        assert_eq!(result1, "container-instance");

        // Verify container was registered
        assert!(env.has_container("test-container").await);
        let (created, reused) = env.get_container_reuse_stats().await;
        assert_eq!(created, 1);
        assert_eq!(reused, 0);

        // Second call should return the SAME container instance (true reuse!)
        let result2 = env
            .get_or_create_container("test-container", || {
                Ok::<String, CleanroomError>("container-instance-2".to_string())
            })
            .await?;
        // This should be the SAME instance, not a new one
        assert_eq!(result2, "container-instance");

        // Verify reuse was tracked
        let (created, reused) = env.get_container_reuse_stats().await;
        assert_eq!(created, 1);
        assert_eq!(reused, 1);
        Ok(())
    }

    #[tokio::test]
    async fn test_check_health_delegates_to_service_registry() -> Result<()> {
        let env = CleanroomEnvironment::new().await?;
        let health_status = env.check_health().await;
        // Should return empty HashMap since no services are registered
        assert!(health_status.is_empty());
        Ok(())
    }
}