swarm-engine-eval 0.1.6

//! Eval Scenario Type Definitions
//!
//! Type definitions for evaluation scenarios. Designed for direct integration with SwarmApp.
//!
//! # Creating New Scenarios
//!
//! **IMPORTANT**: Always start from the canonical example file:
//!
//! ```text
//! crates/swarm-engine-eval/scenarios/EXAMPLE.toml
//! ```
//!
//! Copy EXAMPLE.toml and customize it for your scenario. The example file contains
//! detailed comments explaining every field and their valid values.
//!
//! ## Quick Start
//!
//! ```bash
//! # 1. Copy the example
//! cp crates/swarm-engine-eval/scenarios/EXAMPLE.toml crates/swarm-engine-eval/scenarios/my_scenario.toml
//!
//! # 2. Edit the scenario
//! # Update [meta], [task], [llm], [[actions]], [environment], [conditions], etc.
//!
//! # 3. Run the evaluation
//! cargo run --package swarm-engine-ui -- eval crates/swarm-engine-eval/scenarios/my_scenario.toml -n 1 -v
//! ```
//!
//! ## Key Sections
//!
//! | Section | Required | Description |
//! |---------|----------|-------------|
//! | `[meta]` | Yes | Scenario identification (name, id, version) |
//! | `[task]` | Yes | Goal and expected outcome |
//! | `[llm]` | Yes | LLM provider configuration |
//! | `[[actions.actions]]` | Yes | Available actions for workers |
//! | `[environment]` | Yes | Environment type and parameters |
//! | `[agents]` | Yes | Workers and managers configuration |
//! | `[conditions]` | Yes | Success/failure conditions |
//! | `[[milestones]]` | No | KPI calculation milestones |
//! | `[[variants]]` | No | Parameter variation presets |
//!
//! ## Scenario ID Format
//!
//! The `meta.id` field determines where learning data is stored:
//!
//! ```text
//! "namespace:name:version"
//!
//! Examples:
//!   "user:troubleshooting:v2"     -> learning data: ~/.swarm-engine/learning/scenarios/troubleshooting/
//!   "builtin:resource_gathering:v1" -> learning data: ~/.swarm-engine/learning/scenarios/resource_gathering/
//! ```
//!
//! The middle part (name) is extracted as `learning_key` for data organization.
//!
//! ## LLM Providers
//!
//! | Provider | Alias | Default Endpoint | Notes |
//! |----------|-------|------------------|-------|
//! | `llama-server` | `llamacppserver` | `http://localhost:8080` | **Recommended** - True batch processing |
//! | `ollama` | - | `http://localhost:11434` | Serial processing |

use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Duration;

// Re-exports for backward compatibility
pub use super::actions::ScenarioActions;
pub use super::conditions::EvalConditions;
pub use super::dependency::DependencyGraphConfig;
pub use super::llm::{LlmConfig, LlmConfigOverride};
pub use super::manager::{
    BatchProcessorConfig, ManagerActivationConfig, ManagerConfig, ManagerTemplate,
};
pub use super::milestone::Milestone;

// ============================================================================
// Task Configuration
// ============================================================================

/// Task definition for swarm evaluation
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct TaskConfig {
    /// The goal/objective for the swarm to achieve
    pub goal: String,

    /// Expected result for evaluation (e.g., "src/auth/handler.rs:42")
    #[serde(default)]
    pub expected: Option<String>,

    /// Additional context for the task
    #[serde(default)]
    pub context: TaskContext,
}

/// Additional context passed to the task
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct TaskContext {
    /// Target path for exploration (e.g., codebase root)
    #[serde(default)]
    pub target_path: Option<String>,

    /// Working directory for workers
    #[serde(default)]
    pub working_dir: Option<String>,

    /// Maximum exploration depth
    #[serde(default)]
    pub max_depth: Option<usize>,

    /// Additional key-value context
    #[serde(default, flatten)]
    pub extra: HashMap<String, toml::Value>,
}

// ============================================================================
// Scenario Identification
// ============================================================================

/// シナリオ識別子
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct ScenarioId(pub String);

impl ScenarioId {
    pub fn new(id: impl Into<String>) -> Self {
        Self(id.into())
    }

    pub fn as_str(&self) -> &str {
        &self.0
    }

    /// IDから学習用キーを抽出
    ///
    /// ID形式: `namespace:name:version` (例: `user:troubleshooting:v2`)
    /// → 中央の `name` 部分を返す
    ///
    /// フォールバック: コロンがない場合はIDをそのままファイルシステム安全な形式に変換
    pub fn learning_key(&self) -> String {
        let parts: Vec<&str> = self.0.split(':').collect();
        if parts.len() >= 2 {
            // namespace:name:version → name を返す
            parts[1].to_string()
        } else {
            // フォールバック: スペース・特殊文字をアンダースコアに
            self.0
                .chars()
                .map(|c| {
                    if c.is_alphanumeric() || c == '-' || c == '_' {
                        c
                    } else {
                        '_'
                    }
                })
                .collect()
        }
    }
}

impl std::fmt::Display for ScenarioId {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.0)
    }
}

impl From<&str> for ScenarioId {
    fn from(s: &str) -> Self {
        Self::new(s)
    }
}

impl From<String> for ScenarioId {
    fn from(s: String) -> Self {
        Self::new(s)
    }
}

// ============================================================================
// Scenario Variant
// ============================================================================

/// シナリオバリアント（パラメータのオーバーライド）
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScenarioVariant {
    /// バリアント名（CLI で指定する識別子）
    pub name: String,

    /// 説明
    #[serde(default)]
    pub description: String,

    /// LLM 設定のオーバーライド（provider 切り替え等）
    #[serde(default)]
    pub llm: Option<LlmConfigOverride>,

    /// 環境パラメータのオーバーライド
    #[serde(default)]
    pub environment_params: serde_json::Value,

    /// 依存グラフ設定のオーバーライド（オプション）
    #[serde(default)]
    pub dependency_graph: Option<DependencyGraphConfig>,

    /// AppConfig のオーバーライド（オプション）
    #[serde(default)]
    pub app_config: Option<AppConfigOverride>,

    /// max_ticks のオーバーライド（オプション）
    #[serde(default)]
    pub max_ticks: Option<u64>,

    /// Worker 数のオーバーライド（最初の worker template の count を上書き）
    #[serde(default)]
    pub workers_count: Option<usize>,

    /// Manager 数のオーバーライド（最初の manager template の count を上書き）
    #[serde(default)]
    pub managers_count: Option<usize>,
}

// ============================================================================
// Eval Scenario
// ============================================================================

/// 評価シナリオの完全な定義
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EvalScenario {
    /// メタ情報
    pub meta: ScenarioMeta,

    /// タスク定義 (goal, expected, context)
    #[serde(default)]
    pub task: TaskConfig,

    /// LLM 設定 (provider, model, endpoint, etc.)
    #[serde(default)]
    pub llm: LlmConfig,

    /// Manager 動作設定 (interval, confidence_threshold, etc.)
    #[serde(default)]
    pub manager: ManagerConfig,

    /// BatchProcessor 設定 (parallel, max_concurrency)
    #[serde(default)]
    pub batch_processor: BatchProcessorConfig,

    /// 依存グラフ設定 (アクション間の依存関係)
    #[serde(default)]
    pub dependency_graph: Option<DependencyGraphConfig>,

    /// 利用可能なアクション設定
    #[serde(default)]
    pub actions: ScenarioActions,

    /// SwarmApp 構築設定
    pub app_config: AppConfigTemplate,

    /// 環境設定 (タスク定義)
    pub environment: EnvironmentConfig,

    /// エージェント設定
    pub agents: AgentsConfig,

    /// 成功/失敗条件
    pub conditions: EvalConditions,

    /// マイルストーン (kpi_score 計算用)
    #[serde(default)]
    pub milestones: Vec<Milestone>,

    /// バリアント定義（パラメータの組み合わせ）
    #[serde(default)]
    pub variants: Vec<ScenarioVariant>,
}

impl EvalScenario {
    /// バリアントを適用した新しいシナリオを返す
    pub fn with_variant(&self, variant_name: &str) -> Option<EvalScenario> {
        let variant = self.variants.iter().find(|v| v.name == variant_name)?;

        let mut scenario = self.clone();

        // LLM 設定をマージ
        if let Some(ref llm_override) = variant.llm {
            llm_override.apply_to(&mut scenario.llm);
        }

        // 環境パラメータをマージ
        if !variant.environment_params.is_null() {
            if let serde_json::Value::Object(override_map) = &variant.environment_params {
                if let serde_json::Value::Object(ref mut base_map) = scenario.environment.params {
                    for (key, value) in override_map {
                        base_map.insert(key.clone(), value.clone());
                    }
                }
            }
        }

        // 依存グラフをオーバーライド
        if variant.dependency_graph.is_some() {
            scenario.dependency_graph = variant.dependency_graph.clone();
        }

        // app_config をオーバーライド
        if let Some(ref app_override) = variant.app_config {
            if let Some(ref strategy) = app_override.management_strategy {
                scenario.app_config.management_strategy = strategy.clone();
            }
            if let Some(tick_ms) = app_override.tick_duration_ms {
                scenario.app_config.tick_duration_ms = tick_ms;
            }
            if let Some(enable_exp) = app_override.enable_exploration {
                scenario.app_config.enable_exploration = enable_exp;
            }
        }

        // max_ticks をオーバーライド
        if let Some(max_ticks) = variant.max_ticks {
            scenario.app_config.max_ticks = max_ticks;
        }

        // workers_count をオーバーライド（最初の worker template）
        if let Some(workers_count) = variant.workers_count {
            if let Some(first_worker) = scenario.agents.workers.first_mut() {
                first_worker.count = workers_count;
            }
        }

        // managers_count をオーバーライド（最初の manager template）
        if let Some(managers_count) = variant.managers_count {
            if let Some(first_manager) = scenario.agents.managers.first_mut() {
                first_manager.count = managers_count;
                // id_pattern が未設定の場合、id から自動生成
                if first_manager.id_pattern.is_none() {
                    if let Some(ref id) = first_manager.id {
                        first_manager.id_pattern = Some(format!("{}_{{i}}", id));
                        first_manager.id = None;
                    }
                }
            }
        }

        // メタ情報を更新（バリアント名を追加）
        scenario.meta.name = format!("{} ({})", self.meta.name, variant_name);

        Some(scenario)
    }

    /// 利用可能なバリアント名のリストを返す
    pub fn variant_names(&self) -> Vec<&str> {
        self.variants.iter().map(|v| v.name.as_str()).collect()
    }
}

// ============================================================================
// Scenario Meta
// ============================================================================

/// シナリオのメタ情報
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScenarioMeta {
    /// シナリオ名
    pub name: String,

    /// バージョン (semver形式推奨)
    #[serde(default = "default_version")]
    pub version: String,

    /// 一意識別子
    pub id: ScenarioId,

    /// 説明文
    #[serde(default)]
    pub description: String,

    /// タグ (検索・フィルタ用)
    #[serde(default)]
    pub tags: Vec<String>,
}

fn default_version() -> String {
    "1.0.0".to_string()
}

// ============================================================================
// App Config Template
// ============================================================================

/// SwarmApp 構築用のテンプレート
///
/// 実際の AppConfig を生成するためのテンプレート。
/// seed や LLM provider は評価時に注入。
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AppConfigTemplate {
    /// Tick 間隔 (ミリ秒)
    #[serde(default = "default_tick_duration_ms")]
    pub tick_duration_ms: u64,

    /// 最大 Tick 数
    #[serde(default = "default_max_ticks")]
    pub max_ticks: u64,

    /// Management Strategy 設定
    #[serde(default)]
    pub management_strategy: ManagementStrategyConfig,

    /// ExplorationSpace を有効化
    #[serde(default)]
    pub enable_exploration: bool,
}

fn default_tick_duration_ms() -> u64 {
    10
}

fn default_max_ticks() -> u64 {
    1000
}

impl AppConfigTemplate {
    pub fn tick_duration(&self) -> Duration {
        Duration::from_millis(self.tick_duration_ms)
    }
}

impl Default for AppConfigTemplate {
    fn default() -> Self {
        Self {
            tick_duration_ms: default_tick_duration_ms(),
            max_ticks: default_max_ticks(),
            management_strategy: ManagementStrategyConfig::default(),
            enable_exploration: false,
        }
    }
}

/// AppConfig のオーバーライド用構造体
///
/// variant で指定されたフィールドだけが base にマージされる。
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct AppConfigOverride {
    /// Management Strategy のオーバーライド
    #[serde(default)]
    pub management_strategy: Option<ManagementStrategyConfig>,

    /// tick_duration_ms のオーバーライド
    #[serde(default)]
    pub tick_duration_ms: Option<u64>,

    /// enable_exploration のオーバーライド
    #[serde(default)]
    pub enable_exploration: Option<bool>,
}

// ============================================================================
// Management Strategy
// ============================================================================

/// Management Strategy 設定
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ManagementStrategyConfig {
    /// 毎 Tick 起動（LLM 不要なフロー向け）
    ///
    /// V2 ExplorationSpace など、LLM 呼び出しなしで Guidance を生成できる場合に使用。
    EveryTick {},

    /// インターバルベース
    IntervalBased {
        #[serde(default = "default_max_interval")]
        max_interval: u64,
    },
    /// イベントドリブン
    EventDriven {
        #[serde(default)]
        triggers: Vec<String>,
    },
    /// ハイブリッド
    Hybrid {
        #[serde(default = "default_max_interval")]
        max_interval: u64,
        #[serde(default)]
        triggers: Vec<String>,
    },
    /// 無効化
    #[serde(alias = "disabled")]
    Disabled {},
}

fn default_max_interval() -> u64 {
    20
}

impl Default for ManagementStrategyConfig {
    fn default() -> Self {
        Self::IntervalBased {
            max_interval: default_max_interval(),
        }
    }
}

// ============================================================================
// Environment Configuration
// ============================================================================

/// 環境設定
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EnvironmentConfig {
    /// 環境タイプ識別子 (e.g., "grid_world", "task_queue")
    pub env_type: String,

    /// 環境固有パラメータ
    #[serde(default)]
    pub params: serde_json::Value,

    /// 初期状態設定
    #[serde(default)]
    pub initial_state: Option<InitialStateConfig>,
}

/// 初期状態設定
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum InitialStateConfig {
    /// Seed から決定論的に生成
    #[serde(alias = "seeded_random")]
    SeededRandom {},
    /// 固定状態
    Fixed {
        /// 固定状態の定義
        state: serde_json::Value,
    },
    /// カスタム生成器
    Custom {
        /// 生成器識別子
        generator: String,
        /// 生成器パラメータ
        params: serde_json::Value,
    },
}

// ============================================================================
// Agents Configuration
// ============================================================================

/// エージェント設定
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct AgentsConfig {
    /// Worker テンプレート
    #[serde(default)]
    pub workers: Vec<WorkerTemplate>,

    /// Manager テンプレート
    #[serde(default)]
    pub managers: Vec<ManagerTemplate>,
}

/// Worker テンプレート
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkerTemplate {
    /// ID生成パターン (e.g., "worker_{i}")
    pub id_pattern: String,

    /// 生成数
    #[serde(default = "default_worker_count")]
    pub count: usize,

    /// 役割
    #[serde(default)]
    pub role: String,

    /// Worker 固有設定
    #[serde(default)]
    pub config: serde_json::Value,
}

fn default_worker_count() -> usize {
    1
}

impl WorkerTemplate {
    /// Worker IDを生成
    pub fn generate_ids(&self) -> Vec<String> {
        (0..self.count)
            .map(|i| self.id_pattern.replace("{i}", &i.to_string()))
            .collect()
    }
}

// ============================================================================
// Tests
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;
    use crate::scenario::llm::LlmProvider;

    #[test]
    fn test_scenario_id() {
        let id = ScenarioId::new("test:scenario:v1");
        assert_eq!(id.as_str(), "test:scenario:v1");
    }

    #[test]
    fn test_scenario_id_learning_key() {
        // Standard format: namespace:name:version
        let id = ScenarioId::new("user:troubleshooting:v2");
        assert_eq!(id.learning_key(), "troubleshooting");

        // Builtin format
        let id = ScenarioId::new("builtin:resource_gathering:v1");
        assert_eq!(id.learning_key(), "resource_gathering");

        // Simple format (no colons) - fallback
        let id = ScenarioId::new("simple_scenario");
        assert_eq!(id.learning_key(), "simple_scenario");

        // With spaces (sanitized)
        let id = ScenarioId::new("Service Troubleshooting");
        assert_eq!(id.learning_key(), "Service_Troubleshooting");
    }

    #[test]
    fn test_worker_template_generate_ids() {
        let template = WorkerTemplate {
            id_pattern: "worker_{i}".to_string(),
            count: 3,
            role: "gatherer".to_string(),
            config: serde_json::Value::Null,
        };

        let ids = template.generate_ids();
        assert_eq!(ids, vec!["worker_0", "worker_1", "worker_2"]);
    }

    #[test]
    fn test_app_config_template_default() {
        let config = AppConfigTemplate::default();
        assert_eq!(config.tick_duration_ms, 10);
        assert_eq!(config.max_ticks, 1000);
    }

    #[test]
    fn test_management_strategy_deserialize() {
        let json = r#"{"type": "hybrid", "max_interval": 30, "triggers": ["event_a"]}"#;
        let strategy: ManagementStrategyConfig = serde_json::from_str(json).unwrap();

        match strategy {
            ManagementStrategyConfig::Hybrid {
                max_interval,
                triggers,
            } => {
                assert_eq!(max_interval, 30);
                assert_eq!(triggers, vec!["event_a"]);
            }
            _ => panic!("Expected Hybrid variant"),
        }
    }

    #[test]
    fn test_task_config_default() {
        let task = TaskConfig::default();
        assert!(task.goal.is_empty());
        assert!(task.expected.is_none());
    }

    #[test]
    fn test_task_config_deserialize_toml() {
        let toml_str = r#"
            goal = "Find the function that handles authentication"
            expected = "src/auth/handler.rs:42"
            [context]
            target_path = "/path/to/codebase"
            working_dir = "/path/to/codebase"
            max_depth = 5
        "#;

        let task: TaskConfig = toml::from_str(toml_str).unwrap();
        assert_eq!(task.goal, "Find the function that handles authentication");
        assert_eq!(task.expected, Some("src/auth/handler.rs:42".to_string()));
        assert_eq!(
            task.context.target_path,
            Some("/path/to/codebase".to_string())
        );
        assert_eq!(task.context.max_depth, Some(5));
    }

    #[test]
    fn test_scenario_variant_with_llm_override() {
        let toml_str = r#"
            [meta]
            name = "Test Scenario"
            id = "test:scenario:v1"

            [task]
            goal = "Test goal"

            [llm]
            provider = "ollama"
            model = "llama3:8b"

            [app_config]
            max_ticks = 100

            [environment]
            env_type = "test"

            [agents]

            [conditions]
            on_timeout = "fail"

            [[variants]]
            name = "mistral"
            description = "Use mistral.rs local inference"
            [variants.llm]
            provider = "mistral"
            model = "LiquidAI/LFM2.5-1.2B-Instruct-GGUF"
            gguf_files = ["LFM2.5-1.2B-Instruct-Q4_K_M.gguf"]
        "#;

        let scenario: EvalScenario = toml::from_str(toml_str).unwrap();
        assert_eq!(scenario.llm.provider, LlmProvider::Ollama);
        assert_eq!(scenario.variants.len(), 1);

        // Apply variant
        let mistral_scenario = scenario.with_variant("mistral").unwrap();
        assert_eq!(mistral_scenario.llm.provider, LlmProvider::Mistral);
        assert_eq!(
            mistral_scenario.llm.model,
            "LiquidAI/LFM2.5-1.2B-Instruct-GGUF"
        );
        assert!(mistral_scenario.llm.is_gguf());
        assert_eq!(mistral_scenario.meta.name, "Test Scenario (mistral)");
    }

    #[test]
    fn test_scenario_variant_partial_llm_override() {
        let toml_str = r#"
            [meta]
            name = "Test"
            id = "test:v1"

            [task]
            goal = "Test"

            [llm]
            provider = "ollama"
            model = "llama3:8b"
            temperature = 0.1
            num_ctx = 4096

            [app_config]
            max_ticks = 100

            [environment]
            env_type = "test"

            [agents]

            [conditions]
            on_timeout = "fail"

            [[variants]]
            name = "high_temp"
            [variants.llm]
            temperature = 0.9
        "#;

        let scenario: EvalScenario = toml::from_str(toml_str).unwrap();
        let variant = scenario.with_variant("high_temp").unwrap();

        // temperature should be overridden
        assert!((variant.llm.temperature - 0.9).abs() < f32::EPSILON);
        // other fields should remain unchanged
        assert_eq!(variant.llm.provider, LlmProvider::Ollama);
        assert_eq!(variant.llm.model, "llama3:8b");
        assert_eq!(variant.llm.num_ctx, Some(4096));
    }
}