swarm_engine_eval/validation/

validator.rs

//! Scenario Validator
//!
//! シナリオ定義の整合性をチェック。

use std::fmt;

use crate::scenario::EvalScenario;
use swarm_engine_core::actions::EnvironmentSpecRegistry;

/// バリデーション警告
#[derive(Debug, Clone)]
pub enum ValidationWarning {
    /// サポートされていないアクション
    UnsupportedAction {
        action_name: String,
        env_type: String,
        suggestion: Option<String>,
    },
    /// 不明な Environment タイプ
    UnknownEnvironment { env_type: String },
    /// 必須パラメータの定義が不足
    MissingRequiredParamDefinition {
        action_name: String,
        param_name: String,
    },
    /// アクションが定義されているが Environment がない
    ActionsWithoutEnvironment { action_count: usize },
    /// Continue アクションは特殊なので警告しない（情報のみ）
    ContinueActionNote,
}

impl fmt::Display for ValidationWarning {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::UnsupportedAction {
                action_name,
                env_type,
                suggestion,
            } => {
                write!(
                    f,
                    "Action '{}' is not supported by environment '{}'",
                    action_name, env_type
                )?;
                if let Some(s) = suggestion {
                    write!(f, ". Did you mean '{}'?", s)?;
                }
                Ok(())
            }
            Self::UnknownEnvironment { env_type } => {
                write!(f, "Unknown environment type: '{}'", env_type)
            }
            Self::MissingRequiredParamDefinition {
                action_name,
                param_name,
            } => {
                write!(
                    f,
                    "Action '{}' has required parameter '{}' but it's not defined in scenario",
                    action_name, param_name
                )
            }
            Self::ActionsWithoutEnvironment { action_count } => {
                write!(
                    f,
                    "{} actions defined but no environment specified",
                    action_count
                )
            }
            Self::ContinueActionNote => {
                write!(
                    f,
                    "'Continue' action is a special no-op action, always supported"
                )
            }
        }
    }
}

impl ValidationWarning {
    /// 警告の重大度を取得
    pub fn severity(&self) -> WarningSeverity {
        match self {
            Self::UnsupportedAction { .. } => WarningSeverity::High,
            Self::UnknownEnvironment { .. } => WarningSeverity::High,
            Self::MissingRequiredParamDefinition { .. } => WarningSeverity::Medium,
            Self::ActionsWithoutEnvironment { .. } => WarningSeverity::Low,
            Self::ContinueActionNote => WarningSeverity::Info,
        }
    }
}

/// 警告の重大度
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum WarningSeverity {
    Info,
    Low,
    Medium,
    High,
}

impl fmt::Display for WarningSeverity {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Info => write!(f, "INFO"),
            Self::Low => write!(f, "LOW"),
            Self::Medium => write!(f, "MEDIUM"),
            Self::High => write!(f, "HIGH"),
        }
    }
}

/// シナリオバリデータ
pub struct ScenarioValidator {
    registry: EnvironmentSpecRegistry,
}

impl Default for ScenarioValidator {
    fn default() -> Self {
        Self::new()
    }
}

impl ScenarioValidator {
    /// 新しいバリデータを作成
    pub fn new() -> Self {
        Self {
            registry: EnvironmentSpecRegistry::new(),
        }
    }

    /// シナリオをバリデート
    pub fn validate(&self, scenario: &EvalScenario) -> Vec<ValidationWarning> {
        let mut warnings = Vec::new();

        let env_type = &scenario.environment.env_type;
        let actions = &scenario.actions.actions;

        // Environment タイプのチェック
        let env_spec = match self.registry.get(env_type) {
            Some(spec) => spec,
            None => {
                warnings.push(ValidationWarning::UnknownEnvironment {
                    env_type: env_type.clone(),
                });
                // Environment が不明な場合、アクションのチェックはスキップ
                return warnings;
            }
        };

        // アクションのチェック
        for action_def in actions {
            let action_name = &action_def.name;

            // Continue は特殊なアクション（常にサポート）
            if action_name.to_lowercase() == "continue" {
                continue;
            }

            // Environment でサポートされているかチェック
            if !env_spec.supports_action(action_name) {
                // 類似アクションを提案
                let suggestion = self.find_similar_action(env_spec, action_name);
                warnings.push(ValidationWarning::UnsupportedAction {
                    action_name: action_name.clone(),
                    env_type: env_type.clone(),
                    suggestion,
                });
                continue;
            }

            // 必須パラメータのチェック
            if let Some(action_spec) = env_spec.get_action(action_name) {
                for param_spec in &action_spec.params {
                    if param_spec.required {
                        // シナリオのアクション定義に必須パラメータが含まれているか確認
                        let has_param = action_def
                            .params
                            .iter()
                            .any(|p| p.name.to_lowercase() == param_spec.name.to_lowercase());

                        if !has_param {
                            warnings.push(ValidationWarning::MissingRequiredParamDefinition {
                                action_name: action_name.clone(),
                                param_name: param_spec.name.clone(),
                            });
                        }
                    }
                }
            }
        }

        warnings
    }

    /// 類似アクションを検索
    fn find_similar_action(
        &self,
        env_spec: &swarm_engine_core::actions::EnvironmentSpec,
        name: &str,
    ) -> Option<String> {
        let name_lower = name.to_lowercase();

        // 編集距離が小さいアクションを探す
        let mut best_match: Option<(String, usize)> = None;

        for action in &env_spec.actions {
            let canonical_lower = action.canonical_name.to_lowercase();
            let distance = levenshtein_distance(&name_lower, &canonical_lower);

            // 閾値は3文字以内
            if distance <= 3 && (best_match.is_none() || distance < best_match.as_ref().unwrap().1)
            {
                best_match = Some((action.canonical_name.clone(), distance));
            }

            // エイリアスもチェック
            for alias in &action.aliases {
                let alias_lower = alias.to_lowercase();
                let alias_distance = levenshtein_distance(&name_lower, &alias_lower);
                if alias_distance <= 3
                    && (best_match.is_none() || alias_distance < best_match.as_ref().unwrap().1)
                {
                    best_match = Some((action.canonical_name.clone(), alias_distance));
                }
            }
        }

        best_match.map(|(name, _)| name)
    }

    /// 静的メソッドとしてバリデート（便利メソッド）
    pub fn validate_scenario(scenario: &EvalScenario) -> Vec<ValidationWarning> {
        let validator = Self::new();
        validator.validate(scenario)
    }
}

/// レーベンシュタイン距離を計算
fn levenshtein_distance(s1: &str, s2: &str) -> usize {
    let len1 = s1.chars().count();
    let len2 = s2.chars().count();

    if len1 == 0 {
        return len2;
    }
    if len2 == 0 {
        return len1;
    }

    let s1_chars: Vec<char> = s1.chars().collect();
    let s2_chars: Vec<char> = s2.chars().collect();

    let mut matrix = vec![vec![0usize; len2 + 1]; len1 + 1];

    for (i, row) in matrix.iter_mut().enumerate().take(len1 + 1) {
        row[0] = i;
    }
    for (j, cell) in matrix[0].iter_mut().enumerate().take(len2 + 1) {
        *cell = j;
    }

    for (i, c1) in s1_chars.iter().enumerate() {
        for (j, c2) in s2_chars.iter().enumerate() {
            let cost = if c1 == c2 { 0 } else { 1 };

            matrix[i + 1][j + 1] = (matrix[i][j + 1] + 1)
                .min(matrix[i + 1][j] + 1)
                .min(matrix[i][j] + cost);
        }
    }

    matrix[len1][len2]
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::scenario::actions::{ScenarioActionCategory, ScenarioActionDef};
    use crate::scenario::llm::LlmConfig;
    use crate::scenario::{
        AgentsConfig, AppConfigTemplate, EnvironmentConfig, EvalConditions, EvalScenario,
        ScenarioActions, ScenarioId, ScenarioMeta, TaskConfig,
    };

    fn make_test_scenario(env_type: &str, action_names: Vec<&str>) -> EvalScenario {
        EvalScenario {
            meta: ScenarioMeta {
                name: "Test".to_string(),
                version: "1.0.0".to_string(),
                id: ScenarioId::new("test:test:v1"),
                description: String::new(),
                tags: vec![],
            },
            task: TaskConfig::default(),
            llm: LlmConfig::default(),
            manager: Default::default(),
            batch_processor: Default::default(),
            dependency_graph: None,
            actions: ScenarioActions {
                actions: action_names
                    .into_iter()
                    .map(|name| ScenarioActionDef {
                        name: name.to_string(),
                        description: "Test action".to_string(),
                        params: vec![],
                        category: ScenarioActionCategory::default(),
                        example: None,
                    })
                    .collect(),
            },
            app_config: AppConfigTemplate::default(),
            environment: EnvironmentConfig {
                env_type: env_type.to_string(),
                params: Default::default(),
                initial_state: None,
            },
            agents: AgentsConfig::default(),
            conditions: EvalConditions::default(),
            milestones: vec![],
            variants: vec![],
        }
    }

    #[test]
    fn test_levenshtein_distance() {
        assert_eq!(levenshtein_distance("", ""), 0);
        assert_eq!(levenshtein_distance("a", ""), 1);
        assert_eq!(levenshtein_distance("", "a"), 1);
        assert_eq!(levenshtein_distance("abc", "abc"), 0);
        assert_eq!(levenshtein_distance("abc", "abd"), 1);
        assert_eq!(levenshtein_distance("kitten", "sitting"), 3);
    }

    #[test]
    fn test_warning_severity_ordering() {
        assert!(WarningSeverity::Info < WarningSeverity::Low);
        assert!(WarningSeverity::Low < WarningSeverity::Medium);
        assert!(WarningSeverity::Medium < WarningSeverity::High);
    }

    #[test]
    fn test_validate_valid_scenario() {
        let scenario = make_test_scenario(
            "troubleshooting",
            vec!["CheckStatus", "ReadLogs", "Diagnose", "Restart"],
        );
        let warnings = ScenarioValidator::validate_scenario(&scenario);

        // 有効なシナリオなので警告なし
        let high_warnings: Vec<_> = warnings
            .iter()
            .filter(|w| w.severity() >= WarningSeverity::High)
            .collect();
        assert!(
            high_warnings.is_empty(),
            "Unexpected warnings: {:?}",
            high_warnings
        );
    }

    #[test]
    fn test_validate_unknown_action() {
        let scenario = make_test_scenario("troubleshooting", vec!["CheckStatus", "UnknownAction"]);
        let warnings = ScenarioValidator::validate_scenario(&scenario);

        // UnknownAction は警告が出るはず
        let unsupported: Vec<_> = warnings
            .iter()
            .filter(|w| matches!(w, ValidationWarning::UnsupportedAction { .. }))
            .collect();
        assert_eq!(unsupported.len(), 1);
    }

    #[test]
    fn test_validate_similar_action_suggestion() {
        let scenario = make_test_scenario(
            "troubleshooting",
            vec!["ChckStatus"], // typo: CheckStatus
        );
        let warnings = ScenarioValidator::validate_scenario(&scenario);

        // 類似アクションが提案されるはず
        let unsupported = warnings.iter().find(|w| {
            matches!(
                w,
                ValidationWarning::UnsupportedAction {
                    suggestion: Some(_),
                    ..
                }
            )
        });
        assert!(unsupported.is_some(), "Expected suggestion for typo");
    }

    #[test]
    fn test_validate_unknown_environment() {
        let scenario = make_test_scenario("unknown_env", vec!["SomeAction"]);
        let warnings = ScenarioValidator::validate_scenario(&scenario);

        // 不明な環境タイプの警告が出るはず
        let unknown_env: Vec<_> = warnings
            .iter()
            .filter(|w| matches!(w, ValidationWarning::UnknownEnvironment { .. }))
            .collect();
        assert_eq!(unknown_env.len(), 1);
    }

    #[test]
    fn test_validate_continue_action_ignored() {
        let scenario = make_test_scenario("troubleshooting", vec!["CheckStatus", "Continue"]);
        let warnings = ScenarioValidator::validate_scenario(&scenario);

        // Continue は特殊アクションなので警告なし
        let unsupported: Vec<_> = warnings
            .iter()
            .filter(|w| matches!(w, ValidationWarning::UnsupportedAction { action_name, .. } if action_name == "Continue"))
            .collect();
        assert!(unsupported.is_empty());
    }
}