adversaria 0.1.0

Adversarial Testing Harness for Large Language Models
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143

use crate::core::error::{AdversariaError, Result};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::{Path, PathBuf};

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Config {
    pub version: String,
    pub default_provider: String,
    pub providers: HashMap<String, ProviderConfig>,
    pub suites: SuitesConfig,
    pub reporting: ReportingConfig,
    pub plugins: PluginsConfig,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProviderConfig {
    pub api_key: Option<String>,
    pub api_base: Option<String>,
    pub model: String,
    pub timeout_seconds: Option<u64>,
    pub max_retries: Option<u32>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SuitesConfig {
    pub directory: PathBuf,
    pub enabled_suites: Vec<String>,
    #[serde(default)]
    pub custom_suites: Vec<PathBuf>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReportingConfig {
    pub output_directory: PathBuf,
    pub format: ReportFormat,
    #[serde(default = "default_keep_reports")]
    pub keep_reports: usize,
}

fn default_keep_reports() -> usize {
    100
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ReportFormat {
    Json,
    Yaml,
    Both,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PluginsConfig {
    pub directory: PathBuf,
    #[serde(default)]
    pub enabled: bool,
}

impl Config {
    pub fn load<P: AsRef<Path>>(path: P) -> Result<Self> {
        let content = std::fs::read_to_string(path.as_ref())
            .map_err(|e| AdversariaError::Config(format!("Failed to read config file: {}", e)))?;

        serde_yaml::from_str(&content)
            .map_err(|e| AdversariaError::Config(format!("Failed to parse config file: {}", e)))
    }

    pub fn default_config_path() -> PathBuf {
        PathBuf::from("adversaria.config.yaml")
    }

    pub fn create_default<P: AsRef<Path>>(path: P) -> Result<()> {
        let default_config = Self::default();
        let yaml = serde_yaml::to_string(&default_config)?;
        std::fs::write(path, yaml)?;
        Ok(())
    }
}

impl Default for Config {
    fn default() -> Self {
        let mut providers = HashMap::new();

        providers.insert(
            "openai".to_string(),
            ProviderConfig {
                api_key: None,
                api_base: Some("https://api.openai.com/v1".to_string()),
                model: "gpt-4".to_string(),
                timeout_seconds: Some(30),
                max_retries: Some(3),
            },
        );

        providers.insert(
            "anthropic".to_string(),
            ProviderConfig {
                api_key: None,
                api_base: Some("https://api.anthropic.com/v1".to_string()),
                model: "claude-3-opus-20240229".to_string(),
                timeout_seconds: Some(30),
                max_retries: Some(3),
            },
        );

        providers.insert(
            "ollama".to_string(),
            ProviderConfig {
                api_key: None,
                api_base: Some("http://localhost:11434".to_string()),
                model: "llama2".to_string(),
                timeout_seconds: Some(60),
                max_retries: Some(2),
            },
        );

        Self {
            version: "1.0".to_string(),
            default_provider: "openai".to_string(),
            providers,
            suites: SuitesConfig {
                directory: PathBuf::from("./suites"),
                enabled_suites: vec![
                    "prompt_injection".to_string(),
                    "jailbreak".to_string(),
                    "role_confusion".to_string(),
                    "data_exfiltration".to_string(),
                ],
                custom_suites: vec![],
            },
            reporting: ReportingConfig {
                output_directory: PathBuf::from("./reports"),
                format: ReportFormat::Json,
                keep_reports: 100,
            },
            plugins: PluginsConfig {
                directory: PathBuf::from("./plugins"),
                enabled: true,
            },
        }
    }
}