aither_core/
moderation.rs

1use alloc::vec::Vec;
2use core::future::Future;
3
4/// Trait for content moderation services.
5pub trait Moderation {
6    /// The error type returned by moderation operations.
7    type Error: core::error::Error + Send + Sync + 'static;
8
9    /// Moderates the provided content and returns a result asynchronously.
10    ///
11    /// # Arguments
12    ///
13    /// * `content` - The content to be moderated.
14    fn moderate(
15        &self,
16        content: &str,
17    ) -> impl Future<Output = Result<ModerationResult, Self::Error>> + Send;
18}
19
20/// The result of a moderation operation.
21#[derive(Debug, Clone, PartialEq, PartialOrd)]
22#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
23pub struct ModerationResult {
24    /// Indicates whether the content was flagged.
25    flagged: bool,
26    /// The categories of violations that were detected in the content.
27    /// All categories in this list represent detected violations, with their respective confidence scores.
28    categories: Vec<ModerationCategory>,
29}
30
31impl ModerationResult {
32    /// Creates a new moderation result.
33    ///
34    /// # Arguments
35    ///
36    /// * `flagged` - Whether the content was flagged as violating policies
37    /// * `categories` - List of detected violation categories with confidence scores
38    #[must_use]
39    pub const fn new(flagged: bool, categories: Vec<ModerationCategory>) -> Self {
40        Self {
41            flagged,
42            categories,
43        }
44    }
45
46    /// Returns whether the content was flagged.
47    #[must_use]
48    pub const fn is_flagged(&self) -> bool {
49        self.flagged
50    }
51
52    /// Returns the detected violation categories.
53    #[must_use]
54    pub fn categories(&self) -> &[ModerationCategory] {
55        &self.categories
56    }
57
58    /// Returns the number of detected violations.
59    #[must_use]
60    pub const fn violation_count(&self) -> usize {
61        self.categories.len()
62    }
63
64    /// Returns whether any violations were detected.
65    #[must_use]
66    pub const fn has_violations(&self) -> bool {
67        !self.categories.is_empty()
68    }
69}
70
71/// Categories of content moderation.
72#[derive(Debug, Clone, PartialEq, PartialOrd)]
73#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
74pub enum ModerationCategory {
75    /// Hate category with a confidence score.
76    Hate {
77        /// Confidence score indicating the severity/certainty of hate content detection (0.0-1.0).
78        score: f32,
79    },
80    /// Hate/threatening category with a confidence score.
81    HateThreatening {
82        /// Confidence score indicating the severity/certainty of threatening hate content detection (0.0-1.0).
83        score: f32,
84    },
85    /// Harassment category with a confidence score.
86    Harassment {
87        /// Confidence score indicating the severity/certainty of harassment content detection (0.0-1.0).
88        score: f32,
89    },
90    /// Harassment/threatening category with a confidence score.
91    HarassmentThreatening {
92        /// Confidence score indicating the severity/certainty of threatening harassment content detection (0.0-1.0).
93        score: f32,
94    },
95    /// Sexual category with a confidence score.
96    Sexual {
97        /// Confidence score indicating the severity/certainty of sexual content detection (0.0-1.0).
98        score: f32,
99    },
100    /// Sexual/minors category with a confidence score.
101    SexualMinors {
102        /// Confidence score indicating the severity/certainty of sexual content involving minors (0.0-1.0).
103        score: f32,
104    },
105    /// Violence category with a confidence score.
106    Violence {
107        /// Confidence score indicating the severity/certainty of violence content detection (0.0-1.0).
108        score: f32,
109    },
110    /// Violence/graphic category with a confidence score.
111    ViolenceGraphic {
112        /// Confidence score indicating the severity/certainty of graphic violence content detection (0.0-1.0).
113        score: f32,
114    },
115    /// Illicit category with a confidence score.
116    Illicit {
117        /// Confidence score indicating the severity/certainty of illicit content detection (0.0-1.0).
118        score: f32,
119    },
120    /// Illicit/violent category with a confidence score.
121    IllicitViolent {
122        /// Confidence score indicating the severity/certainty of violent illicit content detection (0.0-1.0).
123        score: f32,
124    },
125    /// Self-harm category with a confidence score.
126    SelfHarm {
127        /// Confidence score indicating the severity/certainty of self-harm content detection (0.0-1.0).
128        score: f32,
129    },
130    /// Self-harm/intent category with a confidence score.
131    SelfHarmIntent {
132        /// Confidence score indicating the severity/certainty of self-harm intent detection (0.0-1.0).
133        score: f32,
134    },
135    /// Self-harm/instructions category with a confidence score.
136    SelfHarmInstructions {
137        /// Confidence score indicating the severity/certainty of self-harm instructions detection (0.0-1.0).
138        score: f32,
139    },
140}
141
142#[cfg(test)]
143mod tests {
144    use super::*;
145    use alloc::{format, vec};
146    use core::convert::Infallible;
147
148    struct MockModeration;
149
150    impl Moderation for MockModeration {
151        type Error = Infallible;
152
153        fn moderate(
154            &self,
155            content: &str,
156        ) -> impl Future<Output = Result<ModerationResult, Self::Error>> + Send {
157            // Mock moderation logic based on content
158            let flagged = content.contains("bad") || content.contains("harmful");
159            let mut categories = Vec::new();
160
161            if content.contains("hate") {
162                categories.push(ModerationCategory::Hate { score: 0.9 });
163            }
164            if content.contains("violence") {
165                categories.push(ModerationCategory::Violence { score: 0.8 });
166            }
167            if content.contains("sexual") {
168                categories.push(ModerationCategory::Sexual { score: 0.7 });
169            }
170            if content.contains("harassment") {
171                categories.push(ModerationCategory::Harassment { score: 0.85 });
172            }
173            if content.contains("self-harm") {
174                categories.push(ModerationCategory::SelfHarm { score: 0.95 });
175            }
176
177            core::future::ready(Ok(ModerationResult::new(flagged, categories)))
178        }
179    }
180
181    #[tokio::test]
182    async fn moderation_clean_content() {
183        let moderation = MockModeration;
184        let result = moderation
185            .moderate("This is a nice and friendly message")
186            .await
187            .unwrap();
188
189        assert!(!result.is_flagged());
190        assert!(!result.has_violations());
191    }
192
193    #[tokio::test]
194    async fn moderation_flagged_content() {
195        let moderation = MockModeration;
196        let result = moderation
197            .moderate("This contains bad content")
198            .await
199            .unwrap();
200
201        assert!(result.is_flagged());
202        assert!(!result.has_violations()); // No specific categories, just flagged
203    }
204
205    #[tokio::test]
206    async fn moderation_hate_content() {
207        let moderation = MockModeration;
208        let result = moderation
209            .moderate("This message contains hate speech")
210            .await
211            .unwrap();
212
213        assert!(!result.is_flagged()); // Not flagged by "bad" keyword
214        assert_eq!(result.violation_count(), 1);
215
216        match &result.categories()[0] {
217            ModerationCategory::Hate { score } => {
218                assert!((score - 0.9).abs() < f32::EPSILON);
219            }
220            _ => panic!("Expected Hate category"),
221        }
222    }
223
224    #[tokio::test]
225    async fn moderation_violence_content() {
226        let moderation = MockModeration;
227        let result = moderation
228            .moderate("This message promotes violence")
229            .await
230            .unwrap();
231
232        assert!(!result.is_flagged());
233        assert_eq!(result.violation_count(), 1);
234
235        match &result.categories()[0] {
236            ModerationCategory::Violence { score } => {
237                assert!((score - 0.8).abs() < f32::EPSILON);
238            }
239            _ => panic!("Expected Violence category"),
240        }
241    }
242
243    #[tokio::test]
244    async fn moderation_multiple_categories() {
245        let moderation = MockModeration;
246        let result = moderation
247            .moderate("This bad message contains hate and violence")
248            .await
249            .unwrap();
250
251        assert!(result.is_flagged());
252        assert_eq!(result.violation_count(), 2);
253
254        // Check that both categories are present
255        let has_hate = result
256            .categories
257            .iter()
258            .any(|cat| matches!(cat, ModerationCategory::Hate { .. }));
259        let has_violence = result
260            .categories
261            .iter()
262            .any(|cat| matches!(cat, ModerationCategory::Violence { .. }));
263
264        assert!(has_hate);
265        assert!(has_violence);
266    }
267
268    #[tokio::test]
269    async fn moderation_all_categories() {
270        let moderation = MockModeration;
271        let result = moderation
272            .moderate("harmful content with hate, violence, sexual, harassment, and self-harm")
273            .await
274            .unwrap();
275
276        assert!(result.is_flagged());
277        assert_eq!(result.violation_count(), 5);
278
279        // Verify all category types are present
280        let mut found_categories = [false; 5]; // hate, violence, sexual, harassment, self-harm
281
282        for category in result.categories() {
283            match category {
284                ModerationCategory::Hate { score } => {
285                    found_categories[0] = true;
286                    assert!((score - 0.9).abs() < f32::EPSILON);
287                }
288                ModerationCategory::Violence { score } => {
289                    found_categories[1] = true;
290                    assert!((score - 0.8).abs() < f32::EPSILON);
291                }
292                ModerationCategory::Sexual { score } => {
293                    found_categories[2] = true;
294                    assert!((score - 0.7).abs() < f32::EPSILON);
295                }
296                ModerationCategory::Harassment { score } => {
297                    found_categories[3] = true;
298                    assert!((score - 0.85).abs() < f32::EPSILON);
299                }
300                ModerationCategory::SelfHarm { score } => {
301                    found_categories[4] = true;
302                    assert!((score - 0.95).abs() < f32::EPSILON);
303                }
304                _ => {}
305            }
306        }
307
308        assert!(
309            found_categories.iter().all(|&found| found),
310            "Not all categories were found"
311        );
312    }
313
314    #[test]
315    fn moderation_result_creation() {
316        let result = ModerationResult::new(
317            true,
318            vec![
319                ModerationCategory::Hate { score: 0.8 },
320                ModerationCategory::Violence { score: 0.9 },
321            ],
322        );
323
324        assert!(result.is_flagged());
325        assert_eq!(result.categories().len(), 2);
326        assert_eq!(result.violation_count(), 2);
327        assert!(result.has_violations());
328    }
329
330    #[test]
331    fn moderation_category_equality() {
332        let cat1 = ModerationCategory::Hate { score: 0.8 };
333        let cat2 = ModerationCategory::Hate { score: 0.8 };
334        let cat3 = ModerationCategory::Hate { score: 0.9 };
335        let cat4 = ModerationCategory::Violence { score: 0.8 };
336
337        assert_eq!(cat1, cat2);
338        assert_ne!(cat1, cat3);
339        assert_ne!(cat1, cat4);
340    }
341
342    #[test]
343    fn moderation_category_clone() {
344        let original = ModerationCategory::Sexual { score: 0.7 };
345        let cloned = original.clone();
346
347        assert_eq!(original, cloned);
348    }
349
350    #[test]
351    fn moderation_category_debug() {
352        let category = ModerationCategory::Harassment { score: 0.85 };
353        let debug_string = format!("{category:?}");
354
355        assert!(debug_string.contains("Harassment"));
356        assert!(debug_string.contains("0.85"));
357    }
358
359    #[tokio::test]
360    async fn moderation_empty_content() {
361        let moderation = MockModeration;
362        let result = moderation.moderate("").await.unwrap();
363
364        assert!(!result.is_flagged());
365        assert!(!result.has_violations());
366    }
367
368    #[tokio::test]
369    async fn moderation_whitespace_content() {
370        let moderation = MockModeration;
371        let result = moderation.moderate("   \n\t  ").await.unwrap();
372
373        assert!(!result.is_flagged());
374        assert!(!result.has_violations());
375    }
376}
aither_core/moderation.rs

aither_core/
moderation.rs
