Skip to main content

llm_test_bench_core/multimodal/
datasets.rs

1// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
2// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
3// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
4// option. This file may not be copied, modified, or distributed
5// except according to those terms.
6
7//! Multi-modal dataset types and utilities.
8
9use serde::{Deserialize, Serialize};
10use std::path::Path;
11use anyhow::Result;
12
13use super::types::{MultiModalRequest, MultiModalContent};
14use super::image::ImageInput;
15use super::audio::AudioInput;
16
17/// Multi-modal task types
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
19#[serde(rename_all = "snake_case")]
20pub enum MultiModalTask {
21    /// Image captioning
22    ImageCaptioning,
23    /// Visual question answering
24    VisualQuestionAnswering,
25    /// Image classification
26    ImageClassification,
27    /// Object detection
28    ObjectDetection,
29    /// OCR (text extraction)
30    Ocr,
31    /// Speech-to-text transcription
32    SpeechToText,
33    /// Text-to-speech synthesis
34    TextToSpeech,
35    /// Audio classification
36    AudioClassification,
37    /// Video understanding
38    VideoUnderstanding,
39    /// Image generation
40    ImageGeneration,
41    /// Audio generation
42    AudioGeneration,
43}
44
45/// Vision-specific task types
46#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
47#[serde(rename_all = "snake_case")]
48pub enum VisionTask {
49    /// Describe what's in an image
50    Captioning,
51    /// Answer questions about an image
52    QuestionAnswering,
53    /// Classify the image into categories
54    Classification,
55    /// Detect and locate objects
56    ObjectDetection,
57    /// Extract text from image
58    Ocr,
59    /// Understand spatial relationships
60    SpatialReasoning,
61}
62
63/// Audio-specific task types
64#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
65#[serde(rename_all = "snake_case")]
66pub enum AudioTask {
67    /// Transcribe speech to text
68    Transcription,
69    /// Synthesize text to speech
70    Synthesis,
71    /// Classify audio (music, speech, etc.)
72    Classification,
73    /// Separate speakers
74    Diarization,
75    /// Translate speech
76    Translation,
77}
78
79/// A single multi-modal example
80#[derive(Debug, Clone, Serialize, Deserialize)]
81pub struct MultiModalExample {
82    /// Unique identifier
83    pub id: String,
84
85    /// Task type
86    pub task: MultiModalTask,
87
88    /// Input content
89    pub input: MultiModalContent,
90
91    /// Expected output/ground truth
92    pub expected_output: String,
93
94    /// Optional metadata
95    #[serde(skip_serializing_if = "Option::is_none")]
96    pub metadata: Option<serde_json::Value>,
97}
98
99impl MultiModalExample {
100    pub fn new(
101        id: impl Into<String>,
102        task: MultiModalTask,
103        input: MultiModalContent,
104        expected_output: impl Into<String>,
105    ) -> Self {
106        Self {
107            id: id.into(),
108            task,
109            input,
110            expected_output: expected_output.into(),
111            metadata: None,
112        }
113    }
114
115    /// Creates a vision captioning example
116    pub async fn vision_captioning(
117        id: impl Into<String>,
118        image_path: impl AsRef<Path>,
119        caption: impl Into<String>,
120    ) -> Result<Self> {
121        let image = ImageInput::from_path(image_path).await?;
122        let mut content = MultiModalContent::new();
123        content.add_image(image);
124
125        Ok(Self::new(
126            id,
127            MultiModalTask::ImageCaptioning,
128            content,
129            caption,
130        ))
131    }
132
133    /// Creates a VQA example
134    pub async fn visual_qa(
135        id: impl Into<String>,
136        image_path: impl AsRef<Path>,
137        question: impl Into<String>,
138        answer: impl Into<String>,
139    ) -> Result<Self> {
140        let image = ImageInput::from_path(image_path).await?;
141        let mut content = MultiModalContent::new();
142        content.add_text(question);
143        content.add_image(image);
144
145        Ok(Self::new(
146            id,
147            MultiModalTask::VisualQuestionAnswering,
148            content,
149            answer,
150        ))
151    }
152
153    /// Creates a speech transcription example
154    pub async fn speech_transcription(
155        id: impl Into<String>,
156        audio_path: impl AsRef<Path>,
157        transcription: impl Into<String>,
158    ) -> Result<Self> {
159        let audio = AudioInput::from_path(audio_path).await?;
160        let mut content = MultiModalContent::new();
161        content.add_audio(audio);
162
163        Ok(Self::new(
164            id,
165            MultiModalTask::SpeechToText,
166            content,
167            transcription,
168        ))
169    }
170
171    /// Converts to a multi-modal request
172    pub fn to_request(&self, model: impl Into<String>) -> MultiModalRequest {
173        MultiModalRequest::new(model)
174            .with_text("")  // Will be populated from content
175    }
176}
177
178/// A collection of multi-modal examples
179#[derive(Debug, Clone, Serialize, Deserialize)]
180pub struct MultiModalDataset {
181    /// Dataset name
182    pub name: String,
183
184    /// Dataset description
185    #[serde(skip_serializing_if = "Option::is_none")]
186    pub description: Option<String>,
187
188    /// Dataset version
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub version: Option<String>,
191
192    /// Examples in the dataset
193    pub examples: Vec<MultiModalExample>,
194
195    /// Task distribution
196    #[serde(skip_serializing_if = "Option::is_none")]
197    pub task_distribution: Option<std::collections::HashMap<String, usize>>,
198}
199
200impl MultiModalDataset {
201    pub fn new(name: impl Into<String>) -> Self {
202        Self {
203            name: name.into(),
204            description: None,
205            version: None,
206            examples: Vec::new(),
207            task_distribution: None,
208        }
209    }
210
211    /// Adds an example to the dataset
212    pub fn add_example(&mut self, example: MultiModalExample) {
213        self.examples.push(example);
214    }
215
216    /// Returns the number of examples
217    pub fn len(&self) -> usize {
218        self.examples.len()
219    }
220
221    /// Returns true if the dataset is empty
222    pub fn is_empty(&self) -> bool {
223        self.examples.is_empty()
224    }
225
226    /// Filters examples by task
227    pub fn filter_by_task(&self, task: MultiModalTask) -> Vec<&MultiModalExample> {
228        self.examples.iter().filter(|e| e.task == task).collect()
229    }
230
231    /// Calculates task distribution
232    pub fn calculate_task_distribution(&mut self) {
233        let mut distribution = std::collections::HashMap::new();
234
235        for example in &self.examples {
236            let task_name = format!("{:?}", example.task);
237            *distribution.entry(task_name).or_insert(0) += 1;
238        }
239
240        self.task_distribution = Some(distribution);
241    }
242
243    /// Loads a dataset from JSON file
244    pub async fn from_json_file(path: impl AsRef<Path>) -> Result<Self> {
245        let content = tokio::fs::read_to_string(path).await?;
246        let dataset: Self = serde_json::from_str(&content)?;
247        Ok(dataset)
248    }
249
250    /// Saves the dataset to a JSON file
251    pub async fn to_json_file(&self, path: impl AsRef<Path>) -> Result<()> {
252        let json = serde_json::to_string_pretty(self)?;
253        tokio::fs::write(path, json).await?;
254        Ok(())
255    }
256}
257
258/// Builder for creating multi-modal datasets
259pub struct DatasetBuilder {
260    name: String,
261    description: Option<String>,
262    version: Option<String>,
263    examples: Vec<MultiModalExample>,
264}
265
266impl DatasetBuilder {
267    pub fn new(name: impl Into<String>) -> Self {
268        Self {
269            name: name.into(),
270            description: None,
271            version: None,
272            examples: Vec::new(),
273        }
274    }
275
276    pub fn with_description(mut self, description: impl Into<String>) -> Self {
277        self.description = Some(description.into());
278        self
279    }
280
281    pub fn with_version(mut self, version: impl Into<String>) -> Self {
282        self.version = Some(version.into());
283        self
284    }
285
286    pub fn add_example(mut self, example: MultiModalExample) -> Self {
287        self.examples.push(example);
288        self
289    }
290
291    pub fn build(self) -> MultiModalDataset {
292        MultiModalDataset {
293            name: self.name,
294            description: self.description,
295            version: self.version,
296            examples: self.examples,
297            task_distribution: None,
298        }
299    }
300}
301
302#[cfg(test)]
303mod tests {
304    use super::*;
305
306    #[test]
307    fn test_multimodal_dataset() {
308        let mut dataset = MultiModalDataset::new("test-dataset");
309        assert_eq!(dataset.name, "test-dataset");
310        assert_eq!(dataset.len(), 0);
311        assert!(dataset.is_empty());
312    }
313
314    #[test]
315    fn test_dataset_builder() {
316        let dataset = DatasetBuilder::new("test")
317            .with_description("Test dataset")
318            .with_version("1.0")
319            .build();
320
321        assert_eq!(dataset.name, "test");
322        assert_eq!(dataset.description, Some("Test dataset".to_string()));
323        assert_eq!(dataset.version, Some("1.0".to_string()));
324    }
325}