llm_test_bench_core/multimodal/
datasets.rs1use serde::{Deserialize, Serialize};
10use std::path::Path;
11use anyhow::Result;
12
13use super::types::{MultiModalRequest, MultiModalContent};
14use super::image::ImageInput;
15use super::audio::AudioInput;
16
17#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
19#[serde(rename_all = "snake_case")]
20pub enum MultiModalTask {
21 ImageCaptioning,
23 VisualQuestionAnswering,
25 ImageClassification,
27 ObjectDetection,
29 Ocr,
31 SpeechToText,
33 TextToSpeech,
35 AudioClassification,
37 VideoUnderstanding,
39 ImageGeneration,
41 AudioGeneration,
43}
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
47#[serde(rename_all = "snake_case")]
48pub enum VisionTask {
49 Captioning,
51 QuestionAnswering,
53 Classification,
55 ObjectDetection,
57 Ocr,
59 SpatialReasoning,
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
65#[serde(rename_all = "snake_case")]
66pub enum AudioTask {
67 Transcription,
69 Synthesis,
71 Classification,
73 Diarization,
75 Translation,
77}
78
79#[derive(Debug, Clone, Serialize, Deserialize)]
81pub struct MultiModalExample {
82 pub id: String,
84
85 pub task: MultiModalTask,
87
88 pub input: MultiModalContent,
90
91 pub expected_output: String,
93
94 #[serde(skip_serializing_if = "Option::is_none")]
96 pub metadata: Option<serde_json::Value>,
97}
98
99impl MultiModalExample {
100 pub fn new(
101 id: impl Into<String>,
102 task: MultiModalTask,
103 input: MultiModalContent,
104 expected_output: impl Into<String>,
105 ) -> Self {
106 Self {
107 id: id.into(),
108 task,
109 input,
110 expected_output: expected_output.into(),
111 metadata: None,
112 }
113 }
114
115 pub async fn vision_captioning(
117 id: impl Into<String>,
118 image_path: impl AsRef<Path>,
119 caption: impl Into<String>,
120 ) -> Result<Self> {
121 let image = ImageInput::from_path(image_path).await?;
122 let mut content = MultiModalContent::new();
123 content.add_image(image);
124
125 Ok(Self::new(
126 id,
127 MultiModalTask::ImageCaptioning,
128 content,
129 caption,
130 ))
131 }
132
133 pub async fn visual_qa(
135 id: impl Into<String>,
136 image_path: impl AsRef<Path>,
137 question: impl Into<String>,
138 answer: impl Into<String>,
139 ) -> Result<Self> {
140 let image = ImageInput::from_path(image_path).await?;
141 let mut content = MultiModalContent::new();
142 content.add_text(question);
143 content.add_image(image);
144
145 Ok(Self::new(
146 id,
147 MultiModalTask::VisualQuestionAnswering,
148 content,
149 answer,
150 ))
151 }
152
153 pub async fn speech_transcription(
155 id: impl Into<String>,
156 audio_path: impl AsRef<Path>,
157 transcription: impl Into<String>,
158 ) -> Result<Self> {
159 let audio = AudioInput::from_path(audio_path).await?;
160 let mut content = MultiModalContent::new();
161 content.add_audio(audio);
162
163 Ok(Self::new(
164 id,
165 MultiModalTask::SpeechToText,
166 content,
167 transcription,
168 ))
169 }
170
171 pub fn to_request(&self, model: impl Into<String>) -> MultiModalRequest {
173 MultiModalRequest::new(model)
174 .with_text("") }
176}
177
178#[derive(Debug, Clone, Serialize, Deserialize)]
180pub struct MultiModalDataset {
181 pub name: String,
183
184 #[serde(skip_serializing_if = "Option::is_none")]
186 pub description: Option<String>,
187
188 #[serde(skip_serializing_if = "Option::is_none")]
190 pub version: Option<String>,
191
192 pub examples: Vec<MultiModalExample>,
194
195 #[serde(skip_serializing_if = "Option::is_none")]
197 pub task_distribution: Option<std::collections::HashMap<String, usize>>,
198}
199
200impl MultiModalDataset {
201 pub fn new(name: impl Into<String>) -> Self {
202 Self {
203 name: name.into(),
204 description: None,
205 version: None,
206 examples: Vec::new(),
207 task_distribution: None,
208 }
209 }
210
211 pub fn add_example(&mut self, example: MultiModalExample) {
213 self.examples.push(example);
214 }
215
216 pub fn len(&self) -> usize {
218 self.examples.len()
219 }
220
221 pub fn is_empty(&self) -> bool {
223 self.examples.is_empty()
224 }
225
226 pub fn filter_by_task(&self, task: MultiModalTask) -> Vec<&MultiModalExample> {
228 self.examples.iter().filter(|e| e.task == task).collect()
229 }
230
231 pub fn calculate_task_distribution(&mut self) {
233 let mut distribution = std::collections::HashMap::new();
234
235 for example in &self.examples {
236 let task_name = format!("{:?}", example.task);
237 *distribution.entry(task_name).or_insert(0) += 1;
238 }
239
240 self.task_distribution = Some(distribution);
241 }
242
243 pub async fn from_json_file(path: impl AsRef<Path>) -> Result<Self> {
245 let content = tokio::fs::read_to_string(path).await?;
246 let dataset: Self = serde_json::from_str(&content)?;
247 Ok(dataset)
248 }
249
250 pub async fn to_json_file(&self, path: impl AsRef<Path>) -> Result<()> {
252 let json = serde_json::to_string_pretty(self)?;
253 tokio::fs::write(path, json).await?;
254 Ok(())
255 }
256}
257
258pub struct DatasetBuilder {
260 name: String,
261 description: Option<String>,
262 version: Option<String>,
263 examples: Vec<MultiModalExample>,
264}
265
266impl DatasetBuilder {
267 pub fn new(name: impl Into<String>) -> Self {
268 Self {
269 name: name.into(),
270 description: None,
271 version: None,
272 examples: Vec::new(),
273 }
274 }
275
276 pub fn with_description(mut self, description: impl Into<String>) -> Self {
277 self.description = Some(description.into());
278 self
279 }
280
281 pub fn with_version(mut self, version: impl Into<String>) -> Self {
282 self.version = Some(version.into());
283 self
284 }
285
286 pub fn add_example(mut self, example: MultiModalExample) -> Self {
287 self.examples.push(example);
288 self
289 }
290
291 pub fn build(self) -> MultiModalDataset {
292 MultiModalDataset {
293 name: self.name,
294 description: self.description,
295 version: self.version,
296 examples: self.examples,
297 task_distribution: None,
298 }
299 }
300}
301
302#[cfg(test)]
303mod tests {
304 use super::*;
305
306 #[test]
307 fn test_multimodal_dataset() {
308 let mut dataset = MultiModalDataset::new("test-dataset");
309 assert_eq!(dataset.name, "test-dataset");
310 assert_eq!(dataset.len(), 0);
311 assert!(dataset.is_empty());
312 }
313
314 #[test]
315 fn test_dataset_builder() {
316 let dataset = DatasetBuilder::new("test")
317 .with_description("Test dataset")
318 .with_version("1.0")
319 .build();
320
321 assert_eq!(dataset.name, "test");
322 assert_eq!(dataset.description, Some("Test dataset".to_string()));
323 assert_eq!(dataset.version, Some("1.0".to_string()));
324 }
325}