sim_lib_agent/
multimodal.rs

1//! Deterministic synthetic vision, speech, and sensor fixtures for local recipes.
2
3use sim_codec_chat::{model_error_expr, model_response_expr};
4use sim_kernel::{Cx, Error, Expr, Result, Symbol};
5use sim_lib_agent_runner_core::{ModelCard, ModelRequest, ModelResponse, ModelRunner};
6use std::collections::BTreeMap;
7
8/// Scripted output for one synthetic image reference.
9#[derive(Clone, Debug, PartialEq, Eq)]
10pub struct FakeVisionFixture {
11    /// Scripted caption returned for the image.
12    pub caption: String,
13    /// Additional deterministic name/value fields.
14    pub fields: Vec<(String, String)>,
15}
16
17impl FakeVisionFixture {
18    /// Builds a vision fixture from a caption.
19    pub fn new(caption: impl Into<String>) -> Self {
20        Self {
21            caption: caption.into(),
22            fields: Vec::new(),
23        }
24    }
25
26    /// Adds one deterministic field to the fixture.
27    pub fn with_field(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
28        self.fields.push((name.into(), value.into()));
29        self
30    }
31}
32
33/// Local runner that maps synthetic image references to scripted vision output.
34#[derive(Clone, Debug)]
35pub struct FakeVisionRunner {
36    model: String,
37    fixtures: BTreeMap<String, FakeVisionFixture>,
38}
39
40impl FakeVisionRunner {
41    /// Creates an empty fake vision runner for the named model surface.
42    pub fn new(model: impl Into<String>) -> Self {
43        Self {
44            model: model.into(),
45            fixtures: BTreeMap::new(),
46        }
47    }
48
49    /// Adds one synthetic image fixture.
50    pub fn with_fixture(
51        mut self,
52        reference: impl Into<String>,
53        fixture: FakeVisionFixture,
54    ) -> Result<Self> {
55        let reference = reference.into();
56        validate_synthetic_reference(&reference)?;
57        self.fixtures.insert(reference, fixture);
58        Ok(self)
59    }
60
61    /// Returns a fixture by reference.
62    pub fn fixture(&self, reference: &str) -> Option<&FakeVisionFixture> {
63        self.fixtures.get(reference)
64    }
65}
66
67impl ModelRunner for FakeVisionRunner {
68    fn card(&self) -> ModelCard {
69        model_card(
70            Symbol::qualified("runner", "fake-vision"),
71            self.model.clone(),
72            Symbol::new("fake-vision"),
73            ["image"],
74            ["text", "fields"],
75        )
76    }
77
78    fn infer(&self, _cx: &mut Cx, request: ModelRequest) -> Result<ModelResponse> {
79        let Some(reference) = request_reference(&request, &["image", "image-ref", "fixture"])
80        else {
81            return error_response(
82                Symbol::qualified("runner", "fake-vision"),
83                &self.model,
84                "fake vision request missing synthetic image reference",
85            );
86        };
87        if let Err(error) = validate_synthetic_reference(reference) {
88            return error_response(
89                Symbol::qualified("runner", "fake-vision"),
90                &self.model,
91                error.to_string(),
92            );
93        }
94        let Some(fixture) = self.fixtures.get(reference) else {
95            return error_response(
96                Symbol::qualified("runner", "fake-vision"),
97                &self.model,
98                format!("fake vision fixture not found: {reference}"),
99            );
100        };
101        response_from_parts(
102            Symbol::qualified("runner", "fake-vision"),
103            &self.model,
104            vec![
105                text_part(&fixture.caption),
106                string_part("fixture", "reference", reference),
107                fields_part(&fixture.fields),
108            ],
109        )
110    }
111}
112
113/// One deterministic speech transcript segment.
114#[derive(Clone, Debug, PartialEq, Eq)]
115pub struct FakeAsrSegment {
116    /// Segment start offset in milliseconds.
117    pub start_ms: u64,
118    /// Segment end offset in milliseconds, at or after the start.
119    pub end_ms: u64,
120    /// Transcribed text for this segment.
121    pub text: String,
122}
123
124impl FakeAsrSegment {
125    /// Builds one transcript segment with fixed offsets.
126    pub fn new(start_ms: u64, end_ms: u64, text: impl Into<String>) -> Result<Self> {
127        if end_ms < start_ms {
128            return Err(Error::Eval(
129                "fake ASR segment end must be at or after start".to_owned(),
130            ));
131        }
132        Ok(Self {
133            start_ms,
134            end_ms,
135            text: text.into(),
136        })
137    }
138}
139
140/// Scripted output for one synthetic audio reference.
141#[derive(Clone, Debug, PartialEq, Eq)]
142pub struct FakeAsrFixture {
143    /// Full scripted transcript for the audio.
144    pub transcript: String,
145    /// Locale tag for the transcript.
146    pub locale: String,
147    /// Per-segment breakdown of the transcript.
148    pub segments: Vec<FakeAsrSegment>,
149}
150
151impl FakeAsrFixture {
152    /// Builds an ASR fixture from a transcript and locale.
153    pub fn new(transcript: impl Into<String>, locale: impl Into<String>) -> Self {
154        Self {
155            transcript: transcript.into(),
156            locale: locale.into(),
157            segments: Vec::new(),
158        }
159    }
160
161    /// Adds one deterministic transcript segment.
162    pub fn with_segment(mut self, segment: FakeAsrSegment) -> Self {
163        self.segments.push(segment);
164        self
165    }
166}
167
168/// Local runner that maps synthetic audio references to scripted transcripts.
169#[derive(Clone, Debug)]
170pub struct FakeAsrRunner {
171    model: String,
172    fixtures: BTreeMap<String, FakeAsrFixture>,
173}
174
175impl FakeAsrRunner {
176    /// Creates an empty fake ASR runner for the named model surface.
177    pub fn new(model: impl Into<String>) -> Self {
178        Self {
179            model: model.into(),
180            fixtures: BTreeMap::new(),
181        }
182    }
183
184    /// Adds one synthetic audio fixture.
185    pub fn with_fixture(
186        mut self,
187        reference: impl Into<String>,
188        fixture: FakeAsrFixture,
189    ) -> Result<Self> {
190        let reference = reference.into();
191        validate_synthetic_reference(&reference)?;
192        self.fixtures.insert(reference, fixture);
193        Ok(self)
194    }
195
196    /// Returns a fixture by reference.
197    pub fn fixture(&self, reference: &str) -> Option<&FakeAsrFixture> {
198        self.fixtures.get(reference)
199    }
200}
201
202impl ModelRunner for FakeAsrRunner {
203    fn card(&self) -> ModelCard {
204        model_card(
205            Symbol::qualified("runner", "fake-asr"),
206            self.model.clone(),
207            Symbol::new("fake-asr"),
208            ["audio"],
209            ["text", "transcript"],
210        )
211    }
212
213    fn infer(&self, _cx: &mut Cx, request: ModelRequest) -> Result<ModelResponse> {
214        let Some(reference) = request_reference(&request, &["audio", "audio-ref", "fixture"])
215        else {
216            return error_response(
217                Symbol::qualified("runner", "fake-asr"),
218                &self.model,
219                "fake ASR request missing synthetic audio reference",
220            );
221        };
222        if let Err(error) = validate_synthetic_reference(reference) {
223            return error_response(
224                Symbol::qualified("runner", "fake-asr"),
225                &self.model,
226                error.to_string(),
227            );
228        }
229        let Some(fixture) = self.fixtures.get(reference) else {
230            return error_response(
231                Symbol::qualified("runner", "fake-asr"),
232                &self.model,
233                format!("fake ASR fixture not found: {reference}"),
234            );
235        };
236        response_from_parts(
237            Symbol::qualified("runner", "fake-asr"),
238            &self.model,
239            vec![
240                text_part(&fixture.transcript),
241                string_part("fixture", "reference", reference),
242                transcript_part(fixture),
243            ],
244        )
245    }
246}
247
248/// One fixed sensor frame in a synthetic sensor stream.
249#[derive(Clone, Debug, PartialEq)]
250pub struct FakeSensorFrame {
251    /// Monotonic sequence number of this frame in the stream.
252    pub sequence: u64,
253    /// Named sensor readings, each a finite value.
254    pub readings: Vec<(String, f64)>,
255}
256
257impl FakeSensorFrame {
258    /// Builds one sensor frame and rejects non-finite readings.
259    pub fn new(sequence: u64, readings: Vec<(String, f64)>) -> Result<Self> {
260        if readings.iter().any(|(_, value)| !value.is_finite()) {
261            return Err(Error::Eval(
262                "fake sensor frame readings must be finite".to_owned(),
263            ));
264        }
265        Ok(Self { sequence, readings })
266    }
267
268    /// Encodes the frame as an expression for recipe fixtures and traces.
269    pub fn as_expr(&self) -> Expr {
270        Expr::Map(vec![
271            number_entry("sequence", self.sequence),
272            (
273                sym("readings"),
274                Expr::Map(
275                    self.readings
276                        .iter()
277                        .map(|(name, value)| {
278                            (
279                                Expr::String(name.clone()),
280                                number_literal("f64", value.to_string()),
281                            )
282                        })
283                        .collect(),
284                ),
285            ),
286        ])
287    }
288}
289
290/// Fixed local stream over synthetic sensor frames.
291#[derive(Clone, Debug, PartialEq)]
292pub struct FakeSensorStream {
293    frames: Vec<FakeSensorFrame>,
294    cursor: usize,
295}
296
297impl FakeSensorStream {
298    /// Builds a fixed sensor stream from pre-authored frames.
299    pub fn new(frames: Vec<FakeSensorFrame>) -> Result<Self> {
300        if frames.is_empty() {
301            return Err(Error::Eval(
302                "fake sensor stream needs at least one frame".to_owned(),
303            ));
304        }
305        Ok(Self { frames, cursor: 0 })
306    }
307
308    /// Returns the next frame and advances the stream.
309    pub fn next_frame(&mut self) -> Option<FakeSensorFrame> {
310        let frame = self.frames.get(self.cursor).cloned()?;
311        self.cursor += 1;
312        Some(frame)
313    }
314
315    /// Resets the cursor to the start of the fixed sequence.
316    pub fn reset(&mut self) {
317        self.cursor = 0;
318    }
319
320    /// Returns all fixed frames without advancing the stream.
321    pub fn frames(&self) -> &[FakeSensorFrame] {
322        &self.frames
323    }
324}
325
326impl Iterator for FakeSensorStream {
327    type Item = FakeSensorFrame;
328
329    fn next(&mut self) -> Option<Self::Item> {
330        self.next_frame()
331    }
332}
333
334fn validate_synthetic_reference(reference: &str) -> Result<()> {
335    if reference.starts_with("fixture:") || reference.starts_with("synthetic:") {
336        return Ok(());
337    }
338    Err(Error::Eval(
339        "fake multimodal fixtures require fixture: or synthetic: references".to_owned(),
340    ))
341}
342
343fn request_reference<'a>(request: &'a ModelRequest, keys: &[&str]) -> Option<&'a str> {
344    expr_reference(&request.task, keys).or_else(|| {
345        request
346            .messages
347            .iter()
348            .find_map(|expr| expr_reference(expr, keys))
349    })
350}
351
352fn expr_reference<'a>(expr: &'a Expr, keys: &[&str]) -> Option<&'a str> {
353    match expr {
354        Expr::String(text) if is_fixture_reference(text) => Some(text.as_str()),
355        Expr::Map(entries) => entries.iter().find_map(|(key, value)| {
356            if key_matches(key, keys)
357                && let Expr::String(reference) = value
358            {
359                return Some(reference.as_str());
360            }
361            expr_reference(value, keys)
362        }),
363        Expr::List(items) | Expr::Vector(items) => {
364            items.iter().find_map(|item| expr_reference(item, keys))
365        }
366        _ => None,
367    }
368}
369
370fn is_fixture_reference(text: &str) -> bool {
371    text.starts_with("fixture:") || text.starts_with("synthetic:")
372}
373
374fn key_matches(expr: &Expr, keys: &[&str]) -> bool {
375    match expr {
376        Expr::Symbol(symbol) => {
377            let name = symbol.as_qualified_str();
378            keys.iter().any(|key| *key == name)
379        }
380        Expr::String(text) => keys.iter().any(|key| *key == text),
381        _ => false,
382    }
383}
384
385fn response_from_parts(runner: Symbol, model: &str, parts: Vec<Expr>) -> Result<ModelResponse> {
386    ModelResponse::try_from(model_response_expr(
387        runner,
388        model,
389        parts,
390        Symbol::new("stop"),
391    ))
392}
393
394fn error_response(
395    runner: Symbol,
396    model: &str,
397    message: impl Into<String>,
398) -> Result<ModelResponse> {
399    ModelResponse::try_from(model_error_expr(runner, model, message))
400}
401
402fn model_card<const IN: usize, const OUT: usize>(
403    runner: Symbol,
404    model: String,
405    provider: Symbol,
406    modalities_in: [&str; IN],
407    modalities_out: [&str; OUT],
408) -> ModelCard {
409    let mut card = ModelCard::new(runner, model, provider, Symbol::new("local"));
410    card.extra = vec![
411        symbol_list_entry("modalities-in", modalities_in),
412        symbol_list_entry("modalities-out", modalities_out),
413        bool_entry("supports-stream", false),
414        bool_entry("supports-tools", false),
415        bool_entry("supports-json", true),
416        bool_entry("supports-shape", false),
417        symbol_entry("health", "ok"),
418    ];
419    card
420}
421
422use sim_codec_chat::text_part;
423
424fn string_part(kind: &str, key: &str, value: &str) -> Expr {
425    Expr::Map(vec![symbol_entry("type", kind), string_entry(key, value)])
426}
427
428fn fields_part(fields: &[(String, String)]) -> Expr {
429    Expr::Map(vec![
430        symbol_entry("type", "fields"),
431        (
432            sym("fields"),
433            Expr::Map(
434                fields
435                    .iter()
436                    .map(|(key, value)| (Expr::String(key.clone()), Expr::String(value.clone())))
437                    .collect(),
438            ),
439        ),
440    ])
441}
442
443fn transcript_part(fixture: &FakeAsrFixture) -> Expr {
444    Expr::Map(vec![
445        symbol_entry("type", "transcript"),
446        string_entry("locale", &fixture.locale),
447        (
448            sym("segments"),
449            Expr::List(fixture.segments.iter().map(segment_expr).collect()),
450        ),
451    ])
452}
453
454fn segment_expr(segment: &FakeAsrSegment) -> Expr {
455    Expr::Map(vec![
456        number_entry("start-ms", segment.start_ms),
457        number_entry("end-ms", segment.end_ms),
458        string_entry("text", &segment.text),
459    ])
460}
461
462fn symbol_list_entry<const N: usize>(key: &str, values: [&str; N]) -> (Expr, Expr) {
463    (
464        Expr::Symbol(Symbol::new(key)),
465        Expr::List(
466            values
467                .into_iter()
468                .map(|value| Expr::Symbol(Symbol::new(value)))
469                .collect(),
470        ),
471    )
472}
473
474fn bool_entry(key: &str, value: bool) -> (Expr, Expr) {
475    (sym(key), Expr::Bool(value))
476}
477
478fn number_entry(key: &str, value: u64) -> (Expr, Expr) {
479    (sym(key), number_literal("i64", value.to_string()))
480}
481
482fn symbol_entry(key: &str, value: &str) -> (Expr, Expr) {
483    (sym(key), Expr::Symbol(Symbol::new(value)))
484}
485
486fn string_entry(key: &str, value: &str) -> (Expr, Expr) {
487    (sym(key), Expr::String(value.to_owned()))
488}
489
490use sim_value::build::sym;
491
492fn number_literal(domain: &str, canonical: String) -> Expr {
493    Expr::Number(sim_kernel::NumberLiteral {
494        domain: Symbol::qualified("numbers", domain),
495        canonical,
496    })
497}
sim_lib_agent/multimodal.rs

sim_lib_agent/
multimodal.rs
