sim_lib_openai_server/runtime/
runners.rs

1use std::{fmt, sync::Arc};
2
3use sim_kernel::{Cx, Error, Result};
4use sim_lib_agent_runner_core::{ModelCard, ModelRequest, ModelResponse, ModelRunner};
5
6/// Route-local registry of model runners exposed through OpenAI model ids.
7#[derive(Clone, Default)]
8pub struct OpenAiRunnerRegistry {
9    entries: Vec<OpenAiRunnerEntry>,
10}
11
12#[derive(Clone)]
13struct OpenAiRunnerEntry {
14    model: String,
15    runner: Arc<dyn ModelRunner>,
16}
17
18impl fmt::Debug for OpenAiRunnerRegistry {
19    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
20        f.debug_struct("OpenAiRunnerRegistry")
21            .field(
22                "models",
23                &self
24                    .entries
25                    .iter()
26                    .map(|entry| entry.model.as_str())
27                    .collect::<Vec<_>>(),
28            )
29            .finish()
30    }
31}
32
33impl OpenAiRunnerRegistry {
34    /// Returns an empty runner registry.
35    pub fn new() -> Self {
36        Self::default()
37    }
38
39    /// Returns the registry with `runner` registered under OpenAI model id `model`.
40    pub fn with_runner(mut self, model: impl Into<String>, runner: Arc<dyn ModelRunner>) -> Self {
41        self.register(model, runner);
42        self
43    }
44
45    /// Registers `runner` under OpenAI model id `model`.
46    pub fn register(&mut self, model: impl Into<String>, runner: Arc<dyn ModelRunner>) {
47        self.entries.push(OpenAiRunnerEntry {
48            model: model.into(),
49            runner,
50        });
51    }
52
53    /// Returns the model card for each registered runner.
54    ///
55    /// When a runner's own card model differs from the registered id, the card
56    /// is relabeled with the registered id and the original is preserved under a
57    /// `runner-model` extra field.
58    pub fn cards(&self) -> Vec<ModelCard> {
59        self.entries
60            .iter()
61            .map(|entry| {
62                let mut card = entry.runner.card();
63                if card.model != entry.model {
64                    card.extra.push((
65                        sim_kernel::Expr::Symbol(sim_kernel::Symbol::new("runner-model")),
66                        sim_kernel::Expr::String(card.model.clone()),
67                    ));
68                    card.model = entry.model.clone();
69                }
70                card
71            })
72            .collect()
73    }
74
75    /// Dispatches `request` to the runner registered for `model`.
76    ///
77    /// Returns a `model_not_found` error when no runner matches `model`.
78    pub fn infer(&self, cx: &mut Cx, model: &str, request: ModelRequest) -> Result<ModelResponse> {
79        let runner = self
80            .entries
81            .iter()
82            .find(|entry| entry.model == model)
83            .map(|entry| entry.runner.clone())
84            .ok_or_else(|| Error::Eval(format!("model_not_found: {model}")))?;
85        runner.infer(cx, request)
86    }
87
88    /// Returns `true` when no runners are registered.
89    pub fn is_empty(&self) -> bool {
90        self.entries.is_empty()
91    }
92}
sim_lib_openai_server/runtime/runners.rs

sim_lib_openai_server/runtime/
runners.rs
