sim_lib_openai_server/runtime/
runners.rs

1use std::{fmt, sync::Arc};
2
3use sim_kernel::{Cx, Error, Result};
4use sim_lib_agent_runner_core::{ModelCard, ModelRequest, ModelResponse, ModelRunner};
5
6/// Route-local registry of model runners exposed through OpenAI model ids.
7#[derive(Clone, Default)]
8pub struct OpenAiRunnerRegistry {
9    entries: Vec<OpenAiRunnerEntry>,
10}
11
12#[derive(Clone)]
13struct OpenAiRunnerEntry {
14    model: String,
15    runner: Arc<dyn ModelRunner>,
16}
17
18impl fmt::Debug for OpenAiRunnerRegistry {
19    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
20        f.debug_struct("OpenAiRunnerRegistry")
21            .field(
22                "models",
23                &self
24                    .entries
25                    .iter()
26                    .map(|entry| entry.model.as_str())
27                    .collect::<Vec<_>>(),
28            )
29            .finish()
30    }
31}
32
33impl OpenAiRunnerRegistry {
34    /// Returns an empty runner registry.
35    pub fn new() -> Self {
36        Self::default()
37    }
38
39    /// Returns the registry with `runner` registered under OpenAI model id `model`.
40    pub fn with_runner(mut self, model: impl Into<String>, runner: Arc<dyn ModelRunner>) -> Self {
41        self.register(model, runner);
42        self
43    }
44
45    /// Registers `runner` under OpenAI model id `model`.
46    pub fn register(&mut self, model: impl Into<String>, runner: Arc<dyn ModelRunner>) {
47        self.entries.push(OpenAiRunnerEntry {
48            model: model.into(),
49            runner,
50        });
51    }
52
53    fn entry_for(&self, model: &str) -> Option<&OpenAiRunnerEntry> {
54        self.entries.iter().find(|entry| entry.model == model)
55    }
56
57    fn card_for_entry(entry: &OpenAiRunnerEntry) -> ModelCard {
58        let mut card = entry.runner.card();
59        if card.model != entry.model {
60            card.extra.push((
61                sim_kernel::Expr::Symbol(sim_kernel::Symbol::new("runner-model")),
62                sim_kernel::Expr::String(card.model.clone()),
63            ));
64            card.model = entry.model.clone();
65        }
66        card
67    }
68
69    /// Returns the model card for each registered runner.
70    ///
71    /// When a runner's own card model differs from the registered id, the card
72    /// is relabeled with the registered id and the original is preserved under a
73    /// `runner-model` extra field.
74    pub fn cards(&self) -> Vec<ModelCard> {
75        self.entries.iter().map(Self::card_for_entry).collect()
76    }
77
78    /// Returns the model card for `model`, preserving registry relabeling.
79    pub fn card_for(&self, model: &str) -> Option<ModelCard> {
80        self.entry_for(model).map(Self::card_for_entry)
81    }
82
83    /// Dispatches `request` to the runner registered for `model`.
84    ///
85    /// Returns a `model_not_found` error when no runner matches `model`.
86    pub fn infer(&self, cx: &mut Cx, model: &str, request: ModelRequest) -> Result<ModelResponse> {
87        let runner = self
88            .entry_for(model)
89            .map(|entry| entry.runner.clone())
90            .ok_or_else(|| Error::Eval(format!("model_not_found: {model}")))?;
91        runner.infer(cx, request)
92    }
93
94    /// Returns `true` when no runners are registered.
95    pub fn is_empty(&self) -> bool {
96        self.entries.is_empty()
97    }
98}
sim_lib_openai_server/runtime/runners.rs

sim_lib_openai_server/runtime/
runners.rs
