sapphire_framework_bridge_api/
embed.rs

1//! Embedding settings: what the `embed.*` settings methods carry.
2//!
3//! A workgroup configures at most one *local* model and at most one *remote* one; each
4//! device switches each slot on or off, and keeps the remote slot's API key to itself.
5//! See `docs/superpowers/specs/2026-10-10-embedding-settings-design.md`.
6
7use std::fmt;
8use std::path::PathBuf;
9
10use serde::{Deserialize, Serialize};
11
12use crate::EmbedInfoResult;
13
14/// The only local model for now.
15pub const LOCAL_MODEL: &str = "Qwen/Qwen3-VL-Embedding-2B";
16/// Upper bound on [`LocalModel::max_tokens`].
17pub const MAX_TOKENS: usize = 8192;
18/// [`LocalModel::dimension`]'s default, and the dimension #183 settled on.
19pub const DEFAULT_DIMENSION: u32 = 1024;
20/// [`LocalModel::max_tokens`]'s default.
21pub const DEFAULT_MAX_TOKENS: usize = 1024;
22
23/// How the local model's weights are held in memory: a device setting, never synced.
24///
25/// The weights are converted while loading, from the same download. Vectors from every
26/// type count as the model's own: the type is no part of the model's identity. Unset, a
27/// device uses `f16` ([`QType::default`]): measured on x86_64, it is as close to `f32` as
28/// makes no difference at half its memory, and faster than `q8_0` and `q4_k`, which trade
29/// speed for memory.
30#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)]
31pub enum QType {
32    /// Full precision; about 3.5 bytes a parameter resident.
33    #[serde(rename = "f32")]
34    F32,
35    /// Half precision: the default.
36    #[serde(rename = "f16")]
37    F16,
38    /// 8-bit blocks; close to `f32` at about a quarter of its memory.
39    #[serde(rename = "q8_0")]
40    Q8_0,
41    /// 4-bit k-quants; the smallest, with a visible loss.
42    #[serde(rename = "q4_k")]
43    Q4K,
44}
45
46impl QType {
47    /// Every type, in order of size.
48    pub const ALL: [QType; 4] = [QType::F32, QType::F16, QType::Q8_0, QType::Q4K];
49
50    /// Whether this build runs the quantized types on vector instructions. candle picks
51    /// those kernels at compile time: NEON on aarch64 always, AVX2 on x86_64 only in a
52    /// build for such CPUs (`-C target-feature=+avx2`). Without them, `q8_0` and `q4_k` run
53    /// on scalar code, more than ten times slower than `f16`.
54    pub const FAST_QUANTIZED: bool = cfg!(any(
55        target_feature = "avx2",
56        target_feature = "neon",
57        target_feature = "simd128"
58    ));
59
60    /// Whether this is a block-quantized type (`q8_0`, `q4_k`).
61    pub fn is_quantized(self) -> bool {
62        matches!(self, QType::Q8_0 | QType::Q4K)
63    }
64
65    /// The name the settings file uses.
66    pub fn as_str(self) -> &'static str {
67        match self {
68            QType::F32 => "f32",
69            QType::F16 => "f16",
70            QType::Q8_0 => "q8_0",
71            QType::Q4K => "q4_k",
72        }
73    }
74}
75
76impl Default for QType {
77    /// `f16`.
78    fn default() -> QType {
79        QType::F16
80    }
81}
82
83impl fmt::Display for QType {
84    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
85        f.write_str(self.as_str())
86    }
87}
88
89impl std::str::FromStr for QType {
90    type Err = String;
91
92    fn from_str(s: &str) -> Result<Self, String> {
93        QType::ALL
94            .into_iter()
95            .find(|q| q.as_str() == s)
96            .ok_or_else(|| {
97                let names: Vec<&str> = QType::ALL.iter().map(|q| q.as_str()).collect();
98                format!("`{s}` is not one of {}", names.join(", "))
99            })
100    }
101}
102
103/// One of the two model slots.
104#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)]
105#[serde(rename_all = "lowercase")]
106pub enum Slot {
107    /// Computed on this device's CPU.
108    Local,
109    /// An OpenAI-compatible `/v1/embeddings` endpoint.
110    Remote,
111}
112
113impl fmt::Display for Slot {
114    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
115        f.write_str(match self {
116            Slot::Local => "local",
117            Slot::Remote => "remote",
118        })
119    }
120}
121
122/// The local slot's model.
123#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
124#[serde(deny_unknown_fields)]
125pub struct LocalModel {
126    /// The model's name; only [`LOCAL_MODEL`] is supported.
127    #[serde(default = "default_local_model")]
128    pub model: String,
129    /// Output dimension; vectors are cut to it (MRL) and normalized again.
130    #[serde(default = "default_dimension")]
131    pub dimension: u32,
132    /// Content tokens kept before the template is applied.
133    #[serde(default = "default_max_tokens")]
134    pub max_tokens: usize,
135}
136
137fn default_local_model() -> String {
138    LOCAL_MODEL.to_owned()
139}
140fn default_dimension() -> u32 {
141    DEFAULT_DIMENSION
142}
143fn default_max_tokens() -> usize {
144    DEFAULT_MAX_TOKENS
145}
146
147impl Default for LocalModel {
148    fn default() -> Self {
149        LocalModel {
150            model: default_local_model(),
151            dimension: DEFAULT_DIMENSION,
152            max_tokens: DEFAULT_MAX_TOKENS,
153        }
154    }
155}
156
157/// The remote slot's model.
158#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
159#[serde(deny_unknown_fields)]
160pub struct RemoteModel {
161    /// The endpoint's base URL; requests go to `<endpoint>/v1/embeddings`.
162    pub endpoint: String,
163    /// The model's name, as the endpoint knows it.
164    pub model: String,
165    /// Output dimension; longer vectors are cut to it (MRL) and normalized again.
166    pub dimension: u32,
167}
168
169/// The two slots: the workgroup's synced `[embed]` models, or a host's own without a workgroup.
170#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
171#[serde(deny_unknown_fields)]
172pub struct ModelSettings {
173    /// The local slot.
174    #[serde(default, skip_serializing_if = "Option::is_none")]
175    pub local: Option<LocalModel>,
176    /// The remote slot.
177    #[serde(default, skip_serializing_if = "Option::is_none")]
178    pub remote: Option<RemoteModel>,
179}
180
181impl ModelSettings {
182    /// Neither slot is configured.
183    pub fn is_empty(&self) -> bool {
184        self.local.is_none() && self.remote.is_none()
185    }
186
187    /// Check both slots; the message names the field at fault, never its value.
188    pub fn validate(&self) -> Result<(), String> {
189        if let Some(local) = &self.local {
190            if local.model != LOCAL_MODEL {
191                return Err(format!("local: only `{LOCAL_MODEL}` is supported"));
192            }
193            if !(64..=2048).contains(&local.dimension) {
194                return Err("local: `dimension` is out of range (64..=2048)".to_owned());
195            }
196            if !(1..=MAX_TOKENS).contains(&local.max_tokens) {
197                return Err(format!(
198                    "local: `max_tokens` is out of range (1..={MAX_TOKENS})"
199                ));
200            }
201        }
202        if let Some(remote) = &self.remote {
203            if !(remote.endpoint.starts_with("http://") || remote.endpoint.starts_with("https://"))
204            {
205                return Err("remote: `endpoint` is not an http(s) URL".to_owned());
206            }
207            if remote.model.trim().is_empty() {
208                return Err("remote: `model` must not be empty".to_owned());
209            }
210            if remote.dimension == 0 {
211                return Err("remote: `dimension` must be at least 1".to_owned());
212            }
213        }
214        Ok(())
215    }
216}
217
218/// One slot's model, for [`EmbedModelSetParams`].
219#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
220#[serde(rename_all = "lowercase")]
221pub enum SlotModel {
222    /// The local slot's model.
223    Local(LocalModel),
224    /// The remote slot's model.
225    Remote(RemoteModel),
226}
227
228impl SlotModel {
229    /// The slot this model belongs in.
230    pub fn slot(&self) -> Slot {
231        match self {
232            SlotModel::Local(_) => Slot::Local,
233            SlotModel::Remote(_) => Slot::Remote,
234        }
235    }
236}
237
238/// This device's own settings, never synced.
239#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
240pub struct DeviceSettings {
241    /// The local slot's switch; `None` is auto (off without AVX2).
242    #[serde(default)]
243    pub local_enabled: Option<bool>,
244    /// The remote slot's switch; `None` is auto (on).
245    #[serde(default)]
246    pub remote_enabled: Option<bool>,
247    /// Where the local model's files are cached, when not the bridge's default.
248    #[serde(default, skip_serializing_if = "Option::is_none")]
249    pub cache_dir: Option<PathBuf>,
250    /// How the local model's weights are held; `None` is [`QType::default`].
251    /// Absent from bridges before 2.4.
252    #[serde(default, skip_serializing_if = "Option::is_none")]
253    pub local_qtype: Option<QType>,
254}
255
256impl DeviceSettings {
257    /// One slot's switch.
258    pub fn enabled(&self, slot: Slot) -> Option<bool> {
259        match slot {
260            Slot::Local => self.local_enabled,
261            Slot::Remote => self.remote_enabled,
262        }
263    }
264}
265
266/// The remote slot's API key. Never printed: `Debug` shows `ApiKey(..)`.
267#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)]
268#[serde(transparent)]
269pub struct ApiKey(String);
270
271impl ApiKey {
272    /// Wrap a key; surrounding whitespace (a trailing newline from a pipe) is dropped.
273    pub fn new(key: impl Into<String>) -> ApiKey {
274        ApiKey(key.into().trim().to_owned())
275    }
276
277    /// The key itself, for the one place that sends it.
278    pub fn expose(&self) -> &str {
279        &self.0
280    }
281
282    /// Whether the key is empty.
283    pub fn is_empty(&self) -> bool {
284        self.0.is_empty()
285    }
286}
287
288impl fmt::Debug for ApiKey {
289    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
290        f.write_str("ApiKey(..)")
291    }
292}
293
294/// Where the model settings in effect come from.
295#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
296#[serde(rename_all = "lowercase")]
297pub enum ModelSource {
298    /// The workgroup's synced `root/config.toml`.
299    Workgroup,
300    /// This host's own `config.toml`, because it has no workgroup.
301    Device,
302}
303
304/// Why a bridge does not embed, or what it lacks while it does.
305#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
306#[serde(rename_all = "snake_case")]
307pub enum EmbedNote {
308    /// No slot is configured.
309    NotConfigured,
310    /// Every configured slot is switched off on this device.
311    DisabledOnDevice,
312    /// Only the local slot is configured, and auto turned it off: no AVX2.
313    NoAvx2,
314    /// The remote slot is active without an API key (the call is still attempted).
315    KeyMissing,
316    /// The settings could not be read or are invalid.
317    Invalid(String),
318}
319
320impl fmt::Display for EmbedNote {
321    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
322        match self {
323            EmbedNote::NotConfigured => f.write_str("no model is configured"),
324            EmbedNote::DisabledOnDevice => f.write_str("turned off on this device"),
325            EmbedNote::NoAvx2 => f.write_str("the local model needs AVX2, which this CPU lacks"),
326            EmbedNote::KeyMissing => f.write_str("no API key is set for the remote model"),
327            EmbedNote::Invalid(why) => write!(f, "invalid settings: {why}"),
328        }
329    }
330}
331
332/// Result of every `embed.*` settings method: the settings, and what they resolve to.
333#[derive(Clone, Debug, Default, Deserialize, Serialize)]
334pub struct EmbedSettingsResult {
335    /// The model settings in effect.
336    pub models: ModelSettings,
337    /// Where they come from; `None` when there are none.
338    #[serde(default)]
339    pub source: Option<ModelSource>,
340    /// This host's own `[embed]` holds models but the workgroup's settings win.
341    #[serde(default)]
342    pub shadowed_device_models: bool,
343    /// This device's own settings, as written.
344    #[serde(default)]
345    pub device: DeviceSettings,
346    /// The local slot's effective switch.
347    #[serde(default)]
348    pub local_enabled: bool,
349    /// The remote slot's effective switch.
350    #[serde(default)]
351    pub remote_enabled: bool,
352    /// The slot the bridge serves.
353    #[serde(default)]
354    pub active: Option<Slot>,
355    /// How the bridge holds the local model's weights: this device's `local_qtype`, or the
356    /// default. Absent from bridges before 2.4.
357    #[serde(default, skip_serializing_if = "Option::is_none")]
358    pub local_qtype: Option<QType>,
359    /// Whether this CPU has AVX2 (always true off x86_64).
360    #[serde(default)]
361    pub avx2: bool,
362    /// Whether an API key is stored on this device.
363    #[serde(default)]
364    pub key_set: bool,
365    /// What `embed.info` answers now.
366    #[serde(default)]
367    pub info: EmbedInfoResult,
368}
369
370impl EmbedSettingsResult {
371    /// One slot's effective switch.
372    pub fn enabled(&self, slot: Slot) -> bool {
373        match slot {
374            Slot::Local => self.local_enabled,
375            Slot::Remote => self.remote_enabled,
376        }
377    }
378}
379
380/// Parameters of [`EMBED_MODEL_SET`](crate::EMBED_MODEL_SET).
381#[derive(Clone, Debug, Deserialize, Serialize)]
382pub struct EmbedModelSetParams {
383    /// The slot to set or clear.
384    pub slot: Slot,
385    /// Its new model; `None` clears the slot. Must belong to `slot`.
386    #[serde(default)]
387    pub model: Option<SlotModel>,
388}
389
390/// Parameters of [`EMBED_DEVICE_SET`](crate::EMBED_DEVICE_SET).
391#[derive(Clone, Debug, Deserialize, Serialize)]
392pub struct EmbedDeviceSetParams {
393    /// The slot to switch.
394    pub slot: Slot,
395    /// `None` is auto.
396    #[serde(default)]
397    pub enabled: Option<bool>,
398}
399
400/// Parameters of [`EMBED_KEY_SET`](crate::EMBED_KEY_SET). `Debug` hides the key.
401#[derive(Clone, Debug, Deserialize, Serialize)]
402pub struct EmbedKeySetParams {
403    /// The key.
404    pub key: ApiKey,
405}
406
407/// One settings request, as either CLI builds it from its arguments.
408///
409/// The bridge's CLI carries it out on the files when no bridge is running; both CLIs send
410/// it to a running bridge with [`BridgeClient::embed_request`](crate::BridgeClient::embed_request).
411#[derive(Clone, Debug)]
412pub enum EmbedRequest {
413    /// Read the settings.
414    Show,
415    /// Set or clear a slot's model.
416    ModelSet(EmbedModelSetParams),
417    /// Switch a slot on this device.
418    DeviceSet(EmbedDeviceSetParams),
419    /// Store the API key.
420    KeySet(ApiKey),
421    /// Remove the API key.
422    KeyClear,
423}
424
425/// What `embedding show` prints, one line each.
426pub fn describe(s: &EmbedSettingsResult) -> Vec<String> {
427    let mut lines = Vec::new();
428    lines.push(match s.source {
429        Some(ModelSource::Workgroup) if s.shadowed_device_models => {
430            "models: the workgroup's (the models in this device's own config.toml are ignored)"
431                .to_owned()
432        }
433        Some(ModelSource::Workgroup) => "models: the workgroup's".to_owned(),
434        Some(ModelSource::Device) => "models: this device's own".to_owned(),
435        None => "models: none".to_owned(),
436    });
437    let switch = |slot: Slot| -> String {
438        let on = if s.enabled(slot) { "on" } else { "off" };
439        match s.device.enabled(slot) {
440            Some(_) => on.to_owned(),
441            None if slot == Slot::Local && !s.avx2 => format!("{on} (auto: no AVX2)"),
442            None => format!("{on} (auto)"),
443        }
444    };
445    lines.push(match &s.models.local {
446        Some(m) => format!(
447            "local:  {} ({} dimensions, {} tokens) — this device: {}, {}",
448            m.model,
449            m.dimension,
450            m.max_tokens,
451            switch(Slot::Local),
452            match (s.device.local_qtype, s.local_qtype) {
453                (Some(q), _) => q.to_string(),
454                (None, Some(q)) => format!("{q} (default)"),
455                (None, None) => format!("{} (default)", QType::default()),
456            }
457        ),
458        None => "local:  not configured".to_owned(),
459    });
460    lines.push(match &s.models.remote {
461        Some(m) => format!(
462            "remote: {} at {} ({} dimensions) — this device: {}, key {}",
463            m.model,
464            m.endpoint,
465            m.dimension,
466            switch(Slot::Remote),
467            if s.key_set { "set" } else { "not set" }
468        ),
469        None => format!(
470            "remote: not configured{}",
471            if s.key_set { " (a key is set)" } else { "" }
472        ),
473    });
474    let state = match (&s.active, &s.info.note) {
475        (Some(slot), None) => format!(
476            "active: {slot}, {}",
477            if s.info.loaded {
478                "loaded"
479            } else {
480                "not loaded"
481            }
482        ),
483        (Some(slot), Some(note)) => format!("active: {slot} — {note}"),
484        (None, Some(note)) => format!("active: none — {note}"),
485        (None, None) => "active: none".to_owned(),
486    };
487    lines.push(state);
488    lines
489}
490
491#[cfg(test)]
492mod tests {
493    use super::*;
494
495    fn remote() -> RemoteModel {
496        RemoteModel {
497            endpoint: "https://api.example.com".into(),
498            model: "m".into(),
499            dimension: 8,
500        }
501    }
502
503    #[test]
504    fn local_defaults_validate() {
505        let s = ModelSettings {
506            local: Some(LocalModel::default()),
507            remote: Some(remote()),
508        };
509        assert_eq!(s.validate(), Ok(()));
510    }
511
512    #[test]
513    fn validation_keeps_the_local_rules() {
514        let mut local = LocalModel {
515            dimension: 4096,
516            ..LocalModel::default()
517        };
518        let check = |l: &LocalModel| {
519            ModelSettings {
520                local: Some(l.clone()),
521                remote: None,
522            }
523            .validate()
524        };
525        assert!(check(&local).is_err());
526        local.dimension = 1024;
527        local.max_tokens = MAX_TOKENS + 1;
528        assert!(check(&local).is_err());
529        local.max_tokens = MAX_TOKENS;
530        assert!(check(&local).is_ok());
531        local.model = "other".into();
532        assert!(check(&local).is_err());
533    }
534
535    #[test]
536    fn validation_does_not_quote_the_value() {
537        let check = |local: Option<LocalModel>, remote: Option<RemoteModel>| {
538            ModelSettings { local, remote }.validate().unwrap_err()
539        };
540        let errs = [
541            check(
542                Some(LocalModel {
543                    model: "MARKER-42".into(),
544                    ..LocalModel::default()
545                }),
546                None,
547            ),
548            check(
549                Some(LocalModel {
550                    dimension: 4242424,
551                    ..LocalModel::default()
552                }),
553                None,
554            ),
555            check(
556                Some(LocalModel {
557                    max_tokens: 4242424,
558                    ..LocalModel::default()
559                }),
560                None,
561            ),
562            check(
563                None,
564                Some(RemoteModel {
565                    endpoint: "MARKER-42".into(),
566                    ..remote()
567                }),
568            ),
569        ];
570        for err in errs {
571            assert!(!err.contains("MARKER-42"), "{err}");
572            assert!(!err.contains("4242424"), "{err}");
573        }
574    }
575
576    #[test]
577    fn qtypes_round_trip_by_name() {
578        for q in QType::ALL {
579            assert_eq!(q.as_str().parse::<QType>(), Ok(q));
580            assert_eq!(
581                serde_json::to_value(q).unwrap(),
582                serde_json::json!(q.as_str())
583            );
584        }
585        assert_eq!(QType::default(), QType::F16);
586        assert!("fp16".parse::<QType>().is_err());
587    }
588
589    #[test]
590    fn validation_checks_the_remote_slot() {
591        let check = |r: RemoteModel| {
592            ModelSettings {
593                local: None,
594                remote: Some(r),
595            }
596            .validate()
597        };
598        assert!(
599            check(RemoteModel {
600                endpoint: "api.example.com".into(),
601                ..remote()
602            })
603            .is_err()
604        );
605        assert!(
606            check(RemoteModel {
607                model: " ".into(),
608                ..remote()
609            })
610            .is_err()
611        );
612        assert!(
613            check(RemoteModel {
614                dimension: 0,
615                ..remote()
616            })
617            .is_err()
618        );
619    }
620
621    #[test]
622    fn the_key_never_prints() {
623        let key = ApiKey::new("sk-secret\n");
624        assert_eq!(key.expose(), "sk-secret");
625        let params = EmbedKeySetParams { key };
626        let shown = format!("{params:?} {:?}", EmbedRequest::KeySet(params.key.clone()));
627        assert!(!shown.contains("sk-secret"), "{shown}");
628        // The wire form carries it, plainly.
629        assert_eq!(
630            serde_json::to_value(&params).unwrap(),
631            serde_json::json!({ "key": "sk-secret" })
632        );
633    }
634
635    #[test]
636    fn slot_models_are_tagged_by_slot() {
637        let v = serde_json::to_value(SlotModel::Remote(remote())).unwrap();
638        assert_eq!(v["remote"]["model"], "m");
639        let back: SlotModel = serde_json::from_value(v).unwrap();
640        assert_eq!(back.slot(), Slot::Remote);
641    }
642
643    #[test]
644    fn an_info_without_a_note_still_parses() {
645        let info: EmbedInfoResult =
646            serde_json::from_value(serde_json::json!({ "enabled": false })).unwrap();
647        assert_eq!(info.note, None);
648    }
649
650    #[test]
651    fn describe_names_the_source_switches_and_state() {
652        let s = EmbedSettingsResult {
653            models: ModelSettings {
654                local: Some(LocalModel::default()),
655                remote: Some(remote()),
656            },
657            source: Some(ModelSource::Workgroup),
658            local_enabled: false,
659            remote_enabled: true,
660            active: Some(Slot::Remote),
661            local_qtype: Some(QType::F16),
662            avx2: false,
663            key_set: false,
664            info: EmbedInfoResult {
665                enabled: true,
666                note: Some(EmbedNote::KeyMissing),
667                ..EmbedInfoResult::default()
668            },
669            ..EmbedSettingsResult::default()
670        };
671        let lines = describe(&s);
672        assert_eq!(lines[0], "models: the workgroup's");
673        assert!(
674            lines[1].ends_with("this device: off (auto: no AVX2), f16 (default)"),
675            "{}",
676            lines[1]
677        );
678        assert!(
679            lines[2].ends_with("this device: on (auto), key not set"),
680            "{}",
681            lines[2]
682        );
683        assert_eq!(
684            lines[3],
685            "active: remote — no API key is set for the remote model"
686        );
687    }
688
689    #[test]
690    fn describe_without_settings() {
691        let s = EmbedSettingsResult {
692            avx2: true,
693            info: EmbedInfoResult {
694                note: Some(EmbedNote::NotConfigured),
695                ..EmbedInfoResult::default()
696            },
697            ..EmbedSettingsResult::default()
698        };
699        assert_eq!(
700            describe(&s),
701            vec![
702                "models: none",
703                "local:  not configured",
704                "remote: not configured",
705                "active: none — no model is configured",
706            ]
707        );
708    }
709}
sapphire_framework_bridge_api/embed.rs

sapphire_framework_bridge_api/
embed.rs
