sapphire_framework_bridge_api/
embed.rs

1//! Embedding settings: what the `embed.*` settings methods carry.
2//!
3//! A workgroup configures at most one *local* model and at most one *remote* one; each
4//! device switches each slot on or off, and keeps the remote slot's API key to itself.
5//! See `docs/superpowers/specs/2026-10-10-embedding-settings-design.md`.
6
7use std::fmt;
8use std::path::PathBuf;
9
10use serde::{Deserialize, Serialize};
11
12use crate::EmbedInfoResult;
13
14/// The only local model for now.
15pub const LOCAL_MODEL: &str = "Qwen/Qwen3-VL-Embedding-2B";
16/// Upper bound on [`LocalModel::max_tokens`].
17pub const MAX_TOKENS: usize = 8192;
18/// [`LocalModel::dimension`]'s default, and the dimension #183 settled on.
19pub const DEFAULT_DIMENSION: u32 = 1024;
20/// [`LocalModel::max_tokens`]'s default.
21pub const DEFAULT_MAX_TOKENS: usize = 1024;
22
23/// How the local model's weights are held in memory: a device setting, never synced.
24///
25/// The weights are converted while loading, from the same download. Vectors from every
26/// type count as the model's own: the type is no part of the model's identity. Unset, a
27/// device uses `f16` ([`QType::default`]): measured on x86_64, it is as close to `f32` as
28/// makes no difference at half its memory, and faster than `q8_0` and `q4_k`, which trade
29/// speed for memory.
30#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)]
31pub enum QType {
32    /// Full precision; about 3.5 bytes a parameter resident.
33    #[serde(rename = "f32")]
34    F32,
35    /// Half precision: the default.
36    #[serde(rename = "f16")]
37    F16,
38    /// 8-bit blocks; close to `f32` at about a quarter of its memory.
39    #[serde(rename = "q8_0")]
40    Q8_0,
41    /// 4-bit k-quants; the smallest, with a visible loss.
42    #[serde(rename = "q4_k")]
43    Q4K,
44}
45
46impl QType {
47    /// Every type, in order of size.
48    pub const ALL: [QType; 4] = [QType::F32, QType::F16, QType::Q8_0, QType::Q4K];
49
50    /// Whether this build runs the quantized types on vector instructions. candle picks
51    /// those kernels at compile time: NEON on aarch64 always, AVX2 on x86_64 only in a
52    /// build for such CPUs (`-C target-feature=+avx2`). Without them, `q8_0` and `q4_k` run
53    /// on scalar code, more than ten times slower than `f16`.
54    pub const FAST_QUANTIZED: bool = cfg!(any(
55        target_feature = "avx2",
56        target_feature = "neon",
57        target_feature = "simd128"
58    ));
59
60    /// Whether this is a block-quantized type (`q8_0`, `q4_k`).
61    pub fn is_quantized(self) -> bool {
62        matches!(self, QType::Q8_0 | QType::Q4K)
63    }
64
65    /// The name the settings file uses.
66    pub fn as_str(self) -> &'static str {
67        match self {
68            QType::F32 => "f32",
69            QType::F16 => "f16",
70            QType::Q8_0 => "q8_0",
71            QType::Q4K => "q4_k",
72        }
73    }
74}
75
76impl Default for QType {
77    /// `f16`.
78    fn default() -> QType {
79        QType::F16
80    }
81}
82
83impl fmt::Display for QType {
84    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
85        f.write_str(self.as_str())
86    }
87}
88
89impl std::str::FromStr for QType {
90    type Err = String;
91
92    fn from_str(s: &str) -> Result<Self, String> {
93        QType::ALL
94            .into_iter()
95            .find(|q| q.as_str() == s)
96            .ok_or_else(|| {
97                let names: Vec<&str> = QType::ALL.iter().map(|q| q.as_str()).collect();
98                format!("`{s}` is not one of {}", names.join(", "))
99            })
100    }
101}
102
103/// One of the two model slots.
104#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)]
105#[serde(rename_all = "lowercase")]
106pub enum Slot {
107    /// Computed on this device's CPU.
108    Local,
109    /// An OpenAI-compatible `/v1/embeddings` endpoint.
110    Remote,
111}
112
113impl fmt::Display for Slot {
114    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
115        f.write_str(match self {
116            Slot::Local => "local",
117            Slot::Remote => "remote",
118        })
119    }
120}
121
122/// The local slot's model.
123#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
124#[serde(deny_unknown_fields)]
125pub struct LocalModel {
126    /// The model's name; only [`LOCAL_MODEL`] is supported.
127    #[serde(default = "default_local_model")]
128    pub model: String,
129    /// Output dimension; vectors are cut to it (MRL) and normalized again.
130    #[serde(default = "default_dimension")]
131    pub dimension: u32,
132    /// Content tokens kept before the template is applied.
133    #[serde(default = "default_max_tokens")]
134    pub max_tokens: usize,
135}
136
137fn default_local_model() -> String {
138    LOCAL_MODEL.to_owned()
139}
140fn default_dimension() -> u32 {
141    DEFAULT_DIMENSION
142}
143fn default_max_tokens() -> usize {
144    DEFAULT_MAX_TOKENS
145}
146
147impl Default for LocalModel {
148    fn default() -> Self {
149        LocalModel {
150            model: default_local_model(),
151            dimension: DEFAULT_DIMENSION,
152            max_tokens: DEFAULT_MAX_TOKENS,
153        }
154    }
155}
156
157/// The remote slot's model.
158#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
159#[serde(deny_unknown_fields)]
160pub struct RemoteModel {
161    /// The endpoint's base URL; requests go to `<endpoint>/v1/embeddings`.
162    pub endpoint: String,
163    /// The model's name, as the endpoint knows it.
164    pub model: String,
165    /// Output dimension; longer vectors are cut to it (MRL) and normalized again.
166    pub dimension: u32,
167}
168
169/// The two slots: the workgroup's synced `embedding.toml`, or a host's own without one.
170#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
171#[serde(deny_unknown_fields)]
172pub struct ModelSettings {
173    /// The local slot.
174    #[serde(default, skip_serializing_if = "Option::is_none")]
175    pub local: Option<LocalModel>,
176    /// The remote slot.
177    #[serde(default, skip_serializing_if = "Option::is_none")]
178    pub remote: Option<RemoteModel>,
179}
180
181impl ModelSettings {
182    /// Neither slot is configured.
183    pub fn is_empty(&self) -> bool {
184        self.local.is_none() && self.remote.is_none()
185    }
186
187    /// Check both slots; the message names the field at fault.
188    pub fn validate(&self) -> Result<(), String> {
189        if let Some(local) = &self.local {
190            if local.model != LOCAL_MODEL {
191                return Err(format!(
192                    "local: only `{LOCAL_MODEL}` is supported, not `{}`",
193                    local.model
194                ));
195            }
196            if !(64..=2048).contains(&local.dimension) {
197                return Err(format!(
198                    "local: `dimension` = {} is out of range (64..=2048)",
199                    local.dimension
200                ));
201            }
202            if !(1..=MAX_TOKENS).contains(&local.max_tokens) {
203                return Err(format!(
204                    "local: `max_tokens` = {} is out of range (1..={MAX_TOKENS})",
205                    local.max_tokens
206                ));
207            }
208        }
209        if let Some(remote) = &self.remote {
210            if !(remote.endpoint.starts_with("http://") || remote.endpoint.starts_with("https://"))
211            {
212                return Err(format!(
213                    "remote: `endpoint` = `{}` is not an http(s) URL",
214                    remote.endpoint
215                ));
216            }
217            if remote.model.trim().is_empty() {
218                return Err("remote: `model` must not be empty".to_owned());
219            }
220            if remote.dimension == 0 {
221                return Err("remote: `dimension` must be at least 1".to_owned());
222            }
223        }
224        Ok(())
225    }
226}
227
228/// One slot's model, for [`EmbedModelSetParams`].
229#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
230#[serde(rename_all = "lowercase")]
231pub enum SlotModel {
232    /// The local slot's model.
233    Local(LocalModel),
234    /// The remote slot's model.
235    Remote(RemoteModel),
236}
237
238impl SlotModel {
239    /// The slot this model belongs in.
240    pub fn slot(&self) -> Slot {
241        match self {
242            SlotModel::Local(_) => Slot::Local,
243            SlotModel::Remote(_) => Slot::Remote,
244        }
245    }
246}
247
248/// This device's own settings, never synced.
249#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
250pub struct DeviceSettings {
251    /// The local slot's switch; `None` is auto (off without AVX2).
252    #[serde(default)]
253    pub local_enabled: Option<bool>,
254    /// The remote slot's switch; `None` is auto (on).
255    #[serde(default)]
256    pub remote_enabled: Option<bool>,
257    /// Where the local model's files are cached, when not the bridge's default.
258    #[serde(default, skip_serializing_if = "Option::is_none")]
259    pub cache_dir: Option<PathBuf>,
260    /// How the local model's weights are held; `None` is [`QType::default`].
261    /// Absent from bridges before 2.4.
262    #[serde(default, skip_serializing_if = "Option::is_none")]
263    pub local_qtype: Option<QType>,
264}
265
266impl DeviceSettings {
267    /// One slot's switch.
268    pub fn enabled(&self, slot: Slot) -> Option<bool> {
269        match slot {
270            Slot::Local => self.local_enabled,
271            Slot::Remote => self.remote_enabled,
272        }
273    }
274}
275
276/// The remote slot's API key. Never printed: `Debug` shows `ApiKey(..)`.
277#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)]
278#[serde(transparent)]
279pub struct ApiKey(String);
280
281impl ApiKey {
282    /// Wrap a key; surrounding whitespace (a trailing newline from a pipe) is dropped.
283    pub fn new(key: impl Into<String>) -> ApiKey {
284        ApiKey(key.into().trim().to_owned())
285    }
286
287    /// The key itself, for the one place that sends it.
288    pub fn expose(&self) -> &str {
289        &self.0
290    }
291
292    /// Whether the key is empty.
293    pub fn is_empty(&self) -> bool {
294        self.0.is_empty()
295    }
296}
297
298impl fmt::Debug for ApiKey {
299    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
300        f.write_str("ApiKey(..)")
301    }
302}
303
304/// Where the model settings in effect come from.
305#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
306#[serde(rename_all = "lowercase")]
307pub enum ModelSource {
308    /// The workgroup's synced `embedding.toml`.
309    Workgroup,
310    /// This host's own `[models]`, because it has no workgroup or the workgroup has none.
311    Device,
312}
313
314/// Why a bridge does not embed, or what it lacks while it does.
315#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
316#[serde(rename_all = "snake_case")]
317pub enum EmbedNote {
318    /// No slot is configured.
319    NotConfigured,
320    /// Every configured slot is switched off on this device.
321    DisabledOnDevice,
322    /// Only the local slot is configured, and auto turned it off: no AVX2.
323    NoAvx2,
324    /// The remote slot is active without an API key (the call is still attempted).
325    KeyMissing,
326    /// The settings could not be read or are invalid.
327    Invalid(String),
328}
329
330impl fmt::Display for EmbedNote {
331    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
332        match self {
333            EmbedNote::NotConfigured => f.write_str("no model is configured"),
334            EmbedNote::DisabledOnDevice => f.write_str("turned off on this device"),
335            EmbedNote::NoAvx2 => f.write_str("the local model needs AVX2, which this CPU lacks"),
336            EmbedNote::KeyMissing => f.write_str("no API key is set for the remote model"),
337            EmbedNote::Invalid(why) => write!(f, "invalid settings: {why}"),
338        }
339    }
340}
341
342/// Result of every `embed.*` settings method: the settings, and what they resolve to.
343#[derive(Clone, Debug, Default, Deserialize, Serialize)]
344pub struct EmbedSettingsResult {
345    /// The model settings in effect.
346    pub models: ModelSettings,
347    /// Where they come from; `None` when there are none.
348    #[serde(default)]
349    pub source: Option<ModelSource>,
350    /// This host's own `[models]` exists but the workgroup's file wins.
351    #[serde(default)]
352    pub shadowed_device_models: bool,
353    /// This device's own settings, as written.
354    #[serde(default)]
355    pub device: DeviceSettings,
356    /// The local slot's effective switch.
357    #[serde(default)]
358    pub local_enabled: bool,
359    /// The remote slot's effective switch.
360    #[serde(default)]
361    pub remote_enabled: bool,
362    /// The slot the bridge serves.
363    #[serde(default)]
364    pub active: Option<Slot>,
365    /// How the bridge holds the local model's weights: this device's `local_qtype`, or the
366    /// default. Absent from bridges before 2.4.
367    #[serde(default, skip_serializing_if = "Option::is_none")]
368    pub local_qtype: Option<QType>,
369    /// Whether this CPU has AVX2 (always true off x86_64).
370    #[serde(default)]
371    pub avx2: bool,
372    /// Whether an API key is stored on this device.
373    #[serde(default)]
374    pub key_set: bool,
375    /// What `embed.info` answers now.
376    #[serde(default)]
377    pub info: EmbedInfoResult,
378}
379
380impl EmbedSettingsResult {
381    /// One slot's effective switch.
382    pub fn enabled(&self, slot: Slot) -> bool {
383        match slot {
384            Slot::Local => self.local_enabled,
385            Slot::Remote => self.remote_enabled,
386        }
387    }
388}
389
390/// Parameters of [`EMBED_MODEL_SET`](crate::EMBED_MODEL_SET).
391#[derive(Clone, Debug, Deserialize, Serialize)]
392pub struct EmbedModelSetParams {
393    /// The slot to set or clear.
394    pub slot: Slot,
395    /// Its new model; `None` clears the slot. Must belong to `slot`.
396    #[serde(default)]
397    pub model: Option<SlotModel>,
398}
399
400/// Parameters of [`EMBED_DEVICE_SET`](crate::EMBED_DEVICE_SET).
401#[derive(Clone, Debug, Deserialize, Serialize)]
402pub struct EmbedDeviceSetParams {
403    /// The slot to switch.
404    pub slot: Slot,
405    /// `None` is auto.
406    #[serde(default)]
407    pub enabled: Option<bool>,
408}
409
410/// Parameters of [`EMBED_KEY_SET`](crate::EMBED_KEY_SET). `Debug` hides the key.
411#[derive(Clone, Debug, Deserialize, Serialize)]
412pub struct EmbedKeySetParams {
413    /// The key.
414    pub key: ApiKey,
415}
416
417/// One settings request, as either CLI builds it from its arguments.
418///
419/// The bridge's CLI carries it out on the files when no bridge is running; both CLIs send
420/// it to a running bridge with [`BridgeClient::embed_request`](crate::BridgeClient::embed_request).
421#[derive(Clone, Debug)]
422pub enum EmbedRequest {
423    /// Read the settings.
424    Show,
425    /// Set or clear a slot's model.
426    ModelSet(EmbedModelSetParams),
427    /// Switch a slot on this device.
428    DeviceSet(EmbedDeviceSetParams),
429    /// Store the API key.
430    KeySet(ApiKey),
431    /// Remove the API key.
432    KeyClear,
433}
434
435/// What `embedding show` prints, one line each.
436pub fn describe(s: &EmbedSettingsResult) -> Vec<String> {
437    let mut lines = Vec::new();
438    lines.push(match s.source {
439        Some(ModelSource::Workgroup) if s.shadowed_device_models => {
440            "models: the workgroup's (this device's own [models] is ignored)".to_owned()
441        }
442        Some(ModelSource::Workgroup) => "models: the workgroup's".to_owned(),
443        Some(ModelSource::Device) => "models: this device's own".to_owned(),
444        None => "models: none".to_owned(),
445    });
446    let switch = |slot: Slot| -> String {
447        let on = if s.enabled(slot) { "on" } else { "off" };
448        match s.device.enabled(slot) {
449            Some(_) => on.to_owned(),
450            None if slot == Slot::Local && !s.avx2 => format!("{on} (auto: no AVX2)"),
451            None => format!("{on} (auto)"),
452        }
453    };
454    lines.push(match &s.models.local {
455        Some(m) => format!(
456            "local:  {} ({} dimensions, {} tokens) — this device: {}, {}",
457            m.model,
458            m.dimension,
459            m.max_tokens,
460            switch(Slot::Local),
461            match (s.device.local_qtype, s.local_qtype) {
462                (Some(q), _) => q.to_string(),
463                (None, Some(q)) => format!("{q} (default)"),
464                (None, None) => format!("{} (default)", QType::default()),
465            }
466        ),
467        None => "local:  not configured".to_owned(),
468    });
469    lines.push(match &s.models.remote {
470        Some(m) => format!(
471            "remote: {} at {} ({} dimensions) — this device: {}, key {}",
472            m.model,
473            m.endpoint,
474            m.dimension,
475            switch(Slot::Remote),
476            if s.key_set { "set" } else { "not set" }
477        ),
478        None => format!(
479            "remote: not configured{}",
480            if s.key_set { " (a key is set)" } else { "" }
481        ),
482    });
483    let state = match (&s.active, &s.info.note) {
484        (Some(slot), None) => format!(
485            "active: {slot}, {}",
486            if s.info.loaded {
487                "loaded"
488            } else {
489                "not loaded"
490            }
491        ),
492        (Some(slot), Some(note)) => format!("active: {slot} — {note}"),
493        (None, Some(note)) => format!("active: none — {note}"),
494        (None, None) => "active: none".to_owned(),
495    };
496    lines.push(state);
497    lines
498}
499
500#[cfg(test)]
501mod tests {
502    use super::*;
503
504    fn remote() -> RemoteModel {
505        RemoteModel {
506            endpoint: "https://api.example.com".into(),
507            model: "m".into(),
508            dimension: 8,
509        }
510    }
511
512    #[test]
513    fn local_defaults_validate() {
514        let s = ModelSettings {
515            local: Some(LocalModel::default()),
516            remote: Some(remote()),
517        };
518        assert_eq!(s.validate(), Ok(()));
519    }
520
521    #[test]
522    fn validation_keeps_the_local_rules() {
523        let mut local = LocalModel {
524            dimension: 4096,
525            ..LocalModel::default()
526        };
527        let check = |l: &LocalModel| {
528            ModelSettings {
529                local: Some(l.clone()),
530                remote: None,
531            }
532            .validate()
533        };
534        assert!(check(&local).is_err());
535        local.dimension = 1024;
536        local.max_tokens = MAX_TOKENS + 1;
537        assert!(check(&local).is_err());
538        local.max_tokens = MAX_TOKENS;
539        assert!(check(&local).is_ok());
540        local.model = "other".into();
541        assert!(check(&local).is_err());
542    }
543
544    #[test]
545    fn qtypes_round_trip_by_name() {
546        for q in QType::ALL {
547            assert_eq!(q.as_str().parse::<QType>(), Ok(q));
548            assert_eq!(
549                serde_json::to_value(q).unwrap(),
550                serde_json::json!(q.as_str())
551            );
552        }
553        assert_eq!(QType::default(), QType::F16);
554        assert!("fp16".parse::<QType>().is_err());
555    }
556
557    #[test]
558    fn validation_checks_the_remote_slot() {
559        let check = |r: RemoteModel| {
560            ModelSettings {
561                local: None,
562                remote: Some(r),
563            }
564            .validate()
565        };
566        assert!(
567            check(RemoteModel {
568                endpoint: "api.example.com".into(),
569                ..remote()
570            })
571            .is_err()
572        );
573        assert!(
574            check(RemoteModel {
575                model: " ".into(),
576                ..remote()
577            })
578            .is_err()
579        );
580        assert!(
581            check(RemoteModel {
582                dimension: 0,
583                ..remote()
584            })
585            .is_err()
586        );
587    }
588
589    #[test]
590    fn the_key_never_prints() {
591        let key = ApiKey::new("sk-secret\n");
592        assert_eq!(key.expose(), "sk-secret");
593        let params = EmbedKeySetParams { key };
594        let shown = format!("{params:?} {:?}", EmbedRequest::KeySet(params.key.clone()));
595        assert!(!shown.contains("sk-secret"), "{shown}");
596        // The wire form carries it, plainly.
597        assert_eq!(
598            serde_json::to_value(&params).unwrap(),
599            serde_json::json!({ "key": "sk-secret" })
600        );
601    }
602
603    #[test]
604    fn slot_models_are_tagged_by_slot() {
605        let v = serde_json::to_value(SlotModel::Remote(remote())).unwrap();
606        assert_eq!(v["remote"]["model"], "m");
607        let back: SlotModel = serde_json::from_value(v).unwrap();
608        assert_eq!(back.slot(), Slot::Remote);
609    }
610
611    #[test]
612    fn an_info_without_a_note_still_parses() {
613        let info: EmbedInfoResult =
614            serde_json::from_value(serde_json::json!({ "enabled": false })).unwrap();
615        assert_eq!(info.note, None);
616    }
617
618    #[test]
619    fn describe_names_the_source_switches_and_state() {
620        let s = EmbedSettingsResult {
621            models: ModelSettings {
622                local: Some(LocalModel::default()),
623                remote: Some(remote()),
624            },
625            source: Some(ModelSource::Workgroup),
626            local_enabled: false,
627            remote_enabled: true,
628            active: Some(Slot::Remote),
629            local_qtype: Some(QType::F16),
630            avx2: false,
631            key_set: false,
632            info: EmbedInfoResult {
633                enabled: true,
634                note: Some(EmbedNote::KeyMissing),
635                ..EmbedInfoResult::default()
636            },
637            ..EmbedSettingsResult::default()
638        };
639        let lines = describe(&s);
640        assert_eq!(lines[0], "models: the workgroup's");
641        assert!(
642            lines[1].ends_with("this device: off (auto: no AVX2), f16 (default)"),
643            "{}",
644            lines[1]
645        );
646        assert!(
647            lines[2].ends_with("this device: on (auto), key not set"),
648            "{}",
649            lines[2]
650        );
651        assert_eq!(
652            lines[3],
653            "active: remote — no API key is set for the remote model"
654        );
655    }
656
657    #[test]
658    fn describe_without_settings() {
659        let s = EmbedSettingsResult {
660            avx2: true,
661            info: EmbedInfoResult {
662                note: Some(EmbedNote::NotConfigured),
663                ..EmbedInfoResult::default()
664            },
665            ..EmbedSettingsResult::default()
666        };
667        assert_eq!(
668            describe(&s),
669            vec![
670                "models: none",
671                "local:  not configured",
672                "remote: not configured",
673                "active: none — no model is configured",
674            ]
675        );
676    }
677}
sapphire_framework_bridge_api/embed.rs

sapphire_framework_bridge_api/
embed.rs
