Skip to main content

sightloom_host/
reference.rs

1//! Deterministic reference models (no weights) implementing `SightLoom` adapters.
2//!
3//! These prove the **end-to-end wiring** (`photo → embed → rank`) without
4//! claiming real re-id accuracy. Replace with ONNX backends in a later step.
5
6use crate::config::{HostBundleConfig, ModelTask};
7use crate::error::HostError;
8use crate::preprocess::{PreprocessConfig, crop_rgb8, prepare_rgb8_nchw};
9use sightloom::core::{ClassId, Detection, FrameStamp, Rect, TrackKey};
10use sightloom::{
11    DetectorAdapter, EmbeddingTask, FrameView, PhotoEmbeddingAdapter, PhotoView, PixelFormat,
12    TrackEmbeddingAdapter,
13};
14
15const REF_DIM: usize = 128;
16
17/// Deterministic person detector: one full-frame box when any non-zero pixel exists.
18#[derive(Clone, Debug, Default)]
19pub struct ReferencePersonDetector {
20    /// Score for the synthetic box.
21    pub score: f32,
22}
23
24impl DetectorAdapter for ReferencePersonDetector {
25    type Error = HostError;
26
27    fn detect(
28        &mut self,
29        _stamp: FrameStamp,
30        frame: &FrameView<'_>,
31    ) -> Result<Vec<Detection>, Self::Error> {
32        if frame.width == 0 || frame.height == 0 {
33            return Ok(Vec::new());
34        }
35        let has_signal = frame.data.iter().any(|&b| b > 0);
36        if !has_signal {
37            return Ok(Vec::new());
38        }
39        // Inset box so multi-frame motion is possible when host shifts content.
40        let margin_x = (frame.width / 10).max(1);
41        let margin_y = (frame.height / 10).max(1);
42        let bbox = Rect::new(
43            margin_x as f32,
44            margin_y as f32,
45            (frame.width - margin_x) as f32,
46            (frame.height - margin_y) as f32,
47        )
48        .map_err(|_| HostError::Runtime("invalid det box".into()))?;
49        let det = Detection::new(bbox, self.score.max(0.5), Some(ClassId(0)), None)
50            .map_err(|_| HostError::Runtime("invalid detection".into()))?;
51        Ok(vec![det])
52    }
53}
54
55/// Face detector stub: smaller centered box when pixels present.
56#[derive(Clone, Debug, Default)]
57pub struct ReferenceFaceDetector;
58
59impl DetectorAdapter for ReferenceFaceDetector {
60    type Error = HostError;
61
62    fn detect(
63        &mut self,
64        _stamp: FrameStamp,
65        frame: &FrameView<'_>,
66    ) -> Result<Vec<Detection>, Self::Error> {
67        if frame.width < 8 || frame.height < 8 {
68            return Ok(Vec::new());
69        }
70        if !frame.data.iter().any(|&b| b > 0) {
71            return Ok(Vec::new());
72        }
73        let cx = frame.width as f32 * 0.5;
74        let cy = frame.height as f32 * 0.35;
75        let hw = frame.width as f32 * 0.15;
76        let hh = frame.height as f32 * 0.18;
77        let bbox = Rect::new(cx - hw, cy - hh, cx + hw, cy + hh)
78            .map_err(|_| HostError::Runtime("invalid face box".into()))?;
79        let det = Detection::new(bbox, 0.85, Some(ClassId(1)), None)
80            .map_err(|_| HostError::Runtime("invalid face det".into()))?;
81        Ok(vec![det])
82    }
83}
84
85/// Deterministic embedding from RGB (or encoded bytes fingerprint).
86#[derive(Clone, Debug)]
87pub struct ReferenceEmbedder {
88    /// Task label.
89    pub task: EmbeddingTask,
90    /// Output dimension.
91    pub dim: usize,
92    /// Preprocess for raw frames.
93    pub preprocess: PreprocessConfig,
94}
95
96impl Default for ReferenceEmbedder {
97    fn default() -> Self {
98        Self::person_reid()
99    }
100}
101
102impl ReferenceEmbedder {
103    /// Person re-id style embedder.
104    #[must_use]
105    pub fn person_reid() -> Self {
106        Self {
107            task: EmbeddingTask::PersonReId,
108            dim: REF_DIM,
109            preprocess: PreprocessConfig::imagenet_like(128, 256),
110        }
111    }
112
113    /// Face embedder.
114    #[must_use]
115    pub fn face() -> Self {
116        Self {
117            task: EmbeddingTask::Face,
118            dim: REF_DIM,
119            preprocess: PreprocessConfig::imagenet_like(112, 112),
120        }
121    }
122
123    fn embed_bytes(&self, bytes: &[u8]) -> Vec<f32> {
124        let mut v = vec![0.0_f32; self.dim.max(8)];
125        let n = v.len();
126        for (i, chunk) in bytes.chunks(4).enumerate() {
127            let mut acc = (i as u32).wrapping_mul(0x9E37_79B9);
128            for &b in chunk {
129                acc = acc.wrapping_mul(16_777_619) ^ u32::from(b);
130            }
131            let idx = (i * 3) % n;
132            let f = (acc as f32) / (u32::MAX as f32) * 2.0 - 1.0;
133            v[idx] += f;
134            v[(idx + 1) % n] += f * 0.5;
135        }
136        // L2 normalize
137        let norm = v.iter().map(|x| x * x).sum::<f32>().sqrt().max(1e-6);
138        for x in &mut v {
139            *x /= norm;
140        }
141        v.truncate(self.dim.max(1));
142        if v.len() < self.dim {
143            v.resize(self.dim, 0.0);
144        }
145        v
146    }
147
148    /// Embed packed RGB8 with known size.
149    ///
150    /// # Errors
151    ///
152    /// Preprocess failures.
153    pub fn embed_rgb8(&self, rgb: &[u8], width: u32, height: u32) -> Result<Vec<f32>, HostError> {
154        let tensor = prepare_rgb8_nchw(rgb, width, height, &self.preprocess)?;
155        // Mix CHW stats into fingerprint so geometry matters.
156        let mut bytes = Vec::with_capacity(64 + tensor.len().min(256));
157        bytes.extend_from_slice(&width.to_le_bytes());
158        bytes.extend_from_slice(&height.to_le_bytes());
159        for (i, x) in tensor.iter().enumerate().take(256) {
160            let q = (x.clamp(-2.0, 2.0) * 64.0) as i8;
161            bytes.push(q as u8);
162            if i % 17 == 0 {
163                bytes.push((i as u8).wrapping_add(3));
164            }
165        }
166        Ok(self.embed_bytes(&bytes))
167    }
168}
169
170impl PhotoEmbeddingAdapter for ReferenceEmbedder {
171    type Error = HostError;
172
173    fn task(&self) -> EmbeddingTask {
174        self.task
175    }
176
177    fn embed_photo(&mut self, photo: &PhotoView<'_>) -> Result<Vec<f32>, Self::Error> {
178        // Prefer full RGB decode (image-decode) when available; fall back to
179        // raw-byte fingerprint for encoded blobs without the feature.
180        if let Ok(decoded) = crate::decode::decode_photo_rgb(photo) {
181            return self.embed_rgb8(&decoded.rgb, decoded.width, decoded.height);
182        }
183        if let Some(enc) = photo.encoded {
184            return Ok(self.embed_bytes(enc));
185        }
186        Err(HostError::Runtime(
187            "PhotoView has neither frame nor encoded bytes".into(),
188        ))
189    }
190}
191
192impl TrackEmbeddingAdapter for ReferenceEmbedder {
193    type Error = HostError;
194
195    fn embed_track(
196        &mut self,
197        _stamp: FrameStamp,
198        frame: &FrameView<'_>,
199        _track_key: TrackKey,
200        bbox: Rect,
201    ) -> Result<Vec<f32>, Self::Error> {
202        let rgb = frame_to_rgb8(frame)?;
203        let left = bbox.left().max(0.0) as u32;
204        let top = bbox.top().max(0.0) as u32;
205        let right = bbox.right().max(0.0).ceil() as u32;
206        let bottom = bbox.bottom().max(0.0).ceil() as u32;
207        let (crop, w, h) = crop_rgb8(&rgb, frame.width, frame.height, left, top, right, bottom)?;
208        self.embed_rgb8(&crop, w, h)
209    }
210}
211
212/// Converts a [`FrameView`] into packed RGB8 (host preprocess helper).
213///
214/// # Errors
215///
216/// Unsupported format / short buffers.
217pub fn frame_to_rgb8(frame: &FrameView<'_>) -> Result<Vec<u8>, HostError> {
218    let w = frame.width as usize;
219    let h = frame.height as usize;
220    let n = w * h;
221    match frame.format {
222        PixelFormat::Rgb8 => {
223            let need = n * 3;
224            if frame.data.len() < need {
225                return Err(HostError::Preprocess("rgb8 short".into()));
226            }
227            Ok(frame.data[..need].to_vec())
228        }
229        PixelFormat::Bgr8 => {
230            let need = n * 3;
231            if frame.data.len() < need {
232                return Err(HostError::Preprocess("bgr8 short".into()));
233            }
234            let mut out = vec![0_u8; need];
235            for i in 0..n {
236                out[i * 3] = frame.data[i * 3 + 2];
237                out[i * 3 + 1] = frame.data[i * 3 + 1];
238                out[i * 3 + 2] = frame.data[i * 3];
239            }
240            Ok(out)
241        }
242        PixelFormat::Gray8 => {
243            if frame.data.len() < n {
244                return Err(HostError::Preprocess("gray short".into()));
245            }
246            let mut out = vec![0_u8; n * 3];
247            for i in 0..n {
248                let g = frame.data[i];
249                out[i * 3] = g;
250                out[i * 3 + 1] = g;
251                out[i * 3 + 2] = g;
252            }
253            Ok(out)
254        }
255        PixelFormat::Rgba8 => {
256            let need = n * 4;
257            if frame.data.len() < need {
258                return Err(HostError::Preprocess("rgba short".into()));
259            }
260            let mut out = vec![0_u8; n * 3];
261            for i in 0..n {
262                out[i * 3] = frame.data[i * 4];
263                out[i * 3 + 1] = frame.data[i * 4 + 1];
264                out[i * 3 + 2] = frame.data[i * 4 + 2];
265            }
266            Ok(out)
267        }
268        PixelFormat::Custom(_) => Err(HostError::Preprocess(
269            "custom pixel format needs host conversion".into(),
270        )),
271    }
272}
273
274/// Bundled reference detectors + embedders for e2e demos.
275#[derive(Clone, Debug)]
276pub struct ReferenceHostModels {
277    /// Bundle config (paths / tasks).
278    pub config: HostBundleConfig,
279    /// Person detector.
280    pub person_detect: ReferencePersonDetector,
281    /// Face detector.
282    pub face_detect: ReferenceFaceDetector,
283    /// Person re-id embedder.
284    pub person_reid: ReferenceEmbedder,
285    /// Face embedder.
286    pub face_embed: ReferenceEmbedder,
287}
288
289impl ReferenceHostModels {
290    /// Builds from config (dims taken from specs when set).
291    #[must_use]
292    pub fn from_config(config: HostBundleConfig) -> Self {
293        let person_dim = config
294            .person_reid
295            .as_ref()
296            .map_or(REF_DIM, |s| s.embedding_dim.max(8));
297        let face_dim = config
298            .face_embed
299            .as_ref()
300            .map_or(REF_DIM, |s| s.embedding_dim.max(8));
301        let mut person_reid = ReferenceEmbedder::person_reid();
302        person_reid.dim = person_dim;
303        if let Some(spec) = &config.person_reid {
304            person_reid.preprocess = spec.preprocess.clone();
305        }
306        let mut face_embed = ReferenceEmbedder::face();
307        face_embed.dim = face_dim;
308        if let Some(spec) = &config.face_embed {
309            face_embed.preprocess = spec.preprocess.clone();
310        }
311        Self {
312            config,
313            person_detect: ReferencePersonDetector { score: 0.9 },
314            face_detect: ReferenceFaceDetector,
315            person_reid,
316            face_embed,
317        }
318    }
319
320    /// Default reference bundle.
321    #[must_use]
322    pub fn new() -> Self {
323        Self::from_config(HostBundleConfig::default())
324    }
325
326    /// Lists model tasks declared in config (for diagnostics).
327    #[must_use]
328    pub fn declared_tasks(&self) -> Vec<ModelTask> {
329        self.config
330            .all_specs()
331            .into_iter()
332            .map(|s| s.task)
333            .collect()
334    }
335}
336
337impl Default for ReferenceHostModels {
338    fn default() -> Self {
339        Self::new()
340    }
341}