1use crate::config::{HostBundleConfig, ModelTask};
7use crate::error::HostError;
8use crate::preprocess::{PreprocessConfig, crop_rgb8, prepare_rgb8_nchw};
9use sightloom::core::{ClassId, Detection, FrameStamp, Rect, TrackKey};
10use sightloom::{
11 DetectorAdapter, EmbeddingTask, FrameView, PhotoEmbeddingAdapter, PhotoView, PixelFormat,
12 TrackEmbeddingAdapter,
13};
14
15const REF_DIM: usize = 128;
16
17#[derive(Clone, Debug, Default)]
19pub struct ReferencePersonDetector {
20 pub score: f32,
22}
23
24impl DetectorAdapter for ReferencePersonDetector {
25 type Error = HostError;
26
27 fn detect(
28 &mut self,
29 _stamp: FrameStamp,
30 frame: &FrameView<'_>,
31 ) -> Result<Vec<Detection>, Self::Error> {
32 if frame.width == 0 || frame.height == 0 {
33 return Ok(Vec::new());
34 }
35 let has_signal = frame.data.iter().any(|&b| b > 0);
36 if !has_signal {
37 return Ok(Vec::new());
38 }
39 let margin_x = (frame.width / 10).max(1);
41 let margin_y = (frame.height / 10).max(1);
42 let bbox = Rect::new(
43 margin_x as f32,
44 margin_y as f32,
45 (frame.width - margin_x) as f32,
46 (frame.height - margin_y) as f32,
47 )
48 .map_err(|_| HostError::Runtime("invalid det box".into()))?;
49 let det = Detection::new(bbox, self.score.max(0.5), Some(ClassId(0)), None)
50 .map_err(|_| HostError::Runtime("invalid detection".into()))?;
51 Ok(vec![det])
52 }
53}
54
55#[derive(Clone, Debug, Default)]
57pub struct ReferenceFaceDetector;
58
59impl DetectorAdapter for ReferenceFaceDetector {
60 type Error = HostError;
61
62 fn detect(
63 &mut self,
64 _stamp: FrameStamp,
65 frame: &FrameView<'_>,
66 ) -> Result<Vec<Detection>, Self::Error> {
67 if frame.width < 8 || frame.height < 8 {
68 return Ok(Vec::new());
69 }
70 if !frame.data.iter().any(|&b| b > 0) {
71 return Ok(Vec::new());
72 }
73 let cx = frame.width as f32 * 0.5;
74 let cy = frame.height as f32 * 0.35;
75 let hw = frame.width as f32 * 0.15;
76 let hh = frame.height as f32 * 0.18;
77 let bbox = Rect::new(cx - hw, cy - hh, cx + hw, cy + hh)
78 .map_err(|_| HostError::Runtime("invalid face box".into()))?;
79 let det = Detection::new(bbox, 0.85, Some(ClassId(1)), None)
80 .map_err(|_| HostError::Runtime("invalid face det".into()))?;
81 Ok(vec![det])
82 }
83}
84
85#[derive(Clone, Debug)]
87pub struct ReferenceEmbedder {
88 pub task: EmbeddingTask,
90 pub dim: usize,
92 pub preprocess: PreprocessConfig,
94}
95
96impl Default for ReferenceEmbedder {
97 fn default() -> Self {
98 Self::person_reid()
99 }
100}
101
102impl ReferenceEmbedder {
103 #[must_use]
105 pub fn person_reid() -> Self {
106 Self {
107 task: EmbeddingTask::PersonReId,
108 dim: REF_DIM,
109 preprocess: PreprocessConfig::imagenet_like(128, 256),
110 }
111 }
112
113 #[must_use]
115 pub fn face() -> Self {
116 Self {
117 task: EmbeddingTask::Face,
118 dim: REF_DIM,
119 preprocess: PreprocessConfig::imagenet_like(112, 112),
120 }
121 }
122
123 fn embed_bytes(&self, bytes: &[u8]) -> Vec<f32> {
124 let mut v = vec![0.0_f32; self.dim.max(8)];
125 let n = v.len();
126 for (i, chunk) in bytes.chunks(4).enumerate() {
127 let mut acc = (i as u32).wrapping_mul(0x9E37_79B9);
128 for &b in chunk {
129 acc = acc.wrapping_mul(16_777_619) ^ u32::from(b);
130 }
131 let idx = (i * 3) % n;
132 let f = (acc as f32) / (u32::MAX as f32) * 2.0 - 1.0;
133 v[idx] += f;
134 v[(idx + 1) % n] += f * 0.5;
135 }
136 let norm = v.iter().map(|x| x * x).sum::<f32>().sqrt().max(1e-6);
138 for x in &mut v {
139 *x /= norm;
140 }
141 v.truncate(self.dim.max(1));
142 if v.len() < self.dim {
143 v.resize(self.dim, 0.0);
144 }
145 v
146 }
147
148 pub fn embed_rgb8(&self, rgb: &[u8], width: u32, height: u32) -> Result<Vec<f32>, HostError> {
154 let tensor = prepare_rgb8_nchw(rgb, width, height, &self.preprocess)?;
155 let mut bytes = Vec::with_capacity(64 + tensor.len().min(256));
157 bytes.extend_from_slice(&width.to_le_bytes());
158 bytes.extend_from_slice(&height.to_le_bytes());
159 for (i, x) in tensor.iter().enumerate().take(256) {
160 let q = (x.clamp(-2.0, 2.0) * 64.0) as i8;
161 bytes.push(q as u8);
162 if i % 17 == 0 {
163 bytes.push((i as u8).wrapping_add(3));
164 }
165 }
166 Ok(self.embed_bytes(&bytes))
167 }
168}
169
170impl PhotoEmbeddingAdapter for ReferenceEmbedder {
171 type Error = HostError;
172
173 fn task(&self) -> EmbeddingTask {
174 self.task
175 }
176
177 fn embed_photo(&mut self, photo: &PhotoView<'_>) -> Result<Vec<f32>, Self::Error> {
178 if let Ok(decoded) = crate::decode::decode_photo_rgb(photo) {
181 return self.embed_rgb8(&decoded.rgb, decoded.width, decoded.height);
182 }
183 if let Some(enc) = photo.encoded {
184 return Ok(self.embed_bytes(enc));
185 }
186 Err(HostError::Runtime(
187 "PhotoView has neither frame nor encoded bytes".into(),
188 ))
189 }
190}
191
192impl TrackEmbeddingAdapter for ReferenceEmbedder {
193 type Error = HostError;
194
195 fn embed_track(
196 &mut self,
197 _stamp: FrameStamp,
198 frame: &FrameView<'_>,
199 _track_key: TrackKey,
200 bbox: Rect,
201 ) -> Result<Vec<f32>, Self::Error> {
202 let rgb = frame_to_rgb8(frame)?;
203 let left = bbox.left().max(0.0) as u32;
204 let top = bbox.top().max(0.0) as u32;
205 let right = bbox.right().max(0.0).ceil() as u32;
206 let bottom = bbox.bottom().max(0.0).ceil() as u32;
207 let (crop, w, h) = crop_rgb8(&rgb, frame.width, frame.height, left, top, right, bottom)?;
208 self.embed_rgb8(&crop, w, h)
209 }
210}
211
212pub fn frame_to_rgb8(frame: &FrameView<'_>) -> Result<Vec<u8>, HostError> {
218 let w = frame.width as usize;
219 let h = frame.height as usize;
220 let n = w * h;
221 match frame.format {
222 PixelFormat::Rgb8 => {
223 let need = n * 3;
224 if frame.data.len() < need {
225 return Err(HostError::Preprocess("rgb8 short".into()));
226 }
227 Ok(frame.data[..need].to_vec())
228 }
229 PixelFormat::Bgr8 => {
230 let need = n * 3;
231 if frame.data.len() < need {
232 return Err(HostError::Preprocess("bgr8 short".into()));
233 }
234 let mut out = vec![0_u8; need];
235 for i in 0..n {
236 out[i * 3] = frame.data[i * 3 + 2];
237 out[i * 3 + 1] = frame.data[i * 3 + 1];
238 out[i * 3 + 2] = frame.data[i * 3];
239 }
240 Ok(out)
241 }
242 PixelFormat::Gray8 => {
243 if frame.data.len() < n {
244 return Err(HostError::Preprocess("gray short".into()));
245 }
246 let mut out = vec![0_u8; n * 3];
247 for i in 0..n {
248 let g = frame.data[i];
249 out[i * 3] = g;
250 out[i * 3 + 1] = g;
251 out[i * 3 + 2] = g;
252 }
253 Ok(out)
254 }
255 PixelFormat::Rgba8 => {
256 let need = n * 4;
257 if frame.data.len() < need {
258 return Err(HostError::Preprocess("rgba short".into()));
259 }
260 let mut out = vec![0_u8; n * 3];
261 for i in 0..n {
262 out[i * 3] = frame.data[i * 4];
263 out[i * 3 + 1] = frame.data[i * 4 + 1];
264 out[i * 3 + 2] = frame.data[i * 4 + 2];
265 }
266 Ok(out)
267 }
268 PixelFormat::Custom(_) => Err(HostError::Preprocess(
269 "custom pixel format needs host conversion".into(),
270 )),
271 }
272}
273
274#[derive(Clone, Debug)]
276pub struct ReferenceHostModels {
277 pub config: HostBundleConfig,
279 pub person_detect: ReferencePersonDetector,
281 pub face_detect: ReferenceFaceDetector,
283 pub person_reid: ReferenceEmbedder,
285 pub face_embed: ReferenceEmbedder,
287}
288
289impl ReferenceHostModels {
290 #[must_use]
292 pub fn from_config(config: HostBundleConfig) -> Self {
293 let person_dim = config
294 .person_reid
295 .as_ref()
296 .map_or(REF_DIM, |s| s.embedding_dim.max(8));
297 let face_dim = config
298 .face_embed
299 .as_ref()
300 .map_or(REF_DIM, |s| s.embedding_dim.max(8));
301 let mut person_reid = ReferenceEmbedder::person_reid();
302 person_reid.dim = person_dim;
303 if let Some(spec) = &config.person_reid {
304 person_reid.preprocess = spec.preprocess.clone();
305 }
306 let mut face_embed = ReferenceEmbedder::face();
307 face_embed.dim = face_dim;
308 if let Some(spec) = &config.face_embed {
309 face_embed.preprocess = spec.preprocess.clone();
310 }
311 Self {
312 config,
313 person_detect: ReferencePersonDetector { score: 0.9 },
314 face_detect: ReferenceFaceDetector,
315 person_reid,
316 face_embed,
317 }
318 }
319
320 #[must_use]
322 pub fn new() -> Self {
323 Self::from_config(HostBundleConfig::default())
324 }
325
326 #[must_use]
328 pub fn declared_tasks(&self) -> Vec<ModelTask> {
329 self.config
330 .all_specs()
331 .into_iter()
332 .map(|s| s.task)
333 .collect()
334 }
335}
336
337impl Default for ReferenceHostModels {
338 fn default() -> Self {
339 Self::new()
340 }
341}