g2g_core/meta.rs
1//! Per-frame metadata system: typed blobs that travel with a [`Frame`] (the
2//! GstMeta / GstAnalyticsRelationMeta analog), and the `AnalyticsMeta` relation
3//! graph for ML detection / classification / tracking results.
4//!
5//! Gated behind the `metadata` cargo feature. When **off** (the default, and the
6//! only configuration the `no_std` / Cortex-M baseline uses) [`FrameMetaSet`] is
7//! a zero-sized unit: the `Frame::meta` field exists for API stability but costs
8//! nothing per frame. When **on** it is a list of typed [`FrameMeta`] trait
9//! objects with attach / typed-get / iterate / propagate, and the standard
10//! [`AnalyticsMeta`] is available for detection pipelines.
11//!
12//! **Why now:** the field was reserved at M88; the trait body and the relation
13//! graph land with the first metadata-producing element (a YOLO-style detection
14//! postprocess), so a real client shapes the API rather than speculation.
15//!
16//! [`Frame`]: crate::frame::Frame
17
18/// One of the eight ways a picture can be turned without resampling it: the
19/// four quarter rotations and the four mirrors (the dihedral group of the
20/// square). Named after GStreamer's `videoflip` methods, and the vocabulary
21/// [`OrientationMeta`] carries.
22///
23/// Not gated on the `metadata` feature: the enum is what `videoflip` runs on
24/// either way. Only the meta wrapper needs the feature.
25#[derive(Debug, Clone, Copy, PartialEq, Eq)]
26pub enum Orientation {
27 /// No turn (GStreamer `none`).
28 Identity,
29 /// A quarter turn clockwise (`clockwise`).
30 Rotate90Cw,
31 /// A half turn (`rotate-180`).
32 Rotate180,
33 /// A quarter turn counter-clockwise (`counterclockwise`).
34 Rotate90Ccw,
35 /// Mirrored left to right, i.e. reflected in the vertical axis
36 /// (`horizontal-flip`).
37 HorizontalMirror,
38 /// Mirrored top to bottom, i.e. reflected in the horizontal axis
39 /// (`vertical-flip`).
40 VerticalMirror,
41 /// Reflected in the diagonal from the upper-left corner
42 /// (`upper-left-diagonal`): the pixel at `(x, y)` reads as `(y, x)`.
43 Transpose,
44 /// Reflected in the diagonal from the upper-right corner
45 /// (`upper-right-diagonal`).
46 Transverse,
47}
48
49impl Orientation {
50 /// This orientation as `(quarter turns clockwise, mirrored first)`: every
51 /// member is a horizontal mirror (or not) followed by a rotation, which is
52 /// what makes [`compose`](Self::compose) a few lines of arithmetic instead
53 /// of a 64-entry table.
54 const fn parts(self) -> (u8, bool) {
55 match self {
56 Orientation::Identity => (0, false),
57 Orientation::Rotate90Cw => (1, false),
58 Orientation::Rotate180 => (2, false),
59 Orientation::Rotate90Ccw => (3, false),
60 Orientation::HorizontalMirror => (0, true),
61 Orientation::Transverse => (1, true),
62 Orientation::VerticalMirror => (2, true),
63 Orientation::Transpose => (3, true),
64 }
65 }
66
67 /// Inverse of [`parts`](Self::parts). `quarter_turns` must already be
68 /// reduced mod 4.
69 const fn from_parts(quarter_turns: u8, mirrored: bool) -> Orientation {
70 match (quarter_turns, mirrored) {
71 (0, false) => Orientation::Identity,
72 (1, false) => Orientation::Rotate90Cw,
73 (2, false) => Orientation::Rotate180,
74 (3, false) => Orientation::Rotate90Ccw,
75 (0, true) => Orientation::HorizontalMirror,
76 (1, true) => Orientation::Transverse,
77 (2, true) => Orientation::VerticalMirror,
78 _ => Orientation::Transpose,
79 }
80 }
81
82 /// Whether this orientation exchanges width and height. True for the two
83 /// quarter rotations and the two diagonal mirrors.
84 pub const fn swaps_dims(self) -> bool {
85 self.parts().0 % 2 == 1
86 }
87
88 /// The single orientation that does what applying `self` and then `then`
89 /// does.
90 pub const fn compose(self, then: Orientation) -> Orientation {
91 let (turns, mirrored) = self.parts();
92 let (then_turns, then_mirrored) = then.parts();
93 // A mirror reverses the sense of any rotation it is applied after, so
94 // pushing `self`'s rotation through `then`'s mirror negates it.
95 let carried = if then_mirrored { 4 - turns } else { turns };
96 Orientation::from_parts((then_turns + carried) % 4, mirrored != then_mirrored)
97 }
98
99 /// The orientation that undoes this one.
100 pub const fn inverse(self) -> Orientation {
101 let (turns, mirrored) = self.parts();
102 if mirrored {
103 // Every mirror is its own inverse.
104 self
105 } else {
106 Orientation::from_parts((4 - turns) % 4, false)
107 }
108 }
109}
110
111// ---- feature off: the zero-sized placeholder ----
112
113/// Per-frame attachable metadata set (feature `metadata` **off**): a zero-sized
114/// unit, so the baseline pays nothing. See the module docs.
115#[cfg(not(feature = "metadata"))]
116#[derive(Debug, Clone, Default, PartialEq, Eq)]
117pub struct FrameMetaSet;
118
119#[cfg(not(feature = "metadata"))]
120impl FrameMetaSet {
121 /// An empty metadata set. `const` so frame construction stays trivial.
122 #[inline]
123 pub const fn new() -> Self {
124 FrameMetaSet
125 }
126}
127
128/// The metadata types a downstream element asks its producers to attach
129/// (feature `metadata` **off**): a zero-sized always-empty set. The plumbing
130/// that carries it ([`AllocationParams`](crate::AllocationParams)) compiles
131/// either way; `request` / `wants` exist only with the feature on.
132#[cfg(not(feature = "metadata"))]
133#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
134pub struct MetaRequests;
135
136#[cfg(not(feature = "metadata"))]
137impl MetaRequests {
138 /// An empty request set.
139 #[inline]
140 pub const fn new() -> Self {
141 MetaRequests
142 }
143
144 /// Always true: without the feature nothing can be requested.
145 #[inline]
146 pub fn is_empty(&self) -> bool {
147 true
148 }
149
150 /// The demand two sibling consumers put on their shared producer, i.e. still
151 /// nothing.
152 #[inline]
153 pub fn join_branches(self, _other: Self) -> Self {
154 self
155 }
156
157 /// The demand this element passes to its producer, i.e. still nothing.
158 #[inline]
159 pub fn carry_upstream(self, _downstream: Self) -> Self {
160 self
161 }
162}
163
164// ---- feature on: the real typed container + analytics graph ----
165
166#[cfg(feature = "metadata")]
167pub use on::*;
168
169#[cfg(feature = "metadata")]
170mod on {
171 use alloc::boxed::Box;
172 use alloc::string::String;
173 use alloc::sync::Arc;
174 use alloc::vec::Vec;
175 use core::any::{Any, TypeId};
176
177 /// How a piece of metadata survives a transform, the GstMeta
178 /// `transform_func` analog. Reported by [`FrameMeta::propagate`].
179 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
180 pub enum Transform {
181 /// A deep copy (e.g. a tee branch clone): meta is duplicated.
182 Copy,
183 /// A geometry resample (videoscale / compositor pad scale).
184 Scale,
185 /// A spatial crop (videocrop).
186 Crop,
187 /// A re-encode to a compressed codec: pixel-derived meta is lost.
188 Encode,
189 }
190
191 /// Whether a meta is kept through a [`Transform`] or dropped.
192 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
193 pub enum Propagation {
194 Keep,
195 Drop,
196 }
197
198 /// A typed, per-frame, attachable piece of metadata (the `GstMeta` analog).
199 ///
200 /// `as_any` enables typed retrieval via downcast (trait upcasting is not on
201 /// the MSRV); `propagate` is the per-transform survival policy. Meta is
202 /// `Send + Sync` so a frame crosses a multi-thread runtime.
203 pub trait FrameMeta: core::fmt::Debug + Send + Sync {
204 fn as_any(&self) -> &dyn Any;
205 fn as_any_mut(&mut self) -> &mut dyn Any;
206 /// A boxed deep copy of this meta, the GstMeta `copy_func` analog. Backs
207 /// the copy-on-write of a shared meta when a tee branch mutates it (see
208 /// [`FrameMetaSet::get_mut`]); each `FrameMeta` impl is its own concrete
209 /// type, so the duplication can only be expressed on the trait.
210 fn clone_box(&self) -> Box<dyn FrameMeta>;
211 /// How this meta survives `transform`. Default keeps it through
212 /// everything; override to drop on transforms that invalidate it.
213 fn propagate(&self, _transform: Transform) -> Propagation {
214 Propagation::Keep
215 }
216 }
217
218 /// A list of typed [`FrameMeta`] attached to a frame. Empty (no allocation)
219 /// on a freshly constructed frame.
220 ///
221 /// Each entry is an [`Arc`] so a fan-out (tee) clone shares the metadata by
222 /// refcount instead of deep-copying it (cheap, and the analytics graph is
223 /// identical on both branches). A branch that mutates one entry pays a
224 /// copy-on-write deep copy only then (see [`get_mut`](Self::get_mut)), so the
225 /// other branch never observes the change.
226 #[derive(Debug, Default, Clone)]
227 pub struct FrameMetaSet(Vec<Arc<dyn FrameMeta>>);
228
229 impl FrameMetaSet {
230 /// An empty metadata set with no backing allocation.
231 #[inline]
232 pub fn new() -> Self {
233 FrameMetaSet(Vec::new())
234 }
235
236 /// Attach one piece of metadata, replacing any entry of the same type
237 /// (in place, so the order of the other entries is unchanged).
238 ///
239 /// A set holds at most one meta per type: [`get`](Self::get) /
240 /// [`get_mut`](Self::get_mut) key by type, so a second entry of a type
241 /// would be unreachable, and an empty one already on the frame would
242 /// hide what an element attaches later.
243 pub fn attach<T: FrameMeta + 'static>(&mut self, meta: T) {
244 match self.0.iter().position(|m| m.as_any().is::<T>()) {
245 Some(idx) => self.0[idx] = Arc::new(meta),
246 None => self.0.push(Arc::new(meta)),
247 }
248 }
249
250 /// The attached meta of type `T`, if any. At most one is ever attached
251 /// (see [`attach`](Self::attach)).
252 pub fn get<T: FrameMeta + 'static>(&self) -> Option<&T> {
253 self.0.iter().find_map(|m| m.as_any().downcast_ref::<T>())
254 }
255
256 /// Mutable access to the attached meta of type `T`, if any (at most one
257 /// is ever attached, see [`attach`](Self::attach)).
258 ///
259 /// Copy-on-write: if the entry is shared with another frame (a tee
260 /// branch holds the same [`Arc`]), it is first deep-copied via
261 /// [`FrameMeta::clone_box`] so this mutation stays private to this frame.
262 /// When the entry is uniquely owned no copy is made.
263 pub fn get_mut<T: FrameMeta + 'static>(&mut self) -> Option<&mut T> {
264 let idx = self.0.iter().position(|m| m.as_any().is::<T>())?;
265 // Ensure unique ownership before handing out a mutable reference.
266 if Arc::get_mut(&mut self.0[idx]).is_none() {
267 self.0[idx] = Arc::from(self.0[idx].clone_box());
268 }
269 Arc::get_mut(&mut self.0[idx])
270 .expect("entry is unique after the COW above")
271 .as_any_mut()
272 .downcast_mut::<T>()
273 }
274
275 /// Iterate every attached meta as a trait object.
276 pub fn iter(&self) -> impl Iterator<Item = &dyn FrameMeta> {
277 self.0.iter().map(|m| m.as_ref())
278 }
279
280 pub fn len(&self) -> usize {
281 self.0.len()
282 }
283
284 pub fn is_empty(&self) -> bool {
285 self.0.is_empty()
286 }
287
288 /// Apply a [`Transform`]: retain only metas whose `propagate` returns
289 /// [`Propagation::Keep`]. An element that resamples / re-encodes calls
290 /// this so stale meta never rides a frame it no longer describes.
291 pub fn propagate(&mut self, transform: Transform) {
292 self.0
293 .retain(|m| m.propagate(transform) == Propagation::Keep);
294 }
295 }
296
297 /// How many distinct meta types one [`MetaRequests`] carries. Requests past
298 /// this are dropped, which costs an optimization, never correctness: a
299 /// producer that sees no request just produces what it always did.
300 pub const MAX_META_REQUESTS: usize = 4;
301
302 /// What one request needs of the *other* consumers reading the same frames,
303 /// which decides how it survives a fan-out or an intermediate hop.
304 ///
305 /// `Ord` ranks [`EveryConsumer`](Self::EveryConsumer) above
306 /// [`AnyConsumer`](Self::AnyConsumer): when two elements request one meta
307 /// under different policies the stricter one stands, since it is the one
308 /// that can be misread.
309 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
310 pub enum RequestPolicy {
311 /// One asking consumer is enough. Attaching the meta costs a consumer
312 /// that did not ask nothing: it reads the same frame it always did and
313 /// ignores the extra ([`AnalyticsMeta`], [`CaptionMeta`],
314 /// [`TimecodeMeta`]).
315 AnyConsumer,
316 /// Every consumer must ask. Honouring the request changes the *buffer*,
317 /// so a consumer that did not ask would misread it: a frame whose rows
318 /// were left padded, read as tightly packed, is corruption rather than a
319 /// missed optimization.
320 EveryConsumer,
321 }
322
323 /// The metadata types a downstream element wants attached to the frames it
324 /// receives, each keyed by [`TypeId`] and carrying its [`RequestPolicy`].
325 /// The pull half of the metadata system (the GStreamer allocation-query
326 /// `add_meta` analog): a consumer declares its requests from
327 /// [`AsyncElement::meta_requests`](crate::AsyncElement::meta_requests), the
328 /// runner carries them up the allocation cascade on
329 /// [`AllocationParams`](crate::AllocationParams), and a producer asks
330 /// [`wants`](Self::wants) when it configures, so optional metadata is
331 /// produced only where somebody reads it.
332 ///
333 /// A small fixed-capacity set, so it rides the `Copy` allocation params
334 /// without an allocation. Entries are kept sorted, so two sets built in
335 /// different orders compare equal (the cascade suppresses a re-propose when
336 /// the params are unchanged).
337 #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
338 pub struct MetaRequests {
339 entries: [Option<(TypeId, RequestPolicy)>; MAX_META_REQUESTS],
340 }
341
342 impl MetaRequests {
343 /// An empty request set: the element wants no optional metadata.
344 pub const fn new() -> Self {
345 MetaRequests {
346 entries: [None; MAX_META_REQUESTS],
347 }
348 }
349
350 /// This set plus a request for meta `T` that one consumer asking is
351 /// enough for ([`RequestPolicy::AnyConsumer`]). Builder form, since a
352 /// request set is usually written inline:
353 /// `MetaRequests::new().request::<AnalyticsMeta>()`.
354 pub fn request<T: FrameMeta + 'static>(self) -> Self {
355 self.with(TypeId::of::<T>(), RequestPolicy::AnyConsumer)
356 }
357
358 /// This set plus a request for meta `T` that is only honoured when every
359 /// consumer sharing the producer asks for it too
360 /// ([`RequestPolicy::EveryConsumer`]). For a meta whose presence changes
361 /// the buffer, which a consumer that did not ask would misread.
362 pub fn request_from_every_consumer<T: FrameMeta + 'static>(self) -> Self {
363 self.with(TypeId::of::<T>(), RequestPolicy::EveryConsumer)
364 }
365
366 /// Whether meta `T` was requested, under either policy.
367 pub fn wants<T: FrameMeta + 'static>(&self) -> bool {
368 self.policy_of(TypeId::of::<T>()).is_some()
369 }
370
371 /// The policy meta `T` was requested under, `None` if it was not.
372 pub fn policy<T: FrameMeta + 'static>(&self) -> Option<RequestPolicy> {
373 self.policy_of(TypeId::of::<T>())
374 }
375
376 /// The demand two *sibling* consumers put on the one producer they share
377 /// (the branches of a tee). An [`AnyConsumer`](RequestPolicy::AnyConsumer)
378 /// request survives from either side; an
379 /// [`EveryConsumer`](RequestPolicy::EveryConsumer) one only when the
380 /// other branch asks for that meta too, so a branch that would misread
381 /// the changed buffer vetoes it.
382 pub fn join_branches(self, other: Self) -> Self {
383 let mut out = Self::new();
384 for (id, policy) in self.iter().chain(other.iter()) {
385 if policy == RequestPolicy::AnyConsumer
386 || (self.policy_of(id).is_some() && other.policy_of(id).is_some())
387 {
388 out = out.with(id, policy);
389 }
390 }
391 out
392 }
393
394 /// The demand this element (`self`, its own requests) passes on to its
395 /// producer, given what arrived from `downstream`. Its own requests
396 /// always travel: it reads the producer's frames itself. A downstream
397 /// [`EveryConsumer`](RequestPolicy::EveryConsumer) request travels only
398 /// when this element asks for that meta too, since the producer's frames
399 /// pass through here first and a hop that cannot read the changed buffer
400 /// vetoes it just as a sibling branch does.
401 pub fn carry_upstream(self, downstream: Self) -> Self {
402 let mut out = self;
403 for (id, policy) in downstream.iter() {
404 if policy == RequestPolicy::AnyConsumer || self.policy_of(id).is_some() {
405 out = out.with(id, policy);
406 }
407 }
408 out
409 }
410
411 pub fn is_empty(&self) -> bool {
412 self.entries[0].is_none()
413 }
414
415 pub fn len(&self) -> usize {
416 self.entries.iter().flatten().count()
417 }
418
419 fn iter(&self) -> impl Iterator<Item = (TypeId, RequestPolicy)> + '_ {
420 self.entries.iter().flatten().copied()
421 }
422
423 fn policy_of(&self, id: TypeId) -> Option<RequestPolicy> {
424 self.iter().find(|(i, _)| *i == id).map(|(_, p)| p)
425 }
426
427 fn with(mut self, id: TypeId, policy: RequestPolicy) -> Self {
428 let mut free = MAX_META_REQUESTS;
429 for (i, slot) in self.entries.iter_mut().enumerate() {
430 match slot {
431 Some((present, held)) if *present == id => {
432 // Two elements asking for one meta under different
433 // policies: the stricter one is the one that can be
434 // misread, so it stands.
435 *held = (*held).max(policy);
436 return self;
437 }
438 Some(_) => {}
439 None => {
440 free = i;
441 break;
442 }
443 }
444 }
445 if free == MAX_META_REQUESTS {
446 return self;
447 }
448 self.entries[free] = Some((id, policy));
449 // The occupied prefix is packed at the front, so sorting it keeps it
450 // packed and makes the set order-independent under `PartialEq`.
451 self.entries[..=free].sort_unstable();
452 self
453 }
454 }
455
456 /// A normalized bounding box: all fields in `[0, 1]` relative to the frame,
457 /// `(x, y)` the top-left corner and `(w, h)` the size. Normalized so a box
458 /// survives a downstream scale / crop without a coordinate rewrite.
459 #[derive(Debug, Clone, Copy, PartialEq)]
460 pub struct BBox {
461 pub x: f32,
462 pub y: f32,
463 pub w: f32,
464 pub h: f32,
465 }
466
467 impl BBox {
468 /// Intersection-over-union with `other`, the NMS overlap metric.
469 pub fn iou(&self, other: &BBox) -> f32 {
470 let ix0 = self.x.max(other.x);
471 let iy0 = self.y.max(other.y);
472 let ix1 = (self.x + self.w).min(other.x + other.w);
473 let iy1 = (self.y + self.h).min(other.y + other.h);
474 let iw = (ix1 - ix0).max(0.0);
475 let ih = (iy1 - iy0).max(0.0);
476 let inter = iw * ih;
477 let union = self.w * self.h + other.w * other.h - inter;
478 if union <= 0.0 {
479 0.0
480 } else {
481 inter / union
482 }
483 }
484 }
485
486 /// A detected object: its box, class label index, and confidence `[0, 1]`.
487 #[derive(Debug, Clone, Copy, PartialEq)]
488 pub struct ObjectDetection {
489 pub bbox: BBox,
490 pub label: u32,
491 pub confidence: f32,
492 }
493
494 /// A whole-region or per-detection classification result.
495 #[derive(Debug, Clone, Copy, PartialEq)]
496 pub struct Classification {
497 pub label: u32,
498 pub confidence: f32,
499 }
500
501 /// A persistent tracking identity across frames.
502 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
503 pub struct Tracking {
504 pub object_id: u64,
505 }
506
507 /// A per-pixel coverage mask: `width` x `height` 8-bit samples with `stride`
508 /// bytes per row (0 = not covered, 255 = fully covered). Its own grid, not
509 /// the frame's, so it stays valid when the frame is scaled.
510 #[derive(Debug, Clone, PartialEq, Eq)]
511 pub struct Mask {
512 width: u32,
513 height: u32,
514 stride: u32,
515 data: Vec<u8>,
516 }
517
518 impl Mask {
519 /// Build a mask over `data`, or `None` if the geometry does not fit it.
520 /// Dimensions reach this from a model output or a wire peer, so they are
521 /// checked here once and the accessors can then index without guessing.
522 pub fn new(width: u32, height: u32, stride: u32, data: Vec<u8>) -> Option<Self> {
523 if stride < width {
524 return None;
525 }
526 let needed = (stride as u64).checked_mul(height as u64)?;
527 if needed > data.len() as u64 {
528 return None;
529 }
530 Some(Mask {
531 width,
532 height,
533 stride,
534 data,
535 })
536 }
537
538 pub fn width(&self) -> u32 {
539 self.width
540 }
541 pub fn height(&self) -> u32 {
542 self.height
543 }
544 pub fn stride(&self) -> u32 {
545 self.stride
546 }
547 pub fn data(&self) -> &[u8] {
548 &self.data
549 }
550
551 /// Coverage at `(x, y)`, `None` outside the mask.
552 pub fn sample(&self, x: u32, y: u32) -> Option<u8> {
553 if x >= self.width || y >= self.height {
554 return None;
555 }
556 let idx = y as usize * self.stride as usize + x as usize;
557 self.data.get(idx).copied()
558 }
559 }
560
561 /// An instance segmentation: the object's normalized box, its class, and the
562 /// coverage mask over that box (the mask grid is the model's own resolution,
563 /// not the frame's).
564 #[derive(Debug, Clone, PartialEq)]
565 pub struct Segmentation {
566 pub bbox: BBox,
567 pub label: u32,
568 pub confidence: f32,
569 pub mask: Mask,
570 }
571
572 /// A region of interest: a normalized rectangle an encoder, a tracker, or a
573 /// downstream analytic should treat specially (the
574 /// `GstVideoRegionOfInterestMeta` analog). `id` names this region across
575 /// frames; `label` is its class index, as on a detection.
576 #[derive(Debug, Clone, Copy, PartialEq)]
577 pub struct Roi {
578 pub bbox: BBox,
579 pub id: u32,
580 pub label: u32,
581 }
582
583 /// A node in the [`AnalyticsMeta`] relation graph.
584 #[derive(Debug, Clone, PartialEq)]
585 pub enum AnalyticsNode {
586 Detection(ObjectDetection),
587 Classification(Classification),
588 Tracking(Tracking),
589 Segmentation(Segmentation),
590 Roi(Roi),
591 }
592
593 /// The kind of a directed edge between two analytics nodes.
594 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
595 pub enum RelationKind {
596 /// A detection has-a classification (detection -> classification).
597 Classifies,
598 /// A detection has-a tracking identity (detection -> tracking).
599 Tracks,
600 /// A generic containment / part-of relation.
601 Contains,
602 }
603
604 /// A directed edge between two nodes by index into [`AnalyticsMeta::nodes`].
605 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
606 pub struct Relation {
607 pub from: usize,
608 pub to: usize,
609 pub kind: RelationKind,
610 }
611
612 /// The per-frame analytics relation graph (the `GstAnalyticsRelationMeta`
613 /// analog): typed detection / classification / tracking nodes plus directed
614 /// relations between them, so downstream elements (overlay, recorder, alarm)
615 /// read results by node kind and traversal instead of decoding raw tensors.
616 #[derive(Debug, Default, Clone, PartialEq)]
617 pub struct AnalyticsMeta {
618 pub nodes: Vec<AnalyticsNode>,
619 pub relations: Vec<Relation>,
620 /// Class names indexed by a node's `label`, so a consumer can show
621 /// "person" rather than `12`. Shared rather than stored per node: the
622 /// names repeat on every detection of every frame, and a node stays
623 /// `Copy`. `None` means the producer published no table.
624 pub class_names: Option<Arc<[Box<str>]>>,
625 }
626
627 impl AnalyticsMeta {
628 pub fn new() -> Self {
629 Self::default()
630 }
631
632 /// Set the class-name table, indexed by label id.
633 pub fn set_class_names<I, S>(&mut self, names: I)
634 where
635 I: IntoIterator<Item = S>,
636 S: Into<Box<str>>,
637 {
638 self.class_names = Some(names.into_iter().map(Into::into).collect());
639 }
640
641 /// The name for a label id. `None` with no table, or for an id past its
642 /// end: that means the table and the producer disagree, and showing a
643 /// neighbour's name would be worse than showing none.
644 pub fn class_name(&self, label: u32) -> Option<&str> {
645 let names = self.class_names.as_ref()?;
646 names.get(label as usize).map(|name| &**name)
647 }
648
649 /// Append a node, returning its index (used to wire relations).
650 pub fn push(&mut self, node: AnalyticsNode) -> usize {
651 self.nodes.push(node);
652 self.nodes.len() - 1
653 }
654
655 /// Append a detection node, returning its index.
656 pub fn add_detection(&mut self, detection: ObjectDetection) -> usize {
657 self.push(AnalyticsNode::Detection(detection))
658 }
659
660 /// Wire a directed relation between two node indices.
661 pub fn relate(&mut self, from: usize, to: usize, kind: RelationKind) {
662 self.relations.push(Relation { from, to, kind });
663 }
664
665 /// Iterate the detection nodes.
666 pub fn detections(&self) -> impl Iterator<Item = &ObjectDetection> {
667 self.nodes.iter().filter_map(|n| match n {
668 AnalyticsNode::Detection(d) => Some(d),
669 _ => None,
670 })
671 }
672
673 /// Iterate the instance-segmentation nodes.
674 pub fn segmentations(&self) -> impl Iterator<Item = &Segmentation> {
675 self.nodes.iter().filter_map(|n| match n {
676 AnalyticsNode::Segmentation(s) => Some(s),
677 _ => None,
678 })
679 }
680
681 /// Iterate the region-of-interest nodes.
682 pub fn rois(&self) -> impl Iterator<Item = &Roi> {
683 self.nodes.iter().filter_map(|n| match n {
684 AnalyticsNode::Roi(r) => Some(r),
685 _ => None,
686 })
687 }
688 }
689
690 impl FrameMeta for AnalyticsMeta {
691 fn as_any(&self) -> &dyn Any {
692 self
693 }
694 fn as_any_mut(&mut self) -> &mut dyn Any {
695 self
696 }
697 fn clone_box(&self) -> Box<dyn FrameMeta> {
698 Box::new(self.clone())
699 }
700 /// Normalized coordinates survive a scale / crop / copy unchanged; a
701 /// re-encode to a compressed codec discards pixel-derived analytics.
702 fn propagate(&self, transform: Transform) -> Propagation {
703 match transform {
704 Transform::Encode => Propagation::Drop,
705 _ => Propagation::Keep,
706 }
707 }
708 }
709
710 /// One inference output riding along with the frame it was computed from.
711 ///
712 /// Carries what a `Caps::Tensor` link would have carried, since it stands in
713 /// for exactly that: the descriptor plus the raw little-endian bytes. `name`
714 /// says which stage produced it, so a frame can hold the outputs of several
715 /// models at once; it comes from the producing element's configuration, not
716 /// from anything read out of a model file, so every inference backend tags
717 /// its output the same way. Empty is the ordinary single-model case.
718 #[derive(Debug, Clone, PartialEq)]
719 pub struct NamedTensor {
720 pub name: String,
721 pub dtype: crate::caps::TensorDType,
722 pub shape: crate::caps::TensorShape,
723 pub layout: crate::caps::TensorLayout,
724 pub data: Vec<u8>,
725 }
726
727 /// The inference outputs attached to a frame, for the elements that keep the
728 /// picture on the wire instead of replacing it with the tensor. That is what
729 /// lets `inference -> post-process -> overlay` stay one straight chain: the
730 /// frame reaching the overlay is still the video the model saw.
731 ///
732 /// Holds every tensor on the frame, since a [`FrameMetaSet`] keys by concrete
733 /// type and would otherwise let a second model's output replace the first.
734 /// These are not detections; those are the post-processor's [`AnalyticsMeta`].
735 #[derive(Debug, Default, Clone, PartialEq)]
736 pub struct TensorMeta(Vec<NamedTensor>);
737
738 impl TensorMeta {
739 /// An empty set, the starting point a producer pushes onto.
740 pub fn new() -> Self {
741 TensorMeta(Vec::new())
742 }
743
744 /// Add one output, replacing any earlier tensor of the same name so a
745 /// re-run of the same stage updates rather than accumulates.
746 pub fn push(&mut self, tensor: NamedTensor) {
747 match self.0.iter().position(|t| t.name == tensor.name) {
748 Some(idx) => self.0[idx] = tensor,
749 None => self.0.push(tensor),
750 }
751 }
752
753 /// Every attached tensor, in the order the stages produced them.
754 pub fn iter(&self) -> impl Iterator<Item = &NamedTensor> {
755 self.0.iter()
756 }
757
758 /// The tensor named `name`.
759 pub fn get(&self, name: &str) -> Option<&NamedTensor> {
760 self.0.iter().find(|t| t.name == name)
761 }
762
763 /// The tensor when the frame carries exactly one, so the ordinary
764 /// single-model pipeline needs no names. `None` when there are several:
765 /// the consumer must then say which it wants rather than be handed an
766 /// arbitrary one.
767 pub fn only(&self) -> Option<&NamedTensor> {
768 match self.0.as_slice() {
769 [one] => Some(one),
770 _ => None,
771 }
772 }
773
774 pub fn is_empty(&self) -> bool {
775 self.0.is_empty()
776 }
777 }
778
779 impl FrameMeta for TensorMeta {
780 fn as_any(&self) -> &dyn Any {
781 self
782 }
783 fn as_any_mut(&mut self) -> &mut dyn Any {
784 self
785 }
786 fn clone_box(&self) -> Box<dyn FrameMeta> {
787 Box::new(self.clone())
788 }
789 /// The tensors describe the picture the models saw, so a re-encode ends
790 /// their usefulness the way it ends an analytics graph's. A scale or crop
791 /// leaves them readable: what they mean is fixed by the model input size
792 /// the post-processor normalizes against, not the frame's current size.
793 fn propagate(&self, transform: Transform) -> Propagation {
794 match transform {
795 Transform::Encode => Propagation::Drop,
796 _ => Propagation::Keep,
797 }
798 }
799 }
800
801 /// One opaque tagged blob: a `header` tag plus a serialized `payload`.
802 #[derive(Debug, Clone, PartialEq, Eq)]
803 pub struct Blob {
804 pub header: String,
805 pub payload: Vec<u8>,
806 }
807
808 /// Opaque tagged side-data carried with a frame (the GstMeta custom-blob
809 /// analog): serialized results a producer attaches and a specific consumer
810 /// decodes by `header`, e.g. an ML embedding's little-endian f32 bytes or a
811 /// JSON record. A single `BlobMeta` holds every blob on a frame, since a
812 /// [`FrameMetaSet`] keys by concrete type.
813 #[derive(Debug, Default, Clone, PartialEq, Eq)]
814 pub struct BlobMeta {
815 pub blobs: Vec<Blob>,
816 }
817
818 impl BlobMeta {
819 pub fn new() -> Self {
820 Self::default()
821 }
822
823 /// Append a tagged blob.
824 pub fn push(&mut self, header: impl Into<String>, payload: Vec<u8>) {
825 self.blobs.push(Blob {
826 header: header.into(),
827 payload,
828 });
829 }
830
831 /// Iterate the carried blobs in attach order.
832 pub fn iter(&self) -> impl Iterator<Item = &Blob> {
833 self.blobs.iter()
834 }
835
836 pub fn is_empty(&self) -> bool {
837 self.blobs.is_empty()
838 }
839
840 pub fn len(&self) -> usize {
841 self.blobs.len()
842 }
843
844 /// The first blob tagged `header`, if any.
845 pub fn get(&self, header: &str) -> Option<&Blob> {
846 self.blobs.iter().find(|b| b.header == header)
847 }
848 }
849
850 /// A [`Blob`] payload decoded by the [`BLOB_DECODERS`] registry.
851 #[derive(Debug, Clone, PartialEq)]
852 pub enum DecodedBlob {
853 /// A little-endian `f32` vector (an ML embedding / feature vector).
854 Embedding(Vec<f32>),
855 /// UTF-8 text.
856 Text(String),
857 }
858
859 /// Turns one known header's payload into a [`DecodedBlob`], or `None` when
860 /// the bytes do not match the shape that header promises.
861 pub type BlobDecoder = fn(&[u8]) -> Option<DecodedBlob>;
862
863 fn decode_embedding(payload: &[u8]) -> Option<DecodedBlob> {
864 if payload.is_empty() || payload.len() % 4 != 0 {
865 return None;
866 }
867 Some(DecodedBlob::Embedding(
868 payload
869 .chunks_exact(4)
870 .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
871 .collect(),
872 ))
873 }
874
875 fn decode_text(payload: &[u8]) -> Option<DecodedBlob> {
876 core::str::from_utf8(payload)
877 .ok()
878 .map(|s| DecodedBlob::Text(String::from(s)))
879 }
880
881 /// The blob headers this workspace knows how to decode, and their decoders.
882 ///
883 /// [`BlobMeta`] is deliberately opaque: a producer and its own consumer agree
884 /// on a header and nobody else has to care. This table is the escape hatch
885 /// for the headers that *are* a shared vocabulary, so a generic consumer (an
886 /// inspector, a bridge, a recorder) can render them without knowing which
887 /// element produced them. These three come from the Python element host,
888 /// where a `gst-python-ml` element tags its results with them.
889 ///
890 /// A plain `const` table, not a registry a plugin mutates: the decoders are
891 /// pure functions, and a global mutable map would need a lock the `no_std`
892 /// baseline does not have.
893 pub const BLOB_DECODERS: &[(&str, BlobDecoder)] = &[
894 ("embedding", decode_embedding as BlobDecoder),
895 ("model_name", decode_text as BlobDecoder),
896 ("device", decode_text as BlobDecoder),
897 ];
898
899 /// The decoder registered for `header`, if it is a known one.
900 pub fn blob_decoder(header: &str) -> Option<BlobDecoder> {
901 BLOB_DECODERS
902 .iter()
903 .find(|(h, _)| *h == header)
904 .map(|(_, d)| *d)
905 }
906
907 /// Decode `blob` if its header is known and its payload matches the shape
908 /// that header promises. `None` for an unknown header (the normal case for
909 /// an application's private side-data) and for a malformed payload.
910 pub fn decode_blob(blob: &Blob) -> Option<DecodedBlob> {
911 blob_decoder(&blob.header).and_then(|d| d(&blob.payload))
912 }
913
914 impl FrameMeta for BlobMeta {
915 fn as_any(&self) -> &dyn Any {
916 self
917 }
918 fn as_any_mut(&mut self) -> &mut dyn Any {
919 self
920 }
921 fn clone_box(&self) -> Box<dyn FrameMeta> {
922 Box::new(self.clone())
923 }
924 // Opaque serialized results are not pixel-coordinate bound, so they
925 // survive every transform (including a re-encode); the default `Keep`
926 // is correct, stated here for intent.
927 fn propagate(&self, _transform: Transform) -> Propagation {
928 Propagation::Keep
929 }
930 }
931
932 /// One closed-caption byte triple: a two-bit `cc_type` and the two caption
933 /// data bytes, the ATSC A/53 `cc_data` element. `cc_type` 0/1 are the two
934 /// CEA-608 line-21 fields, 2/3 CEA-708 DTVCC packet bytes.
935 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
936 pub struct CaptionTriple {
937 pub cc_type: u8,
938 pub b0: u8,
939 pub b1: u8,
940 }
941
942 /// The closed-caption bytes the coded picture this frame came from carried
943 /// (its A/53 `GA94` caption SEI, or a container caption track). Lets a
944 /// decode -> re-encode chain re-author captions the decoder would otherwise
945 /// have dropped with the bitstream.
946 ///
947 /// Only the triples are stored: the rest of the A/53 `cc_data` header is
948 /// either constant (`process_cc_data_flag`, `em_data`) or derived
949 /// (`cc_count` = the triple count), so a rebuilt SEI is byte-identical.
950 #[derive(Debug, Default, Clone, PartialEq, Eq)]
951 pub struct CaptionMeta {
952 pub triples: Vec<CaptionTriple>,
953 }
954
955 impl CaptionMeta {
956 pub fn new() -> Self {
957 Self::default()
958 }
959
960 /// Append one caption triple, in transmission order.
961 pub fn push(&mut self, triple: CaptionTriple) {
962 self.triples.push(triple);
963 }
964
965 /// Iterate the carried triples in transmission order.
966 pub fn iter(&self) -> impl Iterator<Item = &CaptionTriple> {
967 self.triples.iter()
968 }
969
970 pub fn is_empty(&self) -> bool {
971 self.triples.is_empty()
972 }
973
974 pub fn len(&self) -> usize {
975 self.triples.len()
976 }
977 }
978
979 impl FrameMeta for CaptionMeta {
980 fn as_any(&self) -> &dyn Any {
981 self
982 }
983 fn as_any_mut(&mut self) -> &mut dyn Any {
984 self
985 }
986 fn clone_box(&self) -> Box<dyn FrameMeta> {
987 Box::new(self.clone())
988 }
989 /// Captions are text on a timeline, not pixel geometry, so they survive
990 /// a scale / crop / copy *and* a re-encode: the whole point is that a
991 /// caption inserter downstream of an encoder can re-author them into the
992 /// new bitstream.
993 fn propagate(&self, _transform: Transform) -> Propagation {
994 Propagation::Keep
995 }
996 }
997
998 /// A CIE 1931 xy chromaticity.
999 #[derive(Debug, Clone, Copy, PartialEq)]
1000 pub struct Chromaticity {
1001 pub x: f32,
1002 pub y: f32,
1003 }
1004
1005 /// The SMPTE ST 2086 mastering display colour volume: the primaries and white
1006 /// point of the display the content was graded on, and its luminance range in
1007 /// cd/m^2.
1008 #[derive(Debug, Clone, Copy, PartialEq)]
1009 pub struct MasteringDisplay {
1010 /// Display primaries in **R, G, B** order (the SEI codes them G, B, R).
1011 pub display_primaries: [Chromaticity; 3],
1012 pub white_point: Chromaticity,
1013 pub max_luminance: f32,
1014 pub min_luminance: f32,
1015 }
1016
1017 /// HDR10 static metadata as carried by the H.264 / H.265
1018 /// `mastering_display_colour_volume` and `content_light_level_info` SEI
1019 /// messages: how the content was graded, which a display sink hands to the
1020 /// driver so the panel maps the highlights the way the colourist saw them.
1021 ///
1022 /// Each half is independent: a stream may carry either, both, or (then no meta
1023 /// is attached at all) neither. The colour primaries / transfer function /
1024 /// matrix themselves are *not* here: those are CICP codepoints in the SPS VUI
1025 /// that the decode path already resolves for itself.
1026 #[derive(Debug, Default, Clone, Copy, PartialEq)]
1027 pub struct HdrStaticMeta {
1028 pub mastering: Option<MasteringDisplay>,
1029 /// MaxCLL: the brightest single pixel in the stream, cd/m^2.
1030 pub max_content_light_level: Option<u16>,
1031 /// MaxFALL: the brightest frame average in the stream, cd/m^2.
1032 pub max_frame_average_light_level: Option<u16>,
1033 }
1034
1035 impl HdrStaticMeta {
1036 /// Whether anything was actually recovered (an all-empty meta is not
1037 /// worth attaching).
1038 pub fn is_empty(&self) -> bool {
1039 self.mastering.is_none()
1040 && self.max_content_light_level.is_none()
1041 && self.max_frame_average_light_level.is_none()
1042 }
1043 }
1044
1045 impl FrameMeta for HdrStaticMeta {
1046 fn as_any(&self) -> &dyn Any {
1047 self
1048 }
1049 fn as_any_mut(&mut self) -> &mut dyn Any {
1050 self
1051 }
1052 fn clone_box(&self) -> Box<dyn FrameMeta> {
1053 Box::new(*self)
1054 }
1055 /// A grading description of the content, not of the sample grid: it
1056 /// survives every transform, including a re-encode (the new bitstream
1057 /// describes the same graded picture).
1058 fn propagate(&self, _transform: Transform) -> Propagation {
1059 Propagation::Keep
1060 }
1061 }
1062
1063 /// The SMPTE ST 12M timecode a coded picture carries (H.264 `pic_timing` /
1064 /// H.265 `time_code` SEI, or a container timecode track): where this frame
1065 /// sits on the source's own clock, which is what an edit list, a broadcast
1066 /// log, or a burnt-in overlay refers to.
1067 #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
1068 pub struct TimecodeMeta {
1069 pub hours: u8,
1070 pub minutes: u8,
1071 pub seconds: u8,
1072 pub frames: u8,
1073 /// NTSC drop-frame counting (the 29.97 / 59.94 fps count that skips two
1074 /// frame numbers a minute). Rendered with a `;` before the frame count.
1075 pub drop_frame: bool,
1076 /// Frames per second the count runs at, Q16 fixed point like
1077 /// [`Rate::Fixed`](crate::Rate::Fixed). `None` when the source declared
1078 /// none, so a consumer cannot convert the count to a duration.
1079 pub framerate_q16: Option<u32>,
1080 }
1081
1082 impl FrameMeta for TimecodeMeta {
1083 fn as_any(&self) -> &dyn Any {
1084 self
1085 }
1086 fn as_any_mut(&mut self) -> &mut dyn Any {
1087 self
1088 }
1089 fn clone_box(&self) -> Box<dyn FrameMeta> {
1090 Box::new(*self)
1091 }
1092 /// A position on the source's clock: unchanged by any pixel work, and it
1093 /// is exactly what a re-encode should carry into the new bitstream.
1094 fn propagate(&self, _transform: Transform) -> Propagation {
1095 Propagation::Keep
1096 }
1097 }
1098
1099 /// How the picture in this frame's buffer has to be turned to be shown the
1100 /// right way up. The orientation is relative to the buffer **as stored**:
1101 /// the rows and columns are exactly what the producer wrote, and a consumer
1102 /// that works in display coordinates has to apply this itself.
1103 ///
1104 /// It is what lets a rotation stay unrealized. `videoflip` in front of a
1105 /// display sink that can turn the picture for free (a Wayland
1106 /// `set_buffer_transform`, a KMS plane rotation) attaches this instead of
1107 /// remapping every pixel; in front of a sink that cannot, it rotates as it
1108 /// always did and attaches nothing.
1109 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
1110 pub struct OrientationMeta {
1111 pub orientation: super::Orientation,
1112 }
1113
1114 impl FrameMeta for OrientationMeta {
1115 fn as_any(&self) -> &dyn Any {
1116 self
1117 }
1118 fn as_any_mut(&mut self) -> &mut dyn Any {
1119 self
1120 }
1121 fn clone_box(&self) -> Box<dyn FrameMeta> {
1122 Box::new(*self)
1123 }
1124 /// A scale or a colour convert rewrites the samples but keeps every row
1125 /// a row and every column a column, so the turn still applies. A crop
1126 /// picks its rectangle in the *stored* coordinates, so what comes out is
1127 /// no longer the picture this described: keeping the turn would rotate a
1128 /// region the caller chose un-rotated. A re-encode keeps it, since that
1129 /// is the display matrix a container writes.
1130 fn propagate(&self, transform: Transform) -> Propagation {
1131 match transform {
1132 Transform::Crop => Propagation::Drop,
1133 _ => Propagation::Keep,
1134 }
1135 }
1136 }
1137
1138 /// How many planes a [`PlaneLayout`] describes. Four covers every format in
1139 /// the workspace (planar YUV with alpha is the widest).
1140 pub const MAX_PLANES: usize = 4;
1141
1142 /// Where one plane's rows sit in the frame's buffer.
1143 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
1144 pub struct Plane {
1145 /// Byte offset of the plane's first row from the start of the buffer.
1146 pub offset: usize,
1147 /// Bytes from the start of one row to the start of the next. At least
1148 /// the row's own byte width; more when the rows are padded.
1149 pub stride: usize,
1150 }
1151
1152 /// Where each plane's rows really sit in a raw video frame's buffer (the
1153 /// `GstVideoMeta` analog). Without it a raw frame is assumed tightly packed:
1154 /// every row exactly `width * bytes_per_pixel` and every plane immediately
1155 /// after the last. A producer whose rows are padded (a GPU readback at the
1156 /// API's 256-byte row alignment, a capture driver's `bytesperline`) has to
1157 /// repack them into that shape, row by row, before pushing the frame.
1158 ///
1159 /// A consumer that asks for this meta
1160 /// ([`MetaRequests`](crate::meta::MetaRequests)) says it will read rows where
1161 /// they lie, so the producer can hand over the padded buffer as it is and the
1162 /// repack disappears.
1163 ///
1164 /// Only the geometry of the *buffer* is described here, never the picture:
1165 /// width, height and format stay in the caps.
1166 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
1167 pub struct PlaneLayout {
1168 planes: [Plane; MAX_PLANES],
1169 count: usize,
1170 }
1171
1172 impl PlaneLayout {
1173 /// Describe `planes` (1 to [`MAX_PLANES`] of them), or `None` for an
1174 /// empty / oversized list.
1175 pub fn new(planes: &[Plane]) -> Option<Self> {
1176 if planes.is_empty() || planes.len() > MAX_PLANES {
1177 return None;
1178 }
1179 let mut slots = [Plane {
1180 offset: 0,
1181 stride: 0,
1182 }; MAX_PLANES];
1183 slots[..planes.len()].copy_from_slice(planes);
1184 Some(PlaneLayout {
1185 planes: slots,
1186 count: planes.len(),
1187 })
1188 }
1189
1190 /// One plane at `offset` 0 with row pitch `stride`: the packed-format
1191 /// case (RGBA, YUYV), which is most of what pads rows in practice.
1192 pub fn single(stride: usize) -> Self {
1193 PlaneLayout {
1194 planes: [Plane { offset: 0, stride }; MAX_PLANES],
1195 count: 1,
1196 }
1197 }
1198
1199 pub fn count(&self) -> usize {
1200 self.count
1201 }
1202
1203 /// Plane `index`, or `None` past the described ones.
1204 pub fn plane(&self, index: usize) -> Option<Plane> {
1205 (index < self.count).then(|| self.planes[index])
1206 }
1207
1208 /// Byte range of row `row` of plane `index`, `row_bytes` wide. `None`
1209 /// when the plane does not exist, the stride cannot hold the row, or the
1210 /// arithmetic overflows: a layout can come off a wire or a driver, so
1211 /// every offset derived from it is checked here once and a caller can
1212 /// then slice with what it gets back.
1213 pub fn row_range(
1214 &self,
1215 index: usize,
1216 row: usize,
1217 row_bytes: usize,
1218 ) -> Option<core::ops::Range<usize>> {
1219 let plane = self.plane(index)?;
1220 if plane.stride < row_bytes {
1221 return None;
1222 }
1223 let start = plane.offset.checked_add(row.checked_mul(plane.stride)?)?;
1224 let end = start.checked_add(row_bytes)?;
1225 Some(start..end)
1226 }
1227 }
1228
1229 impl FrameMeta for PlaneLayout {
1230 fn as_any(&self) -> &dyn Any {
1231 self
1232 }
1233 fn as_any_mut(&mut self) -> &mut dyn Any {
1234 self
1235 }
1236 fn clone_box(&self) -> Box<dyn FrameMeta> {
1237 Box::new(*self)
1238 }
1239 /// Dropped by every transform: it describes one specific buffer, and an
1240 /// element only declares a [`Transform`] when it writes a *new* one (a
1241 /// videoconvert says `Copy` and still emits its own tightly-packed
1242 /// frame). A tee branch, which shares the very buffer this describes,
1243 /// clones the meta set without applying a transform, so the layout
1244 /// survives a fan-out.
1245 fn propagate(&self, _transform: Transform) -> Propagation {
1246 Propagation::Drop
1247 }
1248 }
1249}
1250
1251#[cfg(all(test, feature = "metadata"))]
1252mod tests {
1253 use super::*;
1254
1255 fn det(x: f32, y: f32, w: f32, h: f32, label: u32, conf: f32) -> ObjectDetection {
1256 ObjectDetection {
1257 bbox: BBox { x, y, w, h },
1258 label,
1259 confidence: conf,
1260 }
1261 }
1262
1263 #[test]
1264 fn attach_and_typed_get_round_trip() {
1265 let mut set = FrameMetaSet::new();
1266 assert!(set.is_empty());
1267 let mut a = AnalyticsMeta::new();
1268 a.add_detection(det(0.1, 0.1, 0.2, 0.2, 7, 0.9));
1269 set.attach(a);
1270 assert_eq!(set.len(), 1);
1271 let got = set.get::<AnalyticsMeta>().expect("AnalyticsMeta attached");
1272 assert_eq!(got.detections().count(), 1);
1273 assert_eq!(got.detections().next().unwrap().label, 7);
1274 }
1275
1276 #[test]
1277 fn attach_replaces_the_same_type_and_keeps_other_types() {
1278 let mut set = FrameMetaSet::new();
1279 set.attach(AnalyticsMeta::new());
1280 set.attach(BlobMeta::new());
1281
1282 let mut second = AnalyticsMeta::new();
1283 second.add_detection(det(0.1, 0.1, 0.2, 0.2, 7, 0.9));
1284 set.attach(second);
1285
1286 assert_eq!(set.len(), 2, "the replacement is not a second entry");
1287 assert_eq!(
1288 set.get::<AnalyticsMeta>().unwrap().detections().count(),
1289 1,
1290 "the meta attached last is the one that can be read back"
1291 );
1292 assert!(
1293 set.get::<BlobMeta>().is_some(),
1294 "another type is untouched by the replacement"
1295 );
1296 }
1297
1298 #[test]
1299 fn get_mut_allows_in_place_update() {
1300 let mut set = FrameMetaSet::new();
1301 set.attach(AnalyticsMeta::new());
1302 set.get_mut::<AnalyticsMeta>()
1303 .unwrap()
1304 .add_detection(det(0.0, 0.0, 0.5, 0.5, 1, 0.5));
1305 assert_eq!(set.get::<AnalyticsMeta>().unwrap().nodes.len(), 1);
1306 }
1307
1308 #[test]
1309 fn propagate_keeps_through_scale_drops_on_encode() {
1310 let mut set = FrameMetaSet::new();
1311 set.attach(AnalyticsMeta::new());
1312 set.propagate(Transform::Scale);
1313 assert_eq!(set.len(), 1, "normalized analytics survive a scale");
1314 set.propagate(Transform::Encode);
1315 assert!(set.is_empty(), "a re-encode drops pixel-derived analytics");
1316 }
1317
1318 #[test]
1319 fn relation_graph_links_detection_to_classification() {
1320 let mut a = AnalyticsMeta::new();
1321 let d = a.add_detection(det(0.2, 0.2, 0.3, 0.3, 2, 0.8));
1322 let c = a.push(AnalyticsNode::Classification(Classification {
1323 label: 42,
1324 confidence: 0.7,
1325 }));
1326 a.relate(d, c, RelationKind::Classifies);
1327 assert_eq!(a.relations.len(), 1);
1328 assert_eq!(
1329 a.relations[0],
1330 Relation {
1331 from: d,
1332 to: c,
1333 kind: RelationKind::Classifies
1334 }
1335 );
1336 }
1337
1338 #[test]
1339 fn class_names_name_a_label_and_refuse_an_unknown_one() {
1340 let mut m = AnalyticsMeta::new();
1341 m.add_detection(det(0.1, 0.1, 0.2, 0.2, 1, 0.9));
1342 assert_eq!(m.class_name(1), None, "no table yet");
1343
1344 m.set_class_names(["person", "bicycle"]);
1345 assert_eq!(m.class_name(0), Some("person"));
1346 assert_eq!(m.class_name(1), Some("bicycle"));
1347 // An id the table does not cover means producer and table disagree, so
1348 // it reads as unnamed rather than borrowing a neighbour's name.
1349 assert_eq!(m.class_name(2), None);
1350
1351 // The table survives the clone a fan-out branch takes.
1352 assert_eq!(m.clone().class_name(0), Some("person"));
1353 }
1354
1355 #[test]
1356 fn clone_shares_then_get_mut_copies_on_write() {
1357 // A tee clone shares the analytics graph by Arc; mutating one side must
1358 // not leak into the other (copy-on-write deep copy on get_mut).
1359 let mut a = FrameMetaSet::new();
1360 a.attach({
1361 let mut m = AnalyticsMeta::new();
1362 m.add_detection(det(0.1, 0.1, 0.2, 0.2, 7, 0.9));
1363 m
1364 });
1365 let mut b = a.clone();
1366 assert_eq!(a.get::<AnalyticsMeta>().unwrap().nodes.len(), 1);
1367 assert_eq!(b.get::<AnalyticsMeta>().unwrap().nodes.len(), 1);
1368
1369 // Mutate the clone: COW splits the shared entry.
1370 b.get_mut::<AnalyticsMeta>()
1371 .unwrap()
1372 .add_detection(det(0.5, 0.5, 0.1, 0.1, 3, 0.8));
1373 assert_eq!(
1374 b.get::<AnalyticsMeta>().unwrap().nodes.len(),
1375 2,
1376 "clone mutated"
1377 );
1378 assert_eq!(
1379 a.get::<AnalyticsMeta>().unwrap().nodes.len(),
1380 1,
1381 "original untouched after copy-on-write"
1382 );
1383 }
1384
1385 #[test]
1386 fn known_blob_headers_decode_to_typed_values() {
1387 let mut m = BlobMeta::new();
1388 m.push("embedding", alloc::vec![0, 0, 0x80, 0x3F, 0, 0, 0, 0x40]); // 1.0, 2.0
1389 m.push("device", alloc::vec![b'c', b'u', b'd', b'a', b':', b'0']);
1390 m.push("private/thing", alloc::vec![0xDE, 0xAD]);
1391
1392 assert_eq!(
1393 decode_blob(m.get("embedding").unwrap()),
1394 Some(DecodedBlob::Embedding(alloc::vec![1.0, 2.0]))
1395 );
1396 assert_eq!(
1397 decode_blob(m.get("device").unwrap()),
1398 Some(DecodedBlob::Text(alloc::string::String::from("cuda:0")))
1399 );
1400 // An unregistered header stays opaque, which is the point of BlobMeta.
1401 assert!(blob_decoder("private/thing").is_none());
1402 assert_eq!(decode_blob(m.get("private/thing").unwrap()), None);
1403 }
1404
1405 #[test]
1406 fn a_payload_that_does_not_match_its_header_does_not_decode() {
1407 // A registered header is a promise about the bytes, not a guarantee: a
1408 // producer that breaks it must fail the decode, not yield garbage.
1409 let ragged = Blob {
1410 header: alloc::string::String::from("embedding"),
1411 payload: alloc::vec![1, 2, 3],
1412 };
1413 assert_eq!(decode_blob(&ragged), None, "not a whole number of f32s");
1414 let empty = Blob {
1415 header: alloc::string::String::from("embedding"),
1416 payload: alloc::vec::Vec::new(),
1417 };
1418 assert_eq!(decode_blob(&empty), None, "an empty vector is not a vector");
1419 let not_utf8 = Blob {
1420 header: alloc::string::String::from("model_name"),
1421 payload: alloc::vec![0xFF, 0xFE],
1422 };
1423 assert_eq!(decode_blob(¬_utf8), None);
1424 }
1425
1426 #[test]
1427 fn iou_is_zero_for_disjoint_and_one_for_identical() {
1428 let a = BBox {
1429 x: 0.0,
1430 y: 0.0,
1431 w: 0.2,
1432 h: 0.2,
1433 };
1434 let b = BBox {
1435 x: 0.5,
1436 y: 0.5,
1437 w: 0.2,
1438 h: 0.2,
1439 };
1440 assert_eq!(a.iou(&b), 0.0, "disjoint boxes do not overlap");
1441 assert!(
1442 (a.iou(&a) - 1.0).abs() < 1e-6,
1443 "identical boxes fully overlap"
1444 );
1445 // Half-overlap: a and c share half their area horizontally.
1446 let c = BBox {
1447 x: 0.1,
1448 y: 0.0,
1449 w: 0.2,
1450 h: 0.2,
1451 };
1452 let iou = a.iou(&c);
1453 assert!(
1454 iou > 0.3 && iou < 0.34,
1455 "half-shifted overlap ~1/3 IoU: {iou}"
1456 );
1457 }
1458}