Skip to main content

g2g_core/
meta.rs

1//! Per-frame metadata system: typed blobs that travel with a [`Frame`] (the
2//! GstMeta / GstAnalyticsRelationMeta analog), and the `AnalyticsMeta` relation
3//! graph for ML detection / classification / tracking results.
4//!
5//! Gated behind the `metadata` cargo feature. When **off** (the default, and the
6//! only configuration the `no_std` / Cortex-M baseline uses) [`FrameMetaSet`] is
7//! a zero-sized unit: the `Frame::meta` field exists for API stability but costs
8//! nothing per frame. When **on** it is a list of typed [`FrameMeta`] trait
9//! objects with attach / typed-get / iterate / propagate, and the standard
10//! [`AnalyticsMeta`] is available for detection pipelines.
11//!
12//! **Why now:** the field was reserved at M88; the trait body and the relation
13//! graph land with the first metadata-producing element (a YOLO-style detection
14//! postprocess), so a real client shapes the API rather than speculation.
15//!
16//! [`Frame`]: crate::frame::Frame
17
18/// One of the eight ways a picture can be turned without resampling it: the
19/// four quarter rotations and the four mirrors (the dihedral group of the
20/// square). Named after GStreamer's `videoflip` methods, and the vocabulary
21/// [`OrientationMeta`] carries.
22///
23/// Not gated on the `metadata` feature: the enum is what `videoflip` runs on
24/// either way. Only the meta wrapper needs the feature.
25#[derive(Debug, Clone, Copy, PartialEq, Eq)]
26pub enum Orientation {
27    /// No turn (GStreamer `none`).
28    Identity,
29    /// A quarter turn clockwise (`clockwise`).
30    Rotate90Cw,
31    /// A half turn (`rotate-180`).
32    Rotate180,
33    /// A quarter turn counter-clockwise (`counterclockwise`).
34    Rotate90Ccw,
35    /// Mirrored left to right, i.e. reflected in the vertical axis
36    /// (`horizontal-flip`).
37    HorizontalMirror,
38    /// Mirrored top to bottom, i.e. reflected in the horizontal axis
39    /// (`vertical-flip`).
40    VerticalMirror,
41    /// Reflected in the diagonal from the upper-left corner
42    /// (`upper-left-diagonal`): the pixel at `(x, y)` reads as `(y, x)`.
43    Transpose,
44    /// Reflected in the diagonal from the upper-right corner
45    /// (`upper-right-diagonal`).
46    Transverse,
47}
48
49impl Orientation {
50    /// This orientation as `(quarter turns clockwise, mirrored first)`: every
51    /// member is a horizontal mirror (or not) followed by a rotation, which is
52    /// what makes [`compose`](Self::compose) a few lines of arithmetic instead
53    /// of a 64-entry table.
54    const fn parts(self) -> (u8, bool) {
55        match self {
56            Orientation::Identity => (0, false),
57            Orientation::Rotate90Cw => (1, false),
58            Orientation::Rotate180 => (2, false),
59            Orientation::Rotate90Ccw => (3, false),
60            Orientation::HorizontalMirror => (0, true),
61            Orientation::Transverse => (1, true),
62            Orientation::VerticalMirror => (2, true),
63            Orientation::Transpose => (3, true),
64        }
65    }
66
67    /// Inverse of [`parts`](Self::parts). `quarter_turns` must already be
68    /// reduced mod 4.
69    const fn from_parts(quarter_turns: u8, mirrored: bool) -> Orientation {
70        match (quarter_turns, mirrored) {
71            (0, false) => Orientation::Identity,
72            (1, false) => Orientation::Rotate90Cw,
73            (2, false) => Orientation::Rotate180,
74            (3, false) => Orientation::Rotate90Ccw,
75            (0, true) => Orientation::HorizontalMirror,
76            (1, true) => Orientation::Transverse,
77            (2, true) => Orientation::VerticalMirror,
78            _ => Orientation::Transpose,
79        }
80    }
81
82    /// Whether this orientation exchanges width and height. True for the two
83    /// quarter rotations and the two diagonal mirrors.
84    pub const fn swaps_dims(self) -> bool {
85        self.parts().0 % 2 == 1
86    }
87
88    /// The single orientation that does what applying `self` and then `then`
89    /// does.
90    pub const fn compose(self, then: Orientation) -> Orientation {
91        let (turns, mirrored) = self.parts();
92        let (then_turns, then_mirrored) = then.parts();
93        // A mirror reverses the sense of any rotation it is applied after, so
94        // pushing `self`'s rotation through `then`'s mirror negates it.
95        let carried = if then_mirrored { 4 - turns } else { turns };
96        Orientation::from_parts((then_turns + carried) % 4, mirrored != then_mirrored)
97    }
98
99    /// The orientation that undoes this one.
100    pub const fn inverse(self) -> Orientation {
101        let (turns, mirrored) = self.parts();
102        if mirrored {
103            // Every mirror is its own inverse.
104            self
105        } else {
106            Orientation::from_parts((4 - turns) % 4, false)
107        }
108    }
109}
110
111// ---- feature off: the zero-sized placeholder ----
112
113/// Per-frame attachable metadata set (feature `metadata` **off**): a zero-sized
114/// unit, so the baseline pays nothing. See the module docs.
115#[cfg(not(feature = "metadata"))]
116#[derive(Debug, Clone, Default, PartialEq, Eq)]
117pub struct FrameMetaSet;
118
119#[cfg(not(feature = "metadata"))]
120impl FrameMetaSet {
121    /// An empty metadata set. `const` so frame construction stays trivial.
122    #[inline]
123    pub const fn new() -> Self {
124        FrameMetaSet
125    }
126}
127
128/// The metadata types a downstream element asks its producers to attach
129/// (feature `metadata` **off**): a zero-sized always-empty set. The plumbing
130/// that carries it ([`AllocationParams`](crate::AllocationParams)) compiles
131/// either way; `request` / `wants` exist only with the feature on.
132#[cfg(not(feature = "metadata"))]
133#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
134pub struct MetaRequests;
135
136#[cfg(not(feature = "metadata"))]
137impl MetaRequests {
138    /// An empty request set.
139    #[inline]
140    pub const fn new() -> Self {
141        MetaRequests
142    }
143
144    /// Always true: without the feature nothing can be requested.
145    #[inline]
146    pub fn is_empty(&self) -> bool {
147        true
148    }
149
150    /// The demand two sibling consumers put on their shared producer, i.e. still
151    /// nothing.
152    #[inline]
153    pub fn join_branches(self, _other: Self) -> Self {
154        self
155    }
156
157    /// The demand this element passes to its producer, i.e. still nothing.
158    #[inline]
159    pub fn carry_upstream(self, _downstream: Self) -> Self {
160        self
161    }
162}
163
164// ---- feature on: the real typed container + analytics graph ----
165
166#[cfg(feature = "metadata")]
167pub use on::*;
168
169#[cfg(feature = "metadata")]
170mod on {
171    use alloc::boxed::Box;
172    use alloc::string::String;
173    use alloc::sync::Arc;
174    use alloc::vec::Vec;
175    use core::any::{Any, TypeId};
176
177    /// How a piece of metadata survives a transform, the GstMeta
178    /// `transform_func` analog. Reported by [`FrameMeta::propagate`].
179    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
180    pub enum Transform {
181        /// A deep copy (e.g. a tee branch clone): meta is duplicated.
182        Copy,
183        /// A geometry resample (videoscale / compositor pad scale).
184        Scale,
185        /// A spatial crop (videocrop).
186        Crop,
187        /// A re-encode to a compressed codec: pixel-derived meta is lost.
188        Encode,
189    }
190
191    /// Whether a meta is kept through a [`Transform`] or dropped.
192    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
193    pub enum Propagation {
194        Keep,
195        Drop,
196    }
197
198    /// A typed, per-frame, attachable piece of metadata (the `GstMeta` analog).
199    ///
200    /// `as_any` enables typed retrieval via downcast (trait upcasting is not on
201    /// the MSRV); `propagate` is the per-transform survival policy. Meta is
202    /// `Send + Sync` so a frame crosses a multi-thread runtime.
203    pub trait FrameMeta: core::fmt::Debug + Send + Sync {
204        fn as_any(&self) -> &dyn Any;
205        fn as_any_mut(&mut self) -> &mut dyn Any;
206        /// A boxed deep copy of this meta, the GstMeta `copy_func` analog. Backs
207        /// the copy-on-write of a shared meta when a tee branch mutates it (see
208        /// [`FrameMetaSet::get_mut`]); each `FrameMeta` impl is its own concrete
209        /// type, so the duplication can only be expressed on the trait.
210        fn clone_box(&self) -> Box<dyn FrameMeta>;
211        /// How this meta survives `transform`. Default keeps it through
212        /// everything; override to drop on transforms that invalidate it.
213        fn propagate(&self, _transform: Transform) -> Propagation {
214            Propagation::Keep
215        }
216    }
217
218    /// A list of typed [`FrameMeta`] attached to a frame. Empty (no allocation)
219    /// on a freshly constructed frame.
220    ///
221    /// Each entry is an [`Arc`] so a fan-out (tee) clone shares the metadata by
222    /// refcount instead of deep-copying it (cheap, and the analytics graph is
223    /// identical on both branches). A branch that mutates one entry pays a
224    /// copy-on-write deep copy only then (see [`get_mut`](Self::get_mut)), so the
225    /// other branch never observes the change.
226    #[derive(Debug, Default, Clone)]
227    pub struct FrameMetaSet(Vec<Arc<dyn FrameMeta>>);
228
229    impl FrameMetaSet {
230        /// An empty metadata set with no backing allocation.
231        #[inline]
232        pub fn new() -> Self {
233            FrameMetaSet(Vec::new())
234        }
235
236        /// Attach one piece of metadata, replacing any entry of the same type
237        /// (in place, so the order of the other entries is unchanged).
238        ///
239        /// A set holds at most one meta per type: [`get`](Self::get) /
240        /// [`get_mut`](Self::get_mut) key by type, so a second entry of a type
241        /// would be unreachable, and an empty one already on the frame would
242        /// hide what an element attaches later.
243        pub fn attach<T: FrameMeta + 'static>(&mut self, meta: T) {
244            match self.0.iter().position(|m| m.as_any().is::<T>()) {
245                Some(idx) => self.0[idx] = Arc::new(meta),
246                None => self.0.push(Arc::new(meta)),
247            }
248        }
249
250        /// The attached meta of type `T`, if any. At most one is ever attached
251        /// (see [`attach`](Self::attach)).
252        pub fn get<T: FrameMeta + 'static>(&self) -> Option<&T> {
253            self.0.iter().find_map(|m| m.as_any().downcast_ref::<T>())
254        }
255
256        /// Mutable access to the attached meta of type `T`, if any (at most one
257        /// is ever attached, see [`attach`](Self::attach)).
258        ///
259        /// Copy-on-write: if the entry is shared with another frame (a tee
260        /// branch holds the same [`Arc`]), it is first deep-copied via
261        /// [`FrameMeta::clone_box`] so this mutation stays private to this frame.
262        /// When the entry is uniquely owned no copy is made.
263        pub fn get_mut<T: FrameMeta + 'static>(&mut self) -> Option<&mut T> {
264            let idx = self.0.iter().position(|m| m.as_any().is::<T>())?;
265            // Ensure unique ownership before handing out a mutable reference.
266            if Arc::get_mut(&mut self.0[idx]).is_none() {
267                self.0[idx] = Arc::from(self.0[idx].clone_box());
268            }
269            Arc::get_mut(&mut self.0[idx])
270                .expect("entry is unique after the COW above")
271                .as_any_mut()
272                .downcast_mut::<T>()
273        }
274
275        /// Iterate every attached meta as a trait object.
276        pub fn iter(&self) -> impl Iterator<Item = &dyn FrameMeta> {
277            self.0.iter().map(|m| m.as_ref())
278        }
279
280        pub fn len(&self) -> usize {
281            self.0.len()
282        }
283
284        pub fn is_empty(&self) -> bool {
285            self.0.is_empty()
286        }
287
288        /// Apply a [`Transform`]: retain only metas whose `propagate` returns
289        /// [`Propagation::Keep`]. An element that resamples / re-encodes calls
290        /// this so stale meta never rides a frame it no longer describes.
291        pub fn propagate(&mut self, transform: Transform) {
292            self.0
293                .retain(|m| m.propagate(transform) == Propagation::Keep);
294        }
295    }
296
297    /// How many distinct meta types one [`MetaRequests`] carries. Requests past
298    /// this are dropped, which costs an optimization, never correctness: a
299    /// producer that sees no request just produces what it always did.
300    pub const MAX_META_REQUESTS: usize = 4;
301
302    /// What one request needs of the *other* consumers reading the same frames,
303    /// which decides how it survives a fan-out or an intermediate hop.
304    ///
305    /// `Ord` ranks [`EveryConsumer`](Self::EveryConsumer) above
306    /// [`AnyConsumer`](Self::AnyConsumer): when two elements request one meta
307    /// under different policies the stricter one stands, since it is the one
308    /// that can be misread.
309    #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
310    pub enum RequestPolicy {
311        /// One asking consumer is enough. Attaching the meta costs a consumer
312        /// that did not ask nothing: it reads the same frame it always did and
313        /// ignores the extra ([`AnalyticsMeta`], [`CaptionMeta`],
314        /// [`TimecodeMeta`]).
315        AnyConsumer,
316        /// Every consumer must ask. Honouring the request changes the *buffer*,
317        /// so a consumer that did not ask would misread it: a frame whose rows
318        /// were left padded, read as tightly packed, is corruption rather than a
319        /// missed optimization.
320        EveryConsumer,
321    }
322
323    /// The metadata types a downstream element wants attached to the frames it
324    /// receives, each keyed by [`TypeId`] and carrying its [`RequestPolicy`].
325    /// The pull half of the metadata system (the GStreamer allocation-query
326    /// `add_meta` analog): a consumer declares its requests from
327    /// [`AsyncElement::meta_requests`](crate::AsyncElement::meta_requests), the
328    /// runner carries them up the allocation cascade on
329    /// [`AllocationParams`](crate::AllocationParams), and a producer asks
330    /// [`wants`](Self::wants) when it configures, so optional metadata is
331    /// produced only where somebody reads it.
332    ///
333    /// A small fixed-capacity set, so it rides the `Copy` allocation params
334    /// without an allocation. Entries are kept sorted, so two sets built in
335    /// different orders compare equal (the cascade suppresses a re-propose when
336    /// the params are unchanged).
337    #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
338    pub struct MetaRequests {
339        entries: [Option<(TypeId, RequestPolicy)>; MAX_META_REQUESTS],
340    }
341
342    impl MetaRequests {
343        /// An empty request set: the element wants no optional metadata.
344        pub const fn new() -> Self {
345            MetaRequests {
346                entries: [None; MAX_META_REQUESTS],
347            }
348        }
349
350        /// This set plus a request for meta `T` that one consumer asking is
351        /// enough for ([`RequestPolicy::AnyConsumer`]). Builder form, since a
352        /// request set is usually written inline:
353        /// `MetaRequests::new().request::<AnalyticsMeta>()`.
354        pub fn request<T: FrameMeta + 'static>(self) -> Self {
355            self.with(TypeId::of::<T>(), RequestPolicy::AnyConsumer)
356        }
357
358        /// This set plus a request for meta `T` that is only honoured when every
359        /// consumer sharing the producer asks for it too
360        /// ([`RequestPolicy::EveryConsumer`]). For a meta whose presence changes
361        /// the buffer, which a consumer that did not ask would misread.
362        pub fn request_from_every_consumer<T: FrameMeta + 'static>(self) -> Self {
363            self.with(TypeId::of::<T>(), RequestPolicy::EveryConsumer)
364        }
365
366        /// Whether meta `T` was requested, under either policy.
367        pub fn wants<T: FrameMeta + 'static>(&self) -> bool {
368            self.policy_of(TypeId::of::<T>()).is_some()
369        }
370
371        /// The policy meta `T` was requested under, `None` if it was not.
372        pub fn policy<T: FrameMeta + 'static>(&self) -> Option<RequestPolicy> {
373            self.policy_of(TypeId::of::<T>())
374        }
375
376        /// The demand two *sibling* consumers put on the one producer they share
377        /// (the branches of a tee). An [`AnyConsumer`](RequestPolicy::AnyConsumer)
378        /// request survives from either side; an
379        /// [`EveryConsumer`](RequestPolicy::EveryConsumer) one only when the
380        /// other branch asks for that meta too, so a branch that would misread
381        /// the changed buffer vetoes it.
382        pub fn join_branches(self, other: Self) -> Self {
383            let mut out = Self::new();
384            for (id, policy) in self.iter().chain(other.iter()) {
385                if policy == RequestPolicy::AnyConsumer
386                    || (self.policy_of(id).is_some() && other.policy_of(id).is_some())
387                {
388                    out = out.with(id, policy);
389                }
390            }
391            out
392        }
393
394        /// The demand this element (`self`, its own requests) passes on to its
395        /// producer, given what arrived from `downstream`. Its own requests
396        /// always travel: it reads the producer's frames itself. A downstream
397        /// [`EveryConsumer`](RequestPolicy::EveryConsumer) request travels only
398        /// when this element asks for that meta too, since the producer's frames
399        /// pass through here first and a hop that cannot read the changed buffer
400        /// vetoes it just as a sibling branch does.
401        pub fn carry_upstream(self, downstream: Self) -> Self {
402            let mut out = self;
403            for (id, policy) in downstream.iter() {
404                if policy == RequestPolicy::AnyConsumer || self.policy_of(id).is_some() {
405                    out = out.with(id, policy);
406                }
407            }
408            out
409        }
410
411        pub fn is_empty(&self) -> bool {
412            self.entries[0].is_none()
413        }
414
415        pub fn len(&self) -> usize {
416            self.entries.iter().flatten().count()
417        }
418
419        fn iter(&self) -> impl Iterator<Item = (TypeId, RequestPolicy)> + '_ {
420            self.entries.iter().flatten().copied()
421        }
422
423        fn policy_of(&self, id: TypeId) -> Option<RequestPolicy> {
424            self.iter().find(|(i, _)| *i == id).map(|(_, p)| p)
425        }
426
427        fn with(mut self, id: TypeId, policy: RequestPolicy) -> Self {
428            let mut free = MAX_META_REQUESTS;
429            for (i, slot) in self.entries.iter_mut().enumerate() {
430                match slot {
431                    Some((present, held)) if *present == id => {
432                        // Two elements asking for one meta under different
433                        // policies: the stricter one is the one that can be
434                        // misread, so it stands.
435                        *held = (*held).max(policy);
436                        return self;
437                    }
438                    Some(_) => {}
439                    None => {
440                        free = i;
441                        break;
442                    }
443                }
444            }
445            if free == MAX_META_REQUESTS {
446                return self;
447            }
448            self.entries[free] = Some((id, policy));
449            // The occupied prefix is packed at the front, so sorting it keeps it
450            // packed and makes the set order-independent under `PartialEq`.
451            self.entries[..=free].sort_unstable();
452            self
453        }
454    }
455
456    /// A normalized bounding box: all fields in `[0, 1]` relative to the frame,
457    /// `(x, y)` the top-left corner and `(w, h)` the size. Normalized so a box
458    /// survives a downstream scale / crop without a coordinate rewrite.
459    #[derive(Debug, Clone, Copy, PartialEq)]
460    pub struct BBox {
461        pub x: f32,
462        pub y: f32,
463        pub w: f32,
464        pub h: f32,
465    }
466
467    impl BBox {
468        /// Intersection-over-union with `other`, the NMS overlap metric.
469        pub fn iou(&self, other: &BBox) -> f32 {
470            let ix0 = self.x.max(other.x);
471            let iy0 = self.y.max(other.y);
472            let ix1 = (self.x + self.w).min(other.x + other.w);
473            let iy1 = (self.y + self.h).min(other.y + other.h);
474            let iw = (ix1 - ix0).max(0.0);
475            let ih = (iy1 - iy0).max(0.0);
476            let inter = iw * ih;
477            let union = self.w * self.h + other.w * other.h - inter;
478            if union <= 0.0 {
479                0.0
480            } else {
481                inter / union
482            }
483        }
484    }
485
486    /// A detected object: its box, class label index, and confidence `[0, 1]`.
487    #[derive(Debug, Clone, Copy, PartialEq)]
488    pub struct ObjectDetection {
489        pub bbox: BBox,
490        pub label: u32,
491        pub confidence: f32,
492    }
493
494    /// A whole-region or per-detection classification result.
495    #[derive(Debug, Clone, Copy, PartialEq)]
496    pub struct Classification {
497        pub label: u32,
498        pub confidence: f32,
499    }
500
501    /// A persistent tracking identity across frames.
502    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
503    pub struct Tracking {
504        pub object_id: u64,
505    }
506
507    /// A per-pixel coverage mask: `width` x `height` 8-bit samples with `stride`
508    /// bytes per row (0 = not covered, 255 = fully covered). Its own grid, not
509    /// the frame's, so it stays valid when the frame is scaled.
510    #[derive(Debug, Clone, PartialEq, Eq)]
511    pub struct Mask {
512        width: u32,
513        height: u32,
514        stride: u32,
515        data: Vec<u8>,
516    }
517
518    impl Mask {
519        /// Build a mask over `data`, or `None` if the geometry does not fit it.
520        /// Dimensions reach this from a model output or a wire peer, so they are
521        /// checked here once and the accessors can then index without guessing.
522        pub fn new(width: u32, height: u32, stride: u32, data: Vec<u8>) -> Option<Self> {
523            if stride < width {
524                return None;
525            }
526            let needed = (stride as u64).checked_mul(height as u64)?;
527            if needed > data.len() as u64 {
528                return None;
529            }
530            Some(Mask {
531                width,
532                height,
533                stride,
534                data,
535            })
536        }
537
538        pub fn width(&self) -> u32 {
539            self.width
540        }
541        pub fn height(&self) -> u32 {
542            self.height
543        }
544        pub fn stride(&self) -> u32 {
545            self.stride
546        }
547        pub fn data(&self) -> &[u8] {
548            &self.data
549        }
550
551        /// Coverage at `(x, y)`, `None` outside the mask.
552        pub fn sample(&self, x: u32, y: u32) -> Option<u8> {
553            if x >= self.width || y >= self.height {
554                return None;
555            }
556            let idx = y as usize * self.stride as usize + x as usize;
557            self.data.get(idx).copied()
558        }
559    }
560
561    /// An instance segmentation: the object's normalized box, its class, and the
562    /// coverage mask over that box (the mask grid is the model's own resolution,
563    /// not the frame's).
564    #[derive(Debug, Clone, PartialEq)]
565    pub struct Segmentation {
566        pub bbox: BBox,
567        pub label: u32,
568        pub confidence: f32,
569        pub mask: Mask,
570    }
571
572    /// A region of interest: a normalized rectangle an encoder, a tracker, or a
573    /// downstream analytic should treat specially (the
574    /// `GstVideoRegionOfInterestMeta` analog). `id` names this region across
575    /// frames; `label` is its class index, as on a detection.
576    #[derive(Debug, Clone, Copy, PartialEq)]
577    pub struct Roi {
578        pub bbox: BBox,
579        pub id: u32,
580        pub label: u32,
581    }
582
583    /// A node in the [`AnalyticsMeta`] relation graph.
584    #[derive(Debug, Clone, PartialEq)]
585    pub enum AnalyticsNode {
586        Detection(ObjectDetection),
587        Classification(Classification),
588        Tracking(Tracking),
589        Segmentation(Segmentation),
590        Roi(Roi),
591    }
592
593    /// The kind of a directed edge between two analytics nodes.
594    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
595    pub enum RelationKind {
596        /// A detection has-a classification (detection -> classification).
597        Classifies,
598        /// A detection has-a tracking identity (detection -> tracking).
599        Tracks,
600        /// A generic containment / part-of relation.
601        Contains,
602    }
603
604    /// A directed edge between two nodes by index into [`AnalyticsMeta::nodes`].
605    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
606    pub struct Relation {
607        pub from: usize,
608        pub to: usize,
609        pub kind: RelationKind,
610    }
611
612    /// The per-frame analytics relation graph (the `GstAnalyticsRelationMeta`
613    /// analog): typed detection / classification / tracking nodes plus directed
614    /// relations between them, so downstream elements (overlay, recorder, alarm)
615    /// read results by node kind and traversal instead of decoding raw tensors.
616    #[derive(Debug, Default, Clone, PartialEq)]
617    pub struct AnalyticsMeta {
618        pub nodes: Vec<AnalyticsNode>,
619        pub relations: Vec<Relation>,
620        /// Class names indexed by a node's `label`, so a consumer can show
621        /// "person" rather than `12`. Shared rather than stored per node: the
622        /// names repeat on every detection of every frame, and a node stays
623        /// `Copy`. `None` means the producer published no table.
624        pub class_names: Option<Arc<[Box<str>]>>,
625    }
626
627    impl AnalyticsMeta {
628        pub fn new() -> Self {
629            Self::default()
630        }
631
632        /// Set the class-name table, indexed by label id.
633        pub fn set_class_names<I, S>(&mut self, names: I)
634        where
635            I: IntoIterator<Item = S>,
636            S: Into<Box<str>>,
637        {
638            self.class_names = Some(names.into_iter().map(Into::into).collect());
639        }
640
641        /// The name for a label id. `None` with no table, or for an id past its
642        /// end: that means the table and the producer disagree, and showing a
643        /// neighbour's name would be worse than showing none.
644        pub fn class_name(&self, label: u32) -> Option<&str> {
645            let names = self.class_names.as_ref()?;
646            names.get(label as usize).map(|name| &**name)
647        }
648
649        /// Append a node, returning its index (used to wire relations).
650        pub fn push(&mut self, node: AnalyticsNode) -> usize {
651            self.nodes.push(node);
652            self.nodes.len() - 1
653        }
654
655        /// Append a detection node, returning its index.
656        pub fn add_detection(&mut self, detection: ObjectDetection) -> usize {
657            self.push(AnalyticsNode::Detection(detection))
658        }
659
660        /// Wire a directed relation between two node indices.
661        pub fn relate(&mut self, from: usize, to: usize, kind: RelationKind) {
662            self.relations.push(Relation { from, to, kind });
663        }
664
665        /// Iterate the detection nodes.
666        pub fn detections(&self) -> impl Iterator<Item = &ObjectDetection> {
667            self.nodes.iter().filter_map(|n| match n {
668                AnalyticsNode::Detection(d) => Some(d),
669                _ => None,
670            })
671        }
672
673        /// Iterate the instance-segmentation nodes.
674        pub fn segmentations(&self) -> impl Iterator<Item = &Segmentation> {
675            self.nodes.iter().filter_map(|n| match n {
676                AnalyticsNode::Segmentation(s) => Some(s),
677                _ => None,
678            })
679        }
680
681        /// Iterate the region-of-interest nodes.
682        pub fn rois(&self) -> impl Iterator<Item = &Roi> {
683            self.nodes.iter().filter_map(|n| match n {
684                AnalyticsNode::Roi(r) => Some(r),
685                _ => None,
686            })
687        }
688    }
689
690    impl FrameMeta for AnalyticsMeta {
691        fn as_any(&self) -> &dyn Any {
692            self
693        }
694        fn as_any_mut(&mut self) -> &mut dyn Any {
695            self
696        }
697        fn clone_box(&self) -> Box<dyn FrameMeta> {
698            Box::new(self.clone())
699        }
700        /// Normalized coordinates survive a scale / crop / copy unchanged; a
701        /// re-encode to a compressed codec discards pixel-derived analytics.
702        fn propagate(&self, transform: Transform) -> Propagation {
703            match transform {
704                Transform::Encode => Propagation::Drop,
705                _ => Propagation::Keep,
706            }
707        }
708    }
709
710    /// One inference output riding along with the frame it was computed from.
711    ///
712    /// Carries what a `Caps::Tensor` link would have carried, since it stands in
713    /// for exactly that: the descriptor plus the raw little-endian bytes. `name`
714    /// says which stage produced it, so a frame can hold the outputs of several
715    /// models at once; it comes from the producing element's configuration, not
716    /// from anything read out of a model file, so every inference backend tags
717    /// its output the same way. Empty is the ordinary single-model case.
718    #[derive(Debug, Clone, PartialEq)]
719    pub struct NamedTensor {
720        pub name: String,
721        pub dtype: crate::caps::TensorDType,
722        pub shape: crate::caps::TensorShape,
723        pub layout: crate::caps::TensorLayout,
724        pub data: Vec<u8>,
725    }
726
727    /// The inference outputs attached to a frame, for the elements that keep the
728    /// picture on the wire instead of replacing it with the tensor. That is what
729    /// lets `inference -> post-process -> overlay` stay one straight chain: the
730    /// frame reaching the overlay is still the video the model saw.
731    ///
732    /// Holds every tensor on the frame, since a [`FrameMetaSet`] keys by concrete
733    /// type and would otherwise let a second model's output replace the first.
734    /// These are not detections; those are the post-processor's [`AnalyticsMeta`].
735    #[derive(Debug, Default, Clone, PartialEq)]
736    pub struct TensorMeta(Vec<NamedTensor>);
737
738    impl TensorMeta {
739        /// An empty set, the starting point a producer pushes onto.
740        pub fn new() -> Self {
741            TensorMeta(Vec::new())
742        }
743
744        /// Add one output, replacing any earlier tensor of the same name so a
745        /// re-run of the same stage updates rather than accumulates.
746        pub fn push(&mut self, tensor: NamedTensor) {
747            match self.0.iter().position(|t| t.name == tensor.name) {
748                Some(idx) => self.0[idx] = tensor,
749                None => self.0.push(tensor),
750            }
751        }
752
753        /// Every attached tensor, in the order the stages produced them.
754        pub fn iter(&self) -> impl Iterator<Item = &NamedTensor> {
755            self.0.iter()
756        }
757
758        /// The tensor named `name`.
759        pub fn get(&self, name: &str) -> Option<&NamedTensor> {
760            self.0.iter().find(|t| t.name == name)
761        }
762
763        /// The tensor when the frame carries exactly one, so the ordinary
764        /// single-model pipeline needs no names. `None` when there are several:
765        /// the consumer must then say which it wants rather than be handed an
766        /// arbitrary one.
767        pub fn only(&self) -> Option<&NamedTensor> {
768            match self.0.as_slice() {
769                [one] => Some(one),
770                _ => None,
771            }
772        }
773
774        pub fn is_empty(&self) -> bool {
775            self.0.is_empty()
776        }
777    }
778
779    impl FrameMeta for TensorMeta {
780        fn as_any(&self) -> &dyn Any {
781            self
782        }
783        fn as_any_mut(&mut self) -> &mut dyn Any {
784            self
785        }
786        fn clone_box(&self) -> Box<dyn FrameMeta> {
787            Box::new(self.clone())
788        }
789        /// The tensors describe the picture the models saw, so a re-encode ends
790        /// their usefulness the way it ends an analytics graph's. A scale or crop
791        /// leaves them readable: what they mean is fixed by the model input size
792        /// the post-processor normalizes against, not the frame's current size.
793        fn propagate(&self, transform: Transform) -> Propagation {
794            match transform {
795                Transform::Encode => Propagation::Drop,
796                _ => Propagation::Keep,
797            }
798        }
799    }
800
801    /// One opaque tagged blob: a `header` tag plus a serialized `payload`.
802    #[derive(Debug, Clone, PartialEq, Eq)]
803    pub struct Blob {
804        pub header: String,
805        pub payload: Vec<u8>,
806    }
807
808    /// Opaque tagged side-data carried with a frame (the GstMeta custom-blob
809    /// analog): serialized results a producer attaches and a specific consumer
810    /// decodes by `header`, e.g. an ML embedding's little-endian f32 bytes or a
811    /// JSON record. A single `BlobMeta` holds every blob on a frame, since a
812    /// [`FrameMetaSet`] keys by concrete type.
813    #[derive(Debug, Default, Clone, PartialEq, Eq)]
814    pub struct BlobMeta {
815        pub blobs: Vec<Blob>,
816    }
817
818    impl BlobMeta {
819        pub fn new() -> Self {
820            Self::default()
821        }
822
823        /// Append a tagged blob.
824        pub fn push(&mut self, header: impl Into<String>, payload: Vec<u8>) {
825            self.blobs.push(Blob {
826                header: header.into(),
827                payload,
828            });
829        }
830
831        /// Iterate the carried blobs in attach order.
832        pub fn iter(&self) -> impl Iterator<Item = &Blob> {
833            self.blobs.iter()
834        }
835
836        pub fn is_empty(&self) -> bool {
837            self.blobs.is_empty()
838        }
839
840        pub fn len(&self) -> usize {
841            self.blobs.len()
842        }
843
844        /// The first blob tagged `header`, if any.
845        pub fn get(&self, header: &str) -> Option<&Blob> {
846            self.blobs.iter().find(|b| b.header == header)
847        }
848    }
849
850    /// A [`Blob`] payload decoded by the [`BLOB_DECODERS`] registry.
851    #[derive(Debug, Clone, PartialEq)]
852    pub enum DecodedBlob {
853        /// A little-endian `f32` vector (an ML embedding / feature vector).
854        Embedding(Vec<f32>),
855        /// UTF-8 text.
856        Text(String),
857    }
858
859    /// Turns one known header's payload into a [`DecodedBlob`], or `None` when
860    /// the bytes do not match the shape that header promises.
861    pub type BlobDecoder = fn(&[u8]) -> Option<DecodedBlob>;
862
863    fn decode_embedding(payload: &[u8]) -> Option<DecodedBlob> {
864        if payload.is_empty() || payload.len() % 4 != 0 {
865            return None;
866        }
867        Some(DecodedBlob::Embedding(
868            payload
869                .chunks_exact(4)
870                .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
871                .collect(),
872        ))
873    }
874
875    fn decode_text(payload: &[u8]) -> Option<DecodedBlob> {
876        core::str::from_utf8(payload)
877            .ok()
878            .map(|s| DecodedBlob::Text(String::from(s)))
879    }
880
881    /// The blob headers this workspace knows how to decode, and their decoders.
882    ///
883    /// [`BlobMeta`] is deliberately opaque: a producer and its own consumer agree
884    /// on a header and nobody else has to care. This table is the escape hatch
885    /// for the headers that *are* a shared vocabulary, so a generic consumer (an
886    /// inspector, a bridge, a recorder) can render them without knowing which
887    /// element produced them. These three come from the Python element host,
888    /// where a `gst-python-ml` element tags its results with them.
889    ///
890    /// A plain `const` table, not a registry a plugin mutates: the decoders are
891    /// pure functions, and a global mutable map would need a lock the `no_std`
892    /// baseline does not have.
893    pub const BLOB_DECODERS: &[(&str, BlobDecoder)] = &[
894        ("embedding", decode_embedding as BlobDecoder),
895        ("model_name", decode_text as BlobDecoder),
896        ("device", decode_text as BlobDecoder),
897    ];
898
899    /// The decoder registered for `header`, if it is a known one.
900    pub fn blob_decoder(header: &str) -> Option<BlobDecoder> {
901        BLOB_DECODERS
902            .iter()
903            .find(|(h, _)| *h == header)
904            .map(|(_, d)| *d)
905    }
906
907    /// Decode `blob` if its header is known and its payload matches the shape
908    /// that header promises. `None` for an unknown header (the normal case for
909    /// an application's private side-data) and for a malformed payload.
910    pub fn decode_blob(blob: &Blob) -> Option<DecodedBlob> {
911        blob_decoder(&blob.header).and_then(|d| d(&blob.payload))
912    }
913
914    impl FrameMeta for BlobMeta {
915        fn as_any(&self) -> &dyn Any {
916            self
917        }
918        fn as_any_mut(&mut self) -> &mut dyn Any {
919            self
920        }
921        fn clone_box(&self) -> Box<dyn FrameMeta> {
922            Box::new(self.clone())
923        }
924        // Opaque serialized results are not pixel-coordinate bound, so they
925        // survive every transform (including a re-encode); the default `Keep`
926        // is correct, stated here for intent.
927        fn propagate(&self, _transform: Transform) -> Propagation {
928            Propagation::Keep
929        }
930    }
931
932    /// One closed-caption byte triple: a two-bit `cc_type` and the two caption
933    /// data bytes, the ATSC A/53 `cc_data` element. `cc_type` 0/1 are the two
934    /// CEA-608 line-21 fields, 2/3 CEA-708 DTVCC packet bytes.
935    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
936    pub struct CaptionTriple {
937        pub cc_type: u8,
938        pub b0: u8,
939        pub b1: u8,
940    }
941
942    /// The closed-caption bytes the coded picture this frame came from carried
943    /// (its A/53 `GA94` caption SEI, or a container caption track). Lets a
944    /// decode -> re-encode chain re-author captions the decoder would otherwise
945    /// have dropped with the bitstream.
946    ///
947    /// Only the triples are stored: the rest of the A/53 `cc_data` header is
948    /// either constant (`process_cc_data_flag`, `em_data`) or derived
949    /// (`cc_count` = the triple count), so a rebuilt SEI is byte-identical.
950    #[derive(Debug, Default, Clone, PartialEq, Eq)]
951    pub struct CaptionMeta {
952        pub triples: Vec<CaptionTriple>,
953    }
954
955    impl CaptionMeta {
956        pub fn new() -> Self {
957            Self::default()
958        }
959
960        /// Append one caption triple, in transmission order.
961        pub fn push(&mut self, triple: CaptionTriple) {
962            self.triples.push(triple);
963        }
964
965        /// Iterate the carried triples in transmission order.
966        pub fn iter(&self) -> impl Iterator<Item = &CaptionTriple> {
967            self.triples.iter()
968        }
969
970        pub fn is_empty(&self) -> bool {
971            self.triples.is_empty()
972        }
973
974        pub fn len(&self) -> usize {
975            self.triples.len()
976        }
977    }
978
979    impl FrameMeta for CaptionMeta {
980        fn as_any(&self) -> &dyn Any {
981            self
982        }
983        fn as_any_mut(&mut self) -> &mut dyn Any {
984            self
985        }
986        fn clone_box(&self) -> Box<dyn FrameMeta> {
987            Box::new(self.clone())
988        }
989        /// Captions are text on a timeline, not pixel geometry, so they survive
990        /// a scale / crop / copy *and* a re-encode: the whole point is that a
991        /// caption inserter downstream of an encoder can re-author them into the
992        /// new bitstream.
993        fn propagate(&self, _transform: Transform) -> Propagation {
994            Propagation::Keep
995        }
996    }
997
998    /// A CIE 1931 xy chromaticity.
999    #[derive(Debug, Clone, Copy, PartialEq)]
1000    pub struct Chromaticity {
1001        pub x: f32,
1002        pub y: f32,
1003    }
1004
1005    /// The SMPTE ST 2086 mastering display colour volume: the primaries and white
1006    /// point of the display the content was graded on, and its luminance range in
1007    /// cd/m^2.
1008    #[derive(Debug, Clone, Copy, PartialEq)]
1009    pub struct MasteringDisplay {
1010        /// Display primaries in **R, G, B** order (the SEI codes them G, B, R).
1011        pub display_primaries: [Chromaticity; 3],
1012        pub white_point: Chromaticity,
1013        pub max_luminance: f32,
1014        pub min_luminance: f32,
1015    }
1016
1017    /// HDR10 static metadata as carried by the H.264 / H.265
1018    /// `mastering_display_colour_volume` and `content_light_level_info` SEI
1019    /// messages: how the content was graded, which a display sink hands to the
1020    /// driver so the panel maps the highlights the way the colourist saw them.
1021    ///
1022    /// Each half is independent: a stream may carry either, both, or (then no meta
1023    /// is attached at all) neither. The colour primaries / transfer function /
1024    /// matrix themselves are *not* here: those are CICP codepoints in the SPS VUI
1025    /// that the decode path already resolves for itself.
1026    #[derive(Debug, Default, Clone, Copy, PartialEq)]
1027    pub struct HdrStaticMeta {
1028        pub mastering: Option<MasteringDisplay>,
1029        /// MaxCLL: the brightest single pixel in the stream, cd/m^2.
1030        pub max_content_light_level: Option<u16>,
1031        /// MaxFALL: the brightest frame average in the stream, cd/m^2.
1032        pub max_frame_average_light_level: Option<u16>,
1033    }
1034
1035    impl HdrStaticMeta {
1036        /// Whether anything was actually recovered (an all-empty meta is not
1037        /// worth attaching).
1038        pub fn is_empty(&self) -> bool {
1039            self.mastering.is_none()
1040                && self.max_content_light_level.is_none()
1041                && self.max_frame_average_light_level.is_none()
1042        }
1043    }
1044
1045    impl FrameMeta for HdrStaticMeta {
1046        fn as_any(&self) -> &dyn Any {
1047            self
1048        }
1049        fn as_any_mut(&mut self) -> &mut dyn Any {
1050            self
1051        }
1052        fn clone_box(&self) -> Box<dyn FrameMeta> {
1053            Box::new(*self)
1054        }
1055        /// A grading description of the content, not of the sample grid: it
1056        /// survives every transform, including a re-encode (the new bitstream
1057        /// describes the same graded picture).
1058        fn propagate(&self, _transform: Transform) -> Propagation {
1059            Propagation::Keep
1060        }
1061    }
1062
1063    /// The SMPTE ST 12M timecode a coded picture carries (H.264 `pic_timing` /
1064    /// H.265 `time_code` SEI, or a container timecode track): where this frame
1065    /// sits on the source's own clock, which is what an edit list, a broadcast
1066    /// log, or a burnt-in overlay refers to.
1067    #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
1068    pub struct TimecodeMeta {
1069        pub hours: u8,
1070        pub minutes: u8,
1071        pub seconds: u8,
1072        pub frames: u8,
1073        /// NTSC drop-frame counting (the 29.97 / 59.94 fps count that skips two
1074        /// frame numbers a minute). Rendered with a `;` before the frame count.
1075        pub drop_frame: bool,
1076        /// Frames per second the count runs at, Q16 fixed point like
1077        /// [`Rate::Fixed`](crate::Rate::Fixed). `None` when the source declared
1078        /// none, so a consumer cannot convert the count to a duration.
1079        pub framerate_q16: Option<u32>,
1080    }
1081
1082    impl FrameMeta for TimecodeMeta {
1083        fn as_any(&self) -> &dyn Any {
1084            self
1085        }
1086        fn as_any_mut(&mut self) -> &mut dyn Any {
1087            self
1088        }
1089        fn clone_box(&self) -> Box<dyn FrameMeta> {
1090            Box::new(*self)
1091        }
1092        /// A position on the source's clock: unchanged by any pixel work, and it
1093        /// is exactly what a re-encode should carry into the new bitstream.
1094        fn propagate(&self, _transform: Transform) -> Propagation {
1095            Propagation::Keep
1096        }
1097    }
1098
1099    /// How the picture in this frame's buffer has to be turned to be shown the
1100    /// right way up. The orientation is relative to the buffer **as stored**:
1101    /// the rows and columns are exactly what the producer wrote, and a consumer
1102    /// that works in display coordinates has to apply this itself.
1103    ///
1104    /// It is what lets a rotation stay unrealized. `videoflip` in front of a
1105    /// display sink that can turn the picture for free (a Wayland
1106    /// `set_buffer_transform`, a KMS plane rotation) attaches this instead of
1107    /// remapping every pixel; in front of a sink that cannot, it rotates as it
1108    /// always did and attaches nothing.
1109    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
1110    pub struct OrientationMeta {
1111        pub orientation: super::Orientation,
1112    }
1113
1114    impl FrameMeta for OrientationMeta {
1115        fn as_any(&self) -> &dyn Any {
1116            self
1117        }
1118        fn as_any_mut(&mut self) -> &mut dyn Any {
1119            self
1120        }
1121        fn clone_box(&self) -> Box<dyn FrameMeta> {
1122            Box::new(*self)
1123        }
1124        /// A scale or a colour convert rewrites the samples but keeps every row
1125        /// a row and every column a column, so the turn still applies. A crop
1126        /// picks its rectangle in the *stored* coordinates, so what comes out is
1127        /// no longer the picture this described: keeping the turn would rotate a
1128        /// region the caller chose un-rotated. A re-encode keeps it, since that
1129        /// is the display matrix a container writes.
1130        fn propagate(&self, transform: Transform) -> Propagation {
1131            match transform {
1132                Transform::Crop => Propagation::Drop,
1133                _ => Propagation::Keep,
1134            }
1135        }
1136    }
1137
1138    /// How many planes a [`PlaneLayout`] describes. Four covers every format in
1139    /// the workspace (planar YUV with alpha is the widest).
1140    pub const MAX_PLANES: usize = 4;
1141
1142    /// Where one plane's rows sit in the frame's buffer.
1143    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
1144    pub struct Plane {
1145        /// Byte offset of the plane's first row from the start of the buffer.
1146        pub offset: usize,
1147        /// Bytes from the start of one row to the start of the next. At least
1148        /// the row's own byte width; more when the rows are padded.
1149        pub stride: usize,
1150    }
1151
1152    /// Where each plane's rows really sit in a raw video frame's buffer (the
1153    /// `GstVideoMeta` analog). Without it a raw frame is assumed tightly packed:
1154    /// every row exactly `width * bytes_per_pixel` and every plane immediately
1155    /// after the last. A producer whose rows are padded (a GPU readback at the
1156    /// API's 256-byte row alignment, a capture driver's `bytesperline`) has to
1157    /// repack them into that shape, row by row, before pushing the frame.
1158    ///
1159    /// A consumer that asks for this meta
1160    /// ([`MetaRequests`](crate::meta::MetaRequests)) says it will read rows where
1161    /// they lie, so the producer can hand over the padded buffer as it is and the
1162    /// repack disappears.
1163    ///
1164    /// Only the geometry of the *buffer* is described here, never the picture:
1165    /// width, height and format stay in the caps.
1166    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
1167    pub struct PlaneLayout {
1168        planes: [Plane; MAX_PLANES],
1169        count: usize,
1170    }
1171
1172    impl PlaneLayout {
1173        /// Describe `planes` (1 to [`MAX_PLANES`] of them), or `None` for an
1174        /// empty / oversized list.
1175        pub fn new(planes: &[Plane]) -> Option<Self> {
1176            if planes.is_empty() || planes.len() > MAX_PLANES {
1177                return None;
1178            }
1179            let mut slots = [Plane {
1180                offset: 0,
1181                stride: 0,
1182            }; MAX_PLANES];
1183            slots[..planes.len()].copy_from_slice(planes);
1184            Some(PlaneLayout {
1185                planes: slots,
1186                count: planes.len(),
1187            })
1188        }
1189
1190        /// One plane at `offset` 0 with row pitch `stride`: the packed-format
1191        /// case (RGBA, YUYV), which is most of what pads rows in practice.
1192        pub fn single(stride: usize) -> Self {
1193            PlaneLayout {
1194                planes: [Plane { offset: 0, stride }; MAX_PLANES],
1195                count: 1,
1196            }
1197        }
1198
1199        pub fn count(&self) -> usize {
1200            self.count
1201        }
1202
1203        /// Plane `index`, or `None` past the described ones.
1204        pub fn plane(&self, index: usize) -> Option<Plane> {
1205            (index < self.count).then(|| self.planes[index])
1206        }
1207
1208        /// Byte range of row `row` of plane `index`, `row_bytes` wide. `None`
1209        /// when the plane does not exist, the stride cannot hold the row, or the
1210        /// arithmetic overflows: a layout can come off a wire or a driver, so
1211        /// every offset derived from it is checked here once and a caller can
1212        /// then slice with what it gets back.
1213        pub fn row_range(
1214            &self,
1215            index: usize,
1216            row: usize,
1217            row_bytes: usize,
1218        ) -> Option<core::ops::Range<usize>> {
1219            let plane = self.plane(index)?;
1220            if plane.stride < row_bytes {
1221                return None;
1222            }
1223            let start = plane.offset.checked_add(row.checked_mul(plane.stride)?)?;
1224            let end = start.checked_add(row_bytes)?;
1225            Some(start..end)
1226        }
1227    }
1228
1229    impl FrameMeta for PlaneLayout {
1230        fn as_any(&self) -> &dyn Any {
1231            self
1232        }
1233        fn as_any_mut(&mut self) -> &mut dyn Any {
1234            self
1235        }
1236        fn clone_box(&self) -> Box<dyn FrameMeta> {
1237            Box::new(*self)
1238        }
1239        /// Dropped by every transform: it describes one specific buffer, and an
1240        /// element only declares a [`Transform`] when it writes a *new* one (a
1241        /// videoconvert says `Copy` and still emits its own tightly-packed
1242        /// frame). A tee branch, which shares the very buffer this describes,
1243        /// clones the meta set without applying a transform, so the layout
1244        /// survives a fan-out.
1245        fn propagate(&self, _transform: Transform) -> Propagation {
1246            Propagation::Drop
1247        }
1248    }
1249}
1250
1251#[cfg(all(test, feature = "metadata"))]
1252mod tests {
1253    use super::*;
1254
1255    fn det(x: f32, y: f32, w: f32, h: f32, label: u32, conf: f32) -> ObjectDetection {
1256        ObjectDetection {
1257            bbox: BBox { x, y, w, h },
1258            label,
1259            confidence: conf,
1260        }
1261    }
1262
1263    #[test]
1264    fn attach_and_typed_get_round_trip() {
1265        let mut set = FrameMetaSet::new();
1266        assert!(set.is_empty());
1267        let mut a = AnalyticsMeta::new();
1268        a.add_detection(det(0.1, 0.1, 0.2, 0.2, 7, 0.9));
1269        set.attach(a);
1270        assert_eq!(set.len(), 1);
1271        let got = set.get::<AnalyticsMeta>().expect("AnalyticsMeta attached");
1272        assert_eq!(got.detections().count(), 1);
1273        assert_eq!(got.detections().next().unwrap().label, 7);
1274    }
1275
1276    #[test]
1277    fn attach_replaces_the_same_type_and_keeps_other_types() {
1278        let mut set = FrameMetaSet::new();
1279        set.attach(AnalyticsMeta::new());
1280        set.attach(BlobMeta::new());
1281
1282        let mut second = AnalyticsMeta::new();
1283        second.add_detection(det(0.1, 0.1, 0.2, 0.2, 7, 0.9));
1284        set.attach(second);
1285
1286        assert_eq!(set.len(), 2, "the replacement is not a second entry");
1287        assert_eq!(
1288            set.get::<AnalyticsMeta>().unwrap().detections().count(),
1289            1,
1290            "the meta attached last is the one that can be read back"
1291        );
1292        assert!(
1293            set.get::<BlobMeta>().is_some(),
1294            "another type is untouched by the replacement"
1295        );
1296    }
1297
1298    #[test]
1299    fn get_mut_allows_in_place_update() {
1300        let mut set = FrameMetaSet::new();
1301        set.attach(AnalyticsMeta::new());
1302        set.get_mut::<AnalyticsMeta>()
1303            .unwrap()
1304            .add_detection(det(0.0, 0.0, 0.5, 0.5, 1, 0.5));
1305        assert_eq!(set.get::<AnalyticsMeta>().unwrap().nodes.len(), 1);
1306    }
1307
1308    #[test]
1309    fn propagate_keeps_through_scale_drops_on_encode() {
1310        let mut set = FrameMetaSet::new();
1311        set.attach(AnalyticsMeta::new());
1312        set.propagate(Transform::Scale);
1313        assert_eq!(set.len(), 1, "normalized analytics survive a scale");
1314        set.propagate(Transform::Encode);
1315        assert!(set.is_empty(), "a re-encode drops pixel-derived analytics");
1316    }
1317
1318    #[test]
1319    fn relation_graph_links_detection_to_classification() {
1320        let mut a = AnalyticsMeta::new();
1321        let d = a.add_detection(det(0.2, 0.2, 0.3, 0.3, 2, 0.8));
1322        let c = a.push(AnalyticsNode::Classification(Classification {
1323            label: 42,
1324            confidence: 0.7,
1325        }));
1326        a.relate(d, c, RelationKind::Classifies);
1327        assert_eq!(a.relations.len(), 1);
1328        assert_eq!(
1329            a.relations[0],
1330            Relation {
1331                from: d,
1332                to: c,
1333                kind: RelationKind::Classifies
1334            }
1335        );
1336    }
1337
1338    #[test]
1339    fn class_names_name_a_label_and_refuse_an_unknown_one() {
1340        let mut m = AnalyticsMeta::new();
1341        m.add_detection(det(0.1, 0.1, 0.2, 0.2, 1, 0.9));
1342        assert_eq!(m.class_name(1), None, "no table yet");
1343
1344        m.set_class_names(["person", "bicycle"]);
1345        assert_eq!(m.class_name(0), Some("person"));
1346        assert_eq!(m.class_name(1), Some("bicycle"));
1347        // An id the table does not cover means producer and table disagree, so
1348        // it reads as unnamed rather than borrowing a neighbour's name.
1349        assert_eq!(m.class_name(2), None);
1350
1351        // The table survives the clone a fan-out branch takes.
1352        assert_eq!(m.clone().class_name(0), Some("person"));
1353    }
1354
1355    #[test]
1356    fn clone_shares_then_get_mut_copies_on_write() {
1357        // A tee clone shares the analytics graph by Arc; mutating one side must
1358        // not leak into the other (copy-on-write deep copy on get_mut).
1359        let mut a = FrameMetaSet::new();
1360        a.attach({
1361            let mut m = AnalyticsMeta::new();
1362            m.add_detection(det(0.1, 0.1, 0.2, 0.2, 7, 0.9));
1363            m
1364        });
1365        let mut b = a.clone();
1366        assert_eq!(a.get::<AnalyticsMeta>().unwrap().nodes.len(), 1);
1367        assert_eq!(b.get::<AnalyticsMeta>().unwrap().nodes.len(), 1);
1368
1369        // Mutate the clone: COW splits the shared entry.
1370        b.get_mut::<AnalyticsMeta>()
1371            .unwrap()
1372            .add_detection(det(0.5, 0.5, 0.1, 0.1, 3, 0.8));
1373        assert_eq!(
1374            b.get::<AnalyticsMeta>().unwrap().nodes.len(),
1375            2,
1376            "clone mutated"
1377        );
1378        assert_eq!(
1379            a.get::<AnalyticsMeta>().unwrap().nodes.len(),
1380            1,
1381            "original untouched after copy-on-write"
1382        );
1383    }
1384
1385    #[test]
1386    fn known_blob_headers_decode_to_typed_values() {
1387        let mut m = BlobMeta::new();
1388        m.push("embedding", alloc::vec![0, 0, 0x80, 0x3F, 0, 0, 0, 0x40]); // 1.0, 2.0
1389        m.push("device", alloc::vec![b'c', b'u', b'd', b'a', b':', b'0']);
1390        m.push("private/thing", alloc::vec![0xDE, 0xAD]);
1391
1392        assert_eq!(
1393            decode_blob(m.get("embedding").unwrap()),
1394            Some(DecodedBlob::Embedding(alloc::vec![1.0, 2.0]))
1395        );
1396        assert_eq!(
1397            decode_blob(m.get("device").unwrap()),
1398            Some(DecodedBlob::Text(alloc::string::String::from("cuda:0")))
1399        );
1400        // An unregistered header stays opaque, which is the point of BlobMeta.
1401        assert!(blob_decoder("private/thing").is_none());
1402        assert_eq!(decode_blob(m.get("private/thing").unwrap()), None);
1403    }
1404
1405    #[test]
1406    fn a_payload_that_does_not_match_its_header_does_not_decode() {
1407        // A registered header is a promise about the bytes, not a guarantee: a
1408        // producer that breaks it must fail the decode, not yield garbage.
1409        let ragged = Blob {
1410            header: alloc::string::String::from("embedding"),
1411            payload: alloc::vec![1, 2, 3],
1412        };
1413        assert_eq!(decode_blob(&ragged), None, "not a whole number of f32s");
1414        let empty = Blob {
1415            header: alloc::string::String::from("embedding"),
1416            payload: alloc::vec::Vec::new(),
1417        };
1418        assert_eq!(decode_blob(&empty), None, "an empty vector is not a vector");
1419        let not_utf8 = Blob {
1420            header: alloc::string::String::from("model_name"),
1421            payload: alloc::vec![0xFF, 0xFE],
1422        };
1423        assert_eq!(decode_blob(&not_utf8), None);
1424    }
1425
1426    #[test]
1427    fn iou_is_zero_for_disjoint_and_one_for_identical() {
1428        let a = BBox {
1429            x: 0.0,
1430            y: 0.0,
1431            w: 0.2,
1432            h: 0.2,
1433        };
1434        let b = BBox {
1435            x: 0.5,
1436            y: 0.5,
1437            w: 0.2,
1438            h: 0.2,
1439        };
1440        assert_eq!(a.iou(&b), 0.0, "disjoint boxes do not overlap");
1441        assert!(
1442            (a.iou(&a) - 1.0).abs() < 1e-6,
1443            "identical boxes fully overlap"
1444        );
1445        // Half-overlap: a and c share half their area horizontally.
1446        let c = BBox {
1447            x: 0.1,
1448            y: 0.0,
1449            w: 0.2,
1450            h: 0.2,
1451        };
1452        let iou = a.iou(&c);
1453        assert!(
1454            iou > 0.3 && iou < 0.34,
1455            "half-shifted overlap ~1/3 IoU: {iou}"
1456        );
1457    }
1458}