eventcv_core/
representation.rs

1use std::{error::Error, fmt};
2
3use crate::{Event, EventStream};
4
5mod averaged_time_surface;
6mod binary;
7mod count;
8mod countmask;
9mod mcts;
10mod point_set;
11mod polarity;
12mod tencode;
13mod time_surface;
14mod voxel;
15
16pub use averaged_time_surface::AveragedTimeSurface;
17pub use binary::Binary;
18pub use count::EventCount;
19pub use countmask::CountMask;
20pub use mcts::Mcts;
21pub use point_set::{EventPointSet, PointSet};
22pub use polarity::Polarity;
23pub use tencode::Tencode;
24pub use time_surface::TimeSurface;
25pub use voxel::VoxelGrid;
26
27pub trait Representation {
28    type Output;
29
30    /// The reference implementation, on the CPU. Always available, always the definition of what
31    /// this representation *is*: the GPU kernels are checked against it, not the other way round.
32    fn generate(&self, stream: &EventStream) -> Result<Self::Output, RepresentationError>;
33
34    /// Runs on `device`.
35    ///
36    /// The default ignores it and runs [`generate`](Self::generate) — a representation with no GPU
37    /// kernel is not an error to ask for on the GPU, it is simply computed on the CPU, and a
38    /// pipeline that sets `device="gpu"` once should not have to know which of its steps have
39    /// kernels. Asking for a GPU that does not *exist* is still an error; that is decided by the
40    /// implementations that do have one.
41    fn generate_on(
42        &self,
43        stream: &EventStream,
44        _device: crate::accel::Device,
45    ) -> Result<Self::Output, RepresentationError> {
46        self.generate(stream)
47    }
48}
49
50/// Runs a kernel over `stream`, or says why it could not.
51///
52/// Every GPU-capable representation funnels through here, so "no adapter", "this build has no GPU
53/// support" and "an accumulator would have overflowed" are worded once. The `gpu` feature being
54/// off collapses this to the error arm, which is why the body is not itself behind a `cfg`.
55#[cfg_attr(not(feature = "gpu"), allow(unused_variables))]
56pub(crate) fn on_gpu(
57    stream: &EventStream,
58    dispatch: &crate::accel::GpuDispatch,
59) -> Result<Vec<i32>, RepresentationError> {
60    #[cfg(feature = "gpu")]
61    {
62        crate::accel::gpu::with_context(|context| {
63            crate::accel::gpu::run(context, stream, dispatch)
64        })
65        .ok_or_else(|| RepresentationError::Device(crate::accel::unavailable_reason()))?
66        .map_err(|error| match error {
67            crate::accel::gpu::GpuError::Saturated => RepresentationError::Device(
68                "a GPU accumulator overflowed: the kernels sum in fixed point so that the result \
69                 does not depend on the order the events arrived, and a cell of this frame exceeds \
70                 what that can hold. Narrow the window, or use device=\"cpu\"."
71                    .to_owned(),
72            ),
73            crate::accel::gpu::GpuError::Driver(message) => {
74                RepresentationError::Device(format!("the GPU driver refused the work: {message}"))
75            }
76        })
77    }
78    #[cfg(not(feature = "gpu"))]
79    {
80        Err(RepresentationError::Device(
81            crate::accel::unavailable_reason(),
82        ))
83    }
84}
85
86#[derive(Clone, Copy, Debug, PartialEq, Eq)]
87pub enum RepresentationKind {
88    AveragedTimeSurface,
89    Binary,
90    Count,
91    CountMask,
92    Flow,
93    /// A plain greyscale image — not derived from events at all. Produced by the frame readers and
94    /// by reconstruction, and consumed by the simulator. Like `Flow` and `Labels` it has no
95    /// `Representation` impl, because there is no stream to generate it from.
96    Intensity,
97    Labels,
98    Mcts,
99    Polarity,
100    Tencode,
101    TimeSurface,
102    Voxel,
103}
104
105impl RepresentationKind {
106    pub fn as_str(self) -> &'static str {
107        match self {
108            Self::AveragedTimeSurface => "atsurf",
109            Self::Binary => "binary",
110            Self::Count => "count",
111            Self::CountMask => "countmask",
112            Self::Flow => "flow",
113            Self::Intensity => "intensity",
114            Self::Labels => "labels",
115            Self::Mcts => "mcts",
116            Self::Polarity => "polarity",
117            Self::Tencode => "tencode",
118            Self::TimeSurface => "tsurf",
119            Self::Voxel => "voxel",
120        }
121    }
122
123    /// The inverse of [`Self::as_str`] — recovers the kind tag stored by the frame writers.
124    pub fn from_tag(tag: &str) -> Option<Self> {
125        Some(match tag {
126            "atsurf" => Self::AveragedTimeSurface,
127            "binary" => Self::Binary,
128            "count" => Self::Count,
129            "countmask" => Self::CountMask,
130            "flow" => Self::Flow,
131            "intensity" => Self::Intensity,
132            "labels" => Self::Labels,
133            "mcts" => Self::Mcts,
134            "polarity" => Self::Polarity,
135            "tencode" => Self::Tencode,
136            "tsurf" => Self::TimeSurface,
137            "voxel" => Self::Voxel,
138            _ => return None,
139        })
140    }
141}
142
143#[derive(Clone, Debug)]
144pub struct EventFrame {
145    pub(crate) data: EventFrameData,
146    pub(crate) channels: usize,
147    pub(crate) width: usize,
148    pub(crate) height: usize,
149    pub(crate) kind: RepresentationKind,
150    pub(crate) channel_names: Vec<String>,
151}
152
153#[derive(Clone, Debug, PartialEq)]
154pub enum EventFrameData {
155    U8(Vec<u8>),
156    U16(Vec<u16>),
157    U64(Vec<u64>),
158    F32(Vec<f32>),
159}
160
161impl EventFrame {
162    /// Reassembles a frame from its stored parts — used by the IO frame readers. Channels
163    /// is `channel_names.len()`, and the data length must equal `channels * width * height`.
164    pub(crate) fn from_parts(
165        data: EventFrameData,
166        width: usize,
167        height: usize,
168        kind: RepresentationKind,
169        channel_names: Vec<String>,
170    ) -> Self {
171        Self {
172            data,
173            channels: channel_names.len(),
174            width,
175            height,
176            kind,
177            channel_names,
178        }
179    }
180
181    /// Builds a single-channel greyscale frame from raw samples, row-major.
182    ///
183    /// The public counterpart to [`EventFrame::from_parts`], which is crate-private and validates
184    /// nothing — so this is the only way anything outside the core (the bindings, the video decoder)
185    /// can construct a frame, and it checks the length rather than trusting the caller. Everything
186    /// downstream indexes `c * width * height + y * width + x` and would read out of bounds on a
187    /// mismatch.
188    pub fn intensity(
189        data: EventFrameData,
190        width: usize,
191        height: usize,
192    ) -> Result<Self, RepresentationError> {
193        let expected = width
194            .checked_mul(height)
195            .ok_or(RepresentationError::SizeOverflow)?;
196        let actual = match &data {
197            EventFrameData::U8(values) => values.len(),
198            EventFrameData::U16(values) => values.len(),
199            EventFrameData::U64(values) => values.len(),
200            EventFrameData::F32(values) => values.len(),
201        };
202        if actual != expected {
203            return Err(RepresentationError::ShapeMismatch {
204                samples: actual,
205                width,
206                height,
207            });
208        }
209        Ok(Self::from_parts(
210            data,
211            width,
212            height,
213            RepresentationKind::Intensity,
214            vec!["intensity".to_owned()],
215        ))
216    }
217
218    pub fn data(&self) -> &EventFrameData {
219        &self.data
220    }
221
222    /// Takes the frame's samples, for a caller that owns the frame and wants the buffer rather
223    /// than a copy of it — handing a just-generated frame straight to numpy, say.
224    pub fn into_data(self) -> EventFrameData {
225        self.data
226    }
227
228    pub fn shape(&self) -> (usize, usize, usize) {
229        (self.channels, self.height, self.width)
230    }
231
232    pub fn channel_names(&self) -> &[String] {
233        &self.channel_names
234    }
235
236    pub fn kind(&self) -> RepresentationKind {
237        self.kind
238    }
239}
240
241impl EventFrameData {
242    /// Number of scalar elements (`channels * width * height` for a well-formed frame).
243    pub(crate) fn len(&self) -> usize {
244        match self {
245            Self::U8(values) => values.len(),
246            Self::U16(values) => values.len(),
247            Self::U64(values) => values.len(),
248            Self::F32(values) => values.len(),
249        }
250    }
251}
252
253pub(crate) fn frame_len(
254    stream: &EventStream,
255    channels: usize,
256) -> Result<(usize, usize, usize), RepresentationError> {
257    let (width, height) = stream.sensor_size();
258    let plane_len = width
259        .checked_mul(height)
260        .ok_or(RepresentationError::SizeOverflow)?;
261    let length = plane_len
262        .checked_mul(channels)
263        .ok_or(RepresentationError::SizeOverflow)?;
264    Ok((width, height, length))
265}
266
267pub(crate) fn event_index(
268    event: Event,
269    width: usize,
270    height: usize,
271) -> Result<usize, RepresentationError> {
272    if event.x >= width || event.y >= height {
273        return Err(RepresentationError::EventOutOfBounds {
274            x: event.x,
275            y: event.y,
276            width,
277            height,
278        });
279    }
280    Ok(event.y * width + event.x)
281}
282
283pub(crate) fn validate_positive(value: f64, name: &'static str) -> Result<(), RepresentationError> {
284    if !value.is_finite() || value <= 0.0 {
285        return Err(RepresentationError::InvalidParameter(name));
286    }
287    Ok(())
288}
289
290pub(crate) fn reference_time(stream: &EventStream) -> Option<u64> {
291    stream.iter().map(|event| event.timestamp).max()
292}
293
294pub(crate) fn age_ms(stream: &EventStream, reference: u64, timestamp: u64) -> f64 {
295    reference.saturating_sub(timestamp) as f64 * stream.timestamp_scale_ms()
296}
297
298#[derive(Debug, PartialEq, Eq)]
299pub enum RepresentationError {
300    SizeOverflow,
301    CountOverflow {
302        x: usize,
303        y: usize,
304    },
305    EventOutOfBounds {
306        x: usize,
307        y: usize,
308        width: usize,
309        height: usize,
310    },
311    InvalidParameter(&'static str),
312    /// A frame's sample count does not match its dimensions. Caught at construction because every
313    /// consumer indexes `c * width * height + y * width + x` and would read out of bounds.
314    ShapeMismatch {
315        samples: usize,
316        width: usize,
317        height: usize,
318    },
319    /// The requested device could not run this — no adapter, no GPU support in the build, or a
320    /// kernel that could not represent the answer. Carries the sentence explaining which.
321    Device(String),
322}
323
324impl fmt::Display for RepresentationError {
325    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
326        match self {
327            Self::SizeOverflow => formatter.write_str("representation dimensions are too large"),
328            Self::CountOverflow { x, y } => {
329                write!(
330                    formatter,
331                    "event count at ({x}, {y}) exceeds uint16 capacity"
332                )
333            }
334            Self::EventOutOfBounds {
335                x,
336                y,
337                width,
338                height,
339            } => write!(
340                formatter,
341                "event coordinate ({x}, {y}) exceeds sensor size {width}x{height}"
342            ),
343            Self::InvalidParameter(name) => match *name {
344                "bins" => formatter.write_str("bins must be at least 1"),
345                "max_window_ms" => {
346                    formatter.write_str("max_window_ms must be finite and at least 1")
347                }
348                "windows" => formatter
349                    .write_str("windows must be a non-empty list of finite, positive spans (ms)"),
350                "pct" => formatter.write_str("pct must be between 0 and 100"),
351                _ => write!(formatter, "{name} must be finite and positive"),
352            },
353            Self::ShapeMismatch {
354                samples,
355                width,
356                height,
357            } => write!(
358                formatter,
359                "frame has {samples} samples but {width}x{height} needs {}",
360                width * height
361            ),
362            Self::Device(message) => formatter.write_str(message),
363        }
364    }
365}
366
367impl Error for RepresentationError {}
368
369#[cfg(test)]
370mod tests {
371    use ndarray::Array2;
372
373    use super::{
374        AveragedTimeSurface, Binary, CountMask, EventCount, EventFrameData, Mcts, PointSet,
375        Representation, RepresentationError, Tencode, TimeSurface, VoxelGrid,
376    };
377    use crate::EventStream;
378
379    fn empty_stream(width: usize, height: usize) -> EventStream {
380        EventStream::from_array2(Array2::zeros((0, 4)), width, height, 0.001)
381    }
382
383    #[test]
384    fn empty_streams_produce_zero_outputs() {
385        let stream = empty_stream(2, 3);
386
387        for frame in [
388            Binary.generate(&stream).unwrap(),
389            EventCount::new(true).generate(&stream).unwrap(),
390            VoxelGrid::default().generate(&stream).unwrap(),
391            TimeSurface::default().generate(&stream).unwrap(),
392            AveragedTimeSurface::default().generate(&stream).unwrap(),
393            Tencode::default().generate(&stream).unwrap(),
394            Mcts::default().generate(&stream).unwrap(),
395            CountMask::default().generate(&stream).unwrap(),
396        ] {
397            match frame.data() {
398                EventFrameData::U8(values) => assert!(values.iter().all(|&value| value == 0)),
399                EventFrameData::F32(values) => assert!(values.iter().all(|&value| value == 0.0)),
400                _ => panic!("unexpected empty representation dtype"),
401            }
402        }
403        assert_eq!(PointSet.generate(&stream).unwrap().shape(), (0, 4));
404    }
405
406    #[test]
407    fn rejects_invalid_parameters_and_size_overflow() {
408        let stream = empty_stream(2, 3);
409
410        assert_eq!(
411            VoxelGrid::new(0, 30.0).generate(&stream).unwrap_err(),
412            RepresentationError::InvalidParameter("bins")
413        );
414        assert_eq!(
415            TimeSurface::new(f64::NAN).generate(&stream).unwrap_err(),
416            RepresentationError::InvalidParameter("tau_ms")
417        );
418        assert_eq!(
419            Tencode::new(0.0).generate(&stream).unwrap_err(),
420            RepresentationError::InvalidParameter("window_ms")
421        );
422        assert_eq!(
423            Mcts::new(0.5).generate(&stream).unwrap_err(),
424            RepresentationError::InvalidParameter("max_window_ms")
425        );
426        assert_eq!(
427            CountMask::new(150.0, false).generate(&stream).unwrap_err(),
428            RepresentationError::InvalidParameter("pct")
429        );
430
431        let oversized = empty_stream(usize::MAX, 2);
432        assert_eq!(
433            Binary.generate(&oversized).unwrap_err(),
434            RepresentationError::SizeOverflow
435        );
436    }
437}
eventcv_core/representation.rs

eventcv_core/
representation.rs
