realsense_rust/frame/
image.rs

1//! Type for representing an Image frame taken from a RealSense camera.
2//!
3//! An "Image" Frame can be one of several things:
4//!
5//! - Depth Frame: A depth frame taken from a synthetic depth camera.
6//! - Disparity Frame: A disparity frame taken from a synthetic depth camera.
7//! - Color Frame: A frame holding color or monochrome data.
8//!
9//! Each frame type can hold data in multiple formats. The data type presented
10//! depends on the settings and flags used at runtime on the RealSense device.
11
12use super::pixel::{get_pixel, PixelKind};
13use super::prelude::{
14    CouldNotGetFrameSensorError, DepthError, DisparityError, FrameCategory, FrameConstructionError,
15    FrameEx, BITS_PER_BYTE,
16};
17use crate::{
18    check_rs2_error,
19    kind::{Rs2Extension, Rs2FrameMetadata, Rs2Option, Rs2StreamKind, Rs2TimestampDomain},
20    sensor::Sensor,
21    stream_profile::StreamProfile,
22};
23use anyhow::Result;
24use num_traits::FromPrimitive;
25use realsense_sys as sys;
26use std::{
27    convert::{TryFrom, TryInto},
28    marker::PhantomData,
29    os::raw::c_int,
30    ptr::{self, NonNull},
31};
32
33/// A unit struct defining a Depth frame.
34#[derive(Debug)]
35pub struct Depth;
36/// A unit struct defining a Disparity frame.
37#[derive(Debug)]
38pub struct Disparity;
39/// A unit struct defining a Color frame.
40#[derive(Debug)]
41pub struct Color;
42/// A unit struct defining an Infrared frame.
43#[derive(Debug)]
44pub struct Infrared;
45/// A unit struct defining a Fisheye frame.
46#[derive(Debug)]
47pub struct Fisheye;
48/// A unit struct defining a Confidence frame.
49#[derive(Debug)]
50pub struct Confidence;
51
52/// Holds the raw data pointer and derived data for an RS2 Image frame.
53///
54/// This generic type isn't particularly useful on it's own. In all cases, you want a specialized
55/// version of this class ([`DepthFrame`], [`ColorFrame`], [`DisparityFrame`]).
56#[derive(Debug)]
57pub struct ImageFrame<Kind> {
58    /// The raw data pointer from the original rs2 frame.
59    frame_ptr: NonNull<sys::rs2_frame>,
60    /// The width of the frame in pixels.
61    width: usize,
62    /// The height of the frame in pixels.
63    height: usize,
64    /// The pixel stride of the frame in bytes.
65    stride: usize,
66    /// The number of bits per pixel.
67    bits_per_pixel: usize,
68    /// The timestamp of the frame.
69    timestamp: f64,
70    /// The RealSense time domain from which the timestamp is derived.
71    timestamp_domain: Rs2TimestampDomain,
72    /// The frame number.
73    frame_number: u64,
74    /// The Stream Profile that created the frame.
75    frame_stream_profile: StreamProfile,
76    /// The size in bytes of the data contained in the frame.
77    data_size_in_bytes: usize,
78    /// The frame data contained in the frame.
79    data: NonNull<std::os::raw::c_void>,
80    /// A boolean used during `Drop` calls. This allows for proper handling of the pointer
81    /// during ownership transfer.
82    should_drop: bool,
83    /// Holds the type metadata of this frame.
84    _phantom: PhantomData<Kind>,
85}
86
87/// A type which acts as an iterator over an image frame of some pixel kind.
88pub struct Iter<'a, K> {
89    /// The image frame to iterate over.
90    pub(crate) frame: &'a ImageFrame<K>,
91
92    /// The current column.
93    pub(crate) column: usize,
94
95    /// The current row.
96    pub(crate) row: usize,
97}
98
99impl<'a, K> Iterator for Iter<'a, K> {
100    type Item = PixelKind<'a>;
101
102    /// Provides a row-major iterator over an entire Image.
103    fn next(&mut self) -> Option<Self::Item> {
104        if self.column >= self.frame.width() || self.row >= self.frame.height() {
105            return None;
106        }
107
108        let next = self.frame.get_unchecked(self.column, self.row);
109
110        self.column += 1;
111
112        if self.column >= self.frame.width() {
113            self.column = 0;
114            self.row += 1;
115        }
116        Some(next)
117    }
118}
119
120/// An ImageFrame type holding the raw pointer and derived metadata for an RS2 Depth frame.
121///
122/// All fields in this struct are initialized during struct creation (via `try_from`).
123/// Everything called from here during runtime should be valid as long as the
124/// Frame is in scope... like normal Rust.
125pub type DepthFrame = ImageFrame<Depth>;
126/// An ImageFrame type holding the raw pointer and derived metadata for an RS2 Disparity frame.
127///
128/// All fields in this struct are initialized during struct creation (via `try_from`).
129/// Everything called from here during runtime should be valid as long as the
130/// Frame is in scope... like normal Rust.
131pub type DisparityFrame = ImageFrame<Disparity>;
132/// An ImageFrame type holding the raw pointer and derived metadata for an RS2 Color frame.
133///
134/// All fields in this struct are initialized during struct creation (via `try_from`).
135/// Everything called from here during runtime should be valid as long as the
136/// Frame is in scope... like normal Rust.
137pub type ColorFrame = ImageFrame<Color>;
138/// An ImageFrame type holding the raw pointer and derived metadata for an RS2 Infrared frame.
139///
140/// All fields in this struct are initialized during struct creation (via `try_from`).
141/// Everything called from here during runtime should be valid as long as the
142/// Frame is in scope... like normal Rust.
143pub type InfraredFrame = ImageFrame<Infrared>;
144/// An ImageFrame type holding the raw pointer and derived metadata for an RS2 Fisheye frame.
145///
146/// All fields in this struct are initialized during struct creation (via `try_from`).
147/// Everything called from here during runtime should be valid as long as the
148/// Frame is in scope... like normal Rust.
149pub type FisheyeFrame = ImageFrame<Fisheye>;
150/// An ImageFrame type holding the raw pointer and derived metadata for an RS2 Confidence frame.
151///
152/// All fields in this struct are initialized during struct creation (via `try_from`).
153/// Everything called from here during runtime should be valid as long as the
154/// Frame is in scope... like normal Rust.
155pub type ConfidenceFrame = ImageFrame<Confidence>;
156
157impl<K> Drop for ImageFrame<K> {
158    fn drop(&mut self) {
159        unsafe {
160            if self.should_drop {
161                sys::rs2_release_frame(self.frame_ptr.as_ptr());
162            }
163        }
164    }
165}
166
167impl<'a, K> IntoIterator for &'a ImageFrame<K> {
168    type Item = <Iter<'a, K> as Iterator>::Item;
169    type IntoIter = Iter<'a, K>;
170
171    fn into_iter(self) -> Self::IntoIter {
172        self.iter()
173    }
174}
175
176unsafe impl<K> Send for ImageFrame<K> {}
177
178impl<K> TryFrom<NonNull<sys::rs2_frame>> for ImageFrame<K> {
179    type Error = anyhow::Error;
180
181    /// Attempt to construct an Image frame of extension K from the raw `rs2_frame`.
182    ///
183    /// All members of the `ImageFrame` struct are validated and populated during this call.
184    ///
185    /// # Errors
186    ///
187    /// There are a number of errors that may occur if the data in the `rs2_frame` is not valid,
188    /// all of type [`FrameConstructionError`].
189    ///
190    /// - [`CouldNotGetWidth`](FrameConstructionError::CouldNotGetWidth)
191    /// - [`CouldNotGetHeight`](FrameConstructionError::CouldNotGetHeight)
192    /// - [`CouldNotGetBitsPerPixel`](FrameConstructionError::CouldNotGetBitsPerPixel)
193    /// - [`CouldNotGetStride`](FrameConstructionError::CouldNotGetStride)
194    /// - [`CouldNotGetTimestamp`](FrameConstructionError::CouldNotGetTimestamp)
195    /// - [`CouldNotGetTimestampDomain`](FrameConstructionError::CouldNotGetTimestampDomain)
196    /// - [`CouldNotGetFrameStreamProfile`](FrameConstructionError::CouldNotGetFrameStreamProfile)
197    /// - [`CouldNotGetDataSize`](FrameConstructionError::CouldNotGetDataSize)
198    /// - [`CouldNotGetData`](FrameConstructionError::CouldNotGetData)
199    ///
200    /// See [`FrameConstructionError`] documentation for more details.
201    fn try_from(frame_ptr: NonNull<sys::rs2_frame>) -> Result<Self, Self::Error> {
202        unsafe {
203            let mut err = ptr::null_mut::<sys::rs2_error>();
204            let width = sys::rs2_get_frame_width(frame_ptr.as_ptr(), &mut err);
205            check_rs2_error!(err, FrameConstructionError::CouldNotGetWidth)?;
206
207            let height = sys::rs2_get_frame_height(frame_ptr.as_ptr(), &mut err);
208            check_rs2_error!(err, FrameConstructionError::CouldNotGetHeight)?;
209
210            let bits_per_pixel = sys::rs2_get_frame_bits_per_pixel(frame_ptr.as_ptr(), &mut err);
211            check_rs2_error!(err, FrameConstructionError::CouldNotGetBitsPerPixel)?;
212
213            let stride = sys::rs2_get_frame_stride_in_bytes(frame_ptr.as_ptr(), &mut err);
214            check_rs2_error!(err, FrameConstructionError::CouldNotGetStride)?;
215
216            let timestamp = sys::rs2_get_frame_timestamp(frame_ptr.as_ptr(), &mut err);
217            check_rs2_error!(err, FrameConstructionError::CouldNotGetTimestamp)?;
218
219            let timestamp_domain =
220                sys::rs2_get_frame_timestamp_domain(frame_ptr.as_ptr(), &mut err);
221            check_rs2_error!(err, FrameConstructionError::CouldNotGetTimestampDomain)?;
222
223            let frame_number = sys::rs2_get_frame_number(frame_ptr.as_ptr(), &mut err);
224            check_rs2_error!(err, FrameConstructionError::CouldNotGetFrameNumber)?;
225
226            let profile_ptr = sys::rs2_get_frame_stream_profile(frame_ptr.as_ptr(), &mut err);
227            check_rs2_error!(err, FrameConstructionError::CouldNotGetFrameStreamProfile)?;
228
229            let nonnull_profile_ptr =
230                NonNull::new(profile_ptr as *mut sys::rs2_stream_profile).unwrap();
231            let profile = StreamProfile::try_from(nonnull_profile_ptr)?;
232
233            let size = sys::rs2_get_frame_data_size(frame_ptr.as_ptr(), &mut err);
234            check_rs2_error!(err, FrameConstructionError::CouldNotGetDataSize)?;
235
236            debug_assert_eq!(size, width * height * bits_per_pixel / BITS_PER_BYTE);
237
238            let data_ptr = sys::rs2_get_frame_data(frame_ptr.as_ptr(), &mut err);
239            check_rs2_error!(err, FrameConstructionError::CouldNotGetData)?;
240
241            let nonnull_data_ptr = NonNull::new(data_ptr as *mut std::os::raw::c_void).unwrap();
242
243            Ok(ImageFrame {
244                frame_ptr,
245                width: width as usize,
246                height: height as usize,
247                stride: stride as usize,
248                bits_per_pixel: bits_per_pixel as usize,
249                timestamp,
250                timestamp_domain: Rs2TimestampDomain::from_i32(timestamp_domain as i32).unwrap(),
251                frame_number,
252                frame_stream_profile: profile,
253                data_size_in_bytes: size as usize,
254                data: nonnull_data_ptr,
255                should_drop: true,
256                _phantom: PhantomData::<K> {},
257            })
258        }
259    }
260}
261
262impl FrameCategory for DepthFrame {
263    fn extension() -> Rs2Extension {
264        Rs2Extension::DepthFrame
265    }
266
267    fn kind() -> Rs2StreamKind {
268        Rs2StreamKind::Depth
269    }
270
271    fn has_correct_kind(&self) -> bool {
272        self.frame_stream_profile.kind() == Self::kind()
273    }
274}
275
276impl FrameCategory for DisparityFrame {
277    fn extension() -> Rs2Extension {
278        Rs2Extension::DisparityFrame
279    }
280
281    fn kind() -> Rs2StreamKind {
282        Rs2StreamKind::Any
283    }
284
285    fn has_correct_kind(&self) -> bool {
286        self.frame_stream_profile.kind() == Self::kind()
287    }
288}
289
290impl FrameCategory for ColorFrame {
291    fn extension() -> Rs2Extension {
292        Rs2Extension::VideoFrame
293    }
294
295    fn kind() -> Rs2StreamKind {
296        Rs2StreamKind::Color
297    }
298
299    fn has_correct_kind(&self) -> bool {
300        self.frame_stream_profile.kind() == Self::kind()
301    }
302}
303
304impl FrameCategory for InfraredFrame {
305    fn extension() -> Rs2Extension {
306        Rs2Extension::VideoFrame
307    }
308
309    fn kind() -> Rs2StreamKind {
310        Rs2StreamKind::Infrared
311    }
312
313    fn has_correct_kind(&self) -> bool {
314        self.frame_stream_profile.kind() == Self::kind()
315    }
316}
317
318impl FrameCategory for FisheyeFrame {
319    fn extension() -> Rs2Extension {
320        Rs2Extension::VideoFrame
321    }
322
323    fn kind() -> Rs2StreamKind {
324        Rs2StreamKind::Fisheye
325    }
326
327    fn has_correct_kind(&self) -> bool {
328        self.frame_stream_profile.kind() == Self::kind()
329    }
330}
331
332impl FrameCategory for ConfidenceFrame {
333    fn extension() -> Rs2Extension {
334        Rs2Extension::VideoFrame
335    }
336
337    fn kind() -> Rs2StreamKind {
338        Rs2StreamKind::Confidence
339    }
340
341    fn has_correct_kind(&self) -> bool {
342        self.frame_stream_profile.kind() == Self::kind()
343    }
344}
345
346impl<T> FrameEx for ImageFrame<T> {
347    fn stream_profile(&self) -> &StreamProfile {
348        &self.frame_stream_profile
349    }
350
351    fn sensor(&self) -> Result<Sensor> {
352        unsafe {
353            let mut err = std::ptr::null_mut::<sys::rs2_error>();
354            let sensor_ptr = sys::rs2_get_frame_sensor(self.frame_ptr.as_ptr(), &mut err);
355            check_rs2_error!(err, CouldNotGetFrameSensorError)?;
356
357            Ok(Sensor::from(NonNull::new(sensor_ptr).unwrap()))
358        }
359    }
360
361    fn timestamp(&self) -> f64 {
362        self.timestamp
363    }
364
365    fn timestamp_domain(&self) -> Rs2TimestampDomain {
366        self.timestamp_domain
367    }
368
369    fn frame_number(&self) -> u64 {
370        self.frame_number
371    }
372
373    fn metadata(&self, metadata_kind: Rs2FrameMetadata) -> Option<std::os::raw::c_longlong> {
374        if !self.supports_metadata(metadata_kind) {
375            return None;
376        }
377
378        unsafe {
379            let mut err = std::ptr::null_mut::<sys::rs2_error>();
380
381            let val = sys::rs2_get_frame_metadata(
382                self.frame_ptr.as_ptr(),
383                #[allow(clippy::useless_conversion)]
384                (metadata_kind as i32).try_into().unwrap(),
385                &mut err,
386            );
387
388            if err.as_ref().is_none() {
389                Some(val)
390            } else {
391                sys::rs2_free_error(err);
392                None
393            }
394        }
395    }
396
397    fn supports_metadata(&self, metadata_kind: Rs2FrameMetadata) -> bool {
398        unsafe {
399            let mut err = std::ptr::null_mut::<sys::rs2_error>();
400
401            let supports_metadata = sys::rs2_supports_frame_metadata(
402                self.frame_ptr.as_ptr(),
403                #[allow(clippy::useless_conversion)]
404                (metadata_kind as i32).try_into().unwrap(),
405                &mut err,
406            );
407
408            if err.as_ref().is_none() {
409                supports_metadata != 0
410            } else {
411                sys::rs2_free_error(err);
412                false
413            }
414        }
415    }
416
417    unsafe fn get_owned_raw(mut self) -> NonNull<sys::rs2_frame> {
418        self.should_drop = false;
419
420        self.frame_ptr
421    }
422}
423
424impl DepthFrame {
425    /// Given the 2D depth coordinate (x,y) provide the corresponding depth in metric units.
426    ///
427    /// # Warning
428    ///
429    /// It is fairly expensive to use this in practice as it will copy the underlying pixel into a
430    /// f32 value that gives you the direct distance. In practice getting
431    /// `DepthFrame::depth_units` and then applying that to the raw data with [`ImageFrame::get`]
432    /// is a much more efficient way to handle this.
433    pub fn distance(&self, col: usize, row: usize) -> Result<f32, DepthError> {
434        unsafe {
435            let mut err = ptr::null_mut::<sys::rs2_error>();
436            let distance = sys::rs2_depth_frame_get_distance(
437                self.frame_ptr.as_ptr(),
438                col as c_int,
439                row as c_int,
440                &mut err,
441            );
442            check_rs2_error!(err, DepthError::CouldNotGetDistance)?;
443            Ok(distance)
444        }
445    }
446
447    /// Get the metric units currently used for reporting depth information.
448    pub fn depth_units(&self) -> Result<f32> {
449        let sensor = self.sensor()?;
450        let depth_units = sensor.get_option(Rs2Option::DepthUnits).ok_or_else(|| {
451            anyhow::anyhow!("Option is not supported on the sensor for this frame type.")
452        })?;
453        Ok(depth_units)
454    }
455}
456
457impl DisparityFrame {
458    /// Given the 2D depth coordinate (x,y) provide the corresponding depth in metric units.
459    ///
460    /// # Warning
461    ///
462    /// Like with depth frames, this method is fairly expensive to use in practice. The disparity
463    /// can be converted to depth fairly easily, but this will effectively copy every pixel if you
464    /// loop through the data with this method for every index.
465    ///
466    /// It is often much more efficient to directly stream the
467    /// [`Rs2Format::Distance`](crate::kind::Rs2Format::Distance) format if you want the distance
468    /// directly, and access the frame data with [`ImageFrame::get`].
469    pub fn distance(&self, col: usize, row: usize) -> Result<f32, DepthError> {
470        unsafe {
471            let mut err = ptr::null_mut::<sys::rs2_error>();
472            let distance = sys::rs2_depth_frame_get_distance(
473                self.frame_ptr.as_ptr(),
474                col as c_int,
475                row as c_int,
476                &mut err,
477            );
478            check_rs2_error!(err, DepthError::CouldNotGetDistance)?;
479            Ok(distance)
480        }
481    }
482
483    /// Get the metric units currently used for reporting depth information.
484    pub fn depth_units(&self) -> Result<f32> {
485        let sensor = self.sensor()?;
486        let depth_units = sensor.get_option(Rs2Option::DepthUnits).ok_or_else(|| {
487            anyhow::anyhow!("Option is not supported on the sensor for this frame type.")
488        })?;
489        Ok(depth_units)
490    }
491
492    /// Get the baseline used during construction of the Disparity frame
493    pub fn baseline(&self) -> Result<f32, DisparityError> {
494        unsafe {
495            let mut err = ptr::null_mut::<sys::rs2_error>();
496            let baseline =
497                sys::rs2_depth_stereo_frame_get_baseline(self.frame_ptr.as_ptr(), &mut err);
498            check_rs2_error!(err, DisparityError)?;
499            Ok(baseline)
500        }
501    }
502}
503
504impl<K> ImageFrame<K> {
505    /// Iterator through every [pixel](crate::frame::PixelKind) of an image frame.
506    pub fn iter(&self) -> Iter<'_, K> {
507        Iter {
508            frame: self,
509            column: 0,
510            row: 0,
511        }
512    }
513
514    /// Get a pixel value from the Video Frame.
515    ///
516    /// # Safety
517    ///
518    /// This makes a call directly to the underlying data pointer inherited from
519    /// the `rs2_frame`.
520    #[inline(always)]
521    pub fn get_unchecked(&self, col: usize, row: usize) -> PixelKind<'_> {
522        unsafe {
523            get_pixel(
524                self.frame_stream_profile.format(),
525                self.data_size_in_bytes,
526                self.data.as_ptr(),
527                self.stride,
528                col,
529                row,
530            )
531        }
532    }
533
534    /// Get the stride of this Video frame's pixel in bytes.
535    pub fn stride(&self) -> usize {
536        self.stride
537    }
538
539    /// Get the bits per pixel.
540    pub fn bits_per_pixel(&self) -> usize {
541        self.bits_per_pixel
542    }
543
544    /// Get the size of the data in this Video frame in bytes.
545    pub fn get_data_size(&self) -> usize {
546        self.data_size_in_bytes
547    }
548
549    /// Get a reference to the raw data held by this Video frame.
550    ///
551    /// # Safety
552    ///
553    /// This is a raw pointer to the underlying data. This data has to be interpreted according to
554    /// the format of the frame itself. In most scenarios you will probably want to just use the
555    /// `get_unchecked` function associated with the [`FrameEx`](crate::frame::FrameEx) trait, but
556    /// this can be useful if you need more immediate access to the underlying pixel data.
557    pub unsafe fn get_data(&self) -> &std::os::raw::c_void {
558        self.data.as_ref()
559    }
560
561    /// Get the width of this Video frame in pixels
562    pub fn width(&self) -> usize {
563        self.width
564    }
565
566    /// Get the height of this Video frame in pixels
567    pub fn height(&self) -> usize {
568        self.height
569    }
570
571    /// Given a row and column index, Get a pixel value from this frame.
572    pub fn get(&self, col: usize, row: usize) -> Option<PixelKind<'_>> {
573        if col >= self.width || row >= self.height {
574            None
575        } else {
576            Some(self.get_unchecked(col, row))
577        }
578    }
579}
580
581#[cfg(test)]
582mod tests {
583    use super::*;
584
585    #[test]
586    fn frame_has_correct_kind() {
587        assert_eq!(ColorFrame::kind(), Rs2StreamKind::Color);
588        assert_eq!(DepthFrame::kind(), Rs2StreamKind::Depth);
589        assert_eq!(DisparityFrame::kind(), Rs2StreamKind::Any);
590        assert_eq!(InfraredFrame::kind(), Rs2StreamKind::Infrared);
591        assert_eq!(FisheyeFrame::kind(), Rs2StreamKind::Fisheye);
592        assert_eq!(ConfidenceFrame::kind(), Rs2StreamKind::Confidence);
593    }
594}
realsense_rust/frame/image.rs

realsense_rust/frame/
image.rs
