re_mcap/parsers/ros2msg/sensor_msgs/
image.rs

1use anyhow::Context as _;
2use re_chunk::{Chunk, ChunkId};
3use re_sdk_types::archetypes::{CoordinateFrame, DepthImage, Image};
4use re_sdk_types::datatypes::{ChannelDatatype, ColorModel, ImageFormat, PixelFormat};
5
6use super::super::Ros2MessageParser;
7use super::super::util::suffix_image_plane_frame_ids;
8use crate::parsers::cdr;
9use crate::parsers::decode::{MessageParser, ParserContext};
10use crate::parsers::ros2msg::definitions::sensor_msgs;
11
12pub struct ImageMessageParser {
13    /// The raw image data blobs.
14    ///
15    /// Note: These blobs are directly moved into a `Blob`, without copying.
16    blobs: Vec<Vec<u8>>,
17    image_formats: Vec<ImageFormat>,
18    is_depth_image: bool,
19    frame_ids: Vec<String>,
20}
21
22impl Ros2MessageParser for ImageMessageParser {
23    fn new(num_rows: usize) -> Self {
24        Self {
25            blobs: Vec::with_capacity(num_rows),
26            image_formats: Vec::with_capacity(num_rows),
27            is_depth_image: false,
28            frame_ids: Vec::with_capacity(num_rows),
29        }
30    }
31}
32
33impl MessageParser for ImageMessageParser {
34    fn append(&mut self, ctx: &mut ParserContext, msg: &mcap::Message<'_>) -> anyhow::Result<()> {
35        re_tracing::profile_function!();
36        let sensor_msgs::Image {
37            header,
38            data,
39            height,
40            width,
41            encoding,
42            ..
43        } = cdr::try_decode_message::<sensor_msgs::Image<'_>>(&msg.data)
44            .context("Failed to decode sensor_msgs::Image message from CDR data")?;
45
46        // add the sensor timestamp to the context, `log_time` and `publish_time` are added automatically
47        ctx.add_timestamp_cell(crate::util::TimestampCell::guess_from_nanos_ros2(
48            header.stamp.as_nanos() as u64,
49        ));
50
51        self.frame_ids.push(header.frame_id);
52
53        let dimensions = [width, height];
54        let img_encoding = decode_image_encoding(&encoding)
55            .with_context(|| format!("Failed to decode image format for encoding '{encoding}' with dimensions {width}x{height}"))?;
56
57        // We assume that images with a single channel encoding (e.g. `16UC1`) are depth images, and all others are regular color images.
58        self.is_depth_image = img_encoding.is_single_channel();
59
60        self.blobs.push(data.into_owned());
61        self.image_formats
62            .push(img_encoding.to_image_format(dimensions));
63
64        Ok(())
65    }
66
67    fn finalize(self: Box<Self>, ctx: ParserContext) -> anyhow::Result<Vec<re_chunk::Chunk>> {
68        re_tracing::profile_function!();
69        let Self {
70            blobs,
71            image_formats,
72            is_depth_image,
73            frame_ids,
74        } = *self;
75
76        let entity_path = ctx.entity_path().clone();
77        let timelines = ctx.build_timelines();
78
79        // TODO(#10726): big assumption here: image format can technically be different for each image on the topic, e.g. depth and color archetypes could be mixed here!
80        let mut chunk_components: Vec<_> = if is_depth_image {
81            DepthImage::update_fields()
82                .with_many_buffer(blobs)
83                .with_many_format(image_formats)
84                .columns_of_unit_batches()?
85                .collect()
86        } else {
87            Image::update_fields()
88                .with_many_buffer(blobs)
89                .with_many_format(image_formats)
90                .columns_of_unit_batches()?
91                .collect()
92        };
93
94        // We need a frame ID for the image plane. This doesn't exist in ROS,
95        // so we use the camera frame ID with a suffix here (see also camera info parser).
96        let image_plane_frame_ids = suffix_image_plane_frame_ids(frame_ids);
97        chunk_components.extend(
98            CoordinateFrame::update_fields()
99                .with_many_frame(image_plane_frame_ids)
100                .columns_of_unit_batches()?,
101        );
102
103        Ok(vec![Chunk::from_auto_row_ids(
104            ChunkId::new(),
105            entity_path.clone(),
106            timelines.clone(),
107            chunk_components.into_iter().collect(),
108        )?])
109    }
110}
111
112/// A raw image encoding string, as used by ROS and Foxglove.
113///
114/// OpenCV-style single-channel encodings (`8UC1`, `16UC1`, etc.) are treated as depth formats.
115#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::EnumString, strum::VariantNames)]
116pub enum ImageEncoding {
117    #[strum(to_string = "rgb8")]
118    Rgb8,
119    #[strum(to_string = "rgba8")]
120    Rgba8,
121    #[strum(to_string = "rgb16")]
122    Rgb16,
123    #[strum(to_string = "rgba16")]
124    Rgba16,
125    #[strum(to_string = "bgr8")]
126    Bgr8,
127    #[strum(to_string = "bgra8")]
128    Bgra8,
129    #[strum(to_string = "bgr16")]
130    Bgr16,
131    #[strum(to_string = "bgra16")]
132    Bgra16,
133    #[strum(to_string = "mono8")]
134    Mono8,
135    #[strum(to_string = "mono16")]
136    Mono16,
137    #[strum(to_string = "yuyv", serialize = "yuv422_yuy2")]
138    Yuyv,
139    #[strum(to_string = "nv12")]
140    Nv12,
141    // OpenCV-style single-channel (depth) formats
142    #[strum(to_string = "8UC1")]
143    Cv8UC1,
144    #[strum(to_string = "8UC3")]
145    Cv8UC3,
146    #[strum(to_string = "8SC1")]
147    Cv8SC1,
148    #[strum(to_string = "16UC1")]
149    Cv16UC1,
150    #[strum(to_string = "16SC1")]
151    Cv16SC1,
152    #[strum(to_string = "32SC1")]
153    Cv32SC1,
154    #[strum(to_string = "32FC1")]
155    Cv32FC1,
156    #[strum(to_string = "64FC1")]
157    Cv64FC1,
158}
159
160impl ImageEncoding {
161    /// All encoding name strings accepted by [`std::str::FromStr`].
162    pub const NAMES: &[&str] = <Self as strum::VariantNames>::VARIANTS;
163
164    /// Returns `true` for OpenCV-style single-channel encodings (e.g. `8UC1`, `16UC1`, `32FC1`).
165    pub fn is_single_channel(self) -> bool {
166        matches!(
167            self,
168            Self::Cv8UC1
169                | Self::Cv8SC1
170                | Self::Cv16UC1
171                | Self::Cv16SC1
172                | Self::Cv32SC1
173                | Self::Cv32FC1
174                | Self::Cv64FC1
175                | Self::Mono8
176                | Self::Mono16
177        )
178    }
179
180    /// Converts this encoding into a Rerun [`ImageFormat`] for the given dimensions.
181    pub fn to_image_format(self, dimensions: [u32; 2]) -> ImageFormat {
182        match self {
183            Self::Rgb8 => ImageFormat::rgb8(dimensions),
184            Self::Rgba8 => ImageFormat::rgba8(dimensions),
185            Self::Rgb16 => {
186                ImageFormat::from_color_model(dimensions, ColorModel::RGB, ChannelDatatype::U16)
187            }
188            Self::Rgba16 => {
189                ImageFormat::from_color_model(dimensions, ColorModel::RGBA, ChannelDatatype::U16)
190            }
191            Self::Bgr8 | Self::Cv8UC3 => {
192                ImageFormat::from_color_model(dimensions, ColorModel::BGR, ChannelDatatype::U8)
193            }
194            Self::Bgra8 => {
195                ImageFormat::from_color_model(dimensions, ColorModel::BGRA, ChannelDatatype::U8)
196            }
197            Self::Bgr16 => {
198                ImageFormat::from_color_model(dimensions, ColorModel::BGR, ChannelDatatype::U16)
199            }
200            Self::Bgra16 => {
201                ImageFormat::from_color_model(dimensions, ColorModel::BGRA, ChannelDatatype::U16)
202            }
203            Self::Mono8 => {
204                ImageFormat::from_color_model(dimensions, ColorModel::L, ChannelDatatype::U8)
205            }
206            Self::Mono16 => {
207                ImageFormat::from_color_model(dimensions, ColorModel::L, ChannelDatatype::U16)
208            }
209            Self::Yuyv => ImageFormat::from_pixel_format(dimensions, PixelFormat::YUY2),
210            Self::Nv12 => ImageFormat::from_pixel_format(dimensions, PixelFormat::NV12),
211            Self::Cv8UC1 => ImageFormat::depth(dimensions, ChannelDatatype::U8),
212            Self::Cv8SC1 => ImageFormat::depth(dimensions, ChannelDatatype::I8),
213            Self::Cv16UC1 => ImageFormat::depth(dimensions, ChannelDatatype::U16),
214            Self::Cv16SC1 => ImageFormat::depth(dimensions, ChannelDatatype::I16),
215            Self::Cv32SC1 => ImageFormat::depth(dimensions, ChannelDatatype::I32),
216            Self::Cv32FC1 => ImageFormat::depth(dimensions, ChannelDatatype::F32),
217            Self::Cv64FC1 => ImageFormat::depth(dimensions, ChannelDatatype::F64),
218        }
219    }
220}
221
222/// Parses a raw image encoding string (shared by ROS and Foxglove) into an [`ImageEncoding`].
223///
224/// Supports common encoding strings such as `rgb8`, `mono16`, `16UC1`, `yuyv`, `nv12`, etc.
225pub fn decode_image_encoding(encoding: &str) -> anyhow::Result<ImageEncoding> {
226    encoding.parse().map_err(|_err| {
227        anyhow::anyhow!(
228            "Unsupported image encoding '{encoding}'. Supported encodings: {:?}",
229            ImageEncoding::NAMES
230        )
231    })
232}
233
234/// Decodes a raw image encoding string (shared by ROS and Foxglove) into a Rerun [`ImageFormat`].
235///
236/// Supports common encoding strings such as `rgb8`, `mono16`, `16UC1`, `yuyv`, `nv12`, etc.
237/// OpenCV-style single-channel encodings (`8UC1`, `16UC1`, etc.) are treated as depth formats.
238pub fn decode_image_format(encoding: &str, dimensions: [u32; 2]) -> anyhow::Result<ImageFormat> {
239    Ok(decode_image_encoding(encoding)?.to_image_format(dimensions))
240}
re_mcap/parsers/ros2msg/sensor_msgs/image.rs

re_mcap/parsers/ros2msg/sensor_msgs/
image.rs
