mujoco_rs/

renderer.rs

//! Module related to implementation of the [`MjRenderer`].
use crate::wrappers::mj_model::traits::ModelType;
use crate::wrappers::mj_visualization::MjvScene;
use crate::wrappers::mj_rendering::MjrContext;

#[cfg(target_os = "linux")]
use crate::renderer::egl::GlStateEgl;

use crate::vis_common::{sync_geoms, flip_image_vertically, write_png};
use crate::builder_setters;
use crate::prelude::*;

use bitflags::bitflags;
#[cfg(all(feature = "renderer-winit-fallback", target_os = "linux"))]
use log::warn;
use log::debug;

use std::io::{self, BufWriter, Write};
use std::fmt::Display;
use std::error::Error;
use std::num::NonZero;
use std::path::Path;
use std::fs::File;

/// Exported `png` crate for convenience.
pub use png;

/// Scale factor for converting normalized [0..1] depth to u16.
const DEPTH_U16_SCALE: f32 = u16::MAX as f32;

#[cfg(feature = "renderer-winit-fallback")]
mod universal;

#[cfg(feature = "renderer-winit-fallback")]
use universal::GlStateWinit;

#[cfg(target_os = "linux")]
mod egl;


/// Default headroom of internal visual-only geom slots, added on top of the model's `ngeom`.
const EXTRA_INTERNAL_VISUAL_GEOMS: u32 = 100;

/// Compute the depth near/far clip planes in metres from the model.
fn model_near_far(model: &MjModel) -> (f32, f32) {
    let map = &model.vis().map;
    let extent = model.stat().extent as f32;
    (map.znear * extent, map.zfar * extent)
}


/// GlState enum wrapper. By default, headless implementation will be used
/// when supported. Only on failure will an invisible winit window be used.
#[derive(Debug)]
#[allow(clippy::large_enum_variant)]
pub(crate) enum GlState {
    #[cfg(feature = "renderer-winit-fallback")] Winit(GlStateWinit),
    #[cfg(target_os = "linux")] Egl(egl::GlStateEgl),
}

impl GlState {
    /// Creates a new [`GlState`], which by default tries to use
    /// an offscreen implementation. As a fallback, winit will be used.
    pub(crate) fn new(width: NonZero<u32>, height: NonZero<u32>) -> Result<Self, RendererError> {
        #[cfg(target_os = "linux")]
        #[allow(unused_variables)]
        let egl_err = match GlStateEgl::new(width, height) {
            Ok(egl_state) => return Ok(Self::Egl(egl_state)),
            Err(e) => e,
        };

        #[cfg(feature = "renderer-winit-fallback")]
        match GlStateWinit::new(width, height) {
            Ok(winit_state) => {
                #[cfg(target_os = "linux")]
                warn!("offscreen rendering is not available ({egl_err}), using an invisible window");
                return Ok(Self::Winit(winit_state));
            },
            #[cfg(not(target_os = "linux"))]
            Err(e) => {
                return Err(e);
            },

            #[cfg(target_os = "linux")]
            _ => {}
        }

        #[cfg(target_os = "linux")]
        Err(RendererError::GlutinError(egl_err))
    }

    /// Makes the internal OpenGL context current for the calling thread.
    pub(crate) fn make_current(&self) -> glutin::error::Result<()> {
        match self {
            #[cfg(target_os = "linux")]
            Self::Egl(egl_state) => egl_state.make_current(),
            #[cfg(feature = "renderer-winit-fallback")]
            Self::Winit(winit_state) => winit_state.make_current()
        }
    }
}


/// A builder for [`MjRenderer`].
#[derive(Debug, Clone)]
pub struct MjRendererBuilder {
    width: u32,
    height: u32,
    num_visual_internal_geom: u32,
    num_visual_user_geom: u32,
    rgb: bool,
    depth: bool,
    png_compression: png::Compression,
    font_scale: MjtFontScale,
    camera: MjvCamera,
    opts: MjvOption,
}

impl MjRendererBuilder {
    /// Create a builder with default configuration.
    /// Defaults are:
    /// - `width` and `height`: `0`. When *both* stay `0`, the model's `offwidth` and `offheight`
    ///   (visual/global settings) are used; setting only one of them gives
    ///   [`RendererError::ZeroDimension`],
    /// - `num_visual_internal_geom`: 100,
    /// - `num_visual_user_geom`: 0,
    /// - `rgb`: true,
    /// - `depth`: false,
    /// - `png_compression`: [`png::Compression::NoCompression`] (fastest, largest files),
    /// - `font_scale`: [`MjtFontScale::mjFONTSCALE_100`],
    /// - `camera`: [`MjvCamera::default()`] (MuJoCo's `mjv_defaultCamera`),
    /// - `opts`: [`MjvOption::default()`] (MuJoCo's `mjv_defaultOption`).
    pub fn new() -> Self {
        Self {
            width: 0, height: 0,
            num_visual_internal_geom: EXTRA_INTERNAL_VISUAL_GEOMS, num_visual_user_geom: 0,
            rgb: true, depth: false, png_compression: png::Compression::NoCompression,
            font_scale: MjtFontScale::mjFONTSCALE_100,
            camera: MjvCamera::default(), opts: MjvOption::default(),
        }
    }

    builder_setters! {
        width: u32; "
image width.

<div class=\"warning\">

The width must be less or equal to the offscreen buffer width,
which can be configured at the top of the model's XML like so:

```xml
<visual>
    <global offwidth=\"1920\" .../>
</visual>
```

</div>";

        height: u32; "\
image height.

<div class=\"warning\">

The height must be less or equal to the offscreen buffer height,
which can be configured at the top of the model's XML like so:

```xml
<visual>
    <global offheight=\"1080\" .../>
</visual>
```

</div>";

        num_visual_internal_geom: u32; "\
            maximum number of additional visual-only internal geoms to allocate for.
            Note that the total number of geoms in the internal scene will be
            `model.ngeom` + `num_visual_internal_geom` + `num_visual_user_geom`.";

        num_visual_user_geom: u32;      "maximum number of additional visual-only user geoms (drawn by the user).";
        rgb: bool;                      "RGB rendering enabled (true) or disabled (false).";
        depth: bool;                    "depth rendering enabled (true) or disabled (false).";
        png_compression: png::Compression; "PNG compression level used by [`MjRenderer::save_rgb`] and [`MjRenderer::save_depth`].";
        font_scale: MjtFontScale;       "font scale of drawn text (with [MjrContext]).";
        camera: MjvCamera;              "camera used for drawing.";
        opts: MjvOption;                "visualization options.";
    }

    /// Builds a [`MjRenderer`].
    ///
    /// This creates the rendering context, which can stop the process; see [`MjrContext::new`].
    ///
    /// # Returns
    /// On success, returns [`Ok`] variant containing the [`MjRenderer`].
    /// # Errors
    /// - [`RendererError::ZeroDimension`] if only one of `width` and `height` is zero, or if both
    ///   are zero and the model's `offwidth` or `offheight` is zero.
    /// - [`RendererError::GlutinError`] if OpenGL initialization fails.
    /// - [`RendererError::EventLoopError`] if the event loop fails to initialize, and
    ///   [`RendererError::GlInitFailed`] if the fallback window initialization fails (feature
    ///   `renderer-winit-fallback`, targets other than Linux).
    ///
    /// # Panics
    /// Panics if the GL display reports no framebuffer configuration for the fallback window
    /// (feature `renderer-winit-fallback`).
    pub fn build(self, model: &MjModel) -> Result<MjRenderer, RendererError> {
        // Assume model's maximum should be used
        let mut height = self.height;
        let mut width = self.width;
        if width == 0 && height == 0 {
            let global = &model.vis().global;
            height = global.offheight as u32;
            width = global.offwidth as u32;
        }

        let gl_state = GlState::new(
            NonZero::new(width).ok_or(RendererError::ZeroDimension)?,
            NonZero::new(height).ok_or(RendererError::ZeroDimension)?,
        )?;

        // Initialize the rendering context to render to the offscreen buffer.
        // SAFETY: gl_state was just created above, establishing a current GL context.
        let mut context = unsafe { MjrContext::new(model) };
        context.offscreen();
        context.change_font(self.font_scale);

        let extra_geom = self.num_visual_internal_geom as usize + self.num_visual_user_geom as usize;
        let (near, far) = model_near_far(model);

        // The 3D scene for visualization
        let scene = MjvScene::new(
            model,
            model.ffi().ngeom as usize + extra_geom
        );

        let user_scene = MjvScene::new(
            model,
            self.num_visual_user_geom as usize
        );

        // Construct the renderer and create allocated buffers.
        let renderer = MjRenderer {
            scene, user_scene, context, camera: self.camera, option: self.opts,
            flags: RendererFlags::empty(), rgb: None, depth: None,
            width: width as usize, height: height as usize, gl_state,
            png_compression: self.png_compression, font_scale: self.font_scale,
            near, far, extra_geom,
        }   // These require special care
            .with_rgb_rendering(self.rgb)
            .with_depth_rendering(self.depth);

        debug!("created an offscreen renderer of {width}x{height}");
        Ok(renderer)
    }
}


/// Delegates to [`MjRendererBuilder::new`].
impl Default for MjRendererBuilder {
    fn default() -> Self {
        Self::new()
    }
}

/// A renderer for rendering 3D scenes.
/// By default, RGB rendering is enabled and depth rendering is disabled.
#[derive(Debug)]
pub struct MjRenderer {
    scene: MjvScene,
    user_scene: MjvScene,
    context: MjrContext,

    /* OpenGL */
    gl_state: GlState,

    /* Configuration */
    camera: MjvCamera,
    option: MjvOption,
    flags: RendererFlags,
    png_compression: png::Compression,
    font_scale: MjtFontScale,

    /* Cached from the current model */
    // Depth near/far clip planes in metres, derived from model.vis().map and model.stat().
    near: f32,
    far: f32,
    // Scene capacity headroom beyond ngeom: preserved across model switches.
    extra_geom: usize,

    /* Storage */
    // Use Box to allow less space to be used
    // when rgb or depth rendering is disabled
    rgb: Option<Box<[u8]>>,
    depth: Option<Box<[f32]>>,

    width: usize,
    height: usize,
}

impl MjRenderer {
    /// Construct a new renderer.
    /// The `max_user_geom` parameter
    /// defines how much space will be allocated for additional, user-defined visual-only geoms.
    /// It can thus be set to 0 if no additional geoms will be drawn by the user.
    /// # Scene allocation
    /// The renderer uses two scenes:
    /// - the internal scene: used by the renderer to draw the model's state.
    /// - the user scene: used by the user to add additional geoms to the internal scene
    ///
    /// The **internal scene** allocates the amount of space needed to fit every pre-existing
    /// model geom + user visual-only geoms + additional visual-only geoms that aren't from the user (e.g., tendons).
    /// By default, the renderer reserves 100 extra geom slots for drawing the additional visual-only geoms.
    /// If that is not enough or it is too much, you can construct [`MjRenderer`] via its builder
    /// ([`MjRenderer::builder`]), which allows more configuration.
    ///
    /// <div class="warning">
    ///
    /// Parameters `width` and `height` must be less or equal to the offscreen buffer size,
    /// which can be configured at the top of the model's XML like so:
    ///
    /// ```xml
    /// <visual>
    ///    <global offwidth="1920" offheight="1080"/>
    /// </visual>
    /// ```
    ///
    /// </div>
    /// # Returns
    /// On success, returns [`Ok`] variant containing the [`MjRenderer`].
    /// # Errors
    /// - [`RendererError::ZeroDimension`] if only one of `width` and `height` is zero, or if both
    ///   are zero and the model's `offwidth` or `offheight` is zero.
    /// - [`RendererError::GlutinError`] if OpenGL initialization fails. On Linux this also reports
    ///   a failed winit fallback, because the initializer keeps the EGL error and drops the
    ///   fallback's own error.
    /// - [`RendererError::EventLoopError`] if the event loop fails to initialize, and
    ///   [`RendererError::GlInitFailed`] if the fallback window initialization fails (feature
    ///   `renderer-winit-fallback`, targets other than Linux).
    ///
    /// # Panics
    /// Panics if the GL display reports no framebuffer configuration for the fallback window
    /// (feature `renderer-winit-fallback`).
    pub fn new(model: &MjModel, width: usize, height: usize, max_user_geom: usize) -> Result<Self, RendererError> {
        // Saturate instead of truncating: `as u32` would turn an over-large value into a small
        // valid one, so the renderer would silently use dimensions the caller never asked for.
        MjRendererBuilder::new()
            .width(width.try_into().unwrap_or(u32::MAX))
            .height(height.try_into().unwrap_or(u32::MAX))
            .num_visual_user_geom(max_user_geom.try_into().unwrap_or(u32::MAX))
            .build(model)
    }

    /// Create a [`MjRendererBuilder`] to configure [`MjRenderer`].
    pub fn builder() -> MjRendererBuilder {
        MjRendererBuilder::new()
    }

    /// Return an immutable reference to the internal scene.
    pub fn scene(&self) -> &MjvScene {
        &self.scene
    }

    /// Return an immutable reference to a user scene for drawing custom visual-only geoms.
    pub fn user_scene(&self) -> &MjvScene {
        &self.user_scene
    }

    /// Return a mutable reference to a user scene for drawing custom visual-only geoms.
    pub fn user_scene_mut(&mut self) -> &mut MjvScene {
        &mut self.user_scene
    }

    /// Return an immutable reference to visualization options.
    pub fn opts(&self) -> &MjvOption {
        &self.option
    }

    /// Return a mutable reference to visualization options.
    pub fn opts_mut(&mut self) -> &mut MjvOption {
        &mut self.option
    }

    /// Return an immutable reference to the camera.
    pub fn camera(&self) -> &MjvCamera {
        &self.camera
    }

    /// Return a mutable reference to the camera.
    pub fn camera_mut(&mut self) -> &mut MjvCamera {
        &mut self.camera
    }

    /// Check if RGB rendering is enabled.
    pub fn rgb_enabled(&self) -> bool {
        self.flags.contains(RendererFlags::RENDER_RGB)
    }

    /// Check if depth rendering is enabled.
    pub fn depth_enabled(&self) -> bool {
        self.flags.contains(RendererFlags::RENDER_DEPTH)
    }

    /// Sets the font scale.
    ///
    /// # Errors
    /// Returns [`RendererError::GlutinError`] if the OpenGL context cannot be made current.
    pub fn set_font_scale(&mut self, font_scale: MjtFontScale) -> Result<(), RendererError> {
        self.gl_state.make_current().map_err(RendererError::GlutinError)?;
        self.font_scale = font_scale;
        self.context.change_font(font_scale);
        Ok(())
    }

    /// Update the visualization options.
    pub fn set_opts(&mut self, options: MjvOption) {
        self.option = options;
    }

    fn update_x_from(
        &self,
        model: &MjModel,
        id: usize,
        upload: fn(&MjrContext, &MjModel, usize) -> Result<(), crate::error::MjrContextError>,
    ) -> Result<(), RendererError>
    {
        self.prepare_upload(model)?;
        upload(&self.context, model, id)?;
        Ok(())
    }

    fn prepare_upload(&self, model: &MjModel) -> Result<(), RendererError> {
        // An upload reads only the asset arrays of `model` and writes only the context.
        if !self.context.is_asset_compatible_with_model(model) {
            return Err(RendererError::IncompatibleModel);
        }
        self.gl_state.make_current().map_err(RendererError::GlutinError)
    }

    fn update_all_from_impl(
        &self,
        model: &MjModel,
        n: usize,
        upload: fn(&MjrContext, &MjModel, usize) -> Result<(), crate::error::MjrContextError>,
    ) -> Result<(), RendererError>
    {
        self.prepare_upload(model)?;
        for id in 0..n {
            upload(&self.context, model, id)?;
        }
        Ok(())
    }

    /// Re-uploads the texture with `texture_id` from `model` to the GPU immediately.
    ///
    /// # Errors
    /// - [`RendererError::IncompatibleModel`] if the mesh, heightfield and texture part of the
    ///   layout of `model` differs from that of the rendering context.
    /// - [`RendererError::ContextError`] if `texture_id >= model.ntex()`.
    /// - [`RendererError::GlutinError`] if the OpenGL context cannot be made current.
    pub fn update_texture_from(&self, model: &MjModel, texture_id: usize) -> Result<(), RendererError> {
        self.update_x_from(model, texture_id, MjrContext::upload_texture)
    }

    /// Re-uploads all textures from `model` to the GPU immediately.
    ///
    /// # Errors
    /// - [`RendererError::IncompatibleModel`] if the mesh, heightfield and texture part of the
    ///   layout of `model` differs from that of the rendering context.
    /// - [`RendererError::GlutinError`] if the OpenGL context cannot be made current.
    pub fn update_textures_from(&self, model: &MjModel) -> Result<(), RendererError> {
        self.update_all_from_impl(model, model.ntex() as usize, MjrContext::upload_texture)
    }

    /// Re-uploads the mesh with `mesh_id` from `model` to the GPU immediately.
    ///
    /// All data arrays read by `mjr_uploadMesh` are copied: vertex positions
    /// (`mesh_vert`), per-vertex normals (`mesh_normal`), UV texture coordinates
    /// (`mesh_texcoord`), face-vertex indices (`mesh_face`), face-normal indices
    /// (`mesh_facenormal`), face-texcoord indices (`mesh_facetexcoord`), and convex
    /// hull graph data (`mesh_graph`).
    ///
    /// # Errors
    /// - [`RendererError::IncompatibleModel`] if the mesh, heightfield and texture part of the
    ///   layout of `model` differs from that of the rendering context.
    /// - [`RendererError::ContextError`] if `mesh_id >= model.nmesh()`.
    /// - [`RendererError::GlutinError`] if the OpenGL context cannot be made current.
    pub fn update_mesh_from(&self, model: &MjModel, mesh_id: usize) -> Result<(), RendererError> {
        self.update_x_from(model, mesh_id, MjrContext::upload_mesh)
    }

    /// Re-uploads all meshes from `model` to the GPU immediately.
    ///
    /// All data arrays read by `mjr_uploadMesh` are bulk-uploaded: vertex positions
    /// (`mesh_vert`), per-vertex normals (`mesh_normal`), UV texture coordinates
    /// (`mesh_texcoord`), face-vertex indices (`mesh_face`), face-normal indices
    /// (`mesh_facenormal`), face-texcoord indices (`mesh_facetexcoord`), and convex
    /// hull graph data (`mesh_graph`).
    ///
    /// # Errors
    /// - [`RendererError::IncompatibleModel`] if the mesh, heightfield and texture part of the
    ///   layout of `model` differs from that of the rendering context.
    /// - [`RendererError::GlutinError`] if the OpenGL context cannot be made current.
    pub fn update_meshes_from(&self, model: &MjModel) -> Result<(), RendererError> {
        self.update_all_from_impl(model, model.nmesh() as usize, MjrContext::upload_mesh)
    }

    /// Re-uploads the heightfield with `hfield_id` from `model` to the GPU immediately.
    ///
    /// # Errors
    /// - [`RendererError::IncompatibleModel`] if the mesh, heightfield and texture part of the
    ///   layout of `model` differs from that of the rendering context.
    /// - [`RendererError::ContextError`] if `hfield_id >= model.nhfield()`.
    /// - [`RendererError::GlutinError`] if the OpenGL context cannot be made current.
    pub fn update_hfield_from(&self, model: &MjModel, hfield_id: usize) -> Result<(), RendererError> {
        self.update_x_from(model, hfield_id, MjrContext::upload_hfield)
    }

    /// Re-uploads all heightfields from `model` to the GPU immediately.
    ///
    /// # Errors
    /// - [`RendererError::IncompatibleModel`] if the mesh, heightfield and texture part of the
    ///   layout of `model` differs from that of the rendering context.
    /// - [`RendererError::GlutinError`] if the OpenGL context cannot be made current.
    pub fn update_hfields_from(&self, model: &MjModel) -> Result<(), RendererError> {
        self.update_all_from_impl(model, model.nhfield() as usize, MjrContext::upload_hfield)
    }

    /// Set the camera used for rendering.
    pub fn set_camera(&mut self, camera: MjvCamera)  {
        self.camera = camera;
    }

    /// Enables/disables RGB rendering.
    pub fn set_rgb_rendering(&mut self, enable: bool) {
        self.flags.set(RendererFlags::RENDER_RGB, enable);
        self.rgb = if enable { Some(vec![0; 3 * self.width * self.height].into_boxed_slice()) } else { None } ;
    }

    /// Enables/disables depth rendering.
    pub fn set_depth_rendering(&mut self, enable: bool) {
        self.flags.set(RendererFlags::RENDER_DEPTH, enable);
        self.depth = if enable { Some(vec![0.0; self.width * self.height].into_boxed_slice()) } else { None } ;
    }

    /// Sets the font scale. To be used on construction.
    ///
    /// # Errors
    /// Returns [`RendererError::GlutinError`] if the OpenGL context cannot be made current.
    pub fn with_font_scale(mut self, font_scale: MjtFontScale) -> Result<Self, RendererError> {
        self.set_font_scale(font_scale)?;
        Ok(self)
    }

    /// Update the visualization options and return `self`. To be used on construction.
    pub fn with_opts(mut self, options: MjvOption) -> Self {
        self.set_opts(options);
        self
    }

    /// Set the camera used for rendering. To be used on construction.
    pub fn with_camera(mut self, camera: MjvCamera) -> Self  {
        self.set_camera(camera);
        self
    }

    /// Enables/disables RGB rendering. To be used on construction.
    pub fn with_rgb_rendering(mut self, enable: bool) -> Self {
        self.set_rgb_rendering(enable);
        self
    }

    /// Enables/disables depth rendering. To be used on construction.
    pub fn with_depth_rendering(mut self, enable: bool) -> Self {
        self.set_depth_rendering(enable);
        self
    }

    /// Set the PNG compression level used by [`MjRenderer::save_rgb`] and [`MjRenderer::save_depth`].
    pub fn set_png_compression(&mut self, compression: png::Compression) {
        self.png_compression = compression;
    }

    /// Set the PNG compression level. To be used on construction.
    pub fn with_png_compression(mut self, compression: png::Compression) -> Self {
        self.set_png_compression(compression);
        self
    }

    /// Update the scene with new data from data.
    /// When `data`'s model is not compatible with the renderer's scene or rendering context, the
    /// internal scene, the user scene and the rendering context are recreated for the new model.
    /// Recreating the context can stop the process; see [`MjrContext::new`].
    ///
    /// # Errors
    /// - [`RendererError::GlutinError`] if the OpenGL context could not be made current
    ///   (only when the [`MjModel`] in `data` is incompatible with the internal [`MjvScene`] or
    ///   [`MjrContext`]).
    ///
    /// # Panics
    /// Panics if the renderer's camera is a fixed camera whose `fixedcamid` is out of range for
    /// `data`'s model.
    pub fn sync_data<M: ModelType>(&mut self, data: &mut MjData<M>) -> Result<(), RendererError> {
        if !(
            self.scene.is_compatible_with_model(data.model()) &&
            self.context.is_compatible_with_model(data.model())
        ) {
            /* Model changed: preserve the extra-geom headroom and user-geom
             * capacity, only substitute the per-model ngeom base count. */
            // Ensure the GL context is current before dropping old GPU resources
            // (MjrContext::drop calls mjr_freeContext) and creating new ones.
            self.gl_state.make_current().map_err(RendererError::GlutinError)?;

            let user_geom_cap = self.user_scene.maxgeom() as usize;
            let new_ngeom = data.model().ffi().ngeom as usize;
            self.scene = MjvScene::new(data.model(), new_ngeom + self.extra_geom);
            self.user_scene = MjvScene::new(data.model(), user_geom_cap);
            (self.near, self.far) = model_near_far(data.model());

            // Recreate the rendering context so that GPU resources (textures,
            // meshes, heightfields, skins) match the new model.
            // SAFETY: the GL context was made current above.
            self.context = unsafe { MjrContext::new(data.model()) };
            self.context.offscreen();
            self.context.change_font(self.font_scale);
        }

        self.scene.update(data, &self.option, &MjvPerturb::default(), &mut self.camera);
        Ok(())
    }

    /// Return a flattened RGB image of the scene.
    pub fn rgb_flat(&self) -> Option<&[u8]> {
        self.rgb.as_deref()
    }

    /// Return an RGB image of the scene. This method accepts two generic parameters <WIDTH, HEIGHT>
    /// that define the shape of the output slice.
    ///
    /// # Panics
    /// - If `WIDTH * HEIGHT` doesn't match the rendered pixel count.
    /// - If RGB rendering is disabled.
    ///
    /// Use [`MjRenderer::try_rgb`] for a fallible alternative.
    pub fn rgb<const WIDTH: usize, const HEIGHT: usize>(&self) -> &[[[u8; 3]; WIDTH]; HEIGHT] {
        self.try_rgb::<WIDTH, HEIGHT>().unwrap()
    }

    /// Fallible version of [`MjRenderer::rgb`].
    /// # Returns
    /// On success, returns [`Ok`] variant containing the rendered RGB image.
    /// # Errors
    /// - [`RendererError::DimensionMismatch`] if `WIDTH * HEIGHT` doesn't match the rendered pixel count.
    /// - [`RendererError::RgbDisabled`] if RGB rendering is disabled.
    pub fn try_rgb<const WIDTH: usize, const HEIGHT: usize>(&self) -> Result<&[[[u8; 3]; WIDTH]; HEIGHT], RendererError> {
        if let Some(flat) = self.rgb_flat() {
            bytemuck::try_from_bytes(flat)
                .map_err(|_| RendererError::DimensionMismatch)
        }
        else {
            Err(RendererError::RgbDisabled)
        }
    }

    /// Return a flattened depth image of the scene: the distance from the camera in metres,
    /// between the near and far clip planes (`vis().map.znear * stat().extent` and
    /// `vis().map.zfar * stat().extent`).
    pub fn depth_flat(&self) -> Option<&[f32]> {
        self.depth.as_deref()
    }

    /// Return a depth image of the scene. This method accepts two generic parameters <WIDTH, HEIGHT>
    /// that define the shape of the output slice.
    ///
    /// # Panics
    /// - If `WIDTH * HEIGHT` doesn't match the rendered pixel count. Only the product is checked,
    ///   so swapped dimensions pass and return a sheared image.
    /// - If depth rendering is disabled.
    ///
    /// Use [`MjRenderer::try_depth`] for a fallible alternative.
    pub fn depth<const WIDTH: usize, const HEIGHT: usize>(&self) -> &[[f32; WIDTH]; HEIGHT] {
        self.try_depth::<WIDTH, HEIGHT>().unwrap()
    }

    /// Fallible version of [`MjRenderer::depth`].
    /// # Returns
    /// On success, returns [`Ok`] variant containing the rendered depth image.
    /// # Errors
    /// - [`RendererError::DimensionMismatch`] if `WIDTH * HEIGHT` doesn't match the rendered pixel
    ///   count (only the product is checked).
    /// - [`RendererError::DepthDisabled`] if depth rendering is disabled.
    pub fn try_depth<const WIDTH: usize, const HEIGHT: usize>(&self) -> Result<&[[f32; WIDTH]; HEIGHT], RendererError> {
        if let Some(flat) = self.depth_flat() {
            let bytes: &[u8] = bytemuck::cast_slice(flat);
            bytemuck::try_from_bytes(bytes)
                .map_err(|_| RendererError::DimensionMismatch)
        }
        else {
            Err(RendererError::DepthDisabled)
        }
    }

    /// Save an RGB image of the scene to a path.
    /// # Returns
    /// `Ok(())` on success.
    /// # Errors
    /// - [`RendererError::RgbDisabled`] when RGB rendering is disabled.
    /// - [`RendererError::IoError`] if a file I/O operation fails.
    pub fn save_rgb<T: AsRef<Path>>(&self, path: T) -> Result<(), RendererError> {
        if let Some(rgb) = &self.rgb {
            write_png(
                path,
                rgb,
                self.width as u32,
                self.height as u32,
                png::ColorType::Rgb,
                png::BitDepth::Eight,
                self.png_compression
            )?;
            Ok(())
        }
        else {
            Err(RendererError::RgbDisabled)
        }
    }

    /// Save a depth image of the scene to a path. The image is 16-bit PNG, which
    /// can be converted into depth (distance) data by dividing the grayscale values by
    /// 65535.0 and applying inverse normalization: `depth = min + (grayscale / 65535.0) * (max - min)`.
    ///
    /// If `normalize` is `true`, then the data is normalized with per-frame min-max normalization.
    /// When `normalize` is `false`, the depth values are mapped using the model's camera
    /// near/far clip planes as the range, providing a fixed (frame-independent) mapping.
    ///
    /// Use of [`MjRenderer::save_depth_raw`] is recommended if performance is critical, as
    /// it skips PNG encoding and also saves the true depth values directly.
    /// # Returns
    /// An [`Ok`]`((min, max))` is returned, where min and max represent the normalization parameters.
    /// # Errors
    /// - [`RendererError::DepthDisabled`] when depth rendering is disabled.
    /// - [`RendererError::IoError`] if a file I/O operation fails.
    pub fn save_depth<T: AsRef<Path>>(&self, path: T, normalize: bool) -> Result<(f32, f32), RendererError> {
        if let Some(depth) = &self.depth {
            let (norm, min, max) =
            if normalize {
                let max = depth.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
                let min = depth.iter().cloned().fold(f32::INFINITY, f32::min);
                let range = max - min;
                if range == 0.0 {
                    (vec![0u8; depth.len() * 2].into_boxed_slice(), min, max)
                } else {
                    (depth.iter().flat_map(|&x| (((x - min) / range * DEPTH_U16_SCALE).clamp(0.0, DEPTH_U16_SCALE) as u16).to_be_bytes()).collect::<Box<_>>(), min, max)
                }
            }
            else {
                // Use model's camera near/far clip planes as the fixed normalization range.
                // After linearization (in render()), depth values are in meters within [near, far].
                let near = self.near;
                let far = self.far;
                (depth.iter().flat_map(|&x| (((x - near) / (far - near) * DEPTH_U16_SCALE).clamp(0.0, DEPTH_U16_SCALE) as u16).to_be_bytes()).collect::<Box<_>>(), near, far)
            };

            write_png(
                path,
                &norm,
                self.width as u32,
                self.height as u32,
                png::ColorType::Grayscale,
                png::BitDepth::Sixteen,
                self.png_compression
            )?;
            Ok((min, max))
        }
        else {
            Err(RendererError::DepthDisabled)
        }
    }

    /// Save the raw depth data to the `path`. The data is encoded
    /// as a sequence of bytes, where groups of four represent a single f32 value
    /// in native byte order.
    /// # Returns
    /// `Ok(())` on success.
    /// # Errors
    /// - [`RendererError::DepthDisabled`] when depth rendering is disabled.
    /// - [`RendererError::IoError`] if a file I/O operation fails.
    pub fn save_depth_raw<T: AsRef<Path>>(&self, path: T) -> Result<(), RendererError> {
        if let Some(depth) = &self.depth {
            let file = File::create(path.as_ref())?;
            let mut writer = BufWriter::new(file);

            let bytes: &[u8] = bytemuck::cast_slice(depth);
            writer.write_all(bytes)?;
            Ok(())
        }
        else {
            Err(RendererError::DepthDisabled)
        }
    }

    /// Draws the scene to internal arrays.
    /// Use [`MjRenderer::rgb`] or [`MjRenderer::depth`] to obtain the rendered image.
    ///
    /// # Note
    /// MuJoCo reports an error and stops the process when the scene holds geoms but
    /// [`MjRenderer::sync_data`] was never called, which leaves `frustum_near` at zero.
    ///
    /// # Errors
    /// - [`RendererError::GlutinError`] if the OpenGL context could not be made current.
    /// - [`RendererError::SceneError`] if the user-scene sync overflows the geom buffer.
    /// - [`RendererError::ContextError`] if reading pixels from the framebuffer fails.
    pub fn render(&mut self) -> Result<(), RendererError> {
        /* Sync user scene geoms into the main scene before rendering */
        sync_geoms(&self.user_scene, &mut self.scene).map_err(RendererError::SceneError)?;

        self.gl_state.make_current().map_err(RendererError::GlutinError)?;
        let vp = MjrRectangle::new(0, 0, self.width as i32, self.height as i32);
        self.scene.render(&vp, &self.context);

        /* Fully flatten everything */
        let flat_rgb = self.rgb.as_deref_mut();
        let flat_depth = self.depth.as_deref_mut();

        /* Read to whatever is enabled */
        self.context.read_pixels(
            flat_rgb,
            flat_depth,
            &vp
        ).map_err(RendererError::ContextError)?;

        /* Flip the read pixels vertically, as OpenGL reads bottom-up */
        if let Some(rgb) = self.rgb.as_deref_mut() {
            flip_image_vertically(rgb, self.height, self.width * 3);
        }

        /* Make depth values be the actual distance in meters and flip them vertically */
        if let Some(depth) = self.depth.as_deref_mut() {
            flip_image_vertically(depth, self.height, self.width);

            let near = self.near;
            let far = self.far;
            for value in depth {
                *value = near / (1.0 - *value * (1.0 - near / far));
            }
        }

        Ok(())
    }
}

/// Errors that can occur during renderer operations.
#[derive(Debug)]
#[non_exhaustive]
pub enum RendererError {
    /// The event loop failed to initialize.
    #[cfg(feature = "renderer-winit-fallback")]
    EventLoopError(winit::error::EventLoopError),
    /// A glutin operation failed.
    GlutinError(glutin::error::Error),
    /// The supplied width or height was zero; the offscreen GL surface requires positive
    /// dimensions.
    ZeroDimension,
    /// OpenGL / window initialization failed.
    #[cfg(feature = "renderer-winit-fallback")]
    GlInitFailed(crate::error::GlInitError),
    /// RGB rendering was not enabled.
    RgbDisabled,
    /// Depth rendering was not enabled.
    DepthDisabled,
    /// The requested `WIDTH`/`HEIGHT` do not match the renderer's dimensions.
    DimensionMismatch,
    /// An I/O error occurred (e.g. while saving to a file).
    IoError(io::Error),
    /// A scene operation failed (e.g. user-scene sync overflowed the geom buffer).
    SceneError(crate::error::MjSceneError),
    /// A rendering-context operation failed (e.g. an asset or aux-buffer ID is out of range).
    ContextError(crate::error::MjrContextError),
    /// The model is not compatible with the renderer's scene or rendering context.
    /// Call [`MjRenderer::sync_data`] first.
    IncompatibleModel,
}

/// Formats a human-readable description of the renderer error.
impl Display for RendererError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            #[cfg(feature = "renderer-winit-fallback")]
            Self::EventLoopError(_) => write!(f, "event loop failed to initialize"),
            Self::GlutinError(_) => write!(f, "glutin operation failed"),
            Self::ZeroDimension => write!(f, "renderer width and height must both be greater than zero"),
            #[cfg(feature = "renderer-winit-fallback")]
            Self::GlInitFailed(_) => write!(f, "GL initialization failed"),
            Self::RgbDisabled => write!(f, "RGB rendering is not enabled (renderer.with_rgb_rendering(true))"),
            Self::DepthDisabled => write!(f, "depth rendering is not enabled (renderer.with_depth_rendering(true))"),
            Self::DimensionMismatch => write!(f, "the input width and height don't match the renderer's configuration"),
            Self::IoError(_) => write!(f, "I/O operation failed"),
            Self::SceneError(_) => write!(f, "scene operation failed"),
            Self::ContextError(_) => write!(f, "rendering context operation failed"),
            Self::IncompatibleModel => write!(f, "the model is not compatible with the scene or the rendering context: call sync_data first"),
        }
    }
}

/// Provides the underlying error source, if any.
impl Error for RendererError {
    fn source(&self) -> Option<&(dyn Error + 'static)> {
        match self {
            #[cfg(feature = "renderer-winit-fallback")]
            Self::EventLoopError(e) => Some(e),
            #[cfg(feature = "renderer-winit-fallback")]
            Self::GlInitFailed(e) => Some(e),
            Self::GlutinError(e) => Some(e),
            Self::IoError(e) => Some(e),
            Self::SceneError(e) => Some(e),
            Self::ContextError(e) => Some(e),
            _ => None,
        }
    }
}

/// Converts an [`io::Error`] into [`RendererError::IoError`].
impl From<io::Error> for RendererError {
    fn from(e: io::Error) -> Self {
        Self::IoError(e)
    }
}

/// Converts a [`png::EncodingError`] into [`RendererError::IoError`].
impl From<png::EncodingError> for RendererError {
    fn from(e: png::EncodingError) -> Self {
        Self::IoError(io::Error::from(e))
    }
}

/// Converts an [`MjSceneError`] into [`RendererError::SceneError`].
impl From<crate::error::MjSceneError> for RendererError {
    fn from(e: crate::error::MjSceneError) -> Self {
        Self::SceneError(e)
    }
}

/// Converts an [`crate::error::MjrContextError`] into [`RendererError::ContextError`].
impl From<crate::error::MjrContextError> for RendererError {
    fn from(e: crate::error::MjrContextError) -> Self {
        Self::ContextError(e)
    }
}

/// Converts a [`GlInitError`] into [`RendererError::GlInitFailed`].
#[cfg(feature = "renderer-winit-fallback")]
impl From<crate::error::GlInitError> for RendererError {
    fn from(e: crate::error::GlInitError) -> Self {
        Self::GlInitFailed(e)
    }
}

bitflags! {
    /// Flags that enable features of the renderer.
    #[derive(Debug)]
    struct RendererFlags: u8 {
        const RENDER_RGB = 1 << 0;
        const RENDER_DEPTH = 1 << 1;
    }
}



/// Makes the OpenGL context current, when it can, before GPU resources are freed.
impl Drop for MjRenderer {
    fn drop(&mut self) {
        // Ensure the GL context is current before the implicit field drops
        // (MjrContext's Drop calls mjr_freeContext which requires an active GL context).
        let _ = self.gl_state.make_current();
    }
}

/*
** Don't run any tests as OpenGL hates if anything
** runs outside the main thread.
*/

#[cfg(test)]
mod test {
    use crate::assert_relative_eq;

    use super::*;

    const MODEL: &str = stringify!(
        <mujoco>

        <visual>
            <global offwidth="1280" offheight="720"/>
        </visual>

        <worldbody>
            <geom name="floor" type="plane" size="10 10 1" euler="0 0 0"/>
            <geom type="box" size="1 10 10" pos="-1 0 0" euler="0 0 0"/>

            <camera name="depth_test" euler="90 90 0" pos="2.25 0 1"/>

        </worldbody>
        </mujoco>
    );

    /// Depth calculation test.
    /// This is only run on Linux due to EGL requirements (winit cannot be used on multiple threads).
    #[test]
    #[cfg(target_os = "linux")]
    fn test_depth() {
        let model = MjModel::from_xml_string(MODEL).expect("could not load the model");
        let mut data = MjData::new(&model);
        data.step();

        let mut renderer = MjRenderer::builder()
            .rgb(false)
            .depth(true)
            .camera(MjvCamera::new_fixed(model.name_to_id(MjtObjBase::mjOBJ_CAMERA, "depth_test").unwrap()))
            .build(&model)
            .unwrap();

        renderer.sync_data(&mut data).unwrap();
        renderer.render().unwrap();
        let min = renderer.depth_flat().unwrap().iter().fold(f32::INFINITY, |a , &b| a.min(b));
        let max = renderer.depth_flat().unwrap().iter().fold(f32::NEG_INFINITY, |a , &b| a.max(b));

        assert_relative_eq!(min, max, epsilon = 1e-4);
        assert_relative_eq!(min, 2.25, epsilon = 1e-4);
    }

    /// An upload accepts a model whose moved vertex changes the mesh hull, and refuses a model
    /// with another mesh count.
    /// This is only run on Linux due to EGL requirements (winit cannot be used on multiple threads).
    #[test]
    #[cfg(target_os = "linux")]
    fn test_upload_gate() {
        let xml = |vertex: &str, extra: &str| format!("<mujoco><asset><mesh name='m' vertex='0 0 0  1 0 0  \
            0 1 0  0 0 1  {vertex}  .5 .5 -.3'/>{extra}</asset><worldbody><geom type='mesh' mesh='m'/>\
            </worldbody></mujoco>");
        let base = MjModel::from_xml_string(&xml("1 1 1", "")).unwrap();
        let moved = MjModel::from_xml_string(&xml("1.2 1.1 1", "")).unwrap();
        let extra = MjModel::from_xml_string(&xml("1 1 1", "<mesh name='n' vertex='0 0 0  1 0 0  0 1 0  0 0 1'/>"))
            .unwrap();
        assert_ne!(base.mesh_graph(), moved.mesh_graph(), "the move must change the hull");

        let renderer = MjRenderer::builder().build(&base).unwrap();
        renderer.update_mesh_from(&moved, 0).unwrap();
        renderer.update_meshes_from(&moved).unwrap();
        assert!(matches!(renderer.update_mesh_from(&extra, 0), Err(RendererError::IncompatibleModel)));
    }

    /// This is only run on Linux due to EGL requirements (winit cannot be used on multiple threads).
    #[test]
    #[cfg(target_os = "linux")]
    #[should_panic(expected = "ASCII")]
    fn test_overlay_non_ascii_panics() {
        let model = MjModel::from_xml_string(MODEL).unwrap();
        let mut renderer = MjRenderer::builder().build(&model).unwrap();
        renderer.context.overlay(
            MjtFont::mjFONT_NORMAL, MjtGridPos::mjGRID_TOPLEFT, MjrRectangle::default(), "\u{e9}", None
        );
    }

    /// Decode a PNG file and return the raw pixel bytes.
    #[cfg(target_os = "linux")]
    fn decode_png_pixels(path: &std::path::Path) -> Vec<u8> {
        let decoder = png::Decoder::new(std::io::BufReader::new(std::fs::File::open(path).unwrap()));
        let mut reader = decoder.read_info().unwrap();
        let mut buf = vec![0u8; reader.output_buffer_size().unwrap()];
        reader.next_frame(&mut buf).unwrap();
        buf
    }

    /// Verify that all compression levels produce identical pixel data.
    /// This is only run on Linux due to EGL requirements (winit cannot be used on multiple threads).
    #[test]
    #[cfg(target_os = "linux")]
    fn test_png_compression_lossless() {
        let model = MjModel::from_xml_string(MODEL).expect("could not load the model");
        let mut data = MjData::new(&model);
        data.step();

        let mut renderer = MjRenderer::builder()
            .rgb(true)
            .depth(false)
            .build(&model)
            .unwrap();

        renderer.sync_data(&mut data).unwrap();
        renderer.render().unwrap();

        let tmp = std::env::temp_dir();
        let path_none = tmp.join("mujoco_rs_test_none.png");
        let path_fast = tmp.join("mujoco_rs_test_fast.png");
        let path_high = tmp.join("mujoco_rs_test_high.png");

        renderer.save_rgb(&path_none).unwrap();
        renderer.set_png_compression(png::Compression::Fast);
        renderer.save_rgb(&path_fast).unwrap();
        renderer.set_png_compression(png::Compression::High);
        renderer.save_rgb(&path_high).unwrap();

        let pixels_none = decode_png_pixels(&path_none);
        let pixels_fast = decode_png_pixels(&path_fast);
        let pixels_high = decode_png_pixels(&path_high);

        assert_eq!(pixels_none, pixels_fast, "Fast compression must produce identical pixels");
        assert_eq!(pixels_none, pixels_high, "High compression must produce identical pixels");

        let _ = std::fs::remove_file(&path_none);
        let _ = std::fs::remove_file(&path_fast);
        let _ = std::fs::remove_file(&path_high);
    }
}