vrm-runtime 0.1.0

Renderer-agnostic VRM 1.0 loader, validator, animation system, and runtime
Documentation
use std::borrow::Cow;
use std::collections::HashSet;
use vrm_spec::vrmc_vrm_1_0 as spec;

use crate::{Avatar, HumanBone, RenderPrimitive};

/// How a mesh is treated by the first-person (FPV) camera.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum FirstPersonFlag {
    /// Render the annotated mesh only for third-person cameras.
    ThirdPersonOnly,
    /// Render the annotated mesh only for first-person cameras.
    FirstPersonOnly,
    /// Render the annotated mesh for both camera modes.
    Both,
    /// Split head-weighted triangles automatically for first-person rendering.
    #[default]
    Auto,
}

/// Camera visibility pass used to select a primitive's index buffer.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CameraMode {
    /// Select indices visible from the avatar's own viewpoint.
    FirstPerson,
    /// Select indices visible from an external camera.
    ThirdPerson,
}

impl From<spec::FirstPersonType> for FirstPersonFlag {
    fn from(flag: spec::FirstPersonType) -> Self {
        match flag {
            spec::FirstPersonType::ThirdPersonOnly => Self::ThirdPersonOnly,
            spec::FirstPersonType::FirstPersonOnly => Self::FirstPersonOnly,
            spec::FirstPersonType::Both => Self::Both,
            spec::FirstPersonType::Auto => Self::Auto,
        }
    }
}

/// An explicit override of the first-person flag for one mesh node.
#[derive(Debug, Clone)]
pub struct MeshAnnotation {
    /// Node whose mesh receives this annotation.
    pub node_index: u32,
    /// Visibility rule applied to the node's mesh.
    pub flag: FirstPersonFlag,
}

/// First-person visibility rules of the avatar.
#[derive(Debug, Clone, Default)]
pub struct FirstPerson {
    /// Explicit per-node mesh annotations from the VRM extension.
    pub mesh_annotations: Vec<MeshAnnotation>,
}

impl FirstPerson {
    pub(crate) fn load_first_person(schema: &spec::FirstPerson) -> Self {
        let mesh_annotations = schema
            .mesh_annotations
            .as_deref()
            .unwrap_or_default()
            .iter()
            .map(|annotation| {
                let node = annotation
                    .node
                    .as_ref()
                    .expect("first-person annotations are validated before loading");
                let flag = FirstPersonFlag::from(annotation.mesh_annotation_type);
                MeshAnnotation {
                    node_index: node.value() as u32,
                    flag,
                }
            })
            .collect();

        Self { mesh_annotations }
    }

    /// Returns the first-person flag for a mesh node.
    ///
    /// Meshes without an explicit annotation default to
    /// [`FirstPersonFlag::Auto`].
    pub fn flag_for(&self, node_index: u32) -> FirstPersonFlag {
        self.mesh_annotations
            .iter()
            .find(|annotation| annotation.node_index == node_index)
            .map(|annotation| annotation.flag)
            .unwrap_or(FirstPersonFlag::Auto)
    }

    /// Collects the node indices of the head bone and all of its descendants.
    ///
    /// These are the joints the [`FirstPersonFlag::Auto`] algorithm uses to
    /// decide whether a vertex belongs to the head part.
    pub fn head_node_indices(&self, avatar: &Avatar) -> HashSet<u32> {
        let Some(head) = avatar.humanoid().bone(HumanBone::Head) else {
            return HashSet::new();
        };

        let mut head_nodes = HashSet::new();
        let mut stack = vec![head.0];
        while let Some(index) = stack.pop() {
            if !head_nodes.insert(index) {
                continue;
            }
            if let Some(node) = avatar.nodes().iter().find(|n| n.index == index) {
                stack.extend(node.children.iter().copied());
            }
        }
        head_nodes
    }

    /// Applies the `auto` first-person split to `primitive`.
    ///
    /// Every vertex is checked for a weight on the head bone (or a child of
    /// the head bone). Triangles that contain at least one such vertex belong
    /// to the head part ([`AutoSplit::third_person_only`]); the remaining
    /// triangles form the `both` part.
    pub fn auto_split(&self, avatar: &Avatar, primitive: &RenderPrimitive) -> AutoSplit {
        let head_nodes = self.head_node_indices(avatar);
        let skin_joints = primitive
            .skin_index()
            .and_then(|skin_index| avatar.skins().get(skin_index))
            .map(crate::Skin::joints);

        let vertex_is_head = |vertex: usize| {
            let Some(skin_joints) = skin_joints else {
                return false;
            };
            primitive.skinning_sets().iter().any(|set| {
                let (Some(joints), Some(weights)) =
                    (set.joints().get(vertex), set.weights().get(vertex))
                else {
                    return false;
                };
                joints.iter().zip(weights).any(|(&joint, &weight)| {
                    weight > 0.0
                        && skin_joints
                            .get(joint as usize)
                            .is_some_and(|node| head_nodes.contains(node))
                })
            })
        };

        let mut split = AutoSplit::default();
        for (triangle_index, triangle) in primitive.indices().chunks_exact(3).enumerate() {
            if triangle.iter().any(|&v| vertex_is_head(v as usize)) {
                split.third_person_only.push(triangle_index as u32);
            } else {
                split.both.push(triangle_index as u32);
            }
        }
        split
    }

    /// Returns the indices visible to the selected camera.
    ///
    /// Most modes borrow the primitive's original index buffer. An `auto`
    /// first-person triangle mesh returns an owned buffer with head-weighted
    /// triangles removed.
    pub fn visible_indices<'a>(
        &self,
        avatar: &Avatar,
        primitive: &'a RenderPrimitive,
        camera: CameraMode,
    ) -> Cow<'a, [u32]> {
        let flag = self.flag_for(primitive.node_index());
        match (camera, flag) {
            (CameraMode::FirstPerson, FirstPersonFlag::ThirdPersonOnly)
            | (CameraMode::ThirdPerson, FirstPersonFlag::FirstPersonOnly) => Cow::Borrowed(&[]),
            (CameraMode::FirstPerson, FirstPersonFlag::Auto)
                if primitive.topology() == crate::PrimitiveTopology::Triangles =>
            {
                let split = self.auto_split(avatar, primitive);
                let mut indices = Vec::with_capacity(split.both.len() * 3);
                for triangle in split.both {
                    let start = triangle as usize * 3;
                    indices.extend_from_slice(&primitive.indices()[start..start + 3]);
                }
                Cow::Owned(indices)
            }
            _ => Cow::Borrowed(primitive.indices()),
        }
    }
}

/// Triangle groups produced by the `auto` first-person split.
///
/// Each entry is a zero-based triangle ordinal. Multiply it by three to get
/// the first corresponding position in [`RenderPrimitive::indices`].
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct AutoSplit {
    /// Triangles containing head-related vertices; hidden from the first-person camera.
    pub third_person_only: Vec<u32>,
    /// The remaining triangles; rendered by every camera.
    pub both: Vec<u32>,
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::{AvatarNode, Humanoid, PrimitiveTopology, Transform};

    fn head_avatar() -> Avatar {
        use glam::{Quat, Vec3};
        Avatar::new()
            .with_humanoid(Humanoid::default().with_bone(HumanBone::Head, 0))
            .with_nodes(vec![
                AvatarNode {
                    index: 0,
                    name: Some("head".to_string()),
                    parent: None,
                    children: vec![1, 2],
                    initial_translation: Vec3::ZERO,
                    initial_rotation: Quat::IDENTITY,
                    initial_scale: Vec3::ONE,
                },
                AvatarNode {
                    index: 1,
                    name: Some("leftEye".to_string()),
                    parent: Some(0),
                    children: vec![],
                    initial_translation: Vec3::ZERO,
                    initial_rotation: Quat::IDENTITY,
                    initial_scale: Vec3::ONE,
                },
                AvatarNode {
                    index: 2,
                    name: Some("rightEye".to_string()),
                    parent: Some(0),
                    children: vec![],
                    initial_translation: Vec3::ZERO,
                    initial_rotation: Quat::IDENTITY,
                    initial_scale: Vec3::ONE,
                },
            ])
    }

    fn primitive_with_triangles(
        node_index: u32,
        joints: Vec<[u16; 4]>,
        weights: Vec<[f32; 4]>,
        skin_index: Option<usize>,
    ) -> RenderPrimitive {
        let vertex_count = joints.len();
        RenderPrimitive::new(
            node_index,
            Transform::IDENTITY,
            vec![[0.0, 0.0, 0.0]; vertex_count],
            vec![[0.0, 0.0, 0.0]; vertex_count],
            (0..vertex_count as u32).collect(),
            joints,
            weights,
            vec![],
            vec![[0.0, 0.0]; vertex_count],
            vec![],
            vec![],
            vec![[0.0, 0.0]; vertex_count],
            None,
            None,
            PrimitiveTopology::Triangles,
            vec![],
            false,
            skin_index,
            0,
        )
    }

    #[test]
    fn head_node_indices_includes_head_and_descendants() {
        let avatar = head_avatar();
        let indices = FirstPerson::default().head_node_indices(&avatar);
        assert_eq!(indices, HashSet::from([0, 1, 2]));
    }

    #[test]
    fn auto_split_separates_head_related_triangles() {
        let avatar = head_avatar().with_skins(vec![crate::Skin::new(
            vec![0, 1, 2, 3],
            vec![glam::Mat4::IDENTITY; 4],
        )]);

        // Triangle 0: vertices skinned to head node 0 and its children (1, 2).
        // Triangle 1: vertices skinned to an unrelated node (3).
        let joints = vec![
            [0, 0, 0, 0],
            [1, 0, 0, 0],
            [2, 0, 0, 0],
            [3, 0, 0, 0],
            [3, 0, 0, 0],
            [3, 0, 0, 0],
        ];
        let weights = vec![[1.0, 0.0, 0.0, 0.0]; 6];
        let primitive = primitive_with_triangles(10, joints, weights, Some(0));

        let split = FirstPerson::default().auto_split(&avatar, &primitive);

        assert_eq!(split.third_person_only, vec![0]);
        assert_eq!(split.both, vec![1]);
    }

    #[test]
    fn visible_indices_apply_camera_annotations() {
        let avatar = head_avatar();
        let primitive = primitive_with_triangles(
            0,
            vec![[0; 4], [0; 4], [0; 4]],
            vec![[0.0; 4], [0.0; 4], [0.0; 4]],
            None,
        );
        let first_person = FirstPerson {
            mesh_annotations: vec![MeshAnnotation {
                node_index: 0,
                flag: FirstPersonFlag::ThirdPersonOnly,
            }],
        };

        assert!(
            first_person
                .visible_indices(&avatar, &primitive, CameraMode::FirstPerson)
                .is_empty()
        );
        assert_eq!(
            first_person.visible_indices(&avatar, &primitive, CameraMode::ThirdPerson),
            primitive.indices()
        );
    }

    #[test]
    fn auto_split_treats_unskinned_mesh_as_both() {
        let avatar = head_avatar();

        let joints = vec![[0, 0, 0, 0]; 6];
        let weights = vec![[0.0, 0.0, 0.0, 0.0]; 6];
        let primitive = primitive_with_triangles(10, joints, weights, None);

        let split = FirstPerson::default().auto_split(&avatar, &primitive);

        assert!(split.third_person_only.is_empty());
        assert_eq!(split.both, vec![0, 1]);
    }

    #[test]
    fn auto_split_maps_joint_slots_through_the_skin() {
        let avatar = head_avatar().with_skins(vec![crate::Skin::new(
            vec![3, 4, 5, 6, 7, 0],
            vec![glam::Mat4::IDENTITY; 6],
        )]);
        let primitive = primitive_with_triangles(
            10,
            vec![[5, 0, 0, 0], [5, 0, 0, 0], [5, 0, 0, 0]],
            vec![[1.0, 0.0, 0.0, 0.0]; 3],
            Some(0),
        );

        let split = FirstPerson::default().auto_split(&avatar, &primitive);

        assert_eq!(split.third_person_only, vec![0]);
        assert!(split.both.is_empty());
    }

    #[test]
    fn auto_split_does_not_treat_a_slot_number_as_a_node_index() {
        let avatar =
            head_avatar().with_skins(vec![crate::Skin::new(vec![3], vec![glam::Mat4::IDENTITY])]);
        let primitive = primitive_with_triangles(
            10,
            vec![[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],
            vec![[1.0, 0.0, 0.0, 0.0]; 3],
            Some(0),
        );

        let split = FirstPerson::default().auto_split(&avatar, &primitive);

        assert!(split.third_person_only.is_empty());
        assert_eq!(split.both, vec![0]);
    }
}