Skip to main content

vrm_runtime/
first_person.rs

1use std::borrow::Cow;
2use std::collections::HashSet;
3use vrm_spec::vrmc_vrm_1_0 as spec;
4
5use crate::{Avatar, HumanBone, RenderPrimitive};
6
7/// How a mesh is treated by the first-person (FPV) camera.
8#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
9pub enum FirstPersonFlag {
10    /// Render the annotated mesh only for third-person cameras.
11    ThirdPersonOnly,
12    /// Render the annotated mesh only for first-person cameras.
13    FirstPersonOnly,
14    /// Render the annotated mesh for both camera modes.
15    Both,
16    /// Split head-weighted triangles automatically for first-person rendering.
17    #[default]
18    Auto,
19}
20
21/// Camera visibility pass used to select a primitive's index buffer.
22#[derive(Debug, Clone, Copy, PartialEq, Eq)]
23pub enum CameraMode {
24    /// Select indices visible from the avatar's own viewpoint.
25    FirstPerson,
26    /// Select indices visible from an external camera.
27    ThirdPerson,
28}
29
30impl From<spec::FirstPersonType> for FirstPersonFlag {
31    fn from(flag: spec::FirstPersonType) -> Self {
32        match flag {
33            spec::FirstPersonType::ThirdPersonOnly => Self::ThirdPersonOnly,
34            spec::FirstPersonType::FirstPersonOnly => Self::FirstPersonOnly,
35            spec::FirstPersonType::Both => Self::Both,
36            spec::FirstPersonType::Auto => Self::Auto,
37        }
38    }
39}
40
41/// An explicit override of the first-person flag for one mesh node.
42#[derive(Debug, Clone)]
43pub struct MeshAnnotation {
44    /// Node whose mesh receives this annotation.
45    pub node_index: u32,
46    /// Visibility rule applied to the node's mesh.
47    pub flag: FirstPersonFlag,
48}
49
50/// First-person visibility rules of the avatar.
51#[derive(Debug, Clone, Default)]
52pub struct FirstPerson {
53    /// Explicit per-node mesh annotations from the VRM extension.
54    pub mesh_annotations: Vec<MeshAnnotation>,
55}
56
57impl FirstPerson {
58    pub(crate) fn load_first_person(schema: &spec::FirstPerson) -> Self {
59        let mesh_annotations = schema
60            .mesh_annotations
61            .as_deref()
62            .unwrap_or_default()
63            .iter()
64            .map(|annotation| {
65                let node = annotation
66                    .node
67                    .as_ref()
68                    .expect("first-person annotations are validated before loading");
69                let flag = FirstPersonFlag::from(annotation.mesh_annotation_type);
70                MeshAnnotation {
71                    node_index: node.value() as u32,
72                    flag,
73                }
74            })
75            .collect();
76
77        Self { mesh_annotations }
78    }
79
80    /// Returns the first-person flag for a mesh node.
81    ///
82    /// Meshes without an explicit annotation default to
83    /// [`FirstPersonFlag::Auto`].
84    pub fn flag_for(&self, node_index: u32) -> FirstPersonFlag {
85        self.mesh_annotations
86            .iter()
87            .find(|annotation| annotation.node_index == node_index)
88            .map(|annotation| annotation.flag)
89            .unwrap_or(FirstPersonFlag::Auto)
90    }
91
92    /// Collects the node indices of the head bone and all of its descendants.
93    ///
94    /// These are the joints the [`FirstPersonFlag::Auto`] algorithm uses to
95    /// decide whether a vertex belongs to the head part.
96    pub fn head_node_indices(&self, avatar: &Avatar) -> HashSet<u32> {
97        let Some(head) = avatar.humanoid().bone(HumanBone::Head) else {
98            return HashSet::new();
99        };
100
101        let mut head_nodes = HashSet::new();
102        let mut stack = vec![head.0];
103        while let Some(index) = stack.pop() {
104            if !head_nodes.insert(index) {
105                continue;
106            }
107            if let Some(node) = avatar.nodes().iter().find(|n| n.index == index) {
108                stack.extend(node.children.iter().copied());
109            }
110        }
111        head_nodes
112    }
113
114    /// Applies the `auto` first-person split to `primitive`.
115    ///
116    /// Every vertex is checked for a weight on the head bone (or a child of
117    /// the head bone). Triangles that contain at least one such vertex belong
118    /// to the head part ([`AutoSplit::third_person_only`]); the remaining
119    /// triangles form the `both` part.
120    pub fn auto_split(&self, avatar: &Avatar, primitive: &RenderPrimitive) -> AutoSplit {
121        let head_nodes = self.head_node_indices(avatar);
122        let skin_joints = primitive
123            .skin_index()
124            .and_then(|skin_index| avatar.skins().get(skin_index))
125            .map(crate::Skin::joints);
126
127        let vertex_is_head = |vertex: usize| {
128            let Some(skin_joints) = skin_joints else {
129                return false;
130            };
131            primitive.skinning_sets().iter().any(|set| {
132                let (Some(joints), Some(weights)) =
133                    (set.joints().get(vertex), set.weights().get(vertex))
134                else {
135                    return false;
136                };
137                joints.iter().zip(weights).any(|(&joint, &weight)| {
138                    weight > 0.0
139                        && skin_joints
140                            .get(joint as usize)
141                            .is_some_and(|node| head_nodes.contains(node))
142                })
143            })
144        };
145
146        let mut split = AutoSplit::default();
147        for (triangle_index, triangle) in primitive.indices().chunks_exact(3).enumerate() {
148            if triangle.iter().any(|&v| vertex_is_head(v as usize)) {
149                split.third_person_only.push(triangle_index as u32);
150            } else {
151                split.both.push(triangle_index as u32);
152            }
153        }
154        split
155    }
156
157    /// Returns the indices visible to the selected camera.
158    ///
159    /// Most modes borrow the primitive's original index buffer. An `auto`
160    /// first-person triangle mesh returns an owned buffer with head-weighted
161    /// triangles removed.
162    pub fn visible_indices<'a>(
163        &self,
164        avatar: &Avatar,
165        primitive: &'a RenderPrimitive,
166        camera: CameraMode,
167    ) -> Cow<'a, [u32]> {
168        let flag = self.flag_for(primitive.node_index());
169        match (camera, flag) {
170            (CameraMode::FirstPerson, FirstPersonFlag::ThirdPersonOnly)
171            | (CameraMode::ThirdPerson, FirstPersonFlag::FirstPersonOnly) => Cow::Borrowed(&[]),
172            (CameraMode::FirstPerson, FirstPersonFlag::Auto)
173                if primitive.topology() == crate::PrimitiveTopology::Triangles =>
174            {
175                let split = self.auto_split(avatar, primitive);
176                let mut indices = Vec::with_capacity(split.both.len() * 3);
177                for triangle in split.both {
178                    let start = triangle as usize * 3;
179                    indices.extend_from_slice(&primitive.indices()[start..start + 3]);
180                }
181                Cow::Owned(indices)
182            }
183            _ => Cow::Borrowed(primitive.indices()),
184        }
185    }
186}
187
188/// Triangle groups produced by the `auto` first-person split.
189///
190/// Each entry is a zero-based triangle ordinal. Multiply it by three to get
191/// the first corresponding position in [`RenderPrimitive::indices`].
192#[derive(Debug, Clone, Default, PartialEq, Eq)]
193pub struct AutoSplit {
194    /// Triangles containing head-related vertices; hidden from the first-person camera.
195    pub third_person_only: Vec<u32>,
196    /// The remaining triangles; rendered by every camera.
197    pub both: Vec<u32>,
198}
199
200#[cfg(test)]
201mod tests {
202    use super::*;
203    use crate::{AvatarNode, Humanoid, PrimitiveTopology, Transform};
204
205    fn head_avatar() -> Avatar {
206        use glam::{Quat, Vec3};
207        Avatar::new()
208            .with_humanoid(Humanoid::default().with_bone(HumanBone::Head, 0))
209            .with_nodes(vec![
210                AvatarNode {
211                    index: 0,
212                    name: Some("head".to_string()),
213                    parent: None,
214                    children: vec![1, 2],
215                    initial_translation: Vec3::ZERO,
216                    initial_rotation: Quat::IDENTITY,
217                    initial_scale: Vec3::ONE,
218                },
219                AvatarNode {
220                    index: 1,
221                    name: Some("leftEye".to_string()),
222                    parent: Some(0),
223                    children: vec![],
224                    initial_translation: Vec3::ZERO,
225                    initial_rotation: Quat::IDENTITY,
226                    initial_scale: Vec3::ONE,
227                },
228                AvatarNode {
229                    index: 2,
230                    name: Some("rightEye".to_string()),
231                    parent: Some(0),
232                    children: vec![],
233                    initial_translation: Vec3::ZERO,
234                    initial_rotation: Quat::IDENTITY,
235                    initial_scale: Vec3::ONE,
236                },
237            ])
238    }
239
240    fn primitive_with_triangles(
241        node_index: u32,
242        joints: Vec<[u16; 4]>,
243        weights: Vec<[f32; 4]>,
244        skin_index: Option<usize>,
245    ) -> RenderPrimitive {
246        let vertex_count = joints.len();
247        RenderPrimitive::new(
248            node_index,
249            Transform::IDENTITY,
250            vec![[0.0, 0.0, 0.0]; vertex_count],
251            vec![[0.0, 0.0, 0.0]; vertex_count],
252            (0..vertex_count as u32).collect(),
253            joints,
254            weights,
255            vec![],
256            vec![[0.0, 0.0]; vertex_count],
257            vec![],
258            vec![],
259            vec![[0.0, 0.0]; vertex_count],
260            None,
261            None,
262            PrimitiveTopology::Triangles,
263            vec![],
264            false,
265            skin_index,
266            0,
267        )
268    }
269
270    #[test]
271    fn head_node_indices_includes_head_and_descendants() {
272        let avatar = head_avatar();
273        let indices = FirstPerson::default().head_node_indices(&avatar);
274        assert_eq!(indices, HashSet::from([0, 1, 2]));
275    }
276
277    #[test]
278    fn auto_split_separates_head_related_triangles() {
279        let avatar = head_avatar().with_skins(vec![crate::Skin::new(
280            vec![0, 1, 2, 3],
281            vec![glam::Mat4::IDENTITY; 4],
282        )]);
283
284        // Triangle 0: vertices skinned to head node 0 and its children (1, 2).
285        // Triangle 1: vertices skinned to an unrelated node (3).
286        let joints = vec![
287            [0, 0, 0, 0],
288            [1, 0, 0, 0],
289            [2, 0, 0, 0],
290            [3, 0, 0, 0],
291            [3, 0, 0, 0],
292            [3, 0, 0, 0],
293        ];
294        let weights = vec![[1.0, 0.0, 0.0, 0.0]; 6];
295        let primitive = primitive_with_triangles(10, joints, weights, Some(0));
296
297        let split = FirstPerson::default().auto_split(&avatar, &primitive);
298
299        assert_eq!(split.third_person_only, vec![0]);
300        assert_eq!(split.both, vec![1]);
301    }
302
303    #[test]
304    fn visible_indices_apply_camera_annotations() {
305        let avatar = head_avatar();
306        let primitive = primitive_with_triangles(
307            0,
308            vec![[0; 4], [0; 4], [0; 4]],
309            vec![[0.0; 4], [0.0; 4], [0.0; 4]],
310            None,
311        );
312        let first_person = FirstPerson {
313            mesh_annotations: vec![MeshAnnotation {
314                node_index: 0,
315                flag: FirstPersonFlag::ThirdPersonOnly,
316            }],
317        };
318
319        assert!(
320            first_person
321                .visible_indices(&avatar, &primitive, CameraMode::FirstPerson)
322                .is_empty()
323        );
324        assert_eq!(
325            first_person.visible_indices(&avatar, &primitive, CameraMode::ThirdPerson),
326            primitive.indices()
327        );
328    }
329
330    #[test]
331    fn auto_split_treats_unskinned_mesh_as_both() {
332        let avatar = head_avatar();
333
334        let joints = vec![[0, 0, 0, 0]; 6];
335        let weights = vec![[0.0, 0.0, 0.0, 0.0]; 6];
336        let primitive = primitive_with_triangles(10, joints, weights, None);
337
338        let split = FirstPerson::default().auto_split(&avatar, &primitive);
339
340        assert!(split.third_person_only.is_empty());
341        assert_eq!(split.both, vec![0, 1]);
342    }
343
344    #[test]
345    fn auto_split_maps_joint_slots_through_the_skin() {
346        let avatar = head_avatar().with_skins(vec![crate::Skin::new(
347            vec![3, 4, 5, 6, 7, 0],
348            vec![glam::Mat4::IDENTITY; 6],
349        )]);
350        let primitive = primitive_with_triangles(
351            10,
352            vec![[5, 0, 0, 0], [5, 0, 0, 0], [5, 0, 0, 0]],
353            vec![[1.0, 0.0, 0.0, 0.0]; 3],
354            Some(0),
355        );
356
357        let split = FirstPerson::default().auto_split(&avatar, &primitive);
358
359        assert_eq!(split.third_person_only, vec![0]);
360        assert!(split.both.is_empty());
361    }
362
363    #[test]
364    fn auto_split_does_not_treat_a_slot_number_as_a_node_index() {
365        let avatar =
366            head_avatar().with_skins(vec![crate::Skin::new(vec![3], vec![glam::Mat4::IDENTITY])]);
367        let primitive = primitive_with_triangles(
368            10,
369            vec![[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],
370            vec![[1.0, 0.0, 0.0, 0.0]; 3],
371            Some(0),
372        );
373
374        let split = FirstPerson::default().auto_split(&avatar, &primitive);
375
376        assert!(split.third_person_only.is_empty());
377        assert_eq!(split.both, vec![0]);
378    }
379}