1use glam::{Quat, Vec3};
2use std::collections::{BTreeMap, HashMap};
3
4use crate::animation::VrmAnimation;
5use crate::expressions::{MaterialColorType, MaterialColorUpdate, TextureTransformUpdate};
6use crate::{Avatar, AvatarAsset, HumanBone, MorphTargetUpdate};
7
8#[derive(Debug, Clone, PartialEq)]
11pub struct BoneUpdate {
12 node_index: u32,
13 rotation: Quat,
14 translation: Option<Vec3>,
15}
16
17impl BoneUpdate {
18 pub fn new(node_index: u32, rotation: Quat) -> Self {
20 Self {
21 node_index,
22 rotation,
23 translation: None,
24 }
25 }
26
27 pub fn with_translation(node_index: u32, rotation: Quat, translation: Vec3) -> Self {
29 Self {
30 node_index,
31 rotation,
32 translation: Some(translation),
33 }
34 }
35
36 pub fn node_index(&self) -> u32 {
38 self.node_index
39 }
40
41 pub fn rotation(&self) -> Quat {
43 self.rotation
44 }
45
46 pub fn translation(&self) -> Option<Vec3> {
48 self.translation
49 }
50}
51
52#[derive(Debug, Default, Clone)]
55pub struct AvatarEvaluation {
56 pub morph_updates: Vec<MorphTargetUpdate>,
58 pub material_color_updates: Vec<MaterialColorUpdate>,
60 pub texture_transform_updates: Vec<TextureTransformUpdate>,
62 pub bone_updates: Vec<BoneUpdate>,
64}
65
66#[derive(Debug, Default, Clone)]
70pub struct AvatarInstance {
71 expression_weights: HashMap<String, f32>,
72 bone_rotations: HashMap<HumanBone, Quat>,
73 bone_translations: HashMap<HumanBone, Vec3>,
74 look_at_target: Option<Vec3>,
75 look_at_rotation: Option<Quat>,
76}
77
78impl AvatarInstance {
79 pub(crate) fn new() -> Self {
80 Self::default()
81 }
82
83 pub fn reset_frame_state(&mut self) {
88 self.expression_weights.clear();
89 self.bone_rotations.clear();
90 self.bone_translations.clear();
91 self.look_at_target = None;
92 self.look_at_rotation = None;
93 }
94
95 pub fn set_expression(&mut self, name: impl Into<String>, weight: f32) {
97 self.expression_weights
98 .insert(name.into(), weight.clamp(0.0, 1.0));
99 }
100
101 pub fn expression_weight(&self, name: &str) -> f32 {
103 self.expression_weights.get(name).copied().unwrap_or(0.0)
104 }
105
106 pub fn clear_expression(&mut self, name: &str) {
108 self.expression_weights.remove(name);
109 }
110
111 pub fn clear_expressions(&mut self) {
113 self.expression_weights.clear();
114 }
115
116 pub fn set_bone_rotation(&mut self, bone: HumanBone, rotation: Quat) {
120 self.bone_rotations.insert(bone, rotation);
121 }
122
123 pub fn bone_rotation(&self, bone: HumanBone) -> Quat {
125 self.bone_rotations
126 .get(&bone)
127 .copied()
128 .unwrap_or(Quat::IDENTITY)
129 }
130
131 pub fn clear_bone_rotation(&mut self, bone: HumanBone) {
133 self.bone_rotations.remove(&bone);
134 }
135
136 pub fn clear_bone_rotations(&mut self) {
138 self.bone_rotations.clear();
139 }
140
141 pub fn set_bone_translation(&mut self, bone: HumanBone, translation: Vec3) {
143 self.bone_translations.insert(bone, translation);
144 }
145
146 pub fn bone_translation(&self, bone: HumanBone) -> Vec3 {
148 self.bone_translations
149 .get(&bone)
150 .copied()
151 .unwrap_or(Vec3::ZERO)
152 }
153
154 pub fn clear_bone_translation(&mut self, bone: HumanBone) {
156 self.bone_translations.remove(&bone);
157 }
158
159 pub fn clear_bone_translations(&mut self) {
161 self.bone_translations.clear();
162 }
163
164 pub fn set_look_at_target(&mut self, target: Option<Vec3>) {
166 self.look_at_target = target;
167 if target.is_some() {
168 self.look_at_rotation = None;
169 }
170 }
171
172 pub fn look_at_target(&self) -> Option<Vec3> {
174 self.look_at_target
175 }
176
177 pub fn set_look_at_rotation(&mut self, rotation: Option<Quat>) {
179 self.look_at_rotation = rotation;
180 if rotation.is_some() {
181 self.look_at_target = None;
182 }
183 }
184
185 pub(crate) fn overlay(&mut self, controls: &Self) {
186 self.expression_weights
187 .extend(controls.expression_weights.clone());
188 self.bone_rotations.extend(controls.bone_rotations.clone());
189 self.bone_translations
190 .extend(controls.bone_translations.clone());
191 if controls.look_at_target.is_some() {
192 self.look_at_target = controls.look_at_target;
193 self.look_at_rotation = None;
194 } else if controls.look_at_rotation.is_some() {
195 self.look_at_rotation = controls.look_at_rotation;
196 self.look_at_target = None;
197 }
198 }
199
200 pub fn apply_animation(&mut self, animation: &VrmAnimation, time: f32, avatar: &AvatarAsset) {
206 self.apply_animation_with_options(animation, time, avatar, true);
207 }
208
209 pub(crate) fn apply_animation_with_options(
210 &mut self,
211 animation: &VrmAnimation,
212 time: f32,
213 avatar: &Avatar,
214 scale_hips_translation: bool,
215 ) {
216 let rest_world = avatar.compute_world_matrices(&[]);
217 let hips_scale = if scale_hips_translation {
218 animation
219 .source_hips_height()
220 .filter(|height| height.is_finite() && height.abs() > f32::EPSILON)
221 .and_then(|source_height| {
222 let hips = avatar.humanoid().bone(HumanBone::Hips)?;
223 let target_height = rest_world.get(hips.0 as usize)?.w_axis.y;
224 target_height
225 .is_finite()
226 .then_some(target_height / source_height)
227 })
228 .unwrap_or(1.0)
229 } else {
230 1.0
231 };
232
233 for (&bone, target) in avatar.humanoid().bones() {
234 let rotation = if animation.has_source_rest_pose() {
235 retargeted_rotation_with_missing_ancestors(
236 animation,
237 time,
238 avatar,
239 &rest_world,
240 bone,
241 target.0 as usize,
242 )
243 } else {
244 animation.sample_programmatic_bone_rotation_delta(bone, time)
245 };
246 if let Some(rotation) = rotation {
247 self.set_bone_rotation(bone, rotation);
248 }
249 }
250
251 for (bone, track) in animation.bone_tracks() {
252 if track.translation().is_none() || avatar.humanoid().bone(bone).is_none() {
253 continue;
254 }
255 if let Some(mut translation) = animation.sample_bone_translation_delta(bone, time) {
256 if bone == HumanBone::Hips {
257 translation *= hips_scale;
258 }
259 self.set_bone_translation(bone, translation);
260 }
261 }
262
263 self.set_look_at_rotation(animation.sample_look_at_rotation(time));
264 for (name, _) in animation.expression_tracks() {
265 self.set_expression(name, animation.sample_expression_weight(name, time));
266 }
267 }
268
269 pub fn evaluate(&self, avatar: &Avatar) -> AvatarEvaluation {
275 let mut expression_weights = self.expression_weights.clone();
278 let mut bone_rotations = self.bone_rotations.clone();
279
280 if let Some(look_at) = avatar.look_at() {
285 let controlled_bones: std::collections::HashSet<_> = bone_rotations
286 .keys()
287 .chain(self.bone_translations.keys())
288 .copied()
289 .collect();
290 let pre_look_at_updates: Vec<BoneUpdate> = controlled_bones
291 .into_iter()
292 .filter_map(|bone| {
293 let node = avatar.humanoid().bone(bone)?;
294 let rotation = bone_rotations.get(&bone).copied().unwrap_or(Quat::IDENTITY);
295 Some(match self.bone_translations.get(&bone) {
296 Some(&translation) => {
297 BoneUpdate::with_translation(node.0, rotation, translation)
298 }
299 None => BoneUpdate::new(node.0, rotation),
300 })
301 })
302 .collect();
303 let current_world = avatar.compute_world_matrices(&pre_look_at_updates);
304 let rest_world = avatar.compute_world_matrices(&[]);
305 let local_target = if let Some(target) = self.look_at_target {
306 avatar.humanoid().bone(HumanBone::Head).and_then(|head| {
307 let current_head = current_world.get(head.0 as usize)?;
308 let rest_head = rest_world.get(head.0 as usize)?;
309 let (_, rest_rotation, _) = rest_head.to_scale_rotation_translation();
310 let look_at_world = *current_head
311 * glam::Mat4::from_translation(look_at.offset_from_head_bone)
312 * glam::Mat4::from_quat(rest_rotation.inverse());
313 Some(look_at_world.inverse().transform_point3(target))
314 })
315 } else {
316 self.look_at_rotation.map(|rotation| rotation * Vec3::Z)
317 };
318
319 if let Some(local_target) = local_target {
320 let (yaw, pitch) = crate::look_at::calc_yaw_pitch(local_target);
323
324 match look_at.look_at_type {
325 crate::LookAtType::Bone => {
327 let (left_rot, right_rot) = look_at.apply_bone(yaw, pitch);
328 bone_rotations.insert(HumanBone::LeftEye, left_rot);
329 bone_rotations.insert(HumanBone::RightEye, right_rot);
330 }
331 crate::LookAtType::Expression => {
334 let (l_left, l_right, l_down, l_up) = look_at.apply_expression(yaw, pitch);
335 if l_left > 0.0 {
336 expression_weights.insert("lookLeft".to_string(), l_left);
337 }
338 if l_right > 0.0 {
339 expression_weights.insert("lookRight".to_string(), l_right);
340 }
341 if l_down > 0.0 {
342 expression_weights.insert("lookDown".to_string(), l_down);
343 }
344 if l_up > 0.0 {
345 expression_weights.insert("lookUp".to_string(), l_up);
346 }
347 }
348 }
349 }
350 }
351
352 let mut mouth_block = false;
359 let mut blink_block = false;
360 let mut look_at_block = false;
361
362 let mut mouth_blend_sum = 0.0f32;
364 let mut blink_blend_sum = 0.0f32;
365 let mut look_at_blend_sum = 0.0f32;
366
367 for (name, &input_weight) in &self.expression_weights {
368 if input_weight <= 0.0 {
370 continue;
371 }
372 let Some(expr) = avatar.expressions().get(name) else {
373 continue;
374 };
375
376 let output_weight = if expr.is_binary() {
379 if input_weight >= 0.5 { 1.0 } else { 0.0 }
380 } else {
381 input_weight
382 };
383
384 match expr.override_mouth() {
385 crate::expressions::ExpressionOverrideType::Block => {
386 mouth_block |= output_weight > 0.0;
387 }
388 crate::expressions::ExpressionOverrideType::Blend => {
389 mouth_blend_sum += output_weight;
390 }
391 _ => {}
392 }
393 match expr.override_blink() {
394 crate::expressions::ExpressionOverrideType::Block => {
395 blink_block |= output_weight > 0.0;
396 }
397 crate::expressions::ExpressionOverrideType::Blend => {
398 blink_blend_sum += output_weight;
399 }
400 _ => {}
401 }
402 match expr.override_look_at() {
403 crate::expressions::ExpressionOverrideType::Block => {
404 look_at_block |= output_weight > 0.0;
405 }
406 crate::expressions::ExpressionOverrideType::Blend => {
407 look_at_blend_sum += output_weight;
408 }
409 _ => {}
410 }
411 }
412
413 let mouth_names = ["aa", "ih", "ou", "ee", "oh"];
417 let blink_names = ["blink", "blinkLeft", "blinkRight"];
418 let look_at_names = ["lookUp", "lookDown", "lookLeft", "lookRight"];
419
420 let attenuation = |sum: f32| (1.0 - sum.min(1.0)).max(0.0);
421 let overridden_load =
422 |name: &str, avatar: &Avatar, block: bool, sum: f32, weight: &mut f32| {
423 if block {
424 *weight = 0.0;
425 } else if sum > 0.0 {
426 let binary_target = avatar
429 .expressions()
430 .get(name)
431 .map(|e| e.is_binary())
432 .unwrap_or(false);
433 *weight = if binary_target {
434 0.0
435 } else {
436 *weight * attenuation(sum)
437 };
438 }
439 };
440
441 for (name, weight) in expression_weights.iter_mut() {
442 if mouth_names.contains(&name.as_str()) {
443 overridden_load(name, avatar, mouth_block, mouth_blend_sum, weight);
444 }
445 if blink_names.contains(&name.as_str()) {
446 overridden_load(name, avatar, blink_block, blink_blend_sum, weight);
447 }
448 if look_at_names.contains(&name.as_str()) {
449 overridden_load(name, avatar, look_at_block, look_at_blend_sum, weight);
450 }
451 }
452
453 let mut weights = BTreeMap::<(u32, u32), f32>::new();
458 let mut material_colors = BTreeMap::<(u32, MaterialColorType), [f32; 4]>::new();
461 let mut texture_transforms = BTreeMap::<u32, ([f32; 2], [f32; 2])>::new();
462
463 for (name, input_weight) in &expression_weights {
464 let Some(expression) = avatar.expressions().get(name) else {
465 continue;
466 };
467
468 let final_weight = if expression.is_binary() {
471 if *input_weight >= 0.5 { 1.0 } else { 0.0 }
472 } else {
473 *input_weight
474 };
475
476 let expr_eval = expression.evaluate(final_weight);
477
478 for update in expr_eval.morph_updates {
481 let key = (update.node_index(), update.target_index());
482 *weights.entry(key).or_insert(0.0) += update.weight();
483 }
484
485 for update in expr_eval.material_color_updates {
488 let key = (update.material_index(), update.color_type());
489 let base = avatar.base_color(update.material_index(), update.color_type());
490 let target = update.target_value();
491 let weight = update.weight();
492 let entry = material_colors.entry(key).or_insert([0.0; 4]);
493 for i in 0..4 {
494 entry[i] += (target[i] - base[i]) * weight;
495 }
496 }
497
498 for update in expr_eval.texture_transform_updates {
501 let key = update.material_index();
502 let scale = update.scale();
503 let offset = update.offset();
504 let weight = update.weight();
505 let entry = texture_transforms
506 .entry(key)
507 .or_insert(([0.0; 2], [0.0; 2]));
508 for i in 0..2 {
509 entry.0[i] += (scale[i] - 1.0) * weight;
510 entry.1[i] += (offset[i] - 0.0) * weight;
511 }
512 }
513 }
514
515 let morph_updates = weights
519 .into_iter()
520 .map(|((node_index, target_index), weight)| {
521 MorphTargetUpdate::new(node_index, target_index, weight.clamp(0.0, 1.0))
522 })
523 .collect();
524
525 let material_color_updates = material_colors
528 .into_iter()
529 .map(|((material_index, color_type), delta)| {
530 let base = avatar.base_color(material_index, color_type);
531 let mut target_value = [0.0; 4];
532 for i in 0..4 {
533 target_value[i] = base[i] + delta[i];
534 }
535 MaterialColorUpdate::new(material_index, color_type, target_value, 1.0)
536 })
537 .collect();
538
539 let texture_transform_updates = texture_transforms
540 .into_iter()
541 .map(|(material_index, (scale_delta, offset_delta))| {
542 let scale = [1.0 + scale_delta[0], 1.0 + scale_delta[1]];
543 let offset = [offset_delta[0], offset_delta[1]];
544 TextureTransformUpdate::new(material_index, scale, offset, 1.0)
545 })
546 .collect();
547
548 let bone_updates = bone_rotations
551 .iter()
552 .filter_map(|(bone, &rotation)| {
553 let node = avatar.humanoid().bone(*bone)?;
554 Some(match self.bone_translations.get(bone) {
555 Some(&translation) => {
556 BoneUpdate::with_translation(node.0, rotation, translation)
557 }
558 None => BoneUpdate::new(node.0, rotation),
559 })
560 })
561 .chain(
562 self.bone_translations
563 .iter()
564 .filter_map(|(bone, &translation)| {
565 if bone_rotations.contains_key(bone) {
566 return None;
567 }
568 let node = avatar.humanoid().bone(*bone)?;
569 Some(BoneUpdate::with_translation(
570 node.0,
571 Quat::IDENTITY,
572 translation,
573 ))
574 }),
575 )
576 .collect();
577
578 AvatarEvaluation {
579 morph_updates,
580 material_color_updates,
581 texture_transform_updates,
582 bone_updates,
583 }
584 }
585}
586
587fn retargeted_rotation_with_missing_ancestors(
588 animation: &VrmAnimation,
589 time: f32,
590 avatar: &Avatar,
591 rest_world: &[glam::Mat4],
592 bone: HumanBone,
593 target_node: usize,
594) -> Option<Quat> {
595 let own_rotation = animation.sample_normalized_source_bone_rotation(bone, time);
596 let mut missing_ancestors = Vec::new();
597 let mut ancestor = bone.parent();
598 while let Some(parent) = ancestor {
599 if avatar.humanoid().bone(parent).is_some() {
600 break;
601 }
602 if animation.bone_track(parent).is_some()
603 && let Some(rotation) = animation.sample_normalized_source_bone_rotation(parent, time)
604 {
605 missing_ancestors.push(rotation);
606 }
607 ancestor = parent.parent();
608 }
609
610 if own_rotation.is_none() && missing_ancestors.is_empty() {
611 return None;
612 }
613 let normalized = missing_ancestors
614 .into_iter()
615 .rev()
616 .fold(Quat::IDENTITY, |rotation, next| rotation * next)
617 * own_rotation.unwrap_or(Quat::IDENTITY);
618 let target_world_rotation = rest_world
619 .get(target_node)?
620 .to_scale_rotation_translation()
621 .1;
622 Some((target_world_rotation.inverse() * normalized * target_world_rotation).normalize())
623}
624
625#[cfg(test)]
626mod tests {
627 use super::*;
628 use crate::{
629 Avatar, AvatarNode, Expression, Expressions, Humanoid, LookAt, LookAtRangeMap, LookAtType,
630 MorphTargetBind,
631 };
632
633 fn look_at_avatar(head_translation: Vec3) -> Avatar {
634 Avatar::new()
635 .with_humanoid(
636 Humanoid::default()
637 .with_bone(HumanBone::Head, 0)
638 .with_bone(HumanBone::LeftEye, 1)
639 .with_bone(HumanBone::RightEye, 2),
640 )
641 .with_nodes(vec![
642 AvatarNode {
643 index: 0,
644 name: Some("head".to_string()),
645 parent: None,
646 children: vec![1, 2],
647 initial_translation: head_translation,
648 initial_rotation: Quat::IDENTITY,
649 initial_scale: Vec3::ONE,
650 },
651 AvatarNode {
652 index: 1,
653 name: Some("leftEye".to_string()),
654 parent: Some(0),
655 children: vec![],
656 initial_translation: Vec3::ZERO,
657 initial_rotation: Quat::IDENTITY,
658 initial_scale: Vec3::ONE,
659 },
660 AvatarNode {
661 index: 2,
662 name: Some("rightEye".to_string()),
663 parent: Some(0),
664 children: vec![],
665 initial_translation: Vec3::ZERO,
666 initial_rotation: Quat::IDENTITY,
667 initial_scale: Vec3::ONE,
668 },
669 ])
670 .with_look_at(LookAt {
671 look_at_type: LookAtType::Bone,
672 offset_from_head_bone: Vec3::ZERO,
673 range_map_horizontal_inner: LookAtRangeMap::default(),
674 range_map_horizontal_outer: LookAtRangeMap::default(),
675 range_map_vertical_down: LookAtRangeMap::default(),
676 range_map_vertical_up: LookAtRangeMap::default(),
677 })
678 }
679
680 #[test]
681 fn instance_stores_expression_weight() {
682 let mut instance = AvatarInstance::new();
683
684 assert_eq!(instance.expression_weight("happy"), 0.0);
685
686 instance.set_expression("happy", 0.7);
687
688 assert_eq!(instance.expression_weight("happy"), 0.7);
689 }
690
691 #[test]
692 fn instance_evaluates_happy_expression() {
693 let happy = Expression::new("happy")
694 .with_morph_target_binds(vec![MorphTargetBind::new(91, 3, 1.0)]);
695
696 let expressions = Expressions::new(vec![happy]);
697
698 let avatar = Avatar::new().with_expressions(expressions);
699
700 let mut instance = avatar.instantiate();
701 instance.set_expression("happy", 0.7);
702
703 let eval = instance.evaluate(&avatar);
704
705 assert_eq!(eval.morph_updates.len(), 1);
706 assert_eq!(eval.morph_updates[0].node_index(), 91);
707 assert_eq!(eval.morph_updates[0].target_index(), 3);
708 assert_eq!(eval.morph_updates[0].weight(), 0.7);
709 assert!(eval.bone_updates.is_empty());
710 }
711
712 #[test]
713 fn look_at_uses_the_head_node_when_the_head_has_no_mesh() {
714 let avatar = look_at_avatar(Vec3::new(10.0, 0.0, 0.0));
715 let mut instance = AvatarInstance::new();
716 instance.set_look_at_target(Some(Vec3::new(10.0, 0.0, 5.0)));
717
718 let evaluation = instance.evaluate(&avatar);
719
720 for eye in [1, 2] {
721 let rotation = evaluation
722 .bone_updates
723 .iter()
724 .find(|update| update.node_index() == eye)
725 .unwrap()
726 .rotation();
727 assert!(rotation.abs_diff_eq(Quat::IDENTITY, 1.0e-5));
728 }
729 }
730
731 #[test]
732 fn look_at_space_follows_the_current_head_rotation() {
733 let avatar = look_at_avatar(Vec3::ZERO);
734 let mut instance = AvatarInstance::new();
735 instance.set_bone_rotation(
736 HumanBone::Head,
737 Quat::from_rotation_y(90.0_f32.to_radians()),
738 );
739 instance.set_look_at_target(Some(Vec3::new(5.0, 0.0, 0.0)));
740
741 let evaluation = instance.evaluate(&avatar);
742
743 for eye in [1, 2] {
744 let rotation = evaluation
745 .bone_updates
746 .iter()
747 .find(|update| update.node_index() == eye)
748 .unwrap()
749 .rotation();
750 assert!(rotation.abs_diff_eq(Quat::IDENTITY, 1.0e-5));
751 }
752 }
753
754 #[test]
755 fn look_at_space_follows_the_current_head_translation() {
756 let avatar = look_at_avatar(Vec3::ZERO);
757 let mut instance = AvatarInstance::new();
758 instance.set_bone_translation(HumanBone::Head, Vec3::new(10.0, 0.0, 0.0));
759 instance.set_look_at_target(Some(Vec3::new(10.0, 0.0, 5.0)));
760
761 let evaluation = instance.evaluate(&avatar);
762
763 for eye in [1, 2] {
764 let rotation = evaluation
765 .bone_updates
766 .iter()
767 .find(|update| update.node_index() == eye)
768 .expect("LookAt must update each eye")
769 .rotation();
770 assert!(rotation.abs_diff_eq(Quat::IDENTITY, 1.0e-5));
771 }
772 }
773}