use glam::{Quat, Vec3};
use std::collections::{BTreeMap, HashMap};
use crate::animation::VrmAnimation;
use crate::expressions::{MaterialColorType, MaterialColorUpdate, TextureTransformUpdate};
use crate::{Avatar, AvatarAsset, HumanBone, MorphTargetUpdate};
#[derive(Debug, Clone, PartialEq)]
pub struct BoneUpdate {
node_index: u32,
rotation: Quat,
translation: Option<Vec3>,
}
impl BoneUpdate {
pub fn new(node_index: u32, rotation: Quat) -> Self {
Self {
node_index,
rotation,
translation: None,
}
}
pub fn with_translation(node_index: u32, rotation: Quat, translation: Vec3) -> Self {
Self {
node_index,
rotation,
translation: Some(translation),
}
}
pub fn node_index(&self) -> u32 {
self.node_index
}
pub fn rotation(&self) -> Quat {
self.rotation
}
pub fn translation(&self) -> Option<Vec3> {
self.translation
}
}
#[derive(Debug, Default, Clone)]
pub struct AvatarEvaluation {
pub morph_updates: Vec<MorphTargetUpdate>,
pub material_color_updates: Vec<MaterialColorUpdate>,
pub texture_transform_updates: Vec<TextureTransformUpdate>,
pub bone_updates: Vec<BoneUpdate>,
}
#[derive(Debug, Default, Clone)]
pub struct AvatarInstance {
expression_weights: HashMap<String, f32>,
bone_rotations: HashMap<HumanBone, Quat>,
bone_translations: HashMap<HumanBone, Vec3>,
look_at_target: Option<Vec3>,
look_at_rotation: Option<Quat>,
}
impl AvatarInstance {
pub(crate) fn new() -> Self {
Self::default()
}
pub fn reset_frame_state(&mut self) {
self.expression_weights.clear();
self.bone_rotations.clear();
self.bone_translations.clear();
self.look_at_target = None;
self.look_at_rotation = None;
}
pub fn set_expression(&mut self, name: impl Into<String>, weight: f32) {
self.expression_weights
.insert(name.into(), weight.clamp(0.0, 1.0));
}
pub fn expression_weight(&self, name: &str) -> f32 {
self.expression_weights.get(name).copied().unwrap_or(0.0)
}
pub fn clear_expression(&mut self, name: &str) {
self.expression_weights.remove(name);
}
pub fn clear_expressions(&mut self) {
self.expression_weights.clear();
}
pub fn set_bone_rotation(&mut self, bone: HumanBone, rotation: Quat) {
self.bone_rotations.insert(bone, rotation);
}
pub fn bone_rotation(&self, bone: HumanBone) -> Quat {
self.bone_rotations
.get(&bone)
.copied()
.unwrap_or(Quat::IDENTITY)
}
pub fn clear_bone_rotation(&mut self, bone: HumanBone) {
self.bone_rotations.remove(&bone);
}
pub fn clear_bone_rotations(&mut self) {
self.bone_rotations.clear();
}
pub fn set_bone_translation(&mut self, bone: HumanBone, translation: Vec3) {
self.bone_translations.insert(bone, translation);
}
pub fn bone_translation(&self, bone: HumanBone) -> Vec3 {
self.bone_translations
.get(&bone)
.copied()
.unwrap_or(Vec3::ZERO)
}
pub fn clear_bone_translation(&mut self, bone: HumanBone) {
self.bone_translations.remove(&bone);
}
pub fn clear_bone_translations(&mut self) {
self.bone_translations.clear();
}
pub fn set_look_at_target(&mut self, target: Option<Vec3>) {
self.look_at_target = target;
if target.is_some() {
self.look_at_rotation = None;
}
}
pub fn look_at_target(&self) -> Option<Vec3> {
self.look_at_target
}
pub fn set_look_at_rotation(&mut self, rotation: Option<Quat>) {
self.look_at_rotation = rotation;
if rotation.is_some() {
self.look_at_target = None;
}
}
pub(crate) fn overlay(&mut self, controls: &Self) {
self.expression_weights
.extend(controls.expression_weights.clone());
self.bone_rotations.extend(controls.bone_rotations.clone());
self.bone_translations
.extend(controls.bone_translations.clone());
if controls.look_at_target.is_some() {
self.look_at_target = controls.look_at_target;
self.look_at_rotation = None;
} else if controls.look_at_rotation.is_some() {
self.look_at_rotation = controls.look_at_rotation;
self.look_at_target = None;
}
}
pub fn apply_animation(&mut self, animation: &VrmAnimation, time: f32, avatar: &AvatarAsset) {
self.apply_animation_with_options(animation, time, avatar, true);
}
pub(crate) fn apply_animation_with_options(
&mut self,
animation: &VrmAnimation,
time: f32,
avatar: &Avatar,
scale_hips_translation: bool,
) {
let rest_world = avatar.compute_world_matrices(&[]);
let hips_scale = if scale_hips_translation {
animation
.source_hips_height()
.filter(|height| height.is_finite() && height.abs() > f32::EPSILON)
.and_then(|source_height| {
let hips = avatar.humanoid().bone(HumanBone::Hips)?;
let target_height = rest_world.get(hips.0 as usize)?.w_axis.y;
target_height
.is_finite()
.then_some(target_height / source_height)
})
.unwrap_or(1.0)
} else {
1.0
};
for (&bone, target) in avatar.humanoid().bones() {
let rotation = if animation.has_source_rest_pose() {
retargeted_rotation_with_missing_ancestors(
animation,
time,
avatar,
&rest_world,
bone,
target.0 as usize,
)
} else {
animation.sample_programmatic_bone_rotation_delta(bone, time)
};
if let Some(rotation) = rotation {
self.set_bone_rotation(bone, rotation);
}
}
for (bone, track) in animation.bone_tracks() {
if track.translation().is_none() || avatar.humanoid().bone(bone).is_none() {
continue;
}
if let Some(mut translation) = animation.sample_bone_translation_delta(bone, time) {
if bone == HumanBone::Hips {
translation *= hips_scale;
}
self.set_bone_translation(bone, translation);
}
}
self.set_look_at_rotation(animation.sample_look_at_rotation(time));
for (name, _) in animation.expression_tracks() {
self.set_expression(name, animation.sample_expression_weight(name, time));
}
}
pub fn evaluate(&self, avatar: &Avatar) -> AvatarEvaluation {
let mut expression_weights = self.expression_weights.clone();
let mut bone_rotations = self.bone_rotations.clone();
if let Some(look_at) = avatar.look_at() {
let controlled_bones: std::collections::HashSet<_> = bone_rotations
.keys()
.chain(self.bone_translations.keys())
.copied()
.collect();
let pre_look_at_updates: Vec<BoneUpdate> = controlled_bones
.into_iter()
.filter_map(|bone| {
let node = avatar.humanoid().bone(bone)?;
let rotation = bone_rotations.get(&bone).copied().unwrap_or(Quat::IDENTITY);
Some(match self.bone_translations.get(&bone) {
Some(&translation) => {
BoneUpdate::with_translation(node.0, rotation, translation)
}
None => BoneUpdate::new(node.0, rotation),
})
})
.collect();
let current_world = avatar.compute_world_matrices(&pre_look_at_updates);
let rest_world = avatar.compute_world_matrices(&[]);
let local_target = if let Some(target) = self.look_at_target {
avatar.humanoid().bone(HumanBone::Head).and_then(|head| {
let current_head = current_world.get(head.0 as usize)?;
let rest_head = rest_world.get(head.0 as usize)?;
let (_, rest_rotation, _) = rest_head.to_scale_rotation_translation();
let look_at_world = *current_head
* glam::Mat4::from_translation(look_at.offset_from_head_bone)
* glam::Mat4::from_quat(rest_rotation.inverse());
Some(look_at_world.inverse().transform_point3(target))
})
} else {
self.look_at_rotation.map(|rotation| rotation * Vec3::Z)
};
if let Some(local_target) = local_target {
let (yaw, pitch) = crate::look_at::calc_yaw_pitch(local_target);
match look_at.look_at_type {
crate::LookAtType::Bone => {
let (left_rot, right_rot) = look_at.apply_bone(yaw, pitch);
bone_rotations.insert(HumanBone::LeftEye, left_rot);
bone_rotations.insert(HumanBone::RightEye, right_rot);
}
crate::LookAtType::Expression => {
let (l_left, l_right, l_down, l_up) = look_at.apply_expression(yaw, pitch);
if l_left > 0.0 {
expression_weights.insert("lookLeft".to_string(), l_left);
}
if l_right > 0.0 {
expression_weights.insert("lookRight".to_string(), l_right);
}
if l_down > 0.0 {
expression_weights.insert("lookDown".to_string(), l_down);
}
if l_up > 0.0 {
expression_weights.insert("lookUp".to_string(), l_up);
}
}
}
}
}
let mut mouth_block = false;
let mut blink_block = false;
let mut look_at_block = false;
let mut mouth_blend_sum = 0.0f32;
let mut blink_blend_sum = 0.0f32;
let mut look_at_blend_sum = 0.0f32;
for (name, &input_weight) in &self.expression_weights {
if input_weight <= 0.0 {
continue;
}
let Some(expr) = avatar.expressions().get(name) else {
continue;
};
let output_weight = if expr.is_binary() {
if input_weight >= 0.5 { 1.0 } else { 0.0 }
} else {
input_weight
};
match expr.override_mouth() {
crate::expressions::ExpressionOverrideType::Block => {
mouth_block |= output_weight > 0.0;
}
crate::expressions::ExpressionOverrideType::Blend => {
mouth_blend_sum += output_weight;
}
_ => {}
}
match expr.override_blink() {
crate::expressions::ExpressionOverrideType::Block => {
blink_block |= output_weight > 0.0;
}
crate::expressions::ExpressionOverrideType::Blend => {
blink_blend_sum += output_weight;
}
_ => {}
}
match expr.override_look_at() {
crate::expressions::ExpressionOverrideType::Block => {
look_at_block |= output_weight > 0.0;
}
crate::expressions::ExpressionOverrideType::Blend => {
look_at_blend_sum += output_weight;
}
_ => {}
}
}
let mouth_names = ["aa", "ih", "ou", "ee", "oh"];
let blink_names = ["blink", "blinkLeft", "blinkRight"];
let look_at_names = ["lookUp", "lookDown", "lookLeft", "lookRight"];
let attenuation = |sum: f32| (1.0 - sum.min(1.0)).max(0.0);
let overridden_load =
|name: &str, avatar: &Avatar, block: bool, sum: f32, weight: &mut f32| {
if block {
*weight = 0.0;
} else if sum > 0.0 {
let binary_target = avatar
.expressions()
.get(name)
.map(|e| e.is_binary())
.unwrap_or(false);
*weight = if binary_target {
0.0
} else {
*weight * attenuation(sum)
};
}
};
for (name, weight) in expression_weights.iter_mut() {
if mouth_names.contains(&name.as_str()) {
overridden_load(name, avatar, mouth_block, mouth_blend_sum, weight);
}
if blink_names.contains(&name.as_str()) {
overridden_load(name, avatar, blink_block, blink_blend_sum, weight);
}
if look_at_names.contains(&name.as_str()) {
overridden_load(name, avatar, look_at_block, look_at_blend_sum, weight);
}
}
let mut weights = BTreeMap::<(u32, u32), f32>::new();
let mut material_colors = BTreeMap::<(u32, MaterialColorType), [f32; 4]>::new();
let mut texture_transforms = BTreeMap::<u32, ([f32; 2], [f32; 2])>::new();
for (name, input_weight) in &expression_weights {
let Some(expression) = avatar.expressions().get(name) else {
continue;
};
let final_weight = if expression.is_binary() {
if *input_weight >= 0.5 { 1.0 } else { 0.0 }
} else {
*input_weight
};
let expr_eval = expression.evaluate(final_weight);
for update in expr_eval.morph_updates {
let key = (update.node_index(), update.target_index());
*weights.entry(key).or_insert(0.0) += update.weight();
}
for update in expr_eval.material_color_updates {
let key = (update.material_index(), update.color_type());
let base = avatar.base_color(update.material_index(), update.color_type());
let target = update.target_value();
let weight = update.weight();
let entry = material_colors.entry(key).or_insert([0.0; 4]);
for i in 0..4 {
entry[i] += (target[i] - base[i]) * weight;
}
}
for update in expr_eval.texture_transform_updates {
let key = update.material_index();
let scale = update.scale();
let offset = update.offset();
let weight = update.weight();
let entry = texture_transforms
.entry(key)
.or_insert(([0.0; 2], [0.0; 2]));
for i in 0..2 {
entry.0[i] += (scale[i] - 1.0) * weight;
entry.1[i] += (offset[i] - 0.0) * weight;
}
}
}
let morph_updates = weights
.into_iter()
.map(|((node_index, target_index), weight)| {
MorphTargetUpdate::new(node_index, target_index, weight.clamp(0.0, 1.0))
})
.collect();
let material_color_updates = material_colors
.into_iter()
.map(|((material_index, color_type), delta)| {
let base = avatar.base_color(material_index, color_type);
let mut target_value = [0.0; 4];
for i in 0..4 {
target_value[i] = base[i] + delta[i];
}
MaterialColorUpdate::new(material_index, color_type, target_value, 1.0)
})
.collect();
let texture_transform_updates = texture_transforms
.into_iter()
.map(|(material_index, (scale_delta, offset_delta))| {
let scale = [1.0 + scale_delta[0], 1.0 + scale_delta[1]];
let offset = [offset_delta[0], offset_delta[1]];
TextureTransformUpdate::new(material_index, scale, offset, 1.0)
})
.collect();
let bone_updates = bone_rotations
.iter()
.filter_map(|(bone, &rotation)| {
let node = avatar.humanoid().bone(*bone)?;
Some(match self.bone_translations.get(bone) {
Some(&translation) => {
BoneUpdate::with_translation(node.0, rotation, translation)
}
None => BoneUpdate::new(node.0, rotation),
})
})
.chain(
self.bone_translations
.iter()
.filter_map(|(bone, &translation)| {
if bone_rotations.contains_key(bone) {
return None;
}
let node = avatar.humanoid().bone(*bone)?;
Some(BoneUpdate::with_translation(
node.0,
Quat::IDENTITY,
translation,
))
}),
)
.collect();
AvatarEvaluation {
morph_updates,
material_color_updates,
texture_transform_updates,
bone_updates,
}
}
}
fn retargeted_rotation_with_missing_ancestors(
animation: &VrmAnimation,
time: f32,
avatar: &Avatar,
rest_world: &[glam::Mat4],
bone: HumanBone,
target_node: usize,
) -> Option<Quat> {
let own_rotation = animation.sample_normalized_source_bone_rotation(bone, time);
let mut missing_ancestors = Vec::new();
let mut ancestor = bone.parent();
while let Some(parent) = ancestor {
if avatar.humanoid().bone(parent).is_some() {
break;
}
if animation.bone_track(parent).is_some()
&& let Some(rotation) = animation.sample_normalized_source_bone_rotation(parent, time)
{
missing_ancestors.push(rotation);
}
ancestor = parent.parent();
}
if own_rotation.is_none() && missing_ancestors.is_empty() {
return None;
}
let normalized = missing_ancestors
.into_iter()
.rev()
.fold(Quat::IDENTITY, |rotation, next| rotation * next)
* own_rotation.unwrap_or(Quat::IDENTITY);
let target_world_rotation = rest_world
.get(target_node)?
.to_scale_rotation_translation()
.1;
Some((target_world_rotation.inverse() * normalized * target_world_rotation).normalize())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
Avatar, AvatarNode, Expression, Expressions, Humanoid, LookAt, LookAtRangeMap, LookAtType,
MorphTargetBind,
};
fn look_at_avatar(head_translation: Vec3) -> Avatar {
Avatar::new()
.with_humanoid(
Humanoid::default()
.with_bone(HumanBone::Head, 0)
.with_bone(HumanBone::LeftEye, 1)
.with_bone(HumanBone::RightEye, 2),
)
.with_nodes(vec![
AvatarNode {
index: 0,
name: Some("head".to_string()),
parent: None,
children: vec![1, 2],
initial_translation: head_translation,
initial_rotation: Quat::IDENTITY,
initial_scale: Vec3::ONE,
},
AvatarNode {
index: 1,
name: Some("leftEye".to_string()),
parent: Some(0),
children: vec![],
initial_translation: Vec3::ZERO,
initial_rotation: Quat::IDENTITY,
initial_scale: Vec3::ONE,
},
AvatarNode {
index: 2,
name: Some("rightEye".to_string()),
parent: Some(0),
children: vec![],
initial_translation: Vec3::ZERO,
initial_rotation: Quat::IDENTITY,
initial_scale: Vec3::ONE,
},
])
.with_look_at(LookAt {
look_at_type: LookAtType::Bone,
offset_from_head_bone: Vec3::ZERO,
range_map_horizontal_inner: LookAtRangeMap::default(),
range_map_horizontal_outer: LookAtRangeMap::default(),
range_map_vertical_down: LookAtRangeMap::default(),
range_map_vertical_up: LookAtRangeMap::default(),
})
}
#[test]
fn instance_stores_expression_weight() {
let mut instance = AvatarInstance::new();
assert_eq!(instance.expression_weight("happy"), 0.0);
instance.set_expression("happy", 0.7);
assert_eq!(instance.expression_weight("happy"), 0.7);
}
#[test]
fn instance_evaluates_happy_expression() {
let happy = Expression::new("happy")
.with_morph_target_binds(vec![MorphTargetBind::new(91, 3, 1.0)]);
let expressions = Expressions::new(vec![happy]);
let avatar = Avatar::new().with_expressions(expressions);
let mut instance = avatar.instantiate();
instance.set_expression("happy", 0.7);
let eval = instance.evaluate(&avatar);
assert_eq!(eval.morph_updates.len(), 1);
assert_eq!(eval.morph_updates[0].node_index(), 91);
assert_eq!(eval.morph_updates[0].target_index(), 3);
assert_eq!(eval.morph_updates[0].weight(), 0.7);
assert!(eval.bone_updates.is_empty());
}
#[test]
fn look_at_uses_the_head_node_when_the_head_has_no_mesh() {
let avatar = look_at_avatar(Vec3::new(10.0, 0.0, 0.0));
let mut instance = AvatarInstance::new();
instance.set_look_at_target(Some(Vec3::new(10.0, 0.0, 5.0)));
let evaluation = instance.evaluate(&avatar);
for eye in [1, 2] {
let rotation = evaluation
.bone_updates
.iter()
.find(|update| update.node_index() == eye)
.unwrap()
.rotation();
assert!(rotation.abs_diff_eq(Quat::IDENTITY, 1.0e-5));
}
}
#[test]
fn look_at_space_follows_the_current_head_rotation() {
let avatar = look_at_avatar(Vec3::ZERO);
let mut instance = AvatarInstance::new();
instance.set_bone_rotation(
HumanBone::Head,
Quat::from_rotation_y(90.0_f32.to_radians()),
);
instance.set_look_at_target(Some(Vec3::new(5.0, 0.0, 0.0)));
let evaluation = instance.evaluate(&avatar);
for eye in [1, 2] {
let rotation = evaluation
.bone_updates
.iter()
.find(|update| update.node_index() == eye)
.unwrap()
.rotation();
assert!(rotation.abs_diff_eq(Quat::IDENTITY, 1.0e-5));
}
}
#[test]
fn look_at_space_follows_the_current_head_translation() {
let avatar = look_at_avatar(Vec3::ZERO);
let mut instance = AvatarInstance::new();
instance.set_bone_translation(HumanBone::Head, Vec3::new(10.0, 0.0, 0.0));
instance.set_look_at_target(Some(Vec3::new(10.0, 0.0, 5.0)));
let evaluation = instance.evaluate(&avatar);
for eye in [1, 2] {
let rotation = evaluation
.bone_updates
.iter()
.find(|update| update.node_index() == eye)
.expect("LookAt must update each eye")
.rotation();
assert!(rotation.abs_diff_eq(Quat::IDENTITY, 1.0e-5));
}
}
}