Skip to main content

metaverse_core/
avatar.rs

1use super::session::Mailbox;
2use crate::session::SendUIMessage;
3use actix::{AsyncContext, Handler, Message, WrapFuture};
4use benthic_protocol::messages::ui::camera_position::CameraPosition;
5use benthic_protocol::messages::ui::mesh_update::{MeshType, MeshUpdate};
6use benthic_protocol::messages::ui::play_animation::PlayAnimation;
7use benthic_protocol::messages::ui::ui_messages::UIMessage;
8use benthic_protocol::session::create_agent_animation_dir;
9use benthic_protocol::skeleton::{JointName, Skeleton};
10use glam::{Quat, Vec3};
11use log::{error, warn};
12use metaverse_avatar::animation::build_animation;
13use metaverse_avatar::avatar::Avatar;
14use metaverse_avatar::avatar::OutfitObject;
15use metaverse_avatar::avatar_object_handler::AvatarState::FullyLoaded;
16use metaverse_avatar::avatar_object_handler::{
17    AvatarType, add_object_to_avatar, finalize_avatar, init_avatar,
18};
19use metaverse_avatar::errors::AvatarError::{self};
20use metaverse_cache::agent::sqlite_update_avatar;
21use metaverse_cache::avatar::cache_current_outfit;
22use metaverse_messages::http::capabilities::Capability;
23use metaverse_messages::udp::agent::avatar_animation::AvatarAnimation;
24use metaverse_messages::udp::agent::avatar_appearance::AvatarAppearance;
25use metaverse_messages::utils::object_types::ObjectType;
26use metaverse_objects::avatar_asset::download_asset_objects;
27use std::collections::BTreeSet;
28use std::path::PathBuf;
29use std::time::Duration;
30use uuid::Uuid;
31
32/// Requests Agent data from the ViewerAsset capability endpoint
33///
34/// These are requested from inventory objects, and not ObjectUpdate packets.
35///
36/// # Cause
37/// - [`HandleNewAvatar`]
38///
39/// # Effects
40///  - Dispatches an [`AddObjectToAvatar`] message on successful download
41#[derive(Debug, Message)]
42#[rtype(result = "()")]
43pub struct DownloadAgentAsset {
44    /// the asset of the object to request
45    pub asset_id: Uuid,
46    /// the ID of the agent the asset belongs to
47    pub agent_id: Uuid,
48    /// the object type, used to retrieve from the capability endpoint
49    pub item_type: ObjectType,
50}
51impl Handler<DownloadAgentAsset> for Mailbox {
52    type Result = ();
53    fn handle(&mut self, msg: DownloadAgentAsset, ctx: &mut Self::Context) -> Self::Result {
54        let Some(session) = self.session.as_mut() else {
55            return;
56        };
57        let db_conn = session.inventory_db_connection.clone();
58        let server_endpoint = session
59            .capability_urls
60            .get(&Capability::ViewerAsset)
61            .unwrap()
62            .to_string();
63        let object_type = msg.item_type;
64        let asset_id = msg.asset_id;
65        let agent_id = msg.agent_id;
66
67        let addr = ctx.address();
68        ctx.spawn(
69            async move {
70                match download_asset_objects(
71                    &server_endpoint,
72                    db_conn,
73                    object_type,
74                    asset_id,
75                    agent_id,
76                )
77                .await
78                {
79                    Ok(json_path) => {
80                        addr.do_send(AddObjectToAvatar {
81                            object: OutfitObject::MeshObject(json_path),
82                            agent_id,
83                        });
84                    }
85                    Err(e) => {
86                        error!("{:?}", e)
87                    }
88                };
89            }
90            .into_actor(self),
91        );
92    }
93}
94
95/// Adds an object to the avatar's outfit list
96///
97/// This updates the in-memory struct containing each player's avatar data, and updates
98/// the global avatar skeleton if the outfit contains skeleton data, and triggers a render
99/// if the avatar's outfit is finished loading in.
100///
101/// # Cause
102/// - [`AddObjectToAvatar`]
103///
104/// # Effects
105/// - Dispatches a [`FinalizeAvatar`] message if the avatar's outfit has all items
106#[derive(Debug, Message)]
107#[rtype(result = "()")]
108pub struct AddObjectToAvatar {
109    /// ID of the agent to add the object to
110    pub agent_id: Uuid,
111    /// Object to add
112    pub object: OutfitObject,
113}
114impl Handler<AddObjectToAvatar> for Mailbox {
115    type Result = ();
116    fn handle(&mut self, msg: AddObjectToAvatar, ctx: &mut Self::Context) -> Self::Result {
117        let Some(session) = self.session.as_mut() else {
118            return;
119        };
120        match add_object_to_avatar(session, msg.agent_id, msg.object) {
121            Ok(FullyLoaded) => {
122                ctx.address().do_send(FinalizeAvatar {
123                    agent_id: msg.agent_id,
124                });
125            }
126            Ok(_) => {}
127            Err(e) => {
128                error!("{:?}", e);
129            }
130        };
131    }
132}
133
134/// Message to spawn a new avatar
135///
136/// Avatars are added to the scene via ObjectUpdate packet with minimal information aside from the
137/// ID. This adds those IDs to the session and begins downloading their appearances, or loads from
138/// the cache.
139///
140/// # Cause
141/// - [`HandleObjectUpdate`]
142///
143/// # Effect
144/// - Resends a [`HandleNewAvatar`] message if the inventory is not yet loaded
145/// - If the avatar is the current player's avatar
146///    - Dispatches a [`CameraPosition`] message
147///    - If the avatar is in the cache:
148///     - Dispatches a [`LoadFromCache`] message to skip asset downloading.
149///    - else:
150///     - Dispatches a [`DownloadAgentAsset`] message for each object in the outfit
151///     - Dispatches a [`AddObjectToAvatar`] message for each bodypart in the outfit
152#[derive(Debug, Message)]
153#[rtype(result = "()")]
154pub struct HandleNewAvatar(pub Avatar);
155impl Handler<HandleNewAvatar> for Mailbox {
156    type Result = ();
157    fn handle(&mut self, msg: HandleNewAvatar, ctx: &mut Self::Context) -> Self::Result {
158        let Some(session) = self.session.as_mut() else {
159            return;
160        };
161        let addr = ctx.address();
162        match init_avatar(session, &msg.0) {
163            Ok(user_type) => {
164                match user_type {
165                    AvatarType::User => {
166                        addr.do_send(SendUIMessage {
167                            // handle the z-y flip
168                            ui_message: UIMessage::new_camera_position(CameraPosition {
169                                position: Vec3::new(
170                                    msg.0.position.x,
171                                    msg.0.position.z,
172                                    msg.0.position.y,
173                                ),
174                            }),
175                        });
176                        let agent_id = session.agent_id;
177                        let db_conn = session.inventory_db_connection.clone();
178                        ctx.spawn(
179                            async move {
180                                match cache_current_outfit(db_conn, agent_id).await {
181                                    Ok((avatar, outfit_items)) => {
182                                        if addr
183                                            .send(SetOutfitSize {
184                                                agent_id,
185                                                outfit_size: outfit_items.len(),
186                                            })
187                                            .await
188                                            .is_err()
189                                        {
190                                            error!("Failed to set outfit size for {:?}", agent_id);
191                                            return;
192                                        }
193
194                                        if let Some(avatar) = avatar {
195                                            addr.do_send(LoadFromCache { avatar });
196                                            return;
197                                        }
198                                        for item in outfit_items {
199                                            match item.item_type {
200                                                ObjectType::Object => {
201                                                    addr.do_send(DownloadAgentAsset {
202                                                        asset_id: item.asset_id,
203                                                        item_type: item.item_type,
204                                                        agent_id,
205                                                    });
206                                                }
207                                                ObjectType::Bodypart => {
208                                                    addr.do_send(AddObjectToAvatar {
209                                                        object: OutfitObject::Bodypart,
210                                                        agent_id,
211                                                    });
212                                                }
213                                                ObjectType::Clothing => {
214                                                    addr.do_send(AddObjectToAvatar {
215                                                        object: OutfitObject::Clothing,
216                                                        agent_id,
217                                                    });
218                                                }
219                                                _ => {
220                                                    addr.do_send(AddObjectToAvatar {
221                                                        object: OutfitObject::Other,
222                                                        agent_id,
223                                                    });
224                                                }
225                                            }
226                                        }
227                                    }
228                                    Err(e) => {
229                                        error!("{:?}", e)
230                                    }
231                                }
232                            }
233                            .into_actor(self),
234                        );
235                    }
236                    AvatarType::NonUser => {
237                        //TODO: handle nonuser updates
238                    }
239                }
240            }
241            Err(e) => match e {
242                AvatarError::InventoryUninitialized { .. } => {
243                    warn!("{:?}, Requeueing avatar download...", e);
244                    ctx.notify_later(msg, Duration::from_secs(1));
245                }
246                e => {
247                    error!("{:?}", e)
248                }
249            },
250        };
251    }
252}
253
254/// Message to finalize the avatar
255///
256/// This is triggered when all of the avatar's objects have loaded in. This finalizes the global
257/// skeleton, writes the JSON for the full baked avatar, generates the mesh from metaverse-mesh,
258/// and triggers a UI update.
259///
260/// # Cause
261/// - [`AddObjectToAvatar`]
262///
263/// # Effects
264/// - Dispatches a [`RenderAvatar`] message to render the finalized avatar
265/// - Dispatches a [`SetAvatarGlbPath`] message to ensure the GLB path gets set
266#[derive(Debug, Message)]
267#[rtype(result = "()")]
268pub struct FinalizeAvatar {
269    /// the avatar object to generate
270    pub agent_id: Uuid,
271}
272impl Handler<FinalizeAvatar> for Mailbox {
273    type Result = ();
274    fn handle(&mut self, msg: FinalizeAvatar, ctx: &mut Self::Context) -> Self::Result {
275        let Some(session) = self.session.as_mut() else {
276            return;
277        };
278
279        let Some(avatar) = session.avatars.get_mut(&msg.agent_id) else {
280            error!("Avatar {:?} not found in avatar cache", msg.agent_id);
281            return;
282        };
283
284        let agent_id = msg.agent_id;
285        let skeleton = avatar.skeleton.clone();
286        let used_joints = avatar.used_joints.clone();
287        let items = avatar.items.clone();
288        let db_conn = session.inventory_db_connection.clone();
289        let position = avatar.position;
290        let mut avatar = avatar.clone();
291
292        let addr = ctx.address();
293        ctx.spawn(
294            async move {
295                match finalize_avatar(agent_id, skeleton, used_joints.clone(), items).await {
296                    Ok((glb_path, skeleton)) => {
297                        addr.do_send(SetAvatarGlbPath {
298                            path: glb_path.clone(),
299                            agent_id,
300                        });
301                        addr.do_send(SetAvatarFinalSkeleton {
302                            final_skeleton: skeleton,
303                            agent_id,
304                        });
305                        // edit the temp avatar for the sqlite update.
306                        avatar.path = Some(glb_path.clone());
307                        if let Err(e) = sqlite_update_avatar(&db_conn, avatar).await {
308                            error!("{:?}", e);
309                        }
310                        addr.do_send(RenderAvatar {
311                            message: MeshUpdate {
312                                position,
313                                scale: Vec3::ONE,
314                                rotation: Quat::IDENTITY,
315                                parent: None,
316                                scene_id: None,
317                                path: glb_path,
318                                mesh_type: MeshType::Avatar,
319                                id: Some(msg.agent_id),
320                            },
321                            agent_id,
322                            skeleton: used_joints,
323                        });
324                    }
325                    Err(e) => {
326                        error!("{:?}", e)
327                    }
328                };
329            }
330            .into_actor(self),
331        );
332    }
333}
334
335#[derive(Debug, Message)]
336#[rtype(result = "()")]
337struct LoadFromCache {
338    avatar: Avatar,
339}
340impl Handler<LoadFromCache> for Mailbox {
341    type Result = ();
342    fn handle(&mut self, msg: LoadFromCache, ctx: &mut Self::Context) -> Self::Result {
343        let Some(session) = self.session.as_mut() else {
344            return;
345        };
346        // insert the avatar to the session
347        session
348            .avatars
349            .insert(msg.avatar.agent_id, msg.avatar.clone());
350
351        // render the cached avatar
352        ctx.address().do_send(RenderAvatar {
353            message: MeshUpdate {
354                position: msg.avatar.position,
355                scale: Vec3::ONE,
356                rotation: Quat::IDENTITY,
357                parent: None,
358                scene_id: None,
359                path: msg.avatar.path.unwrap(),
360                mesh_type: MeshType::Avatar,
361                id: Some(msg.avatar.agent_id),
362            },
363            agent_id: msg.avatar.agent_id,
364            skeleton: msg.avatar.used_joints,
365        });
366    }
367}
368
369/// Message to handle an updated animation
370///
371/// # Cause
372/// - Avatar Appearance packet received from UDP socket
373///
374/// # Effect
375/// - Dispatches a [`PlayAnimation`] message to inform the UI of a new animation
376#[derive(Debug, Message)]
377#[rtype(result = "()")]
378pub struct HandleNewAvatarAnimation {
379    /// the avatar appearance data to handle
380    pub avatar_animation: AvatarAnimation,
381}
382impl Handler<HandleNewAvatarAnimation> for Mailbox {
383    type Result = ();
384    fn handle(&mut self, msg: HandleNewAvatarAnimation, ctx: &mut Self::Context) -> Self::Result {
385        let Some(session) = self.session.as_mut() else {
386            return;
387        };
388
389        let avatar = match session.avatars.get(&msg.avatar_animation.sender_id) {
390            Some(avatar) => {
391                if !avatar.fully_loaded {
392                    warn!(
393                        "Animation targeting player {:?} is not yet fully loaded. Queueing animation...",
394                        msg.avatar_animation.sender_id
395                    );
396                    ctx.notify_later(msg, Duration::from_secs(1));
397                    return;
398                }
399                avatar
400            }
401            None => {
402                warn!(
403                    "Animation targeting player {:?}, not yet in scene. Queueing animation...",
404                    msg.avatar_animation.sender_id
405                );
406                ctx.notify_later(msg, Duration::from_secs(1));
407                return;
408            }
409        };
410
411        let animations = msg.avatar_animation.animations;
412        let used_joints = avatar.used_joints.clone();
413        let target_skeleton = avatar.skeleton.clone();
414        let Ok(out_dir) = create_agent_animation_dir(&avatar.agent_id) else {
415            error!("Failed to create agent animation directory");
416            return;
417        };
418
419        let sender_id = msg.avatar_animation.sender_id;
420        let addr = ctx.address();
421        ctx.spawn(
422            async move {
423                match build_animation(animations, used_joints, target_skeleton, out_dir).await {
424                    Ok(mesh_paths) => {
425                        for path in mesh_paths {
426                            addr.do_send(SendUIMessage {
427                                ui_message: UIMessage::new_play_animation(PlayAnimation {
428                                    player_id: sender_id,
429                                    animation_path: path,
430                                }),
431                            });
432                        }
433                    }
434                    Err(e) => {
435                        error!("{:?}", e)
436                    }
437                };
438            }
439            .into_actor(self),
440        );
441    }
442}
443
444/// Message to handle an updated avatar appearance
445///
446/// TODO: currently a stub
447///
448/// # Cause
449/// - Avatar Appearance packet received from UDP socket
450#[derive(Debug, Message)]
451#[rtype(result = "()")]
452pub struct HandleNewAvatarAppearance {
453    /// the avatar appearance data to handle
454    pub avatar_appearance: AvatarAppearance,
455}
456impl Handler<HandleNewAvatarAppearance> for Mailbox {
457    type Result = ();
458    fn handle(
459        &mut self,
460        _msg: HandleNewAvatarAppearance,
461        _ctx: &mut Self::Context,
462    ) -> Self::Result {
463        // TODO: implement this. AvatarAppearance packets change skeleton joint positions to change
464        // the appearance of the avatar.
465        warn!("AvatarAppearance packet received. Currently unimplemented");
466    }
467}
468
469/// Helper message to ensure all avatars are marked as fully loaded once the mesh render is triggered.
470///
471/// # Cause
472/// - [`FinalizeAvatar`]
473/// - [`LoadFromCache`]
474///
475/// # Effects
476/// - Dispatches a [`MeshUpdate`] message to inform the UI of an update
477#[derive(Debug, Message)]
478#[rtype(result = "()")]
479pub struct RenderAvatar {
480    message: MeshUpdate,
481    agent_id: Uuid,
482    skeleton: BTreeSet<JointName>,
483}
484impl Handler<RenderAvatar> for Mailbox {
485    type Result = ();
486    fn handle(&mut self, msg: RenderAvatar, ctx: &mut Self::Context) -> Self::Result {
487        if let Some(session) = self.session.as_mut()
488            && let Some(avatar) = session.avatars.get_mut(&msg.agent_id)
489        {
490            avatar.used_joints = msg.skeleton;
491            let addr = ctx.address();
492
493            addr.do_send(SendUIMessage {
494                ui_message: UIMessage::new_mesh_update(msg.message),
495            });
496
497            avatar.fully_loaded = true;
498        }
499    }
500}
501
502/// Message to set the outfit size of the avatar
503///
504/// This must be set in order to trigger a render. Without this, the avatar doesn't know when the
505/// outfit is finished loading in.
506///
507/// # Cause
508/// - [`HandleNewAvatar`]
509#[derive(Debug, Message)]
510#[rtype(result = "()")]
511pub struct SetOutfitSize {
512    /// agent ID to set the outfit size of
513    pub agent_id: Uuid,
514    /// size of the outfit
515    pub outfit_size: usize,
516}
517impl Handler<SetOutfitSize> for Mailbox {
518    type Result = ();
519    fn handle(&mut self, msg: SetOutfitSize, _ctx: &mut Self::Context) -> Self::Result {
520        if let Some(session) = self.session.as_mut()
521            && let Some(avatar) = session.avatars.get_mut(&msg.agent_id)
522        {
523            avatar.outfit_size = msg.outfit_size;
524        }
525    }
526}
527
528/// Simple helper function to ensure that the avatar's GLB path is set correctly
529///
530/// #Cause
531/// - [`FinalizeAvatar`]
532#[derive(Debug, Message)]
533#[rtype(result = "()")]
534pub struct SetAvatarGlbPath {
535    /// GLB path
536    pub path: PathBuf,
537    /// Agent ID to set
538    pub agent_id: Uuid,
539}
540impl Handler<SetAvatarGlbPath> for Mailbox {
541    type Result = ();
542    fn handle(&mut self, msg: SetAvatarGlbPath, _ctx: &mut Self::Context) -> Self::Result {
543        if let Some(session) = self.session.as_mut()
544            && let Some(avatar) = session.avatars.get_mut(&msg.agent_id)
545        {
546            avatar.path = Some(msg.path)
547        }
548    }
549}
550/// Simple helper function to ensure that the avatar's skeleton is updated after finalizing
551///
552/// #Cause
553/// - [`FinalizeAvatar`]
554#[derive(Debug, Message)]
555#[rtype(result = "()")]
556pub struct SetAvatarFinalSkeleton {
557    /// Skeleton
558    pub final_skeleton: Skeleton,
559    /// Agent ID to set
560    pub agent_id: Uuid,
561}
562impl Handler<SetAvatarFinalSkeleton> for Mailbox {
563    type Result = ();
564    fn handle(&mut self, msg: SetAvatarFinalSkeleton, _ctx: &mut Self::Context) -> Self::Result {
565        if let Some(session) = self.session.as_mut()
566            && let Some(avatar) = session.avatars.get_mut(&msg.agent_id)
567        {
568            avatar.skeleton = msg.final_skeleton
569        }
570    }
571}