use super::remote_server_top_level_impl_framebuffer::{FrameStatus, Framebuffer};
use crate::embedding::offscreen::{OffscreenTopLevelImplBase, OffscreenTopLevelImplOverrides};
use crate::platform::ITopLevelImpl;
use crate::remote::UiThreadHandle;
use ferroui_base::input::raw::{
IRawInputEventArgs, RawKeyEventArgs, RawKeyEventType, RawMouseWheelEventArgs, RawPointerEventArgs,
RawPointerEventType, RawTextInputEventArgs,
};
use ferroui_base::input::{
IInputDevice, IInputRoot, IKeyboardDevice, IMouseDevice, Key, KeyDeviceType, MouseDevice, PhysicalKey,
RawInputModifiers,
};
use ferroui_base::platform::surfaces::{
FramebufferLockProperties, IFramebufferPlatformSurface, IFramebufferRenderTarget, IPlatformRenderSurface,
IPlatformRenderSurfaceRenderTarget,
};
use ferroui_base::platform::{ILockedFramebuffer, PlatformRenderTargetState, RenderTargetSceneInfo};
use ferroui_base::threading::DispatcherPriority;
use ferroui_base::{FerroLocator, LocatorExtensions, Point, Rect, Size, Vector};
use ferroui_remote_protocol::input::{
InputModifiers, KeyEventMessage, MouseButton as ProtocolMouseButton, PointerMovedEventMessage,
PointerPressedEventMessage, PointerReleasedEventMessage, ScrollEventMessage, TextInputEventMessage,
};
use ferroui_remote_protocol::viewport::{
ClientRenderInfoMessage, ClientSupportedPixelFormatsMessage, ClientViewportAllocatedMessage, FrameReceivedMessage,
MeasureViewportMessage, PixelFormat as ProtocolPixelFormat,
};
use ferroui_remote_protocol::{message_handler, IFerroRemoteTransportConnection, Message};
use std::any::{Any, TypeId};
use std::rc::{Rc, Weak};
use std::sync::{Arc, Mutex, MutexGuard, PoisonError, Weak as ArcWeak};
pub type RemoteServerUiThreadHandle = UiThreadHandle<Weak<RemoteServerTopLevelImpl>>;
pub type OnMessageOverride =
Arc<dyn Fn(&RemoteServerUiThreadHandle, &dyn IFerroRemoteTransportConnection, &Message) -> bool + Send + Sync>;
pub trait RemoteServerTopLevelImplOverrides {
fn measure(&self, base: &RemoteServerTopLevelImpl, constraint: Size) -> Size {
base.base_measure(constraint)
}
fn desktop_scaling(&self, base: &RemoteServerTopLevelImpl) -> f64 {
base.render_scaling()
}
fn try_get_feature(&self, _base: &RemoteServerTopLevelImpl, _feature_type: TypeId) -> Option<Rc<dyn Any>> {
None
}
}
struct State {
framebuffer: Arc<Framebuffer>,
last_sent_frame: i64,
last_received_frame: i64,
next_frame_number: i64,
render_requested: bool,
pending_allocation: Option<ClientViewportAllocatedMessage>,
format: Option<ProtocolPixelFormat>,
client_size: Size,
render_scaling: f64,
is_disposed: bool,
}
struct RemoteServerSurface {
this: ArcWeak<RemoteServerSurface>,
transport: Arc<dyn IFerroRemoteTransportConnection>,
lock: Mutex<State>,
handle: RemoteServerUiThreadHandle,
on_message_override: Option<OnMessageOverride>,
}
pub struct RemoteServerTopLevelImpl {
base: Rc<OffscreenTopLevelImplBase>,
overrides: Option<Rc<dyn RemoteServerTopLevelImplOverrides>>,
surface: Arc<RemoteServerSurface>,
mouse_device: Rc<MouseDevice>,
keyboard_device: Rc<dyn IKeyboardDevice>,
}
struct OffscreenOverrides {
owner: Weak<RemoteServerTopLevelImpl>,
surface: Arc<RemoteServerSurface>,
mouse_device: Rc<MouseDevice>,
}
impl OffscreenTopLevelImplOverrides for OffscreenOverrides {
fn surfaces(&self) -> Vec<Arc<dyn IPlatformRenderSurface>> {
let surface: Arc<dyn IPlatformRenderSurface> = self.surface.clone();
vec![surface]
}
fn mouse_device(&self) -> Rc<dyn IMouseDevice> {
self.mouse_device.clone()
}
fn dispose(&self, base: &OffscreenTopLevelImplBase) {
self.surface.state().is_disposed = true;
self.surface.handle.unregister();
base.base_dispose();
}
fn desktop_scaling(&self, base: &OffscreenTopLevelImplBase) -> f64 {
match self.owner.upgrade() {
Some(owner) => owner.desktop_scaling(),
None => base.render_scaling(),
}
}
fn try_get_feature(&self, feature_type: TypeId) -> Option<Rc<dyn Any>> {
let owner = self.owner.upgrade()?;
owner.overrides.as_ref()?.try_get_feature(&owner, feature_type)
}
}
fn get_ferro_event_type(button: ProtocolMouseButton, pressed: bool) -> RawPointerEventType {
match button {
ProtocolMouseButton::Left => {
if pressed {
RawPointerEventType::LeftButtonDown
} else {
RawPointerEventType::LeftButtonUp
}
}
ProtocolMouseButton::Middle => {
if pressed {
RawPointerEventType::MiddleButtonDown
} else {
RawPointerEventType::MiddleButtonUp
}
}
ProtocolMouseButton::Right => {
if pressed {
RawPointerEventType::RightButtonDown
} else {
RawPointerEventType::RightButtonUp
}
}
_ => RawPointerEventType::Move,
}
}
fn get_ferro_raw_input_modifiers(modifiers: Option<&[InputModifiers]>) -> RawInputModifiers {
let mut result = RawInputModifiers::NONE;
let Some(modifiers) = modifiers else {
return result;
};
for modifier in modifiers {
match modifier {
InputModifiers::Control => result |= RawInputModifiers::CONTROL,
InputModifiers::Alt => result |= RawInputModifiers::ALT,
InputModifiers::Shift => result |= RawInputModifiers::SHIFT,
InputModifiers::Windows => result |= RawInputModifiers::META,
InputModifiers::LeftMouseButton => result |= RawInputModifiers::LEFT_MOUSE_BUTTON,
InputModifiers::MiddleMouseButton => result |= RawInputModifiers::MIDDLE_MOUSE_BUTTON,
InputModifiers::RightMouseButton => result |= RawInputModifiers::RIGHT_MOUSE_BUTTON,
}
}
result
}
fn try_get_valid_pixel_format(formats: Option<&[ProtocolPixelFormat]>) -> Option<ProtocolPixelFormat> {
if let Some(formats) = formats {
for format in formats {
let value = *format as i32;
if value >= 0 && value <= ProtocolPixelFormat::MaxValue as i32 {
return Some(*format);
}
}
}
None
}
impl RemoteServerSurface {
fn state(&self) -> MutexGuard<'_, State> {
self.lock.lock().unwrap_or_else(PoisonError::into_inner)
}
fn on_message(&self, transport: &dyn IFerroRemoteTransportConnection, obj: &Message) {
if let Some(on_message_override) = &self.on_message_override {
if on_message_override(&self.handle, transport, obj) {
return;
}
}
let mut state = self.state();
if let Some(last_frame) = obj.downcast_ref::<FrameReceivedMessage>() {
state.last_received_frame = last_frame.sequence_id.max(state.last_received_frame);
self.handle.post(
|this| {
this.with(|this| {
if this.surface.state().render_requested {
this.render_and_send_frame_if_needed();
} else {
this.surface.send_last_frame_if_needed();
}
})
},
DispatcherPriority::DEFAULT,
);
} else if let Some(render_info) = obj.downcast_ref::<ClientRenderInfoMessage>() {
let dpi_x = render_info.dpi_x;
self.handle.post(
move |this| {
this.with(|this| {
this.set_render_scaling(dpi_x / 96.0);
this.render_and_send_frame_if_needed();
})
},
DispatcherPriority::DEFAULT,
);
} else if let Some(supported_formats) = obj.downcast_ref::<ClientSupportedPixelFormatsMessage>() {
state.format = try_get_valid_pixel_format(supported_formats.formats.as_deref());
self.handle.post(|this| this.with(|this| this.render_and_send_frame_if_needed()), DispatcherPriority::DEFAULT);
} else if let Some(measure) = obj.downcast_ref::<MeasureViewportMessage>() {
let constraint = Size::new(measure.width, measure.height);
self.handle.post(
move |this| {
this.with(|this| {
let m = this.measure(constraint);
this.surface.transport.send(Arc::new(MeasureViewportMessage { width: m.width, height: m.height }));
})
},
DispatcherPriority::DEFAULT,
);
} else if let Some(allocated) = obj.downcast_ref::<ClientViewportAllocatedMessage>() {
if state.pending_allocation.is_none() {
self.handle.post(
|this| {
this.with(|this| {
let allocation = this.surface.state().pending_allocation.take();
let Some(allocation) = allocation else {
return;
};
this.set_render_scaling(allocation.dpi_x / 96.0);
this.set_client_size(Size::new(allocation.width, allocation.height));
this.render_and_send_frame_if_needed();
})
},
DispatcherPriority::DEFAULT,
);
}
state.pending_allocation = Some(allocated.clone());
} else if let Some(pointer) = obj.downcast_ref::<PointerMovedEventMessage>() {
let position = Point::new(pointer.x, pointer.y);
let modifiers = get_ferro_raw_input_modifiers(pointer.modifiers.as_deref());
self.handle.post(
move |this| this.with(|this| this.raise_pointer(RawPointerEventType::Move, position, modifiers)),
DispatcherPriority::INPUT,
);
} else if let Some(pressed) = obj.downcast_ref::<PointerPressedEventMessage>() {
let type_ = get_ferro_event_type(pressed.button, true);
let position = Point::new(pressed.x, pressed.y);
let modifiers = get_ferro_raw_input_modifiers(pressed.modifiers.as_deref());
self.handle.post(
move |this| this.with(|this| this.raise_pointer(type_, position, modifiers)),
DispatcherPriority::INPUT,
);
} else if let Some(released) = obj.downcast_ref::<PointerReleasedEventMessage>() {
let type_ = get_ferro_event_type(released.button, false);
let position = Point::new(released.x, released.y);
let modifiers = get_ferro_raw_input_modifiers(released.modifiers.as_deref());
self.handle.post(
move |this| this.with(|this| this.raise_pointer(type_, position, modifiers)),
DispatcherPriority::INPUT,
);
} else if let Some(scroll) = obj.downcast_ref::<ScrollEventMessage>() {
let position = Point::new(scroll.x, scroll.y);
let delta = Vector::new(scroll.delta_x, scroll.delta_y);
let modifiers = get_ferro_raw_input_modifiers(scroll.modifiers.as_deref());
self.handle.post(
move |this| {
this.with(|this| {
let Some(input) = this.base.input() else {
return;
};
input(Rc::new(RawMouseWheelEventArgs::new(
this.mouse_input_device(),
0,
this.required_input_root(),
position,
delta,
modifiers,
)));
})
},
DispatcherPriority::INPUT,
);
} else if let Some(key) = obj.downcast_ref::<KeyEventMessage>() {
let key = key.clone();
self.handle.post(
move |this| {
this.with(|this| {
this.run_jobs_above_input();
let Some(input) = this.base.input() else {
return;
};
input(Rc::new(RawKeyEventArgs::new(
this.keyboard_input_device(),
0,
this.required_input_root(),
if key.is_down { RawKeyEventType::KeyDown } else { RawKeyEventType::KeyUp },
Key::from_value(key.key as i32).unwrap_or(Key::None),
get_ferro_raw_input_modifiers(key.modifiers.as_deref()),
PhysicalKey::from_value(key.physical_key as i32).unwrap_or(PhysicalKey::None),
key.key_symbol.clone(),
KeyDeviceType::Keyboard,
)));
})
},
DispatcherPriority::INPUT,
);
} else if let Some(text) = obj.downcast_ref::<TextInputEventMessage>() {
let text = text.text.clone();
self.handle.post(
move |this| {
this.with(|this| {
this.run_jobs_above_input();
let Some(input) = this.base.input() else {
return;
};
input(Rc::new(RawTextInputEventArgs::new(
this.keyboard_input_device(),
0,
this.required_input_root(),
text,
)));
})
},
DispatcherPriority::INPUT,
);
}
}
fn get_or_create_framebuffer(&self) -> Arc<Framebuffer> {
let mut state = self.state();
match state.format {
None => {
if state.framebuffer.stride() > 0 {
state.framebuffer = Framebuffer::empty();
}
}
Some(format) => {
if state.framebuffer.format() != format
|| state.framebuffer.client_size() != state.client_size
|| state.framebuffer.render_scaling() != state.render_scaling
{
state.framebuffer = Arc::new(Framebuffer::new(format, state.client_size, state.render_scaling));
}
}
}
state.framebuffer.clone()
}
fn send_last_frame_if_needed(&self) {
let framebuffer;
let sequence_id;
{
let mut state = self.state();
if state.is_disposed {
return;
}
if state.last_received_frame != state.last_sent_frame || state.framebuffer.get_status() != FrameStatus::Rendered
{
return;
}
framebuffer = state.framebuffer.clone();
state.last_sent_frame = state.next_frame_number;
state.next_frame_number += 1;
sequence_id = state.last_sent_frame;
}
self.transport.send(Arc::new(framebuffer.to_message(sequence_id)));
}
}
impl IPlatformRenderSurface for RemoteServerSurface {
fn as_framebuffer_surface(&self) -> Option<&dyn IFramebufferPlatformSurface> {
Some(self)
}
fn as_any(&self) -> &dyn Any {
self
}
}
impl IFramebufferPlatformSurface for RemoteServerSurface {
fn create_framebuffer_render_target(&self) -> Rc<dyn IFramebufferRenderTarget> {
let surface = self.this.upgrade().expect("the surface is alive while it is used");
Rc::new(RemoteServerFramebufferRenderTarget { surface })
}
}
struct RemoteServerFramebufferRenderTarget {
surface: Arc<RemoteServerSurface>,
}
impl IPlatformRenderSurfaceRenderTarget for RemoteServerFramebufferRenderTarget {
fn state(&self) -> PlatformRenderTargetState {
if self.surface.get_or_create_framebuffer().stride() > 0 {
PlatformRenderTargetState::READY
} else {
PlatformRenderTargetState::NOT_READY_TRY_LATER
}
}
}
impl IFramebufferRenderTarget for RemoteServerFramebufferRenderTarget {
fn lock(&self, _scene_info: &RenderTargetSceneInfo) -> (Rc<dyn ILockedFramebuffer>, FramebufferLockProperties) {
let unlocked = self.surface.clone();
let framebuffer =
self.surface.get_or_create_framebuffer().lock(Box::new(move || unlocked.send_last_frame_if_needed()));
(framebuffer, FramebufferLockProperties::default())
}
fn dispose(&self) {}
}
trait PostedTarget {
fn with(&self, action: impl FnOnce(&Rc<RemoteServerTopLevelImpl>));
}
impl PostedTarget for Weak<RemoteServerTopLevelImpl> {
fn with(&self, action: impl FnOnce(&Rc<RemoteServerTopLevelImpl>)) {
if let Some(this) = self.upgrade() {
action(&this);
}
}
}
impl RemoteServerTopLevelImpl {
pub fn new(transport: Arc<dyn IFerroRemoteTransportConnection>) -> Rc<RemoteServerTopLevelImpl> {
Self::with_overrides(transport, None, None)
}
pub fn with_overrides(
transport: Arc<dyn IFerroRemoteTransportConnection>,
overrides: Option<Rc<dyn RemoteServerTopLevelImplOverrides>>,
on_message_override: Option<OnMessageOverride>,
) -> Rc<RemoteServerTopLevelImpl> {
let keyboard_device = FerroLocator::current().get_required_service::<dyn IKeyboardDevice>();
let mouse_device = MouseDevice::new();
let this = Rc::new_cyclic(|this: &Weak<RemoteServerTopLevelImpl>| {
let handle = UiThreadHandle::register(this.clone(), |this| this.strong_count() > 0);
let surface = Arc::new_cyclic(|surface: &ArcWeak<RemoteServerSurface>| RemoteServerSurface {
this: surface.clone(),
transport: transport.clone(),
lock: Mutex::new(State {
framebuffer: Framebuffer::empty(),
last_sent_frame: -1,
last_received_frame: -1,
next_frame_number: 1,
render_requested: false,
pending_allocation: None,
format: None,
client_size: Size::default(),
render_scaling: 1.0,
is_disposed: false,
}),
handle,
on_message_override,
});
let base = OffscreenTopLevelImplBase::new(Rc::new(OffscreenOverrides {
owner: this.clone(),
surface: surface.clone(),
mouse_device: mouse_device.clone(),
}));
RemoteServerTopLevelImpl { base, overrides, surface, mouse_device, keyboard_device }
});
let surface = Arc::downgrade(&this.surface);
transport.on_message(message_handler(move |transport, obj| {
if let Some(surface) = surface.upgrade() {
surface.on_message(transport, obj);
}
}));
this
}
pub fn base(&self) -> &Rc<OffscreenTopLevelImplBase> {
&self.base
}
pub fn as_top_level_impl(&self) -> Rc<dyn ITopLevelImpl> {
self.base.clone()
}
pub fn is_disposed(&self) -> bool {
self.base.is_disposed()
}
pub fn input_root(&self) -> Option<Rc<dyn IInputRoot>> {
self.base.input_root()
}
pub fn client_size(&self) -> Size {
self.base.client_size()
}
pub fn set_client_size(&self, value: Size) {
self.surface.state().client_size = value;
self.base.set_client_size(value);
}
pub fn render_scaling(&self) -> f64 {
self.base.render_scaling()
}
pub fn set_render_scaling(&self, value: f64) {
self.surface.state().render_scaling = value;
self.base.set_render_scaling(value);
}
pub fn desktop_scaling(&self) -> f64 {
match &self.overrides {
Some(overrides) => overrides.desktop_scaling(self),
None => self.render_scaling(),
}
}
pub fn measure(&self, constraint: Size) -> Size {
match &self.overrides {
Some(overrides) => overrides.measure(self, constraint),
None => self.base_measure(constraint),
}
}
pub fn base_measure(&self, constraint: Size) -> Size {
let l = self.required_input_root().root_element();
l.measure(constraint);
l.desired_size()
}
pub fn mouse_device(&self) -> Rc<dyn IMouseDevice> {
self.mouse_device.clone()
}
pub fn keyboard_device(&self) -> Rc<dyn IKeyboardDevice> {
self.keyboard_device.clone()
}
pub fn surfaces(&self) -> Vec<Arc<dyn IPlatformRenderSurface>> {
self.base.surfaces()
}
pub fn render_and_send_frame_if_needed(&self) {
if self.is_disposed() {
return;
}
{
let mut state = self.surface.state();
if state.format.is_none() {
return;
}
if state.last_received_frame != state.last_sent_frame {
state.render_requested = true;
return;
}
state.render_requested = false;
state.client_size = self.base.client_size();
state.render_scaling = self.base.render_scaling();
}
let framebuffer = self.surface.get_or_create_framebuffer();
if framebuffer.stride() > 0 {
if let Some(paint) = self.base.paint() {
paint(Rect::from_size(framebuffer.client_size()));
}
}
self.surface.send_last_frame_if_needed();
}
fn required_input_root(&self) -> Rc<dyn IInputRoot> {
self.base.input_root().expect("Value cannot be null. (Parameter 'root')")
}
fn mouse_input_device(&self) -> Rc<dyn IInputDevice> {
let device: Rc<dyn IInputDevice> = self.mouse_device.clone();
device
}
fn keyboard_input_device(&self) -> Rc<dyn IInputDevice> {
let device: Rc<dyn IInputDevice> = self.keyboard_device.clone();
device
}
fn run_jobs_above_input(&self) {
let priority = DispatcherPriority::from_value(DispatcherPriority::INPUT.value() + 1);
self.surface.handle.dispatcher().run_jobs(Some(priority));
}
fn raise_pointer(&self, type_: RawPointerEventType, position: Point, modifiers: RawInputModifiers) {
let Some(input) = self.base.input() else {
return;
};
let args: Rc<dyn IRawInputEventArgs> = Rc::new(RawPointerEventArgs::new(
self.mouse_input_device(),
0,
self.required_input_root(),
type_,
position,
modifiers,
));
input(args);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_mouse_button_maps_to_the_raw_event_of_the_press_or_the_release() {
assert_eq!(RawPointerEventType::LeftButtonDown, get_ferro_event_type(ProtocolMouseButton::Left, true));
assert_eq!(RawPointerEventType::LeftButtonUp, get_ferro_event_type(ProtocolMouseButton::Left, false));
assert_eq!(RawPointerEventType::MiddleButtonDown, get_ferro_event_type(ProtocolMouseButton::Middle, true));
assert_eq!(RawPointerEventType::MiddleButtonUp, get_ferro_event_type(ProtocolMouseButton::Middle, false));
assert_eq!(RawPointerEventType::RightButtonDown, get_ferro_event_type(ProtocolMouseButton::Right, true));
assert_eq!(RawPointerEventType::RightButtonUp, get_ferro_event_type(ProtocolMouseButton::Right, false));
assert_eq!(RawPointerEventType::Move, get_ferro_event_type(ProtocolMouseButton::None, true));
}
#[test]
fn the_modifiers_of_a_message_become_raw_modifiers() {
assert_eq!(RawInputModifiers::NONE, get_ferro_raw_input_modifiers(None));
assert_eq!(RawInputModifiers::NONE, get_ferro_raw_input_modifiers(Some(&[])));
assert_eq!(
RawInputModifiers::CONTROL
| RawInputModifiers::ALT
| RawInputModifiers::SHIFT
| RawInputModifiers::META
| RawInputModifiers::LEFT_MOUSE_BUTTON
| RawInputModifiers::MIDDLE_MOUSE_BUTTON
| RawInputModifiers::RIGHT_MOUSE_BUTTON,
get_ferro_raw_input_modifiers(Some(&[
InputModifiers::Control,
InputModifiers::Alt,
InputModifiers::Shift,
InputModifiers::Windows,
InputModifiers::LeftMouseButton,
InputModifiers::MiddleMouseButton,
InputModifiers::RightMouseButton,
]))
);
}
#[test]
fn the_first_pixel_format_of_the_client_is_taken() {
assert_eq!(None, try_get_valid_pixel_format(None));
assert_eq!(None, try_get_valid_pixel_format(Some(&[])));
assert_eq!(
Some(ProtocolPixelFormat::Bgra8888),
try_get_valid_pixel_format(Some(&[ProtocolPixelFormat::Bgra8888, ProtocolPixelFormat::Rgba8888]))
);
}
#[test]
fn the_shared_part_is_shared_between_the_threads() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<RemoteServerSurface>();
}
}