netget/client/vnc/

actions.rs

//! VNC client protocol actions implementation

use crate::llm::actions::{
    client_trait::{Client, ClientActionResult},
    protocol_trait::Protocol,
    ActionDefinition, Parameter, ParameterDefinition,
};
use crate::protocol::EventType;
use crate::state::app_state::AppState;
use anyhow::{Context, Result};
use serde_json::json;
use std::sync::LazyLock;

/// VNC client connected event
pub static VNC_CLIENT_CONNECTED_EVENT: LazyLock<EventType> = LazyLock::new(|| {
    EventType::new(
        "vnc_connected",
        "VNC client successfully connected and authenticated",
        json!({
            "type": "request_framebuffer_update",
            "incremental": true
        }),
    )
    .with_parameters(vec![
        Parameter {
            name: "remote_addr".to_string(),
            type_hint: "string".to_string(),
            description: "Remote VNC server address".to_string(),
            required: true,
        },
        Parameter {
            name: "width".to_string(),
            type_hint: "number".to_string(),
            description: "Framebuffer width in pixels".to_string(),
            required: true,
        },
        Parameter {
            name: "height".to_string(),
            type_hint: "number".to_string(),
            description: "Framebuffer height in pixels".to_string(),
            required: true,
        },
        Parameter {
            name: "server_name".to_string(),
            type_hint: "string".to_string(),
            description: "VNC server name/description".to_string(),
            required: true,
        },
    ])
});

/// VNC framebuffer update received event
pub static VNC_CLIENT_FRAMEBUFFER_UPDATE_EVENT: LazyLock<EventType> = LazyLock::new(|| {
    EventType::new(
        "vnc_framebuffer_update",
        "Framebuffer update received from VNC server",
        json!({
            "type": "send_pointer_event",
            "x": 100,
            "y": 200,
            "button_mask": 1
        }),
    )
    .with_parameters(vec![
        Parameter {
            name: "rectangles".to_string(),
            type_hint: "number".to_string(),
            description: "Number of rectangles in this update".to_string(),
            required: true,
        },
        Parameter {
            name: "update_summary".to_string(),
            type_hint: "string".to_string(),
            description: "Human-readable summary of the update".to_string(),
            required: true,
        },
    ])
});

/// VNC server clipboard text event
pub static VNC_CLIENT_SERVER_CUT_TEXT_EVENT: LazyLock<EventType> = LazyLock::new(|| {
    EventType::new(
        "vnc_server_cut_text",
        "Server sent clipboard text",
        json!({"type": "placeholder", "event_id": "vnc_server_cut_text"}),
    )
    .with_parameters(vec![Parameter {
        name: "text".to_string(),
        type_hint: "string".to_string(),
        description: "Clipboard text from server".to_string(),
        required: true,
    }])
});

/// VNC client protocol action handler
pub struct VncClientProtocol;

impl VncClientProtocol {
    pub fn new() -> Self {
        Self
    }
}

// Implement Protocol trait (common functionality)
impl Protocol for VncClientProtocol {
    fn get_async_actions(&self, _state: &AppState) -> Vec<ActionDefinition> {
        vec![
            ActionDefinition {
                name: "request_framebuffer_update".to_string(),
                description: "Request a framebuffer update from the VNC server".to_string(),
                parameters: vec![
                    Parameter {
                        name: "incremental".to_string(),
                        type_hint: "boolean".to_string(),
                        description: "If true, only send changes since last update".to_string(),
                        required: false,
                    },
                    Parameter {
                        name: "x".to_string(),
                        type_hint: "number".to_string(),
                        description: "X coordinate of update region (default: 0)".to_string(),
                        required: false,
                    },
                    Parameter {
                        name: "y".to_string(),
                        type_hint: "number".to_string(),
                        description: "Y coordinate of update region (default: 0)".to_string(),
                        required: false,
                    },
                    Parameter {
                        name: "width".to_string(),
                        type_hint: "number".to_string(),
                        description: "Width of update region (default: full width)".to_string(),
                        required: false,
                    },
                    Parameter {
                        name: "height".to_string(),
                        type_hint: "number".to_string(),
                        description: "Height of update region (default: full height)".to_string(),
                        required: false,
                    },
                ],
                example: json!({
                    "type": "request_framebuffer_update",
                    "incremental": true
                }),
                log_template: None,
            },
            ActionDefinition {
                name: "send_pointer_event".to_string(),
                description: "Send a mouse pointer event (move, click, release)".to_string(),
                parameters: vec![
                    Parameter {
                        name: "x".to_string(),
                        type_hint: "number".to_string(),
                        description: "X coordinate of pointer".to_string(),
                        required: true,
                    },
                    Parameter {
                        name: "y".to_string(),
                        type_hint: "number".to_string(),
                        description: "Y coordinate of pointer".to_string(),
                        required: true,
                    },
                    Parameter {
                        name: "button_mask".to_string(),
                        type_hint: "number".to_string(),
                        description: "Button mask (0=no buttons, 1=left, 2=middle, 4=right)"
                            .to_string(),
                        required: true,
                    },
                ],
                example: json!({
                    "type": "send_pointer_event",
                    "x": 100,
                    "y": 200,
                    "button_mask": 1
                }),
                log_template: None,
            },
            ActionDefinition {
                name: "send_key_event".to_string(),
                description: "Send a keyboard event (key press or release)".to_string(),
                parameters: vec![
                    Parameter {
                        name: "key".to_string(),
                        type_hint: "string".to_string(),
                        description: "The key to send: a single character such as \"a\" or \"A\",                             a named key such as \"Enter\", \"Escape\", \"Tab\", \"Backspace\", an                             arrow (\"Left\"/\"Up\"/\"Right\"/\"Down\"), a modifier (\"Shift\",                             \"Control\", \"Alt\", \"Meta\") or a function key (\"F1\"..\"F35\"). An                             X11 keysym number is also accepted. An unrecognised value is an                             error rather than a silently-sent keysym 0"
                            .to_string(),
                        required: true,
                    },
                    Parameter {
                        name: "down".to_string(),
                        type_hint: "boolean".to_string(),
                        description: "True for key press, false for key release".to_string(),
                        required: true,
                    },
                ],
                example: json!({
                    "type": "send_key_event",
                    "key": "a",
                    "down": true
                }),
                log_template: None,
            },
            ActionDefinition {
                name: "send_client_cut_text".to_string(),
                description: "Send clipboard text to the server".to_string(),
                parameters: vec![Parameter {
                    name: "text".to_string(),
                    type_hint: "string".to_string(),
                    description: "Text to send to clipboard".to_string(),
                    required: true,
                }],
                example: json!({
                    "type": "send_client_cut_text",
                    "text": "Hello, VNC!"
                }),
                log_template: None,
            },
            ActionDefinition {
                name: "disconnect".to_string(),
                description: "Disconnect from the VNC server".to_string(),
                parameters: vec![],
                example: json!({
                    "type": "disconnect"
                }),
                log_template: None,
            },
        ]
    }
    fn get_sync_actions(&self) -> Vec<ActionDefinition> {
        vec![
            ActionDefinition {
                name: "request_framebuffer_update".to_string(),
                description: "Request a framebuffer update in response to server event".to_string(),
                parameters: vec![Parameter {
                    name: "incremental".to_string(),
                    type_hint: "boolean".to_string(),
                    description: "If true, only send changes since last update".to_string(),
                    required: false,
                }],
                example: json!({
                    "type": "request_framebuffer_update",
                    "incremental": true
                }),
                log_template: None,
            },
            ActionDefinition {
                name: "send_pointer_event".to_string(),
                description: "Send pointer event in response to framebuffer update".to_string(),
                parameters: vec![
                    Parameter {
                        name: "x".to_string(),
                        type_hint: "number".to_string(),
                        description: "X coordinate".to_string(),
                        required: true,
                    },
                    Parameter {
                        name: "y".to_string(),
                        type_hint: "number".to_string(),
                        description: "Y coordinate".to_string(),
                        required: true,
                    },
                    Parameter {
                        name: "button_mask".to_string(),
                        type_hint: "number".to_string(),
                        description: "Button mask".to_string(),
                        required: true,
                    },
                ],
                example: json!({
                    "type": "send_pointer_event",
                    "x": 100,
                    "y": 200,
                    "button_mask": 1
                }),
                log_template: None,
            },
            // send_key_event and send_client_cut_text were declared ONLY as async actions, so
            // answering a network event the model could move the pointer but could not press a
            // key or set the clipboard — the executor rejects an action the event never
            // advertised. Both have working sync executors, so the gap was purely in what was
            // offered. Found by a test whose key press never reached the server.
            ActionDefinition {
                name: "send_key_event".to_string(),
                description: "Send a key press or release to the VNC server".to_string(),
                parameters: vec![
                    Parameter {
                        name: "key".to_string(),
                        type_hint: "string".to_string(),
                        description: "The key: a single character such as \"a\", a named key                             such as \"Enter\"/\"Escape\"/\"Tab\", an arrow, a modifier, or                             \"F1\"..\"F35\". An X11 keysym number is also accepted"
                            .to_string(),
                        required: true,
                    },
                    Parameter {
                        name: "down".to_string(),
                        type_hint: "boolean".to_string(),
                        description: "True for key press, false for key release".to_string(),
                        required: true,
                    },
                ],
                example: json!({"type": "send_key_event", "key": "a", "down": true}),
                log_template: None,
            },
            ActionDefinition {
                name: "send_client_cut_text".to_string(),
                description: "Set the VNC server's clipboard to this text".to_string(),
                parameters: vec![Parameter {
                    name: "text".to_string(),
                    type_hint: "string".to_string(),
                    description: "Clipboard contents to send".to_string(),
                    required: true,
                }],
                example: json!({"type": "send_client_cut_text", "text": "hello"}),
                log_template: None,
            },
            ActionDefinition {
                name: "wait_for_more".to_string(),
                description: "Wait for more updates before taking action".to_string(),
                parameters: vec![],
                example: json!({
                    "type": "wait_for_more"
                }),
                log_template: None,
            },
        ]
    }
    fn protocol_name(&self) -> &'static str {
        "VNC"
    }
    fn get_event_types(&self) -> Vec<EventType> {
        vec![
            EventType::new(
                "vnc_connected",
                "Triggered when VNC client connects and authenticates",
                json!({"type": "placeholder", "event_id": "vnc_connected"}),
            ),
            EventType::new(
                "vnc_framebuffer_update",
                "Triggered when framebuffer update is received",
                json!({"type": "placeholder", "event_id": "vnc_framebuffer_update"}),
            ),
            EventType::new(
                "vnc_server_cut_text",
                "Triggered when server sends clipboard text",
                json!({"type": "placeholder", "event_id": "vnc_server_cut_text"}),
            ),
        ]
    }
    fn stack_name(&self) -> &'static str {
        "ETH>IP>TCP>RFB"
    }
    fn keywords(&self) -> Vec<&'static str> {
        vec!["vnc", "vnc client", "remote desktop", "rfb"]
    }
    fn metadata(&self) -> crate::protocol::metadata::ProtocolMetadataV2 {
        use crate::protocol::metadata::{DevelopmentState, ProtocolMetadataV2};

        ProtocolMetadataV2::builder()
            .state(DevelopmentState::Experimental)
            .implementation("Custom RFB protocol implementation")
            .llm_control("Control mouse, keyboard, and screen updates")
            .e2e_testing("x11vnc or TigerVNC server")
            .build()
    }
    fn description(&self) -> &'static str {
        "VNC (Remote Framebuffer) client for remote desktop control"
    }
    fn example_prompt(&self) -> &'static str {
        "Connect to VNC at localhost:5900 and click at position (100, 200)"
    }
    fn group_name(&self) -> &'static str {
        "Specialized"
    }
    fn get_startup_parameters(&self) -> Vec<ParameterDefinition> {
        vec![ParameterDefinition {
            name: "password".to_string(),
            type_hint: "string".to_string(),
            description: "VNC password (optional, for VNC authentication)".to_string(),
            required: false,
            example: json!("mypassword"),
            default: None,
        }]
    }
    fn get_startup_examples(&self) -> crate::llm::actions::StartupExamples {
        use crate::llm::actions::StartupExamples;
        use serde_json::json;

        StartupExamples::new(
            // LLM mode: LLM controls VNC interaction
            json!({
                "type": "open_client",
                "remote_addr": "localhost:5900",
                "base_stack": "vnc",
                "instruction": "Click at position (100, 200) and type 'Hello'"
            }),
            // Script mode: Code-based VNC control
            json!({
                "type": "open_client",
                "remote_addr": "localhost:5900",
                "base_stack": "vnc",
                "event_handlers": [{
                    "event_pattern": "vnc_framebuffer_update",
                    "handler": {
                        "type": "script",
                        "language": "python",
                        "code": "<vnc_client_handler>"
                    }
                }]
            }),
            // Static mode: Fixed pointer event
            json!({
                "type": "open_client",
                "remote_addr": "localhost:5900",
                "base_stack": "vnc",
                "event_handlers": [
                    {
                        "event_pattern": "vnc_connected",
                        "handler": {
                            "type": "static",
                            "actions": [{
                                "type": "request_framebuffer_update",
                                "incremental": true
                            }]
                        }
                    },
                    {
                        "event_pattern": "vnc_framebuffer_update",
                        "handler": {
                            "type": "static",
                            "actions": [{
                                "type": "send_pointer_event",
                                "x": 100,
                                "y": 200,
                                "button_mask": 1
                            }]
                        }
                    }
                ]
            }),
        )
    }
}

// Implement Client trait (client-specific functionality)
impl Client for VncClientProtocol {
    fn connect(
        &self,
        ctx: crate::protocol::ConnectContext,
    ) -> std::pin::Pin<
        Box<dyn std::future::Future<Output = anyhow::Result<std::net::SocketAddr>> + Send>,
    > {
        Box::pin(async move {
            use crate::client::vnc::VncClient;
            VncClient::connect_with_llm_actions(
                ctx.remote_addr,
                ctx.llm_client,
                ctx.state,
                ctx.status_tx,
                ctx.client_id,
                ctx.startup_params,
            )
            .await
        })
    }
    fn execute_action(&self, action: serde_json::Value) -> Result<ClientActionResult> {
        let action_type = action
            .get("type")
            .and_then(|v| v.as_str())
            .context("Missing 'type' field in action")?;

        match action_type {
            "request_framebuffer_update" => {
                let incremental = action
                    .get("incremental")
                    .and_then(|v| v.as_bool())
                    .unwrap_or(true);
                let x = super::rfb_u16(&action, "x", 0)?;
                let y = super::rfb_u16(&action, "y", 0)?;
                let width = super::rfb_u16(&action, "width", u16::MAX)?;
                let height = super::rfb_u16(&action, "height", u16::MAX)?;

                Ok(ClientActionResult::Custom {
                    name: "request_framebuffer_update".to_string(),
                    data: json!({
                        "incremental": incremental,
                        "x": x,
                        "y": y,
                        "width": width,
                        "height": height,
                    }),
                })
            }
            "send_pointer_event" => {
                for key in ["x", "y", "button_mask"] {
                    if action.get(key).is_none_or(serde_json::Value::is_null) {
                        anyhow::bail!("Missing '{key}' field");
                    }
                }
                let x = super::rfb_u16(&action, "x", 0)?;
                let y = super::rfb_u16(&action, "y", 0)?;
                let button_mask = super::rfb_u8(&action, "button_mask", 0)?;

                Ok(ClientActionResult::Custom {
                    name: "send_pointer_event".to_string(),
                    data: json!({
                        "x": x,
                        "y": y,
                        "button_mask": button_mask,
                    }),
                })
            }
            "send_key_event" => {
                // Accepts a name ("a", "Enter", "F1") or a keysym number. This is where the
                // action is validated before dispatch, so requiring as_u64() here rejected
                // every named key regardless of what the executor could handle.
                let key = crate::client::vnc::resolve_keysym(
                    action.get("key").unwrap_or(&serde_json::Value::Null),
                )
                .context(
                    "Missing or invalid 'key' field: use a single character like \"a\", a name                      like \"Enter\"/\"Escape\"/\"F1\", or an X11 keysym number",
                )?;
                let down = action
                    .get("down")
                    .and_then(|v| v.as_bool())
                    .context("Missing or invalid 'down' field")?;

                Ok(ClientActionResult::Custom {
                    name: "send_key_event".to_string(),
                    data: json!({
                        "key": key,
                        "down": down,
                    }),
                })
            }
            "send_client_cut_text" => {
                let text = action
                    .get("text")
                    .and_then(|v| v.as_str())
                    .context("Missing or invalid 'text' field")?;

                Ok(ClientActionResult::Custom {
                    name: "send_client_cut_text".to_string(),
                    data: json!({
                        "text": text,
                    }),
                })
            }
            "disconnect" => Ok(ClientActionResult::Disconnect),
            "wait_for_more" => Ok(ClientActionResult::WaitForMore),
            _ => Err(anyhow::anyhow!(
                "Unknown VNC client action: {}",
                action_type
            )),
        }
    }
}