rightkit_agent_tools/backends/
control.rs

1//! Tools for one Right Suite app, reached through its in-app control service
2//! (`rightkit-qa-driver`'s `embedded` server, a `rightkit-daemon` host named
3//! `control-<app>`). Real webview input, DOM/AX query, screenshot, and the app's
4//! own typed commands (`command{name,args}`), all over the user-only socket/pipe
5//! with the service's per-app request allowlist on the far side.
6
7use crate::exposure::{self, GENERIC_READ_METHODS};
8use crate::spec::*;
9use rightkit_daemon::{Client, ServiceError, Suite};
10use serde_json::{json, Value};
11use std::sync::{Arc, Mutex};
12use std::time::Duration;
13
14/// The control methods of the in-app server, in the server's own order
15/// (`rightkit_qa_driver::embedded::METHODS`). A conformance test asserts this list
16/// against the QA crate's dispatch table so the two cannot drift.
17pub const CONTROL_METHODS: &[&str] = &[
18    "health",
19    "click",
20    "move",
21    "drag",
22    "pointer",
23    "wheel",
24    "key",
25    "type",
26    "eval",
27    "dom",
28    "ax",
29    "screenshot",
30    "set_viewport",
31    "focus",
32    "command",
33    "wait",
34    "batch",
35    "lease_acquire",
36    "lease_release",
37    "lease_reconcile",
38    "lease_status",
39];
40
41/// The `app` label this crate's clients present to services (their allowlist key).
42pub const CLIENT_APP: &str = "right-ctl";
43
44#[derive(Debug, Clone, PartialEq, Eq)]
45pub enum CommandPolicy {
46    /// Any command the service itself allows (still subject to the hard exposure rules).
47    Any,
48    /// Only these command names.
49    Only(Vec<String>),
50}
51
52/// Per-app client-side allowlist: which control methods and app commands agents
53/// may use. The service enforces its own allowlist as well; this one is the
54/// agent-facing layer on top.
55#[derive(Debug, Clone)]
56pub struct AppProfile {
57    pub app: String,
58    /// Service name under `<suite>/runtime/`, default `control-<app>`.
59    pub service: String,
60    pub methods: Vec<String>,
61    pub commands: CommandPolicy,
62    /// Sensitive apps (HeardRight) expose no generic read channel.
63    pub sensitive: bool,
64    /// Dictation apps (HeardRight) also refuse the dictation, delivery and history names
65    /// ([`exposure::forbidden_with`]). Other apps keep those words.
66    pub dictation: bool,
67}
68
69impl AppProfile {
70    /// Every control method; any app command.
71    pub fn standard(app: &str) -> Self {
72        Self {
73            app: app.into(),
74            service: format!("control-{app}"),
75            methods: CONTROL_METHODS.iter().map(|s| s.to_string()).collect(),
76            commands: CommandPolicy::Any,
77            sensitive: false,
78            dictation: false,
79        }
80    }
81
82    /// An app whose own data must not leave through generic channels: `health`
83    /// and an explicit command list only. Commands the hard rules forbid are
84    /// dropped from the list.
85    pub fn sensitive(app: &str, commands: &[&str]) -> Self {
86        let commands = commands
87            .iter()
88            .filter(|c| exposure::forbidden(c).is_none())
89            .map(|c| c.to_string())
90            .collect();
91        Self {
92            app: app.into(),
93            service: format!("control-{app}"),
94            methods: vec!["health".into(), "command".into()],
95            commands: CommandPolicy::Only(commands),
96            sensitive: true,
97            dictation: false,
98        }
99    }
100
101    /// Apply the dictation exposure layer to this app: dictation, delivery and history names
102    /// are refused, and any listed command they cover is dropped.
103    pub fn with_dictation_rules(mut self) -> Self {
104        self.dictation = true;
105        if let CommandPolicy::Only(names) = &mut self.commands {
106            names.retain(|c| exposure::forbidden_with(c, true).is_none());
107        }
108        self
109    }
110
111    /// Whether the hard exposure rules refuse command `name` for this app.
112    pub fn command_forbidden(&self, name: &str) -> bool {
113        exposure::forbidden_with(name, self.dictation).is_some()
114    }
115
116    /// The built-in profile for `app`: HeardRight is sensitive, everything else standard.
117    pub fn for_app(app: &str) -> Self {
118        match app {
119            "heardright" => Self::sensitive(
120                "heardright",
121                &["health", "version", "list_models", "model_status"],
122            )
123            .with_dictation_rules(),
124            _ => Self::standard(app),
125        }
126    }
127
128    pub fn with_methods(mut self, methods: &[&str]) -> Self {
129        self.methods = methods.iter().map(|s| s.to_string()).collect();
130        self
131    }
132
133    pub fn with_commands(mut self, policy: CommandPolicy) -> Self {
134        self.commands = policy;
135        self
136    }
137
138    /// Allowlist patterns (see `Toolbox::allow`) that grant exactly this profile.
139    pub fn allow_patterns(&self) -> Vec<String> {
140        let mut out = vec![];
141        for m in &self.methods {
142            if m == "command" {
143                match &self.commands {
144                    CommandPolicy::Any => out.push(format!("{}.command:*", self.app)),
145                    CommandPolicy::Only(names) => {
146                        out.extend(names.iter().map(|n| format!("{}.command:{n}", self.app)))
147                    }
148                }
149            } else {
150                out.push(format!("{}.{m}", self.app));
151            }
152        }
153        out
154    }
155
156    fn method_enabled(&self, m: &str) -> bool {
157        self.methods.iter().any(|x| x == m)
158            && !(self.sensitive && GENERIC_READ_METHODS.contains(&m))
159    }
160}
161
162fn num(desc: &str) -> Value {
163    json!({"type": "number", "description": desc})
164}
165
166/// JSON schema + description of one control method.
167fn method_schema(method: &str) -> (&'static str, Effect, Value) {
168    match method {
169        "health" => ("Is the app's control server up? Returns its status.", Effect::Observe, json!({"type": "object", "properties": {}})),
170        "click" => (
171            "Real click in the webview (viewport CSS pixels, origin top-left). Optional paceMs spaces the native events (default 0).",
172            Effect::Act,
173            json!({"type": "object", "properties": {"x": num("x"), "y": num("y"), "button": {"type": "string", "enum": ["left", "right"]}, "count": {"type": "integer"}, "paceMs": {"type": "integer"}}, "required": ["x", "y"]}),
174        ),
175        "move" => ("Move the pointer to x,y (hover).", Effect::Act, json!({"type": "object", "properties": {"x": num("x"), "y": num("y")}, "required": ["x", "y"]})),
176        "drag" => (
177            "Drag from (x1,y1) to (x2,y2).",
178            Effect::Act,
179            json!({"type": "object", "properties": {"x1": num("x1"), "y1": num("y1"), "x2": num("x2"), "y2": num("y2"), "steps": {"type": "integer"}}, "required": ["x1", "y1", "x2", "y2"]}),
180        ),
181        "wheel" => ("Scroll wheel at x,y by dx,dy.", Effect::Act, json!({"type": "object", "properties": {"x": num("x"), "y": num("y"), "dx": num("dx"), "dy": num("dy")}, "required": ["x", "y", "dx", "dy"]})),
182        "key" => ("Press a key chord such as `cmd+k` or `enter`.", Effect::Act, json!({"type": "object", "properties": {"key": {"type": "string"}}, "required": ["key"]})),
183        "type" => ("Type text into the focused element with real key events.", Effect::Act, json!({"type": "object", "properties": {"text": {"type": "string"}}, "required": ["text"]})),
184        "eval" => ("Evaluate JavaScript in the webview and return the result.", Effect::Act, json!({"type": "object", "properties": {"js": {"type": "string"}}, "required": ["js"]})),
185        "dom" => ("Query DOM nodes by CSS selector.", Effect::Observe, json!({"type": "object", "properties": {"selector": {"type": "string"}}, "required": ["selector"]})),
186        "ax" => ("Accessibility tree of the app window.", Effect::Observe, json!({"type": "object", "properties": {}})),
187        "screenshot" => (
188            "Screenshot of the webview (password and secret elements are masked). Returns {path,bytes,masked}, or with inline=true {bytes,masked,encoding:\"base64\",data} and no file.",
189            Effect::Capture,
190            json!({"type": "object", "properties": {"mask": {"type": "array"}, "inline": {"type": "boolean"}}}),
191        ),
192        "wait" => (
193            "Wait in the page, in ONE call, until a JavaScript function body returns something truthy (or timeoutMs, default 10000). Returns {matched,value|last,error,elapsedMs}. Prefer this to polling eval.",
194            Effect::Act,
195            json!({"type": "object", "properties": {"js": {"type": "string"}, "timeoutMs": {"type": "integer"}, "intervalMs": {"type": "integer"}}, "required": ["js"]}),
196        ),
197        "batch" => (
198            "Run many control steps in ONE call and stop at the first failure: steps=[{method,params}] with any of this app's methods except batch. Returns {completed,results,stepMs} or {ok:false,failedStep,error,results}. The way to drive an app: one call per step is slow.",
199            Effect::Act,
200            json!({"type": "object", "properties": {"steps": {"type": "array", "items": {"type": "object", "properties": {"method": {"type": "string"}, "params": {"type": "object"}}, "required": ["method"]}}}, "required": ["steps"]}),
201        ),
202        "command" => (
203            "Run one of the app's own typed commands by name; `args` is the command's argument string.",
204            Effect::Act,
205            json!({"type": "object", "properties": {"name": {"type": "string"}, "args": {"type": "string"}}, "required": ["name"]}),
206        ),
207        "pointer" => (
208            "One raw pointer step in the webview (viewport CSS pixels, origin top-left): phase down, drag, up or move at x,y. Optional button, modifiers and paceMs (native-event spacing, default 0).",
209            Effect::Act,
210            json!({"type": "object", "properties": {"phase": {"type": "string", "enum": ["down", "drag", "up", "move"]}, "x": num("x"), "y": num("y"), "button": {"type": "string", "enum": ["left", "right"]}, "modifiers": {"type": "array"}, "paceMs": {"type": "integer"}}, "required": ["phase", "x", "y"]}),
211        ),
212        "set_viewport" => (
213            "Resize the webview viewport to width x height CSS pixels without activating the window; returns the size read back after settling (optional settleMs bound).",
214            Effect::Act,
215            json!({"type": "object", "properties": {"width": {"type": "integer"}, "height": {"type": "integer"}, "settleMs": {"type": "integer"}}, "required": ["width", "height"]}),
216        ),
217        "focus" => ("Focus the app window.", Effect::Act, json!({"type": "object", "properties": {}})),
218        "lease_acquire" => (
219            "Acquire the server's input lease; returns {leaseEpoch,nextInputSequence,fencedEpoch}. Optional reconcile (delivered|lost) first settles an unacknowledged input of a previous lease. The leaseEpoch is the ownership token lease_release and lease_reconcile must present.",
220            Effect::Act,
221            json!({"type": "object", "properties": {"reconcile": {"type": "string", "enum": ["delivered", "lost"]}}}),
222        ),
223        "lease_release" => (
224            "Release the input lease held under leaseEpoch; a stale epoch is refused by the server.",
225            Effect::Act,
226            json!({"type": "object", "properties": {"leaseEpoch": {"type": "integer"}}, "required": ["leaseEpoch"]}),
227        ),
228        "lease_reconcile" => (
229            "Reconcile the input lease held under leaseEpoch: decision delivered counts its unacknowledged input as delivered, lost as never landed.",
230            Effect::Act,
231            json!({"type": "object", "properties": {"leaseEpoch": {"type": "integer"}, "decision": {"type": "string", "enum": ["delivered", "lost"]}}, "required": ["leaseEpoch", "decision"]}),
232        ),
233        "lease_status" => ("Snapshot of the server's input lease (holder epoch and sequencing state).", Effect::Observe, json!({"type": "object", "properties": {}})),
234        _ => ("", Effect::Act, json!({"type": "object"})),
235    }
236}
237
238struct ControlTool {
239    spec: ToolSpec,
240    suite: Suite,
241    profile: AppProfile,
242    method: String,
243    /// One connection shared by every tool of the profile (connecting costs a handshake per call).
244    client: Arc<Mutex<Option<Arc<Client>>>>,
245}
246
247fn map_err(e: ServiceError) -> ToolError {
248    ToolError {
249        code: e.code,
250        message: e.message,
251        retryable: e.retryable,
252    }
253}
254
255impl Tool for ControlTool {
256    fn spec(&self) -> &ToolSpec {
257        &self.spec
258    }
259
260    fn subject(&self, args: &Value) -> Option<String> {
261        (self.method == "command").then(|| args["name"].as_str().unwrap_or("").to_string())
262    }
263
264    fn call(&self, ctx: &CallCtx, args: Value) -> Result<Value, ToolError> {
265        if ctx.cancel.is_cancelled() {
266            return Err(ToolError::new(ERR_CANCELLED, "cancelled before dispatch"));
267        }
268        if self.method == "batch" {
269            check_batch(&self.profile, &args)?;
270        }
271        let params = if self.method == "command" {
272            let name = args["name"].as_str().unwrap_or_default();
273            let allowed = match &self.profile.commands {
274                CommandPolicy::Any => true,
275                CommandPolicy::Only(l) => l.iter().any(|c| c == name),
276            };
277            if !allowed || self.profile.command_forbidden(name) {
278                return Err(ToolError::new(
279                    ERR_NOT_ALLOWED,
280                    format!("command `{name}` is not exposed for {}", self.profile.app),
281                ));
282            }
283            let a = match &args["args"] {
284                Value::Null => String::new(),
285                Value::String(s) => s.clone(),
286                other => other.to_string(),
287            };
288            json!({"name": name, "args": a})
289        } else {
290            args
291        };
292        // `batch` and `wait` run longer than one step: give them the time they ask for.
293        let long = match self.method.as_str() {
294            "wait" => Some(
295                Duration::from_millis(args_timeout(&params, "timeoutMs", 10_000))
296                    + Duration::from_secs(10),
297            ),
298            "batch" => Some(Duration::from_secs(
299                30 * params["steps"]
300                    .as_array()
301                    .map_or(1, |s| s.len().clamp(1, 20)) as u64,
302            )),
303            _ => None,
304        };
305        if let Some(t) = long {
306            let client =
307                Client::connect_with(&self.suite, &self.profile.service, CLIENT_APP, Some(t))
308                    .map_err(map_err)?;
309            return client.call(&self.method, params).map_err(map_err);
310        }
311        let client = {
312            let mut slot = self.client.lock().unwrap_or_else(|e| e.into_inner());
313            match slot.as_ref() {
314                Some(c) => c.clone(),
315                None => {
316                    let c = Arc::new(
317                        Client::connect_with(
318                            &self.suite,
319                            &self.profile.service,
320                            CLIENT_APP,
321                            Some(Duration::from_secs(30)),
322                        )
323                        .map_err(map_err)?,
324                    );
325                    *slot = Some(c.clone());
326                    c
327                }
328            }
329        };
330        match client.call(&self.method, params) {
331            Ok(v) => Ok(v),
332            Err(e) => {
333                // Reconnect next time: the connection may be the thing that failed.
334                *self.client.lock().unwrap_or_else(|e| e.into_inner()) = None;
335                Err(map_err(e))
336            }
337        }
338    }
339}
340
341fn args_timeout(params: &Value, key: &str, default: u64) -> u64 {
342    params[key].as_u64().unwrap_or(default).min(600_000)
343}
344
345/// `batch` is one call carrying many methods, so the profile's limits apply to each step: a
346/// method the profile does not enable (a sensitive app's `eval`/`dom`/`ax`/`screenshot`), a
347/// nested `batch`, or a `command` the command policy refuses is rejected before anything runs.
348fn check_batch(profile: &AppProfile, args: &Value) -> Result<(), ToolError> {
349    let steps = args["steps"]
350        .as_array()
351        .ok_or_else(|| ToolError::new(ERR_INVALID_ARGS, "batch needs a `steps` array"))?;
352    for (i, step) in steps.iter().enumerate() {
353        let m = step["method"].as_str().unwrap_or_default();
354        if m == "batch" || !profile.method_enabled(m) {
355            return Err(ToolError::new(
356                ERR_NOT_ALLOWED,
357                format!("batch step {i}: `{m}` is not exposed for {}", profile.app),
358            ));
359        }
360        if m == "command" {
361            let name = step["params"]["name"].as_str().unwrap_or_default();
362            let allowed = match &profile.commands {
363                CommandPolicy::Any => true,
364                CommandPolicy::Only(l) => l.iter().any(|c| c == name),
365            };
366            if !allowed || profile.command_forbidden(name) {
367                return Err(ToolError::new(
368                    ERR_NOT_ALLOWED,
369                    format!(
370                        "batch step {i}: command `{name}` is not exposed for {}",
371                        profile.app
372                    ),
373                ));
374            }
375        }
376    }
377    Ok(())
378}
379
380/// Tools for `profile` on `suite`: one per enabled control method, named `<app>.<method>`.
381pub fn control_tools(suite: &Suite, profile: &AppProfile) -> Vec<Arc<dyn Tool>> {
382    let client: Arc<Mutex<Option<Arc<Client>>>> = Arc::new(Mutex::new(None));
383    CONTROL_METHODS
384        .iter()
385        .filter(|m| profile.method_enabled(m))
386        .map(|m| {
387            let (desc, effect, schema) = method_schema(m);
388            let description = format!("{desc} (app: {})", profile.app);
389            Arc::new(ControlTool {
390                spec: ToolSpec::new(format!("{}.{m}", profile.app), description, effect, schema),
391                suite: suite.clone(),
392                profile: profile.clone(),
393                method: m.to_string(),
394                client: client.clone(),
395            }) as Arc<dyn Tool>
396        })
397        .collect()
398}
399
400#[cfg(test)]
401mod tests {
402    use super::*;
403
404    fn steps(methods: &[&str]) -> Value {
405        json!({"steps": methods.iter().map(|m| json!({"method": m, "params": {}})).collect::<Vec<_>>()})
406    }
407
408    #[test]
409    fn standard_profile_may_batch_its_methods_but_not_nest() {
410        let p = AppProfile::standard("demo");
411        assert!(p.methods.iter().any(|m| m == "batch") && p.methods.iter().any(|m| m == "wait"));
412        assert!(check_batch(&p, &steps(&["click", "type", "screenshot", "wait"])).is_ok());
413        assert!(check_batch(&p, &steps(&["batch"])).is_err());
414        assert!(check_batch(&p, &json!({})).is_err());
415    }
416
417    #[test]
418    fn the_seven_newer_methods_are_standard_with_schemas() {
419        let p = AppProfile::standard("demo");
420        for m in [
421            "pointer",
422            "set_viewport",
423            "focus",
424            "lease_acquire",
425            "lease_release",
426            "lease_reconcile",
427            "lease_status",
428        ] {
429            assert!(CONTROL_METHODS.contains(&m), "{m}");
430            assert!(p.method_enabled(m), "{m}");
431            let (desc, _, schema) = method_schema(m);
432            assert!(!desc.is_empty(), "{m}");
433            assert_eq!(schema["type"], "object", "{m}");
434        }
435        assert!(check_batch(
436            &p,
437            &steps(&["pointer", "set_viewport", "focus", "lease_status"])
438        )
439        .is_ok());
440        // Lease ownership is in the schemas: release/reconcile cannot omit the epoch.
441        for m in ["lease_release", "lease_reconcile"] {
442            let (_, _, schema) = method_schema(m);
443            assert!(
444                schema["required"]
445                    .as_array()
446                    .is_some_and(|r| r.iter().any(|v| v == "leaseEpoch")),
447                "{m}"
448            );
449        }
450        // A sensitive profile still gets none of them.
451        let p = AppProfile::for_app("heardright");
452        for m in ["pointer", "set_viewport", "focus", "lease_acquire"] {
453            assert!(!p.method_enabled(m), "{m}");
454        }
455    }
456
457    #[test]
458    fn dictation_names_are_refused_only_for_a_dictation_profile() {
459        let heard = AppProfile::for_app("heardright");
460        assert!(heard.dictation);
461        for name in ["history", "delivery_status", "start_dictation"] {
462            assert!(heard.command_forbidden(name), "{name}");
463        }
464        let gen = AppProfile::standard("genright");
465        assert!(!gen.dictation);
466        for name in ["history", "delivery_status"] {
467            assert!(!gen.command_forbidden(name), "{name}");
468        }
469        assert!(gen.command_forbidden("paste_text"));
470        let listed = AppProfile::sensitive("x", &["history", "health"]).with_dictation_rules();
471        assert!(
472            matches!(&listed.commands, CommandPolicy::Only(l) if l == &vec!["health".to_string()])
473        );
474    }
475
476    #[test]
477    fn a_batch_cannot_smuggle_generic_reads_past_a_sensitive_profile() {
478        let p = AppProfile::for_app("heardright");
479        for m in ["eval", "dom", "ax", "screenshot", "wait", "batch", "click"] {
480            assert!(!p.method_enabled(m), "{m}");
481            assert!(check_batch(&p, &steps(&[m])).is_err(), "{m}");
482        }
483        let ok = json!({"steps": [{"method": "command", "params": {"name": "version"}}]});
484        assert!(check_batch(&p, &ok).is_ok());
485        let bad = json!({"steps": [{"method": "command", "params": {"name": "transcribe_all"}}]});
486        assert!(check_batch(&p, &bad).is_err());
487    }
488}
rightkit_agent_tools/backends/control.rs

rightkit_agent_tools/backends/
control.rs
