agent_first_http/sdk/fetch/artifacts/
observation.rs

1//! Agent-readable page snapshot (`observation.json`).
2//!
3//! Mechanical projection of the accessibility tree + DOM geometry per
4//! `architecture.md §8`. Disallowed: intent labels like "login", "captcha",
5//! importance ranking. This module owns the
6//! schema and the no-intent-label invariant.
7
8use std::path::PathBuf;
9
10use serde::{Deserialize, Serialize};
11
12use crate::sdk::fetch::writer;
13use crate::shared::artifacts::{Artifact, ArtifactPaths};
14use crate::shared::error::{Error, ErrorCode};
15
16pub const OBSERVATION_SCHEMA_VERSION: u32 = 2;
17
18#[derive(Debug, Clone, Serialize, Deserialize)]
19pub struct Observation {
20    pub schema_version: u32,
21    #[serde(alias = "url")]
22    pub page_url: String,
23    pub title: String,
24    pub viewport: Viewport,
25    pub frames: Vec<Frame>,
26    pub nodes: Vec<Node>,
27    #[serde(default)]
28    pub forms: Vec<Form>,
29    #[serde(skip_serializing_if = "Option::is_none")]
30    pub focused_ref: Option<String>,
31    #[serde(skip_serializing_if = "Option::is_none")]
32    pub truncated: Option<ObservationTruncation>,
33}
34
35#[derive(Debug, Clone, Serialize, Deserialize)]
36pub struct Viewport {
37    pub width: u32,
38    pub height: u32,
39    pub device_scale_factor: f32,
40}
41
42#[derive(Debug, Clone, Serialize, Deserialize)]
43pub struct Frame {
44    pub frame_id: String,
45    #[serde(alias = "url")]
46    pub frame_url: String,
47}
48
49#[derive(Debug, Clone, Serialize, Deserialize)]
50pub struct Node {
51    /// Per-snapshot ref; not a durable selector.
52    pub r#ref: String,
53    pub frame_id: String,
54    pub role: String,
55    #[serde(skip_serializing_if = "Option::is_none")]
56    pub name: Option<String>,
57    #[serde(skip_serializing_if = "Option::is_none")]
58    pub text: Option<String>,
59    pub visible: bool,
60    pub enabled: bool,
61    #[serde(skip_serializing_if = "Option::is_none")]
62    pub bbox: Option<BBox>,
63    #[serde(default, skip_serializing_if = "Vec::is_empty")]
64    pub actions: Vec<String>,
65    #[serde(skip_serializing_if = "Option::is_none")]
66    pub href: Option<String>,
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub src: Option<String>,
69    #[serde(skip_serializing_if = "Option::is_none")]
70    pub frame_ref: Option<String>,
71    #[serde(skip_serializing_if = "Option::is_none")]
72    pub input_type: Option<String>,
73    #[serde(skip_serializing_if = "Option::is_none")]
74    pub checked: Option<bool>,
75    #[serde(skip_serializing_if = "Option::is_none")]
76    pub selected: Option<bool>,
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub focused: Option<bool>,
79    #[serde(skip_serializing_if = "Option::is_none")]
80    pub value_redacted: Option<bool>,
81    #[serde(skip_serializing_if = "Option::is_none")]
82    pub selector_hint: Option<String>,
83    #[serde(skip_serializing_if = "Option::is_none")]
84    pub selector_hint_unique: Option<bool>,
85}
86
87#[derive(Debug, Clone, Serialize, Deserialize)]
88pub struct BBox {
89    pub x: f32,
90    pub y: f32,
91    pub width: f32,
92    pub height: f32,
93}
94
95#[derive(Debug, Clone, Serialize, Deserialize)]
96pub struct Form {
97    pub r#ref: String,
98    #[serde(skip_serializing_if = "Option::is_none")]
99    pub action: Option<String>,
100    #[serde(default, skip_serializing_if = "Vec::is_empty")]
101    pub field_refs: Vec<String>,
102}
103
104#[derive(Debug, Clone, Serialize, Deserialize)]
105pub struct ObservationTruncation {
106    pub reason: String,
107    pub node_limit: usize,
108    pub scan_limit: usize,
109    pub scanned: usize,
110    pub emitted_nodes: usize,
111}
112
113/// Capture an `Observation` from a CDP session. Projects the accessibility
114/// tree + page title + viewport into the mechanical schema; never adds
115/// intent labels or importance ranks (see `DISALLOWED_LABELS`).
116pub async fn capture(
117    conn: &crate::sdk::cdp::ws_client::Connection,
118    session_id: &str,
119    url: &str,
120) -> Result<Observation, Error> {
121    // Page metadata.
122    let meta = conn
123        .send(
124            "Runtime.evaluate",
125            &serde_json::json!({
126                "expression": "JSON.stringify({title: document.title, w: window.innerWidth, h: window.innerHeight, dpr: window.devicePixelRatio})",
127                "returnByValue": true,
128            }),
129            Some(session_id),
130        )
131        .await?;
132    let title;
133    let viewport;
134    if let Some(s) = meta["result"]["value"].as_str() {
135        let v: serde_json::Value = serde_json::from_str(s).unwrap_or(serde_json::Value::Null);
136        title = v["title"].as_str().unwrap_or("").to_string();
137        viewport = Viewport {
138            width: v["w"].as_u64().unwrap_or(0) as u32,
139            height: v["h"].as_u64().unwrap_or(0) as u32,
140            device_scale_factor: v["dpr"].as_f64().unwrap_or(1.0) as f32,
141        };
142    } else {
143        title = String::new();
144        viewport = Viewport {
145            width: 0,
146            height: 0,
147            device_scale_factor: 1.0,
148        };
149    }
150
151    // Mechanical DOM projection for agent planning. This intentionally
152    // captures roles/states/geometry/actions without ranking or intent labels.
153    let dom = conn
154        .send(
155            "Runtime.evaluate",
156            &serde_json::json!({
157                "expression": OBSERVATION_JS,
158                "returnByValue": true,
159            }),
160            Some(session_id),
161        )
162        .await?;
163    let mut nodes: Vec<Node> = Vec::new();
164    let mut forms: Vec<Form> = Vec::new();
165    let mut frames: Vec<Frame> = vec![Frame {
166        frame_id: "main".into(),
167        frame_url: crate::shared::redact::redact_url(url),
168    }];
169    let mut focused_ref: Option<String> = None;
170    let mut truncated: Option<ObservationTruncation> = None;
171    if let Some(s) = dom["result"]["value"].as_str() {
172        let v: serde_json::Value = serde_json::from_str(s).unwrap_or(serde_json::Value::Null);
173        if let Ok(parsed_nodes) =
174            serde_json::from_value::<Vec<Node>>(v.get("nodes").cloned().unwrap_or_default())
175        {
176            nodes = parsed_nodes;
177        }
178        if let Ok(parsed_forms) =
179            serde_json::from_value::<Vec<Form>>(v.get("forms").cloned().unwrap_or_default())
180        {
181            forms = parsed_forms;
182        }
183        if let Ok(parsed_frames) =
184            serde_json::from_value::<Vec<Frame>>(v.get("frames").cloned().unwrap_or_default())
185            && !parsed_frames.is_empty()
186        {
187            frames = parsed_frames
188                .into_iter()
189                .map(|mut frame| {
190                    frame.frame_url = crate::shared::redact::redact_url(&frame.frame_url);
191                    frame
192                })
193                .collect();
194        }
195        focused_ref = v
196            .get("focused_ref")
197            .and_then(|v| v.as_str())
198            .map(str::to_string);
199        if let Ok(parsed_truncation) = serde_json::from_value::<ObservationTruncation>(
200            v.get("truncated").cloned().unwrap_or_default(),
201        ) {
202            truncated = Some(parsed_truncation);
203        }
204    }
205
206    Ok(Observation {
207        schema_version: OBSERVATION_SCHEMA_VERSION,
208        page_url: crate::shared::redact::redact_url(url),
209        title,
210        viewport,
211        frames,
212        nodes,
213        forms,
214        focused_ref,
215        truncated,
216    })
217}
218
219const OBSERVATION_JS: &str = include_str!("../../../../assets/observation/snapshot.js");
220
221pub async fn write(paths: &ArtifactPaths, obs: &Observation) -> Result<PathBuf, Error> {
222    let target = paths.file_for(Artifact::Observation);
223    let bytes = serde_json::to_vec_pretty(obs).map_err(|e| {
224        Error::new(
225            ErrorCode::InternalError,
226            format!("serialize observation: {e}"),
227        )
228    })?;
229    writer::write_bytes(&target, &bytes).await?;
230    Ok(target)
231}
232
233/// Disallowed substrings in any string field of an `Observation`. Tests
234/// use this to enforce `design.md §"Observation is mechanical, not
235/// interpretive"`.
236pub const DISALLOWED_LABELS: &[&str] =
237    &["login", "captcha", "paywall", "important", "best", "likely"];
238
239#[cfg(test)]
240mod tests {
241    use super::*;
242
243    #[test]
244    fn schema_round_trips() {
245        let obs = Observation {
246            schema_version: OBSERVATION_SCHEMA_VERSION,
247            page_url: "https://example.com/".into(),
248            title: "Example".into(),
249            viewport: Viewport {
250                width: 1280,
251                height: 720,
252                device_scale_factor: 1.0,
253            },
254            frames: vec![Frame {
255                frame_id: "main".into(),
256                frame_url: "https://example.com/".into(),
257            }],
258            nodes: vec![Node {
259                r#ref: "obs-1".into(),
260                frame_id: "main".into(),
261                role: "button".into(),
262                name: Some("Submit".into()),
263                text: Some("Submit".into()),
264                visible: true,
265                enabled: true,
266                bbox: Some(BBox {
267                    x: 0.0,
268                    y: 0.0,
269                    width: 80.0,
270                    height: 32.0,
271                }),
272                actions: vec!["click".into()],
273                href: None,
274                src: None,
275                frame_ref: None,
276                input_type: None,
277                checked: None,
278                selected: None,
279                focused: None,
280                value_redacted: None,
281                selector_hint: None,
282                selector_hint_unique: None,
283            }],
284            forms: vec![],
285            focused_ref: None,
286            truncated: None,
287        };
288        let json = serde_json::to_string(&obs).unwrap_or_default();
289        let parsed: Observation = serde_json::from_str(&json).unwrap();
290        assert_eq!(parsed.nodes.len(), 1);
291        assert_eq!(parsed.nodes[0].role, "button");
292    }
293
294    #[test]
295    fn disallowed_labels_present_in_constant() {
296        for label in DISALLOWED_LABELS {
297            assert!(!label.is_empty());
298        }
299    }
300}
agent_first_http/sdk/fetch/artifacts/observation.rs

agent_first_http/sdk/fetch/artifacts/
observation.rs
