agent_first_http/sdk/fetch/artifacts/
observation.rs

1//! Agent-readable page snapshot (`observation.json`).
2//!
3//! Mechanical projection of the accessibility tree + DOM geometry per
4//! `architecture.md §8`. Disallowed: intent labels like "login", "captcha",
5//! importance ranking. This module owns the
6//! schema and the no-intent-label invariant.
7
8use std::path::PathBuf;
9
10use serde::{Deserialize, Serialize};
11
12use crate::sdk::fetch::writer;
13use crate::shared::artifacts::{Artifact, ArtifactPaths};
14use crate::shared::error::{Error, ErrorCode};
15
16#[derive(Debug, Clone, Serialize, Deserialize)]
17pub struct Observation {
18    pub schema_version: u32,
19    pub url: String,
20    pub title: String,
21    pub viewport: Viewport,
22    pub frames: Vec<Frame>,
23    pub nodes: Vec<Node>,
24    #[serde(default)]
25    pub forms: Vec<Form>,
26    #[serde(skip_serializing_if = "Option::is_none")]
27    pub focused_ref: Option<String>,
28    #[serde(skip_serializing_if = "Option::is_none")]
29    pub truncated: Option<ObservationTruncation>,
30}
31
32#[derive(Debug, Clone, Serialize, Deserialize)]
33pub struct Viewport {
34    pub width: u32,
35    pub height: u32,
36    pub device_scale_factor: f32,
37}
38
39#[derive(Debug, Clone, Serialize, Deserialize)]
40pub struct Frame {
41    pub frame_id: String,
42    pub url: String,
43}
44
45#[derive(Debug, Clone, Serialize, Deserialize)]
46pub struct Node {
47    /// Per-snapshot ref; not a durable selector.
48    pub r#ref: String,
49    pub frame_id: String,
50    pub role: String,
51    #[serde(skip_serializing_if = "Option::is_none")]
52    pub name: Option<String>,
53    #[serde(skip_serializing_if = "Option::is_none")]
54    pub text: Option<String>,
55    pub visible: bool,
56    pub enabled: bool,
57    #[serde(skip_serializing_if = "Option::is_none")]
58    pub bbox: Option<BBox>,
59    #[serde(default, skip_serializing_if = "Vec::is_empty")]
60    pub actions: Vec<String>,
61    #[serde(skip_serializing_if = "Option::is_none")]
62    pub href: Option<String>,
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub src: Option<String>,
65    #[serde(skip_serializing_if = "Option::is_none")]
66    pub frame_ref: Option<String>,
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub input_type: Option<String>,
69    #[serde(skip_serializing_if = "Option::is_none")]
70    pub checked: Option<bool>,
71    #[serde(skip_serializing_if = "Option::is_none")]
72    pub selected: Option<bool>,
73    #[serde(skip_serializing_if = "Option::is_none")]
74    pub focused: Option<bool>,
75    #[serde(skip_serializing_if = "Option::is_none")]
76    pub value_redacted: Option<bool>,
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub selector_hint: Option<String>,
79    #[serde(skip_serializing_if = "Option::is_none")]
80    pub selector_hint_unique: Option<bool>,
81}
82
83#[derive(Debug, Clone, Serialize, Deserialize)]
84pub struct BBox {
85    pub x: f32,
86    pub y: f32,
87    pub width: f32,
88    pub height: f32,
89}
90
91#[derive(Debug, Clone, Serialize, Deserialize)]
92pub struct Form {
93    pub r#ref: String,
94    #[serde(skip_serializing_if = "Option::is_none")]
95    pub action: Option<String>,
96    #[serde(default, skip_serializing_if = "Vec::is_empty")]
97    pub field_refs: Vec<String>,
98}
99
100#[derive(Debug, Clone, Serialize, Deserialize)]
101pub struct ObservationTruncation {
102    pub reason: String,
103    pub node_limit: usize,
104    pub scan_limit: usize,
105    pub scanned: usize,
106    pub emitted_nodes: usize,
107}
108
109/// Capture an `Observation` from a CDP session. Projects the accessibility
110/// tree + page title + viewport into the mechanical schema; never adds
111/// intent labels or importance ranks (see `DISALLOWED_LABELS`).
112pub async fn capture(
113    conn: &crate::sdk::cdp::ws_client::Connection,
114    session_id: &str,
115    url: &str,
116) -> Result<Observation, Error> {
117    // Page metadata.
118    let meta = conn
119        .send(
120            "Runtime.evaluate",
121            &serde_json::json!({
122                "expression": "JSON.stringify({title: document.title, w: window.innerWidth, h: window.innerHeight, dpr: window.devicePixelRatio})",
123                "returnByValue": true,
124            }),
125            Some(session_id),
126        )
127        .await?;
128    let title;
129    let viewport;
130    if let Some(s) = meta["result"]["value"].as_str() {
131        let v: serde_json::Value = serde_json::from_str(s).unwrap_or(serde_json::Value::Null);
132        title = v["title"].as_str().unwrap_or("").to_string();
133        viewport = Viewport {
134            width: v["w"].as_u64().unwrap_or(0) as u32,
135            height: v["h"].as_u64().unwrap_or(0) as u32,
136            device_scale_factor: v["dpr"].as_f64().unwrap_or(1.0) as f32,
137        };
138    } else {
139        title = String::new();
140        viewport = Viewport {
141            width: 0,
142            height: 0,
143            device_scale_factor: 1.0,
144        };
145    }
146
147    // Mechanical DOM projection for agent planning. This intentionally
148    // captures roles/states/geometry/actions without ranking or intent labels.
149    let dom = conn
150        .send(
151            "Runtime.evaluate",
152            &serde_json::json!({
153                "expression": OBSERVATION_JS,
154                "returnByValue": true,
155            }),
156            Some(session_id),
157        )
158        .await?;
159    let mut nodes: Vec<Node> = Vec::new();
160    let mut forms: Vec<Form> = Vec::new();
161    let mut frames: Vec<Frame> = vec![Frame {
162        frame_id: "main".into(),
163        url: url.to_string(),
164    }];
165    let mut focused_ref: Option<String> = None;
166    let mut truncated: Option<ObservationTruncation> = None;
167    if let Some(s) = dom["result"]["value"].as_str() {
168        let v: serde_json::Value = serde_json::from_str(s).unwrap_or(serde_json::Value::Null);
169        if let Ok(parsed_nodes) =
170            serde_json::from_value::<Vec<Node>>(v.get("nodes").cloned().unwrap_or_default())
171        {
172            nodes = parsed_nodes;
173        }
174        if let Ok(parsed_forms) =
175            serde_json::from_value::<Vec<Form>>(v.get("forms").cloned().unwrap_or_default())
176        {
177            forms = parsed_forms;
178        }
179        if let Ok(parsed_frames) =
180            serde_json::from_value::<Vec<Frame>>(v.get("frames").cloned().unwrap_or_default())
181        {
182            if !parsed_frames.is_empty() {
183                frames = parsed_frames;
184            }
185        }
186        focused_ref = v
187            .get("focused_ref")
188            .and_then(|v| v.as_str())
189            .map(str::to_string);
190        if let Ok(parsed_truncation) = serde_json::from_value::<ObservationTruncation>(
191            v.get("truncated").cloned().unwrap_or_default(),
192        ) {
193            truncated = Some(parsed_truncation);
194        }
195    }
196
197    Ok(Observation {
198        schema_version: 1,
199        url: url.to_string(),
200        title,
201        viewport,
202        frames,
203        nodes,
204        forms,
205        focused_ref,
206        truncated,
207    })
208}
209
210const OBSERVATION_JS: &str = include_str!("../../../../assets/observation/snapshot.js");
211
212pub async fn write(paths: &ArtifactPaths, obs: &Observation) -> Result<PathBuf, Error> {
213    let target = paths.file_for(Artifact::Observation);
214    let bytes = serde_json::to_vec_pretty(obs).map_err(|e| {
215        Error::new(
216            ErrorCode::InternalError,
217            format!("serialize observation: {e}"),
218        )
219    })?;
220    writer::write_bytes(&target, &bytes).await?;
221    Ok(target)
222}
223
224/// Disallowed substrings in any string field of an `Observation`. Tests
225/// use this to enforce `design.md §"Observation is mechanical, not
226/// interpretive"`.
227pub const DISALLOWED_LABELS: &[&str] =
228    &["login", "captcha", "paywall", "important", "best", "likely"];
229
230#[cfg(test)]
231mod tests {
232    use super::*;
233
234    #[test]
235    fn schema_round_trips() {
236        let obs = Observation {
237            schema_version: 1,
238            url: "https://example.com/".into(),
239            title: "Example".into(),
240            viewport: Viewport {
241                width: 1280,
242                height: 720,
243                device_scale_factor: 1.0,
244            },
245            frames: vec![Frame {
246                frame_id: "main".into(),
247                url: "https://example.com/".into(),
248            }],
249            nodes: vec![Node {
250                r#ref: "obs-1".into(),
251                frame_id: "main".into(),
252                role: "button".into(),
253                name: Some("Submit".into()),
254                text: Some("Submit".into()),
255                visible: true,
256                enabled: true,
257                bbox: Some(BBox {
258                    x: 0.0,
259                    y: 0.0,
260                    width: 80.0,
261                    height: 32.0,
262                }),
263                actions: vec!["click".into()],
264                href: None,
265                src: None,
266                frame_ref: None,
267                input_type: None,
268                checked: None,
269                selected: None,
270                focused: None,
271                value_redacted: None,
272                selector_hint: None,
273                selector_hint_unique: None,
274            }],
275            forms: vec![],
276            focused_ref: None,
277            truncated: None,
278        };
279        let json = serde_json::to_string(&obs).unwrap_or_default();
280        let parsed: Observation = serde_json::from_str(&json).unwrap();
281        assert_eq!(parsed.nodes.len(), 1);
282        assert_eq!(parsed.nodes[0].role, "button");
283    }
284
285    #[test]
286    fn disallowed_labels_present_in_constant() {
287        for label in DISALLOWED_LABELS {
288            assert!(!label.is_empty());
289        }
290    }
291}
agent_first_http/sdk/fetch/artifacts/observation.rs

agent_first_http/sdk/fetch/artifacts/
observation.rs
