agent_first_http/sdk/fetch/artifacts/
observation.rs

1//! Agent-readable page snapshot (`observation.json`).
2//!
3//! Mechanical projection of the accessibility tree + DOM geometry per
4//! `architecture.md §8`. Disallowed: intent labels like "login", "captcha",
5//! importance ranking. This module owns the
6//! schema and the no-intent-label invariant.
7
8use std::path::PathBuf;
9
10use serde::{Deserialize, Serialize};
11
12use crate::sdk::fetch::writer;
13use crate::shared::artifacts::{Artifact, ArtifactPaths};
14use crate::shared::error::{Error, ErrorCode};
15
16#[derive(Debug, Clone, Serialize, Deserialize)]
17pub struct Observation {
18    pub schema_version: u32,
19    pub url: String,
20    pub title: String,
21    pub viewport: Viewport,
22    pub frames: Vec<Frame>,
23    pub nodes: Vec<Node>,
24    #[serde(default)]
25    pub forms: Vec<Form>,
26    #[serde(skip_serializing_if = "Option::is_none")]
27    pub focused_ref: Option<String>,
28    #[serde(skip_serializing_if = "Option::is_none")]
29    pub truncated: Option<ObservationTruncation>,
30}
31
32#[derive(Debug, Clone, Serialize, Deserialize)]
33pub struct Viewport {
34    pub width: u32,
35    pub height: u32,
36    pub device_scale_factor: f32,
37}
38
39#[derive(Debug, Clone, Serialize, Deserialize)]
40pub struct Frame {
41    pub frame_id: String,
42    pub url: String,
43}
44
45#[derive(Debug, Clone, Serialize, Deserialize)]
46pub struct Node {
47    /// Per-snapshot ref; not a durable selector.
48    pub r#ref: String,
49    pub frame_id: String,
50    pub role: String,
51    #[serde(skip_serializing_if = "Option::is_none")]
52    pub name: Option<String>,
53    #[serde(skip_serializing_if = "Option::is_none")]
54    pub text: Option<String>,
55    pub visible: bool,
56    pub enabled: bool,
57    #[serde(skip_serializing_if = "Option::is_none")]
58    pub bbox: Option<BBox>,
59    #[serde(default, skip_serializing_if = "Vec::is_empty")]
60    pub actions: Vec<String>,
61    #[serde(skip_serializing_if = "Option::is_none")]
62    pub href: Option<String>,
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub src: Option<String>,
65    #[serde(skip_serializing_if = "Option::is_none")]
66    pub frame_ref: Option<String>,
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub input_type: Option<String>,
69    #[serde(skip_serializing_if = "Option::is_none")]
70    pub checked: Option<bool>,
71    #[serde(skip_serializing_if = "Option::is_none")]
72    pub selected: Option<bool>,
73    #[serde(skip_serializing_if = "Option::is_none")]
74    pub focused: Option<bool>,
75    #[serde(skip_serializing_if = "Option::is_none")]
76    pub value_redacted: Option<bool>,
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub selector_hint: Option<String>,
79    #[serde(skip_serializing_if = "Option::is_none")]
80    pub selector_hint_unique: Option<bool>,
81}
82
83#[derive(Debug, Clone, Serialize, Deserialize)]
84pub struct BBox {
85    pub x: f32,
86    pub y: f32,
87    pub width: f32,
88    pub height: f32,
89}
90
91#[derive(Debug, Clone, Serialize, Deserialize)]
92pub struct Form {
93    pub r#ref: String,
94    #[serde(skip_serializing_if = "Option::is_none")]
95    pub action: Option<String>,
96    #[serde(default, skip_serializing_if = "Vec::is_empty")]
97    pub field_refs: Vec<String>,
98}
99
100#[derive(Debug, Clone, Serialize, Deserialize)]
101pub struct ObservationTruncation {
102    pub reason: String,
103    pub node_limit: usize,
104    pub scan_limit: usize,
105    pub scanned: usize,
106    pub emitted_nodes: usize,
107}
108
109/// Capture an `Observation` from a CDP session. Projects the accessibility
110/// tree + page title + viewport into the mechanical schema; never adds
111/// intent labels or importance ranks (see `DISALLOWED_LABELS`).
112pub async fn capture(
113    conn: &crate::sdk::cdp::ws_client::Connection,
114    session_id: &str,
115    url: &str,
116) -> Result<Observation, Error> {
117    // Page metadata.
118    let meta = conn
119        .send(
120            "Runtime.evaluate",
121            &serde_json::json!({
122                "expression": "JSON.stringify({title: document.title, w: window.innerWidth, h: window.innerHeight, dpr: window.devicePixelRatio})",
123                "returnByValue": true,
124            }),
125            Some(session_id),
126        )
127        .await?;
128    let title;
129    let viewport;
130    if let Some(s) = meta["result"]["value"].as_str() {
131        let v: serde_json::Value = serde_json::from_str(s).unwrap_or(serde_json::Value::Null);
132        title = v["title"].as_str().unwrap_or("").to_string();
133        viewport = Viewport {
134            width: v["w"].as_u64().unwrap_or(0) as u32,
135            height: v["h"].as_u64().unwrap_or(0) as u32,
136            device_scale_factor: v["dpr"].as_f64().unwrap_or(1.0) as f32,
137        };
138    } else {
139        title = String::new();
140        viewport = Viewport {
141            width: 0,
142            height: 0,
143            device_scale_factor: 1.0,
144        };
145    }
146
147    // Mechanical DOM projection for agent planning. This intentionally
148    // captures roles/states/geometry/actions without ranking or intent labels.
149    let dom = conn
150        .send(
151            "Runtime.evaluate",
152            &serde_json::json!({
153                "expression": OBSERVATION_JS,
154                "returnByValue": true,
155            }),
156            Some(session_id),
157        )
158        .await?;
159    let mut nodes: Vec<Node> = Vec::new();
160    let mut forms: Vec<Form> = Vec::new();
161    let mut frames: Vec<Frame> = vec![Frame {
162        frame_id: "main".into(),
163        url: url.to_string(),
164    }];
165    let mut focused_ref: Option<String> = None;
166    let mut truncated: Option<ObservationTruncation> = None;
167    if let Some(s) = dom["result"]["value"].as_str() {
168        let v: serde_json::Value = serde_json::from_str(s).unwrap_or(serde_json::Value::Null);
169        if let Ok(parsed_nodes) =
170            serde_json::from_value::<Vec<Node>>(v.get("nodes").cloned().unwrap_or_default())
171        {
172            nodes = parsed_nodes;
173        }
174        if let Ok(parsed_forms) =
175            serde_json::from_value::<Vec<Form>>(v.get("forms").cloned().unwrap_or_default())
176        {
177            forms = parsed_forms;
178        }
179        if let Ok(parsed_frames) =
180            serde_json::from_value::<Vec<Frame>>(v.get("frames").cloned().unwrap_or_default())
181            && !parsed_frames.is_empty()
182        {
183            frames = parsed_frames;
184        }
185        focused_ref = v
186            .get("focused_ref")
187            .and_then(|v| v.as_str())
188            .map(str::to_string);
189        if let Ok(parsed_truncation) = serde_json::from_value::<ObservationTruncation>(
190            v.get("truncated").cloned().unwrap_or_default(),
191        ) {
192            truncated = Some(parsed_truncation);
193        }
194    }
195
196    Ok(Observation {
197        schema_version: 1,
198        url: url.to_string(),
199        title,
200        viewport,
201        frames,
202        nodes,
203        forms,
204        focused_ref,
205        truncated,
206    })
207}
208
209const OBSERVATION_JS: &str = include_str!("../../../../assets/observation/snapshot.js");
210
211pub async fn write(paths: &ArtifactPaths, obs: &Observation) -> Result<PathBuf, Error> {
212    let target = paths.file_for(Artifact::Observation);
213    let bytes = serde_json::to_vec_pretty(obs).map_err(|e| {
214        Error::new(
215            ErrorCode::InternalError,
216            format!("serialize observation: {e}"),
217        )
218    })?;
219    writer::write_bytes(&target, &bytes).await?;
220    Ok(target)
221}
222
223/// Disallowed substrings in any string field of an `Observation`. Tests
224/// use this to enforce `design.md §"Observation is mechanical, not
225/// interpretive"`.
226pub const DISALLOWED_LABELS: &[&str] =
227    &["login", "captcha", "paywall", "important", "best", "likely"];
228
229#[cfg(test)]
230mod tests {
231    use super::*;
232
233    #[test]
234    fn schema_round_trips() {
235        let obs = Observation {
236            schema_version: 1,
237            url: "https://example.com/".into(),
238            title: "Example".into(),
239            viewport: Viewport {
240                width: 1280,
241                height: 720,
242                device_scale_factor: 1.0,
243            },
244            frames: vec![Frame {
245                frame_id: "main".into(),
246                url: "https://example.com/".into(),
247            }],
248            nodes: vec![Node {
249                r#ref: "obs-1".into(),
250                frame_id: "main".into(),
251                role: "button".into(),
252                name: Some("Submit".into()),
253                text: Some("Submit".into()),
254                visible: true,
255                enabled: true,
256                bbox: Some(BBox {
257                    x: 0.0,
258                    y: 0.0,
259                    width: 80.0,
260                    height: 32.0,
261                }),
262                actions: vec!["click".into()],
263                href: None,
264                src: None,
265                frame_ref: None,
266                input_type: None,
267                checked: None,
268                selected: None,
269                focused: None,
270                value_redacted: None,
271                selector_hint: None,
272                selector_hint_unique: None,
273            }],
274            forms: vec![],
275            focused_ref: None,
276            truncated: None,
277        };
278        let json = serde_json::to_string(&obs).unwrap_or_default();
279        let parsed: Observation = serde_json::from_str(&json).unwrap();
280        assert_eq!(parsed.nodes.len(), 1);
281        assert_eq!(parsed.nodes[0].role, "button");
282    }
283
284    #[test]
285    fn disallowed_labels_present_in_constant() {
286        for label in DISALLOWED_LABELS {
287            assert!(!label.is_empty());
288        }
289    }
290}
agent_first_http/sdk/fetch/artifacts/observation.rs

agent_first_http/sdk/fetch/artifacts/
observation.rs
