pagesift_engine/engine/
autoschema.rs

1//! 解析一次 → 缓存 → 自愈 → LLM 兜底（0.97）。
2//!
3//! - **页面快照**：每个有文字或关键属性的元素一行：短 id、角色、可见文字、关键属性。纯 HTTP 页从
4//!   pagesift 的 DOM 推出；`act` 和 `/interact` 在真实页面上用 `Accessibility.getFullAXTree`（`interact/act.rs`）。
5//!   快照的 token 上限 = 模型上下文 − 提示 − 输出上限（[`snapshot_budget`]）。
6//! - **生成 schema**：`{"type": "extract", "intent": "…"}` 没有 schema 时，意图加快照交给 LLM，要它输出
7//!   pagesift 的 schema JSON（不是代码），当场在页面上验证；为空或不合法时把错误反馈给模型，只重试一次。
8//!   缓存键是“注册域 + URL 形状 + 意图哈希”，存在 `pagesift-store`。
9//! - **语义字段**：字段只写 `describe` 时，第一次用时由 LLM 解析出选择器并写回缓存。
10//! - **自愈顺序**：缓存的选择器（精确）→ 0.93 的 `locate` → LLM 重新解析；走到哪一步写进 warnings。
11//! - **observe**：从 HTTP 抓到的页面给出候选元素（选择器、角色、名称）。`act` 不走这里：
12//!   它要在同一个真实页面上选和做，见 `interact/act.rs`。
13
14use pagesift_api::types::ScrapeOptions;
15use pagesift_api::{ApiError, ErrorCode};
16use pagesift_html::{Html, NodeRef};
17use pagesift_llm::{ChatClient, ChatMessage, ChatRequest, ResponseFormat, Role};
18use serde_json::{Value, json};
19
20use super::extract::llm_error;
21
22/// 快照一行里文字的最大字符数：超过的截断（快照按 token 总预算裁剪，这里只防一个元素吃掉全部预算）。
23/// [推导] 取总预算的 1/64（下限一句话的量级 80 字符）：64 个元素以上才会触发截断
24pub(super) fn per_node_chars(budget_tokens: usize) -> usize {
25    (budget_tokens / 64).max(80) // [推导] 见上
26}
27
28/// 快照的 token 预算：上下文长度减去提示和输出上限。
29pub fn snapshot_budget(context_tokens: usize, prompt_tokens: usize, max_output: usize) -> usize {
30    context_tokens.saturating_sub(prompt_tokens + max_output)
31}
32
33fn role(tag: &str) -> &'static str {
34    // [协议] 隐式 ARIA 角色（WAI-ARIA in HTML，W3C Recommendation）
35    match tag {
36        "a" => "link",
37        "button" => "button",
38        "input" | "textarea" => "textbox",
39        "select" => "combobox",
40        "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => "heading",
41        "img" => "img",
42        "li" => "listitem",
43        "ul" | "ol" => "list",
44        "table" => "table",
45        "tr" => "row",
46        "td" | "th" => "cell",
47        "nav" => "navigation",
48        "main" => "main",
49        "article" => "article",
50        "form" => "form",
51        "p" => "paragraph",
52        _ => "generic",
53    }
54}
55
56const KEY_ATTRS: &[&str] = &[
57    "id",
58    "class",
59    "href",
60    "src",
61    "alt",
62    "name",
63    "type",
64    "aria-label",
65    "title",
66    "itemprop",
67    "data-testid",
68    "role",
69];
70
71/// 快照：每行 `[n] role "文字" attr=value …`，按文档顺序，总 token 不超过 `budget`。
72/// 返回快照文字和 id → 元素的对照（observe 用）。
73pub fn snapshot<'a>(doc: &'a Html<'a>, budget: usize) -> (String, Vec<NodeRef<'a>>) {
74    let cap = per_node_chars(budget);
75    let mut out = String::new();
76    let mut nodes = Vec::new();
77    let mut used = 0usize;
78    for n in doc.root().descendants() {
79        let Some(el) = n.as_element() else { continue };
80        if matches!(
81            el.name(),
82            "script" | "style" | "noscript" | "template" | "head" | "svg"
83        ) {
84            continue;
85        }
86        let own: String = n
87            .children()
88            .filter_map(|c| c.as_text())
89            .collect::<Vec<_>>()
90            .join(" ");
91        let text: String = own.split_whitespace().collect::<Vec<_>>().join(" ");
92        let attrs: Vec<String> = KEY_ATTRS
93            .iter()
94            .filter_map(|k| el.attr(k).map(|v| format!("{k}={}", truncate(v, cap / 2))))
95            .collect();
96        if text.is_empty() && attrs.is_empty() {
97            continue;
98        }
99        let line = format!(
100            "[{}] {} <{}> {:?} {}\n",
101            nodes.len(),
102            role(el.name()),
103            el.name(),
104            truncate(&text, cap),
105            attrs.join(" ")
106        );
107        let t = model_tokens(&line);
108        if used + t > budget {
109            break;
110        }
111        used += t;
112        out.push_str(&line);
113        nodes.push(n);
114    }
115    (out, nodes)
116}
117
118/// 模型分词器下的 token 数估计：按字节数 / 3。实测（2026-11-03，Groq `openai/gpt-oss-120b`）：
119/// quotes.toscrape.com 的快照 6033 字节计 1969 个 token，3.06 字节/token，取整到 3（偏保守）。
120/// 快照里多是标签、属性和网址，按词数会严重低估；常说的“4 个字符一个 token”是英文散文的均值，也偏少。
121pub fn model_tokens(s: &str) -> usize {
122    s.len().div_ceil(3) // [推导] 见上，实测 3.06 字节/token 向下取整
123}
124
125fn truncate(s: &str, n: usize) -> String {
126    s.chars().take(n).collect()
127}
128
129/// 生成 schema 的系统提示（预算估算也用它的长度）。
130pub const SCHEMA_SYSTEM_TEXT: &str = SCHEMA_SYSTEM;
131const SCHEMA_SYSTEM: &str = "You write CSS extraction schemas for pagesift. Reply with one JSON object only: \
132{\"baseSelector\": \"<CSS selector of each repeated item, or of the single item>\", \"fields\": [{\"name\": \"<field>\", \
133\"selector\": \"<CSS selector relative to the item>\", \"type\": \"text\" | \"attribute\" | \"html\", \"attribute\": \"<attr, when type is attribute>\"}]}. \
134Use selectors that exist in the page snapshot (tags, ids, classes, attributes shown there). Prefer stable attributes and \
135semantic class names over positions. Do not invent fields that are not on the page.";
136
137/// LLM 调用的计数（重复运行应接近 0）。
138#[derive(Debug, Default, Clone, Copy)]
139pub struct LlmUse {
140    pub calls: usize,
141    pub tokens: usize,
142}
143
144/// `max_out`：输出上限，等于快照预算（`structured.rs` 的对称划分：输入、输出各一半）。
145pub(super) async fn ask(
146    llm: &ChatClient,
147    system: &str,
148    user: String,
149    max_out: usize,
150    used: &mut LlmUse,
151) -> Result<Value, ApiError> {
152    // 快照预算可能大于模型的输出上限（预算只把输出上限从输入里扣掉），超了服务回 400
153    let max_out = match llm.model_limits().await {
154        Some((_, Some(limit))) => max_out.min(limit),
155        _ => max_out,
156    };
157    let req = ChatRequest {
158        messages: vec![
159            ChatMessage::new(Role::System, system),
160            ChatMessage::new(Role::User, user.clone()),
161        ],
162        temperature: Some(0.0),
163        max_tokens: Some(max_out.min(u32::MAX as usize) as u32),
164        response_format: Some(ResponseFormat::JsonObject),
165    };
166    used.calls += 1;
167    let resp = llm.chat(&req).await.map_err(llm_error)?;
168    used.tokens += pagesift_extract::TokenIndex::new(system).len()
169        + pagesift_extract::TokenIndex::new(&user).len()
170        + pagesift_extract::TokenIndex::new(&resp.content).len();
171    let body = resp.content.trim();
172    let body = body
173        .strip_prefix("```json")
174        .or_else(|| body.strip_prefix("```"))
175        .map(|b| b.trim_end_matches("```").trim())
176        .unwrap_or(body);
177    serde_json::from_str(body).map_err(|e| {
178        ApiError::new(
179            ErrorCode::UpstreamError,
180            format!("the LLM did not return JSON: {e}"),
181        )
182    })
183}
184
185/// 在页面上验证 schema：能编译、至少抽出一个列表项、列表项里至少一个字段有值。
186pub fn validate_on(doc: &Html<'_>, schema: &Value) -> Result<Vec<Value>, String> {
187    let s = pagesift_extract::Schema::from_json(schema).map_err(|e| e.to_string())?;
188    let items = pagesift_extract::JsonCssExtractor::new(&s)
189        .map_err(|e| e.to_string())?
190        .extract_document(doc);
191    if items.is_empty() {
192        return Err(format!(
193            "the schema extracted nothing: baseSelector {:?} matched no element with any field",
194            s.base_selector
195        ));
196    }
197    Ok(items)
198}
199
200/// 意图 → schema：生成、验证，不合格时把错误反馈给模型重试一次。
201pub async fn generate(
202    llm: &ChatClient,
203    intent: &str,
204    url: &str,
205    html: &str,
206    budget: usize,
207    used: &mut LlmUse,
208) -> Result<Value, ApiError> {
209    let doc = Html::parse(html);
210    let (snap, _) = snapshot(&doc, budget);
211    let prompt = format!("URL: {url}\nWhat to extract: {intent}\n\nPage snapshot:\n{snap}");
212    let first = ask(llm, SCHEMA_SYSTEM, prompt.clone(), budget, used).await?;
213    match validate_on(&doc, &first) {
214        Ok(_) => Ok(first),
215        Err(why) => {
216            let retry = format!(
217                "{prompt}\n\nYour previous schema was:\n{first}\nIt failed on this page: {why}\nReturn a corrected schema."
218            );
219            let second = ask(llm, SCHEMA_SYSTEM, retry, budget, used).await?;
220            validate_on(&doc, &second).map_err(|why| {
221                ApiError::new(
222                    ErrorCode::UpstreamError,
223                    format!(
224                        "the generated schema does not work on the page after one correction: {why}"
225                    ),
226                )
227            })?;
228            Ok(second)
229        }
230    }
231}
232
233const FIELD_SYSTEM: &str = "You find CSS selectors for pagesift. Reply with one JSON object only: \
234{\"selector\": \"<CSS selector relative to the item element>\", \"type\": \"text\" | \"attribute\" | \"html\", \"attribute\": \"<attr or null>\"}. \
235Use only tags, classes and attributes that appear in the snapshot of the item.";
236
237/// 语义字段或失效字段：在第一个列表项的快照里让 LLM 找选择器。
238pub async fn resolve_field(
239    llm: &ChatClient,
240    describe: &str,
241    item_html: &str,
242    budget: usize,
243    used: &mut LlmUse,
244) -> Result<Value, ApiError> {
245    let doc = Html::parse(item_html);
246    let (snap, _) = snapshot(&doc, budget);
247    ask(
248        llm,
249        FIELD_SYSTEM,
250        format!("Field: {describe}\n\nItem snapshot:\n{snap}"),
251        budget,
252        used,
253    )
254    .await
255}
256
257const BASE_SYSTEM: &str = "You find CSS selectors for pagesift. Reply with one JSON object only: \
258{\"selector\": \"<CSS selector matching every repeated item that holds these fields, or the single item>\"}. \
259Use only tags, classes and attributes that appear in the page snapshot.";
260
261/// `baseSelector` 失效：在整页快照里让 LLM 找列表项。
262pub async fn resolve_base(
263    llm: &ChatClient,
264    fields: &str,
265    html: &str,
266    budget: usize,
267    used: &mut LlmUse,
268) -> Result<Value, ApiError> {
269    let doc = Html::parse(html);
270    let (snap, _) = snapshot(&doc, budget);
271    ask(
272        llm,
273        BASE_SYSTEM,
274        format!("Fields of each item: {fields}\n\nPage snapshot:\n{snap}"),
275        budget,
276        used,
277    )
278    .await
279}
280
281/// observe：意图对应的候选元素（选择器、角色、名称），选择器在页面上验证过恰好命中一个元素。
282pub async fn observe(
283    llm: &ChatClient,
284    instruction: &str,
285    html: &str,
286    budget: usize,
287    used: &mut LlmUse,
288) -> Result<Vec<Value>, ApiError> {
289    let doc = Html::parse(html);
290    let (snap, nodes) = snapshot(&doc, budget);
291    let v = ask(
292        llm,
293        "You pick page elements for an action. Reply with JSON {\"ids\": [<snapshot ids, best first>]}.",
294        format!("Instruction: {instruction}\n\nPage snapshot:\n{snap}"),
295        budget,
296        used,
297    )
298    .await?;
299    let mut out = Vec::new();
300    for id in v["ids"]
301        .as_array()
302        .into_iter()
303        .flatten()
304        .filter_map(Value::as_u64)
305    {
306        let Some(n) = nodes.get(id as usize) else {
307            continue;
308        };
309        let Some(sel) = pagesift_extract::locate::robust_selector(*n, doc.root()) else {
310            continue;
311        };
312        let el = n.as_element();
313        let name = el
314            .and_then(|e| e.attr("aria-label").or(e.attr("title")))
315            .map(str::to_string)
316            .unwrap_or_else(|| n.text().split_whitespace().collect::<Vec<_>>().join(" "));
317        out.push(
318            json!({ "selector": sel, "role": role(el.map_or("", |e| e.name())), "name": name }),
319        );
320    }
321    Ok(out)
322}
323
324/// `[secrets]` 的一项。
325#[derive(Debug, Clone, Default)]
326pub struct ActionSecret {
327    pub value: String,
328    /// 只有这些主机（含子域名）上的页面才能拿到值。
329    pub hosts: Vec<String>,
330}
331
332impl ActionSecret {
333    fn allows(&self, host: &str) -> bool {
334        self.hosts.iter().any(|h| {
335            let h = h.trim_end_matches('.');
336            host.eq_ignore_ascii_case(h)
337                || host.len().checked_sub(h.len() + 1).is_some_and(|i| {
338                    host.as_bytes()[i] == b'.' && host[i + 1..].eq_ignore_ascii_case(h)
339                })
340        })
341    }
342}
343
344/// act 的动作里 `{{secret:NAME}}` 换成本地配置的凭证；替换在发给浏览器之前做，模型看不到值。
345/// `page_url` 是值要填进去的页面。名字不存在和主机不符报同样的错：调用方不能用它探测配了哪些名字。
346pub fn fill_secrets(
347    actions: &mut [Value],
348    credentials: &std::collections::BTreeMap<String, ActionSecret>,
349    page_url: &str,
350) -> Result<(), ApiError> {
351    let parsed = url::Url::parse(page_url).ok();
352    let host = parsed
353        .as_ref()
354        .and_then(url::Url::host_str)
355        .unwrap_or_default();
356    let host = host.trim_end_matches('.');
357    for a in actions {
358        let Some(text) = a.get("text").and_then(Value::as_str) else {
359            continue;
360        };
361        let mut out = String::new();
362        let mut rest = text;
363        while let Some(i) = rest.find("{{secret:") {
364            out.push_str(&rest[..i]);
365            let after = &rest[i + "{{secret:".len()..];
366            let Some(j) = after.find("}}") else {
367                return Err(ApiError::invalid("unterminated {{secret:...}} placeholder"));
368            };
369            let name = &after[..j];
370            let v = credentials
371                .get(name)
372                .filter(|c| c.allows(host))
373                .ok_or_else(|| {
374                    ApiError::invalid(format!("secret {name:?} is not available on this page"))
375                })?;
376            out.push_str(&v.value);
377            rest = &after[j + 2..];
378        }
379        out.push_str(rest);
380        a["text"] = json!(out);
381    }
382    Ok(())
383}
384
385/// `intent` 的缓存键：注册域 + URL 形状 + 意图的 BLAKE3 前 16 个十六进制字符。
386pub fn cache_key(url: &str, intent: &str) -> Option<String> {
387    keyed("autoschema", url, intent)
388}
389
390/// `{kind}/注册域 + URL 形状#指令哈希`：同一类页面、同一句话共用一条缓存。
391pub fn keyed(kind: &str, url: &str, intent: &str) -> Option<String> {
392    let u = url::Url::parse(url).ok()?;
393    let site = pagesift_content::site_key(u.host_str()?);
394    let shape = pagesift_frontier::focus::url_shape(&u);
395    let h = blake3::hash(intent.trim().to_lowercase().as_bytes()).to_hex();
396    Some(format!("{kind}/{site}{shape}#{}", &h[..16]))
397}
398
399/// 选项里的意图（`{"type": "extract", "intent": "..."}`，没有 `schema` 时）。
400pub fn intent_of(opts: &ScrapeOptions) -> Option<String> {
401    let f = opts.formats.iter().find(|f| f.kind == "extract")?;
402    if f.options.contains_key("schema") {
403        return None;
404    }
405    f.options.get("intent")?.as_str().map(str::to_string)
406}
407
408impl super::Scraper {
409    /// observe：页面上和指令对应的候选元素（选择器、角色、名称），最合适的在前。
410    pub async fn observe(
411        &self,
412        url: &str,
413        instruction: &str,
414    ) -> Result<(Vec<Value>, LlmUse), ApiError> {
415        let llm = self.llm.as_ref().ok_or_else(|| {
416            ApiError::unsupported("observe needs an LLM: set [llm] in pagesift.toml")
417        })?;
418        let opts: ScrapeOptions = serde_json::from_value(json!({ "formats": ["rawHtml"] }))
419            .map_err(|e| ApiError::invalid(e.to_string()))?;
420        let doc = self.scrape(url, &opts).await?;
421        let html = doc.raw_html.unwrap_or_default();
422        let mut used = LlmUse::default();
423        let budget = self.snapshot_budget_pub().await;
424        let c = observe(llm, instruction, &html, budget, &mut used).await?;
425        Ok((c, used))
426    }
427}
428
429#[cfg(test)]
430mod tests {
431    use super::*;
432
433    #[test]
434    fn snapshot_respects_budget_and_secrets_stay_local() {
435        let html = "<ul>".to_string()
436            + &(0..200)
437                .map(|i| format!("<li class='item'><a href='/p/{i}'>Product {i}</a></li>"))
438                .collect::<String>()
439            + "</ul>";
440        let doc = Html::parse(&html);
441        let (snap, nodes) = snapshot(&doc, 300);
442        assert!(model_tokens(&snap) <= 300);
443        assert!(!nodes.is_empty() && nodes.len() < 200);
444        assert!(snap.contains("link <a> \"Product 0\" href=/p/0"), "{snap}");
445        let mut acts = vec![json!({"type": "write", "text": "user {{secret:PW}}!"})];
446        let creds = [(
447            "PW".to_string(),
448            ActionSecret {
449                value: "hunter2".to_string(),
450                hosts: vec!["example.com".to_string()],
451            },
452        )]
453        .into();
454        fill_secrets(&mut acts, &creds, "https://login.example.com/").unwrap();
455        assert_eq!(acts[0]["text"], "user hunter2!");
456        let mut bad = vec![json!({"type": "write", "text": "{{secret:NOPE}}"})];
457        let unknown = fill_secrets(&mut bad, &creds, "https://example.com/").unwrap_err();
458        // 主机不符和名字不存在说的一样；页面主机只是后缀相同也不算
459        for page in ["https://evil.test/", "https://notexample.com/", "not a url"] {
460            let mut acts = vec![json!({"type": "write", "text": "{{secret:PW}}"})];
461            let refused = fill_secrets(&mut acts, &creds, page).unwrap_err();
462            assert_eq!(acts[0]["text"], "{{secret:PW}}", "{page}");
463            assert!(
464                refused.message.contains("not available on this page"),
465                "{page}"
466            );
467            assert!(unknown.message.contains("not available on this page"));
468        }
469        let a = cache_key("https://www.shop.example.co.uk/item/123?ref=x", "Prices").unwrap();
470        let b = cache_key("https://a.example.co.uk/item/9?ref=1", " prices ").unwrap();
471        assert_eq!(a, b);
472        assert!(
473            a.starts_with("autoschema/example.co.uk/item/{n}?ref#"),
474            "{a}"
475        );
476    }
477}
pagesift_engine/engine/autoschema.rs

pagesift_engine/engine/
autoschema.rs
