pagesift_engine/engine/
similar.rs

1//! `find_similar` 的对外入口：抓一页，从例子元素找结构相同的兄弟。
2
3use pagesift_api::ApiError;
4use pagesift_api::types::{ScrapeOptions, SimilarData, SimilarItem};
5use pagesift_html::{Html, Selector};
6
7use super::Scraper;
8use super::cpu::cpu_task;
9
10impl Scraper {
11    /// `selector` 取第一个命中的元素当例子；`opts` 要包含 `rawHtml` 格式。
12    pub async fn find_similar(
13        &self,
14        url: &str,
15        selector: &str,
16        opts: &ScrapeOptions,
17    ) -> Result<SimilarData, ApiError> {
18        // 写错的选择器在抓取前就报
19        Selector::parse(selector)
20            .map_err(|e| ApiError::invalid(format!("selector {selector:?}: {e}")))?;
21        let doc = self.scrape(url, opts).await?;
22        let html = doc.raw_html.unwrap_or_default();
23        let selector = selector.to_string();
24        cpu_task(move || similar_in(&html, &selector)).await?
25    }
26}
27
28fn similar_in(html: &str, selector: &str) -> Result<SimilarData, ApiError> {
29    let doc = Html::parse(html);
30    let sel = Selector::parse(selector)
31        .map_err(|e| ApiError::invalid(format!("selector {selector:?}: {e}")))?;
32    let example = doc.select(&sel).next().ok_or_else(|| {
33        ApiError::invalid(format!("selector {selector:?} matches nothing on the page"))
34    })?;
35    let Some(found) = pagesift_extract::locate::find_similar(example, doc.root()) else {
36        return Ok(SimilarData::default());
37    };
38    Ok(SimilarData {
39        selector: found.selector,
40        depth: Some(found.depth),
41        items: found
42            .items
43            .iter()
44            .map(|n| SimilarItem {
45                text: n.text().split_whitespace().collect::<Vec<_>>().join(" "),
46                html: n.html(),
47            })
48            .collect(),
49    })
50}
pagesift_engine/engine/similar.rs

pagesift_engine/engine/
similar.rs
