web2llm 0.4.0

Fetch web pages and convert to clean Markdown for LLM pipelines
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

//! Content extraction and DOM-to-Markdown conversion engine.
//!
//! This module provides the high-level `PageElements` type, which handles
//! DOM parsing, title extraction, and the final conversion into structurally-aware
//! Markdown results using the internal scoring engine.

pub(crate) mod scorer;

use url::Url;

use crate::config::Web2llmConfig;
use crate::error::{Result, Web2llmError};
use crate::fetch::{FetchMode, get_html};
use crate::output::PageResult;

#[cfg(feature = "rendered")]
use tokio::sync::OnceCell;

/// A parsed representation of a web page's DOM, optimized for extraction.
///
/// `PageElements` stores the raw HTML content and metadata (URL, title) and provides
/// the machinery to identify "meaty" content and convert it into Markdown chunks.
pub struct PageElements {
    html_content: String,
    url: Url,
    title: String,
}

impl PageElements {
    /// Parses the HTML content of a page from the given URL using the specified fetch mode.
    ///
    /// This method performs the initial DOM parse to extract the page title
    /// and prepares the internal state for full content extraction.
    pub(crate) async fn parse(
        url: Url,
        client: &reqwest::Client,
        mode: FetchMode,
        #[cfg(feature = "rendered")] browser: &OnceCell<chromiumoxide::Browser>,
    ) -> Result<Self> {
        #[cfg(feature = "rendered")]
        let (html_content, _is_dynamic) = get_html(&url, client, mode, browser).await?;

        #[cfg(not(feature = "rendered"))]
        let (html_content, _is_dynamic) = get_html(&url, client, mode).await?;

        // Extract title quickly
        let title = {
            let dom = tl::parse(&html_content, tl::ParserOptions::default()).unwrap();
            let parser = dom.parser();
            dom.query_selector("title")
                .and_then(|mut iter| iter.next())
                .map(|node| node.get(parser).unwrap().inner_text(parser).into_owned())
                .unwrap_or_default()
        };

        Ok(Self {
            html_content,
            url,
            title,
        })
    }

    /// Processes the DOM using the scoring engine and returns a `PageResult`.
    ///
    /// This is the primary entry point for the extraction pipeline. It identifies
    /// the main content areas, prunes noise, and chunks the resulting Markdown
    /// according to the provided configuration.
    pub(crate) fn into_result(self, config: &Web2llmConfig) -> Result<PageResult> {
        let dom = tl::parse(&self.html_content, tl::ParserOptions::default()).unwrap();
        let parser = dom.parser();

        let body_handle = dom
            .query_selector("body")
            .and_then(|mut iter| iter.next())
            .ok_or(Web2llmError::EmptyContent)?;

        let chunks = scorer::process(body_handle, parser, config)?;

        if chunks.is_empty() {
            return Err(Web2llmError::EmptyContent);
        }

        Ok(PageResult::new(self.url.as_str(), &self.title, chunks))
    }

    /// Extracts all unique, absolute URLs from `<a>` tags within the page.
    pub fn get_urls(&self) -> Vec<String> {
        let dom = tl::parse(&self.html_content, tl::ParserOptions::default()).unwrap();
        let parser = dom.parser();

        let mut urls: Vec<String> = Vec::new();
        if let Some(iter) = dom.query_selector("a[href]") {
            for node_handle in iter {
                if let Some(href) = node_handle
                    .get(parser)
                    .and_then(|n| n.as_tag())
                    .and_then(|t| t.attributes().get("href").flatten())
                {
                    let href_str = std::str::from_utf8(href.as_bytes()).unwrap_or("");
                    if let Ok(joined) = self.url.join(href_str) {
                        urls.push(joined.to_string());
                    }
                }
            }
        }

        urls.sort();
        urls.dedup();
        urls
    }
}