web2llm 0.4.0

Fetch web pages and convert to clean Markdown for LLM pipelines
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133

use web2llm::{Web2llm, Web2llmConfig};
use wiremock::matchers::method;
use wiremock::{Mock, MockServer, ResponseTemplate};

fn test_client() -> Web2llm {
    let config = Web2llmConfig {
        block_private_hosts: false,
        ..Default::default()
    };
    Web2llm::new(config).unwrap()
}

/// Verifies that an article scores higher than a nav element,
/// and only the article is included in the output.
#[tokio::test]
async fn test_article_scores_higher_than_nav() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .respond_with(ResponseTemplate::new(200).set_body_string(
            r#"
            <html><body>
                <nav>
                    <ul><li>Home</li><li>About</li><li>Contact</li></ul>
                </nav>
                <article>
                    This is the main content of the page. It has more words than the navigation bar.
                    Therefore, it should receive a higher score and be the only part included.
                </article>
            </body></html>
        "#,
        ))
        .mount(&server)
        .await;

    let result = test_client().fetch(&server.uri()).await.unwrap();
    assert!(result.markdown().contains("main content"));
    assert!(!result.markdown().contains("Home"));
}

/// Verifies that if multiple content sections score highly, they are
/// both included in the final Markdown.
#[tokio::test]
async fn test_two_content_sections_both_included() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .respond_with(ResponseTemplate::new(200).set_body_string(
            r#"
            <html><body>
                <section id="one">
                    This is the first section of high-quality content that should be kept.
                </section>
                <section id="two">
                    This is the second section of high-quality content that should also be kept.
                </section>
            </body></html>
        "#,
        ))
        .mount(&server)
        .await;

    let result = test_client().fetch(&server.uri()).await.unwrap();
    assert!(result.markdown().contains("first section"));
    assert!(result.markdown().contains("second section"));
}

/// Verifies that headers and footers are excluded when there's better
/// content available in an article.
#[tokio::test]
async fn test_footer_excluded_when_article_present() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .respond_with(ResponseTemplate::new(200).set_body_string(
            r#"
            <html><body>
                <header><h1>Site Title</h1></header>
                <article>
                    This is the actual article content that we want to extract from the page.
                    It should be kept because it scores higher than the boilerplate.
                </article>
                <footer>Copyright 2024. All rights reserved. No part of this site...</footer>
            </body></html>
        "#,
        ))
        .mount(&server)
        .await;

    let result = test_client().fetch(&server.uri()).await.unwrap();
    assert!(result.markdown().contains("actual article content"));
    assert!(!result.markdown().contains("Copyright"));
}

/// Verifies that a single short article is still returned even if there's
/// nothing to compare it against, as long as it's not noise.
#[tokio::test]
async fn test_single_short_article_is_returned() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .respond_with(ResponseTemplate::new(200).set_body_string(
            r#"
            <html><body>
                <article>
                    Just some short but meaningful text content.
                </article>
            </body></html>
        "#,
        ))
        .mount(&server)
        .await;

    let result = test_client().fetch(&server.uri()).await.unwrap();
    assert!(result.markdown().contains("meaningful text"));
}

/// Verifies that if all content scores poorly (below the absolute threshold),
/// an EmptyContent error is returned.
#[tokio::test]
async fn test_only_noise_returns_empty_content() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .respond_with(ResponseTemplate::new(200).set_body_string(
            r#"
            <html><body>
                <nav>Short</nav>
                <footer>Short</footer>
            </body></html>
        "#,
        ))
        .mount(&server)
        .await;

    let result = test_client().fetch(&server.uri()).await;
    assert!(result.is_err());
}