oratos-html 0.2.0

HTML loading and page extraction for Oratos
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

use oratos_html::{load_pages, CrawlOptions, LoadOptions};
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};

const REMOTE_HTML: &str = r#"<!DOCTYPE html>
<html lang="en">
<head><title>Remote Page</title></head>
<body><main><h1>Hello</h1><p>Content served by the mock HTTP server for integration testing.</p></main>
</body></html>"#;

#[tokio::test]
async fn load_pages_fetches_html_over_http() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/page.html"))
        .respond_with(ResponseTemplate::new(200).set_body_string(REMOTE_HTML))
        .mount(&server)
        .await;

    let url = format!("{}/page.html", server.uri());
    let pages = load_pages(&url, &LoadOptions::default())
        .await
        .expect("load_pages");

    assert_eq!(pages.len(), 1);
    assert_eq!(pages[0].title.as_deref(), Some("Remote Page"));
    assert_eq!(pages[0].url_or_path, url);
    assert!(pages[0].has_main_landmark);
}

#[tokio::test]
async fn load_pages_strips_url_fragment_before_fetch() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/"))
        .respond_with(ResponseTemplate::new(200).set_body_string(REMOTE_HTML))
        .mount(&server)
        .await;

    let url = format!("{}/#section", server.uri());
    let pages = load_pages(&url, &LoadOptions::default())
        .await
        .expect("load_pages");

    assert_eq!(pages.len(), 1);
    assert!(!pages[0].url_or_path.contains('#'));
}

const CRAWL_INDEX: &str = r#"<!DOCTYPE html>
<html lang="en"><head><title>Home</title></head>
<body><main><h1>Home</h1><a href="/about.html">About</a></main></body></html>"#;

const CRAWL_ABOUT: &str = r#"<!DOCTYPE html>
<html lang="en"><head><title>About</title></head>
<body><main><h1>About page</h1></main></body></html>"#;

#[tokio::test]
async fn load_pages_crawls_same_origin_links() {
    let server = MockServer::start().await;
    for p in ["/", "/index.html", "/about.html"] {
        let body = if p.contains("about") {
            CRAWL_ABOUT
        } else {
            CRAWL_INDEX
        };
        Mock::given(method("GET"))
            .and(path(p))
            .respond_with(ResponseTemplate::new(200).set_body_string(body))
            .mount(&server)
            .await;
    }

    let options = LoadOptions {
        crawl: Some(CrawlOptions {
            max_pages: 5,
            max_depth: 2,
            respect_robots: false,
            use_sitemap: false,
        }),
        ..Default::default()
    };
    let pages = load_pages(&server.uri(), &options)
        .await
        .expect("crawl load");

    assert!(pages.len() >= 2);
    assert!(pages.iter().any(|p| p.title.as_deref() == Some("About")));
}