trek-rs 0.2.1

A web content extraction library that removes clutter from web pages
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

//! Image element processing

use lol_html::html_content::Element;
use tracing::debug;

const MIN_DIMENSION: u32 = 50;

/// Process image elements
pub fn process_image(el: &mut Element) {
    if el.tag_name() == "img" {
        // Get dimensions to check if it's too small
        let width = el
            .get_attribute("width")
            .and_then(|w| w.parse::<u32>().ok())
            .unwrap_or(0);

        let height = el
            .get_attribute("height")
            .and_then(|h| h.parse::<u32>().ok())
            .unwrap_or(0);

        // Remove small images (likely icons or spacers)
        if (width > 0 && width < MIN_DIMENSION) || (height > 0 && height < MIN_DIMENSION) {
            debug!("Removing small image: {}x{}", width, height);
            el.remove();
            return;
        }

        // Remove tracking pixels
        if let Some(src) = el.get_attribute("src") {
            if src.contains("pixel")
                || src.contains("tracking")
                || src.contains("analytics")
                || src.contains("1x1")
            {
                debug!("Removing tracking pixel");
                el.remove();
                return;
            }
        }

        // Preserve only essential attributes
        let attrs_to_keep = ["src", "alt", "width", "height", "srcset"];
        let attrs_to_remove: Vec<String> = el
            .attributes()
            .iter()
            .filter(|attr| !attrs_to_keep.contains(&attr.name().as_str()))
            .map(lol_html::html_content::Attribute::name)
            .collect();

        for attr in attrs_to_remove {
            el.remove_attribute(&attr);
        }

        // Ensure alt attribute exists
        if el.get_attribute("alt").is_none() {
            let _ = el.set_attribute("alt", "");
        }
    }
}

/// Check if an image element should be preserved
pub fn should_preserve_image(el: &Element) -> bool {
    if el.tag_name() != "img" {
        return true;
    }

    // Check dimensions
    let width = el
        .get_attribute("width")
        .and_then(|w| w.parse::<u32>().ok())
        .unwrap_or(100); // Default to 100 if not specified

    let height = el
        .get_attribute("height")
        .and_then(|h| h.parse::<u32>().ok())
        .unwrap_or(100); // Default to 100 if not specified

    // Preserve images larger than 50x50
    width >= 50 && height >= 50
}

#[cfg(test)]
mod tests {

    #[test]
    fn test_should_preserve_image() {
        // Test with mock element
        let large_img = MockElement {
            tag: "img",
            width: Some("100"),
            height: Some("100"),
        };
        assert!(should_preserve_test_image(&large_img));

        let small_img = MockElement {
            tag: "img",
            width: Some("30"),
            height: Some("30"),
        };
        assert!(!should_preserve_test_image(&small_img));

        let div_element = MockElement {
            tag: "div",
            width: None,
            height: None,
        };
        assert!(should_preserve_test_image(&div_element)); // Non-img elements are preserved
    }

    struct MockElement {
        tag: &'static str,
        width: Option<&'static str>,
        height: Option<&'static str>,
    }

    fn should_preserve_test_image(el: &MockElement) -> bool {
        if el.tag != "img" {
            return true;
        }

        let width = el.width.and_then(|w| w.parse::<u32>().ok()).unwrap_or(100);

        let height = el.height.and_then(|h| h.parse::<u32>().ok()).unwrap_or(100);

        width >= 50 && height >= 50
    }
}