trek-rs 0.2.0

A web content extraction library that removes clutter from web pages
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

//! Footnote processing

use lol_html::html_content::Element;
use tracing::debug;

/// Process footnote elements
pub fn process_footnote(el: &mut Element) {
    // Check if this is a footnote reference
    if is_footnote_reference(el) {
        debug!("Processing footnote reference");
        // Add data attribute to mark it as a footnote
        el.set_attribute("data-footnote", "true").ok();
    }

    // Check if this is a footnote list
    if is_footnote_list(el) {
        debug!("Processing footnote list");
        el.set_attribute("data-footnote-list", "true").ok();
    }
}

/// Check if an element is a footnote reference
pub fn is_footnote_reference(element: &Element) -> bool {
    // Check for common footnote patterns in href
    if element.tag_name() == "a" {
        if let Some(href) = element.get_attribute("href") {
            if href.starts_with("#fn")
                || href.starts_with("#cite")
                || href.starts_with("#reference")
                || href.starts_with("#footnote")
            {
                return true;
            }
        }
    }

    // Check class names
    if let Some(class) = element.get_attribute("class") {
        if class.contains("footnote") || class.contains("reference") || class.contains("citation") {
            return true;
        }
    }

    false
}

/// Check if an element is a footnote list
pub fn is_footnote_list(element: &Element) -> bool {
    let tag = element.tag_name();

    // Check common footnote list patterns
    if tag == "ol" || tag == "ul" || tag == "div" {
        if let Some(class) = element.get_attribute("class") {
            if class.contains("footnotes")
                || class.contains("references")
                || class.contains("citations")
                || class.contains("endnotes")
            {
                return true;
            }
        }

        if let Some(id) = element.get_attribute("id") {
            if id.contains("footnotes")
                || id.contains("references")
                || id.contains("citations")
                || id.contains("endnotes")
            {
                return true;
            }
        }
    }

    false
}

#[cfg(test)]
mod tests {

    #[test]
    fn test_has_footnote_pattern() {
        let el_with_footnote = MockElement {
            class: Some("footnote"),
            id: None,
        };
        assert!(has_footnote_pattern(&el_with_footnote));

        let el_with_endnote = MockElement {
            class: Some("endnote-reference"),
            id: None,
        };
        assert!(has_footnote_pattern(&el_with_endnote));

        let el_without = MockElement {
            class: Some("content"),
            id: None,
        };
        assert!(!has_footnote_pattern(&el_without));
    }

    // Mock element for testing
    struct MockElement {
        class: Option<&'static str>,
        id: Option<&'static str>,
    }

    impl MockElement {
        fn get_attribute(&self, name: &str) -> Option<String> {
            match name {
                "class" => self.class.map(String::from),
                "id" => self.id.map(String::from),
                _ => None,
            }
        }
    }

    // Make has_footnote_pattern testable
    fn has_footnote_pattern(el: &MockElement) -> bool {
        const PATTERNS: &[&str] = &["footnote", "endnote", "reference"];

        if let Some(class) = el.get_attribute("class") {
            if PATTERNS.iter().any(|pattern| class.contains(pattern)) {
                return true;
            }
        }

        if let Some(id) = el.get_attribute("id") {
            if PATTERNS.iter().any(|pattern| id.contains(pattern)) {
                return true;
            }
        }

        false
    }
}