readabilityrs 0.1.4

A Rust port of Mozilla's Readability library for extracting article content from web pages
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

use regex::Regex;
use std::sync::LazyLock;

use crate::markdown::options::MarkdownOptions;
use crate::markdown::rules::text::escape_url_destination;

static YOUTUBE_RE: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"(?i)youtube\.com|youtu\.be").unwrap());
static TWITTER_RE: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"(?i)twitter\.com|x\.com/\w+/status").unwrap());

/// Convert `<iframe>` to markdown placeholder.
pub fn convert_iframe(src: &str, opts: &MarkdownOptions) -> String {
    if src.is_empty() {
        return String::new();
    }
    if opts.sanitize_urls && crate::content_extractor::is_dangerous_url(src) {
        return String::new();
    }

    let label = if YOUTUBE_RE.is_match(src) {
        "Video"
    } else if TWITTER_RE.is_match(src) {
        "Tweet"
    } else {
        "Embed"
    };

    format!("[{}]({})", label, escape_url_destination(src))
}

/// Convert `<video>` to markdown placeholder.
pub fn convert_video(src: &str, opts: &MarkdownOptions) -> String {
    if src.is_empty() {
        return String::new();
    }
    if opts.sanitize_urls && crate::content_extractor::is_dangerous_url(src) {
        return String::new();
    }
    format!("[Video]({})", escape_url_destination(src))
}

/// Convert `<audio>` to markdown placeholder.
pub fn convert_audio(src: &str, opts: &MarkdownOptions) -> String {
    if src.is_empty() {
        return String::new();
    }
    if opts.sanitize_urls && crate::content_extractor::is_dangerous_url(src) {
        return String::new();
    }
    format!("[Audio]({})", escape_url_destination(src))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn opts() -> MarkdownOptions {
        MarkdownOptions::default()
    }

    #[test]
    fn test_youtube_iframe() {
        let result = convert_iframe("https://www.youtube.com/embed/abc123", &opts());
        assert_eq!(result, "[Video](https://www.youtube.com/embed/abc123)");
    }

    #[test]
    fn test_twitter_iframe() {
        let result = convert_iframe("https://twitter.com/user/status/123", &opts());
        assert_eq!(result, "[Tweet](https://twitter.com/user/status/123)");
    }

    #[test]
    fn test_generic_iframe() {
        let result = convert_iframe("https://example.com/widget", &opts());
        assert_eq!(result, "[Embed](https://example.com/widget)");
    }

    #[test]
    fn test_video_src_with_paren_wrapped() {
        let result = convert_video("http://e.com/) [x](http://evil.com)", &opts());
        assert_eq!(result, "[Video](<http://e.com/) [x](http://evil.com)>)");
    }

    #[test]
    fn test_iframe_dropped_when_sanitizing_dangerous_src() {
        let opts = MarkdownOptions {
            sanitize_urls: true,
            ..Default::default()
        };
        assert_eq!(convert_iframe("javascript:evil()", &opts), "");
    }

    #[test]
    fn test_video_dropped_when_sanitizing_dangerous_src() {
        let opts = MarkdownOptions {
            sanitize_urls: true,
            ..Default::default()
        };
        assert_eq!(convert_video("javascript:evil()", &opts), "");
    }
}