mle 0.28.0

The markup link extractor (mle) extracts links from markup files (Markdown and HTML).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134

// SPDX-FileCopyrightText: 2022 - 2025 Robin Vobruba <hoijui.quaero@gmail.com>
// SPDX-FileCopyrightText: 2020 Armin Becher <becherarmin@gmail.com>
//
// SPDX-License-Identifier: AGPL-3.0-or-later

mod html;
mod markdown;

use crate::anchor::Anchor;
use crate::config::Extractor as Config;
use crate::link::Link;
use crate::markup::{self, File};

pub struct ParseRes {
    pub links: Vec<Link>,
    pub anchors: Vec<Anchor>,
}

impl ParseRes {
    #[must_use]
    pub fn invalid_links(&self) -> Vec<&Link> {
        self.links
            .iter()
            .filter(|&link| link.is_invalid())
            .collect()
    }
}

pub fn remove_anchor(link: &mut String) -> Option<String> {
    link.find('#').map(|anchor_pos| {
        // let anchor = link.rsplit(pat: P)(suffix: P)(new_len: usize)
        let anchor: String = link.drain(anchor_pos..).skip(1).collect();
        // link.truncate(anchor_pos);
        anchor
    })
}

/// Finds links (and optionally anchors),
/// using the markup file specific link extractor internally.
///
/// # Errors
///
/// If fetching the markup file content failed.
pub async fn scan_for_links<LR: AsyncFnMut(Link), AR: AsyncFnMut(Anchor)>(
    file: &File<'_>,
    conf: &Config,
    links_receiver: &mut LR,
    anchors_receiver: &mut AR,
) -> std::io::Result<()> {
    let link_extractor = link_extractor_factory(file.markup_type);

    log::debug!(
        "Scanning file at location '{:#?}' for links ...",
        file.locator
    );
    link_extractor
        .find_links_and_anchors(file, conf, links_receiver, anchors_receiver)
        .await
}

/// Finds links (and optionally anchors),
/// using the markup file specific link extractor internally.
///
/// # Errors
///
/// If fetching the markup file content failed.
pub async fn gather_links(file: &File<'_>, conf: &Config) -> std::io::Result<ParseRes> {
    let mut links = vec![];
    let mut anchors = vec![];
    let links_receiver = &mut async |link: Link| {
        for link_ignorer in &conf.ignore_links {
            let link_as_str = link.target.to_string();
            if link_ignorer.matches(&link_as_str) {
                return;
            }
        }
        links.push(link);
    };
    let anchors_receiver = &mut async |anchor: Anchor| {
        anchors.push(anchor);
    };
    scan_for_links(file, conf, links_receiver, anchors_receiver).await?;
    Ok(ParseRes { links, anchors })
}

enum LinkExtractorCont {
    Markdown(markdown::LinkExtractor),
    Html(html::LinkExtractor),
}

impl LinkExtractor for LinkExtractorCont {
    async fn find_links_and_anchors<LR: AsyncFnMut(Link), AR: AsyncFnMut(Anchor)>(
        &self,
        file: &File<'_>,
        conf: &Config,
        links_receiver: &mut LR,
        anchors_receiver: &mut AR,
    ) -> std::io::Result<()> {
        match self {
            Self::Markdown(internal) => {
                internal
                    .find_links_and_anchors(file, conf, links_receiver, anchors_receiver)
                    .await
            }
            Self::Html(internal) => {
                internal
                    .find_links_and_anchors(file, conf, links_receiver, anchors_receiver)
                    .await
            }
        }
    }
}

const fn link_extractor_factory(markup_type: markup::Type) -> LinkExtractorCont {
    match markup_type {
        markup::Type::Markdown => LinkExtractorCont::Markdown(markdown::LinkExtractor()),
        markup::Type::Html => LinkExtractorCont::Html(html::LinkExtractor()),
    }
}

pub(crate) trait LinkExtractor {
    /// Finds links (and optionally anchors),
    /// using the markup file specific link extractor internally.
    ///
    /// # Errors
    /// If fetching the markup file content failed.
    async fn find_links_and_anchors<LR: AsyncFnMut(Link), AR: AsyncFnMut(Anchor)>(
        &self,
        file: &File<'_>,
        conf: &Config,
        links_receiver: &mut LR,
        anchors_receiver: &mut AR,
    ) -> std::io::Result<()>;
}