pagesift_engine/
discovery.rs

1//! Read-only document navigation from HTTP Link fields. Candidates still pass
2//! through the normal crawler scope, robots and budget checks.
3use url::Url;
4
5// A comma in a URI-reference or a quoted parameter is not a link separator.
6fn split_field(value: &str, delimiter: char) -> Vec<&str> {
7    let (mut quoted, mut escaped, mut angle, mut start) = (false, false, false, 0);
8    let mut parts = Vec::new();
9    for (i, c) in value.char_indices() {
10        if escaped {
11            escaped = false;
12        } else if quoted && c == '\\' {
13            escaped = true;
14        } else if c == '"' && !angle {
15            quoted = !quoted;
16        } else if !quoted {
17            match c {
18                '<' => angle = true,
19                '>' => angle = false,
20                _ if c == delimiter && !angle => {
21                    parts.push(&value[start..i]);
22                    start = i + c.len_utf8();
23                }
24                _ => {}
25            }
26        }
27    }
28    if !quoted && !angle {
29        parts.push(&value[start..]);
30    }
31    parts
32}
33
34pub fn header_links<'a>(base: &Url, fields: impl Iterator<Item = &'a str>) -> Vec<String> {
35    let mut out = Vec::new();
36    for field in fields {
37        for item in split_field(field, ',') {
38            let item = item.trim();
39            let Some(rest) = item.strip_prefix('<') else {
40                continue;
41            };
42            let Some((target, params)) = rest.split_once('>') else {
43                continue;
44            };
45            if !params.trim_start().starts_with(';') {
46                continue;
47            }
48            let mut navigation = false;
49            let mut anchored = false;
50            for param in split_field(params, ';') {
51                let Some((name, value)) = param.trim().split_once('=') else {
52                    continue;
53                };
54                let name = name.trim();
55                let value = value.trim();
56                // An anchor changes the link context: do not treat it as a
57                // navigation edge from the fetched document.
58                anchored |= name.eq_ignore_ascii_case("anchor");
59                if name.eq_ignore_ascii_case("rel") {
60                    let value = value
61                        .strip_prefix('"')
62                        .and_then(|v| v.strip_suffix('"'))
63                        .unwrap_or(value);
64                    navigation |= value
65                        .split_ascii_whitespace()
66                        .any(|r| r.eq_ignore_ascii_case("next") || r.eq_ignore_ascii_case("prev"));
67                }
68            }
69            if !navigation || anchored || target.trim().is_empty() || target.starts_with('#') {
70                continue;
71            }
72            if let Ok(mut url) = base.join(target)
73                && matches!(url.scheme(), "http" | "https")
74            {
75                url.set_fragment(None);
76                let url = url.to_string();
77                if !out.contains(&url) {
78                    out.push(url);
79                }
80            }
81        }
82    }
83    out
84}
85
86#[cfg(test)]
87mod tests {
88    use super::*;
89    #[test]
90    fn navigation_respects_field_syntax_and_context() {
91        let base = Url::parse("https://example.com/articles/1").unwrap();
92        let fields = [
93            r#"<2?a=1,2>; title="a,b;c"; rel="next", </style>; rel=preload"#,
94            r#"</other>; rel="prev"; anchor="/different", <javascript:bad>; rel=next"#,
95            r#"</previous#part>; REL="PREV", </previous>; rel=prev, </canonical>; rel=canonical"#,
96            r#"</bad>; rel="next"; title="unterminated"#,
97        ];
98        assert_eq!(
99            header_links(&base, fields.into_iter()),
100            [
101                "https://example.com/articles/2?a=1,2",
102                "https://example.com/previous"
103            ]
104        );
105    }
106}
pagesift_engine/discovery.rs

pagesift_engine/
discovery.rs
