fixred 1.1.3

Command line tool to fix outdated URLs in files with redirected ones
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148

use anyhow::Result;
use chashmap::CHashMap;
use curl::easy::Easy;
use log::{debug, warn};

pub trait Resolver: Default + Sync {
    fn shallow(&mut self, b: bool);
    fn resolve(&self, url: &str) -> Option<String>;
}

#[derive(Default)]
pub struct CurlResolver {
    shallow: bool,
    cache: CHashMap<String, Option<String>>,
}

impl CurlResolver {
    fn try_resolve(&self, url: &str) -> Result<Option<String>> {
        debug!("Resolving {}", url);

        if let Some(u) = self.cache.get(url) {
            debug!("Cache hit: {} -> {:?}", url, *u);
            return Ok(u.clone());
        }

        // https://datatracker.ietf.org/doc/html/rfc3986#section-3
        let fragment = url.find('#').map(|i| &url[i + 1..]);

        debug!("Sending HEAD request to {}", url);
        let mut curl = Easy::new();
        curl.nobody(true)?;
        curl.url(url)?;
        let resolved = if self.shallow {
            curl.perform()?;
            curl.redirect_url()? // Get the first redirect URL
        } else {
            curl.follow_location(true)?;
            curl.perform()?;
            curl.effective_url()?
        };
        let red = resolved.and_then(|u| {
            (u != url).then(|| {
                if let Some(fragment) = fragment {
                    format!("{}#{}", u, fragment)
                } else {
                    u.to_string()
                }
            })
        });
        debug!("Resolved redirect: {} -> {:?}", url, red);
        self.cache.insert(url.to_string(), red.clone());
        Ok(red)
    }
}

impl Resolver for CurlResolver {
    fn shallow(&mut self, enabled: bool) {
        self.shallow = enabled;
    }

    fn resolve(&self, url: &str) -> Option<String> {
        // Do not return error on resolving URLs because it is normal case that broken URL is passed to this function.
        match self.try_resolve(url) {
            Ok(ret) => ret,
            Err(err) => {
                warn!("Could not resolve {:?}: {}", url, err);
                None
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn resolve_url_with_cache() {
        // Redirect: github.com/rhysd/ -> github.com/vim-crystal/ -> raw.githubusercontent
        let url = "https://github.com/rhysd/vim-crystal/raw/master/README.md";

        let res = CurlResolver::default();
        let resolved = res.try_resolve(url).unwrap();
        let resolved = resolved.unwrap();
        assert!(
            resolved.starts_with("https://raw.githubusercontent.com/vim-crystal/"),
            "URL: {}",
            resolved
        );

        assert_eq!(*res.cache.get(url).unwrap(), Some(resolved.clone()));

        let cached = res.try_resolve(url).unwrap();
        assert_eq!(resolved, cached.unwrap());
    }

    #[test]
    fn resolve_shallow_redirect() {
        // Redirect: github.com/rhysd/ -> github.com/vim-crystal/ -> raw.githubusercontent
        let url = "https://github.com/rhysd/vim-crystal/raw/master/README.md";

        let mut res = CurlResolver::default();
        res.shallow(true);
        let resolved = res.try_resolve(url).unwrap();
        let resolved = resolved.unwrap();
        assert!(
            resolved.starts_with("https://github.com/vim-crystal/vim-crystal/"),
            "URL: {}",
            resolved
        );
    }

    #[test]
    fn resolve_url_not_found() {
        // Redirect: github.com/rhysd/ -> github.com/vim-crystal/ -> raw.githubusercontent
        let url = "https://github.com/rhysd/this-repo-does-not-exist";

        let res = CurlResolver::default();
        let resolved = res.resolve(url);
        assert_eq!(resolved, None);

        assert_eq!(*res.cache.get(url).unwrap(), None);

        let cached = res.resolve(url);
        assert_eq!(resolved, cached);
    }

    #[test]
    fn resolve_url_with_fragment() {
        // Redirect: github.com/rhysd/ -> github.com/vim-crystal
        let url = "https://github.com/rhysd/vim-crystal#readme";

        let res = CurlResolver::default();
        let resolved = res.resolve(url).unwrap();
        assert!(
            resolved.ends_with("/vim-crystal#readme"),
            "URL: {}",
            resolved
        );
    }

    #[test]
    fn url_parse_error() {
        let res = CurlResolver::default();
        let resolved = res.try_resolve("https://");
        assert!(resolved.is_err(), "{:?}", resolved);
    }
}