Skip to main content

crossmap_cli/
read_source.rs

1//! Reading the text a command-line argument names: standard input for `-`, a URL when the argument starts with a
2//! scheme, otherwise a file path.
3
4use std::io::Read as _;
5use std::time::Duration;
6
7use base64::Engine as _;
8use percent_encoding::percent_decode_str;
9
10use crate::argument::{STDIN, SUPPORTED_SCHEMES, scheme_of};
11
12/// How long one remote read may take, from sending the request to the last byte of the body. A command has no one
13/// watching a progress indicator, so a server that accepts the connection and then stalls would otherwise hang it for
14/// as long as the socket stays open. Generous enough for a slow link to deliver a document within
15/// [`MAX_RESPONSE_BYTES`].
16pub const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
17
18const BYTES_PER_MIB: u64 = 1024 * 1024;
19
20/// The most bytes a remote body may hold: 64 MiB. The body is buffered whole and then parsed as one JSON text, so the
21/// peak memory is a multiple of its size, and a server (or a redirect to one) that streams without end would otherwise
22/// grow the process until it is killed. A crosswalk is a few kilobytes and a document or sample set is data a person
23/// chose to apply it to, so the bound sits well above any plausible input.
24pub const MAX_RESPONSE_BYTES: u64 = 64 * BYTES_PER_MIB;
25
26/// The bounds on a remote read.
27#[derive(Clone, Copy, Debug)]
28pub struct ReadLimits {
29    /// The time one read may take.
30    pub timeout: Duration,
31    /// The most bytes a body may hold.
32    pub max_bytes: u64,
33}
34
35impl Default for ReadLimits {
36    fn default() -> Self {
37        ReadLimits {
38            timeout: FETCH_TIMEOUT,
39            max_bytes: MAX_RESPONSE_BYTES,
40        }
41    }
42}
43
44/// Reads the text an argument names. `http:` and `https:` are fetched, following redirects, within `limits`;
45/// `data:` is decoded; `file:` is turned into a path and read as paths are. Any other scheme is refused.
46///
47/// # Errors
48///
49/// The reason the source is unreadable, unsupported, slow or larger than the cap. The message does not repeat the
50/// argument; the caller names it.
51pub fn read_source(argument: &str, limits: ReadLimits) -> Result<String, String> {
52    if argument == STDIN {
53        let mut bytes = Vec::new();
54        std::io::stdin()
55            .read_to_end(&mut bytes)
56            .map_err(|error| error.to_string())?;
57        return Ok(String::from_utf8_lossy(&bytes).into_owned());
58    }
59    let Some(scheme) = scheme_of(argument) else {
60        return read_file(argument);
61    };
62    let scheme = scheme.to_ascii_lowercase();
63    match scheme.as_str() {
64        "file" => read_file(&file_url_path(argument)?),
65        "data" => read_data_url(argument),
66        "http" | "https" => fetch_text(argument, limits),
67        _ => {
68            let supported: Vec<String> = SUPPORTED_SCHEMES
69                .iter()
70                .map(|name| format!("{name}:"))
71                .collect();
72            Err(format!(
73                "unsupported scheme {scheme}: (supported: {}, plain paths and -)",
74                supported.join(", ")
75            ))
76        }
77    }
78}
79
80/// A file's text, with invalid UTF-8 replaced as Node's `readFileSync(path, "utf8")` replaces it.
81fn read_file(path: &str) -> Result<String, String> {
82    let bytes = std::fs::read(path).map_err(|error| error.to_string())?;
83    Ok(String::from_utf8_lossy(&bytes).into_owned())
84}
85
86/// The path a `file:` URL names on this platform, as Node's `fileURLToPath` gives it there: the Windows rules on
87/// Windows, the POSIX rules everywhere else.
88fn file_url_path(url: &str) -> Result<String, String> {
89    let (host, path) = file_url_parts(url);
90    if cfg!(windows) {
91        windows_file_path(host, path)
92    } else {
93        posix_file_path(host, path)
94    }
95}
96
97/// A `file:` URL's host and path, without its query or fragment. `localhost` is the empty host, as the URL standard
98/// parses it, and a URL with no `//` has an empty host.
99fn file_url_parts(url: &str) -> (&str, &str) {
100    let rest = &url["file:".len()..];
101    let rest = rest.split(['?', '#']).next().unwrap_or("");
102    let Some(after) = rest.strip_prefix("//") else {
103        return ("", rest);
104    };
105    let (host, path) = after
106        .find('/')
107        .map_or((after, ""), |slash| after.split_at(slash));
108    if host.eq_ignore_ascii_case("localhost") {
109        ("", path)
110    } else {
111        (host, path)
112    }
113}
114
115/// A POSIX path names no host, so the host must be empty, and the percent-decoded path is the file's path.
116fn posix_file_path(host: &str, path: &str) -> Result<String, String> {
117    if !host.is_empty() {
118        return Err(format!(
119            "File URL host must be \"localhost\" or empty on this platform: {host}"
120        ));
121    }
122    if path.is_empty() {
123        return Err("File URL path must be absolute".to_owned());
124    }
125    if path.contains("%2F") || path.contains("%2f") {
126        return Err("File URL path must not include encoded / characters".to_owned());
127    }
128    percent_decode(path)
129}
130
131/// A Windows path: with a host, the UNC path `\\host\path`; without one, the path must start with a drive letter
132/// (`/C:/...` gives `C:\...`). Every `/` becomes `\`, so an encoded `/` or `\` is refused as ambiguous.
133fn windows_file_path(host: &str, path: &str) -> Result<String, String> {
134    let lower = path.to_ascii_lowercase();
135    if lower.contains("%2f") || lower.contains("%5c") {
136        return Err("File URL path must not include encoded \\ or / characters".to_owned());
137    }
138    let path = percent_decode(&path.replace('/', "\\"))?;
139    if !host.is_empty() {
140        return Ok(format!("\\\\{host}{path}"));
141    }
142    let local = path.strip_prefix('\\').unwrap_or(&path);
143    let mut characters = local.chars();
144    if characters
145        .next()
146        .is_some_and(|letter| letter.is_ascii_alphabetic())
147        && characters.next() == Some(':')
148    {
149        Ok(local.to_owned())
150    } else {
151        Err("File URL path must be absolute".to_owned())
152    }
153}
154
155fn percent_decode(path: &str) -> Result<String, String> {
156    percent_decode_str(path)
157        .decode_utf8()
158        .map(|decoded| decoded.into_owned())
159        .map_err(|error| error.to_string())
160}
161
162/// The body of a `data:` URL, as `fetch` decodes it: percent-decoded, then base64-decoded when the media type ends in
163/// `;base64`.
164fn read_data_url(url: &str) -> Result<String, String> {
165    let rest = &url["data:".len()..];
166    let Some((meta, body)) = rest.split_once(',') else {
167        return Err("invalid data: URL, no comma".to_owned());
168    };
169    let body = body.split('#').next().unwrap_or("");
170    let bytes: Vec<u8> = percent_decode_str(body).collect();
171    let is_base64 = meta
172        .rsplit(';')
173        .next()
174        .is_some_and(|last| last.trim().eq_ignore_ascii_case("base64"));
175    let bytes = if is_base64 {
176        let compact: Vec<u8> = bytes
177            .into_iter()
178            .filter(|byte| !byte.is_ascii_whitespace())
179            .collect();
180        base64::engine::general_purpose::STANDARD
181            .decode(&compact)
182            .or_else(|_| base64::engine::general_purpose::STANDARD_NO_PAD.decode(&compact))
183            .map_err(|error| format!("invalid base64 in data: URL ({error})"))?
184    } else {
185        bytes
186    };
187    String::from_utf8(bytes).map_err(|error| format!("the data is not UTF-8 ({error})"))
188}
189
190fn fetch_text(url: &str, limits: ReadLimits) -> Result<String, String> {
191    let agent: ureq::Agent = ureq::Agent::config_builder()
192        .timeout_global(Some(limits.timeout))
193        .http_status_as_error(false)
194        .proxy(None)
195        .build()
196        .into();
197    let mut response = agent.get(url).call().map_err(|error| match error {
198        ureq::Error::Timeout(_) => format!("no response within {} ms", limits.timeout.as_millis()),
199        other => other.to_string(),
200    })?;
201    let status = response.status();
202    if !status.is_success() {
203        let reason = status.canonical_reason().unwrap_or("");
204        return Err(format!("HTTP {} {reason}", status.as_u16())
205            .trim_end()
206            .to_owned());
207    }
208    let declared = response
209        .headers()
210        .get("content-length")
211        .and_then(|length| length.to_str().ok())
212        .and_then(|length| length.trim().parse::<u64>().ok());
213    if let Some(declared) = declared
214        && declared > limits.max_bytes
215    {
216        return Err(format!(
217            "the response is {declared} bytes, over the {} byte limit",
218            limits.max_bytes
219        ));
220    }
221    let bytes = response
222        .body_mut()
223        .with_config()
224        .limit(limits.max_bytes)
225        .read_to_vec()
226        .map_err(|error| match error {
227            ureq::Error::Timeout(_) => format!(
228                "the body was not complete within {} ms",
229                limits.timeout.as_millis()
230            ),
231            ureq::Error::BodyExceedsLimit(_) => {
232                format!("the response is over the {} byte limit", limits.max_bytes)
233            }
234            other => other.to_string(),
235        })?;
236    String::from_utf8(bytes).map_err(|error| format!("the response is not UTF-8 ({error})"))
237}
238
239#[cfg(test)]
240mod tests {
241    use super::*;
242
243    fn absolute() -> Result<String, String> {
244        Err("File URL path must be absolute".to_owned())
245    }
246
247    #[test]
248    fn a_file_url_splits_into_host_and_path_without_query_or_fragment() {
249        assert_eq!(
250            file_url_parts("file:///tmp/a.json?x#y"),
251            ("", "/tmp/a.json")
252        );
253        assert_eq!(
254            file_url_parts("file://LOCALHOST/tmp/a.json"),
255            ("", "/tmp/a.json")
256        );
257        assert_eq!(
258            file_url_parts("file://server/share/a.json"),
259            ("server", "/share/a.json")
260        );
261        assert_eq!(file_url_parts("file:/tmp/a.json"), ("", "/tmp/a.json"));
262    }
263
264    // Each expectation is what Node's `fileURLToPath` returns or throws for the same URL on Linux or macOS.
265    #[test]
266    fn a_posix_path_is_the_decoded_url_path_with_no_host() {
267        let posix = |url| {
268            let (host, path) = file_url_parts(url);
269            posix_file_path(host, path)
270        };
271        assert_eq!(
272            posix("file:///tmp/a%20b.json"),
273            Ok("/tmp/a b.json".to_owned())
274        );
275        assert_eq!(
276            posix("file://localhost/tmp/a.json"),
277            Ok("/tmp/a.json".to_owned())
278        );
279        assert_eq!(
280            posix("file://example.test/x.json"),
281            Err(
282                "File URL host must be \"localhost\" or empty on this platform: example.test"
283                    .to_owned()
284            )
285        );
286        assert_eq!(
287            posix("file:///tmp/a%2Fb.json"),
288            Err("File URL path must not include encoded / characters".to_owned())
289        );
290    }
291
292    // Each expectation is what Node's `fileURLToPath` returns or throws for the same URL on Windows.
293    #[test]
294    fn a_windows_path_needs_a_drive_letter_or_names_a_unc_share() {
295        let windows = |url| {
296            let (host, path) = file_url_parts(url);
297            windows_file_path(host, path)
298        };
299        assert_eq!(
300            windows("file:///C:/Users/a%20b/cw.json"),
301            Ok("C:\\Users\\a b\\cw.json".to_owned())
302        );
303        assert_eq!(
304            windows("file://localhost/d:/x.json"),
305            Ok("d:\\x.json".to_owned())
306        );
307        assert_eq!(
308            windows("file://server/share/x.json"),
309            Ok("\\\\server\\share\\x.json".to_owned())
310        );
311        assert_eq!(windows("file:///tmp/x.json"), absolute());
312        assert_eq!(windows("file:///1:/x.json"), absolute());
313        assert_eq!(
314            windows("file:///C:/a%5Cb.json"),
315            Err("File URL path must not include encoded \\ or / characters".to_owned())
316        );
317        assert_eq!(
318            windows("file:///C:/a%2fb.json"),
319            Err("File URL path must not include encoded \\ or / characters".to_owned())
320        );
321    }
322}