crossmap_cli/
read_source.rs1use std::io::Read as _;
5use std::time::Duration;
6
7use base64::Engine as _;
8use percent_encoding::percent_decode_str;
9
10use crate::argument::{STDIN, SUPPORTED_SCHEMES, scheme_of};
11
12pub const FETCH_TIMEOUT: Duration = Duration::from_secs(30);
17
18const BYTES_PER_MIB: u64 = 1024 * 1024;
19
20pub const MAX_RESPONSE_BYTES: u64 = 64 * BYTES_PER_MIB;
25
26#[derive(Clone, Copy, Debug)]
28pub struct ReadLimits {
29 pub timeout: Duration,
31 pub max_bytes: u64,
33}
34
35impl Default for ReadLimits {
36 fn default() -> Self {
37 ReadLimits {
38 timeout: FETCH_TIMEOUT,
39 max_bytes: MAX_RESPONSE_BYTES,
40 }
41 }
42}
43
44pub fn read_source(argument: &str, limits: ReadLimits) -> Result<String, String> {
52 if argument == STDIN {
53 let mut bytes = Vec::new();
54 std::io::stdin()
55 .read_to_end(&mut bytes)
56 .map_err(|error| error.to_string())?;
57 return Ok(String::from_utf8_lossy(&bytes).into_owned());
58 }
59 let Some(scheme) = scheme_of(argument) else {
60 return read_file(argument);
61 };
62 let scheme = scheme.to_ascii_lowercase();
63 match scheme.as_str() {
64 "file" => read_file(&file_url_path(argument)?),
65 "data" => read_data_url(argument),
66 "http" | "https" => fetch_text(argument, limits),
67 _ => {
68 let supported: Vec<String> = SUPPORTED_SCHEMES
69 .iter()
70 .map(|name| format!("{name}:"))
71 .collect();
72 Err(format!(
73 "unsupported scheme {scheme}: (supported: {}, plain paths and -)",
74 supported.join(", ")
75 ))
76 }
77 }
78}
79
80fn read_file(path: &str) -> Result<String, String> {
82 let bytes = std::fs::read(path).map_err(|error| error.to_string())?;
83 Ok(String::from_utf8_lossy(&bytes).into_owned())
84}
85
86fn file_url_path(url: &str) -> Result<String, String> {
89 let (host, path) = file_url_parts(url);
90 if cfg!(windows) {
91 windows_file_path(host, path)
92 } else {
93 posix_file_path(host, path)
94 }
95}
96
97fn file_url_parts(url: &str) -> (&str, &str) {
100 let rest = &url["file:".len()..];
101 let rest = rest.split(['?', '#']).next().unwrap_or("");
102 let Some(after) = rest.strip_prefix("//") else {
103 return ("", rest);
104 };
105 let (host, path) = after
106 .find('/')
107 .map_or((after, ""), |slash| after.split_at(slash));
108 if host.eq_ignore_ascii_case("localhost") {
109 ("", path)
110 } else {
111 (host, path)
112 }
113}
114
115fn posix_file_path(host: &str, path: &str) -> Result<String, String> {
117 if !host.is_empty() {
118 return Err(format!(
119 "File URL host must be \"localhost\" or empty on this platform: {host}"
120 ));
121 }
122 if path.is_empty() {
123 return Err("File URL path must be absolute".to_owned());
124 }
125 if path.contains("%2F") || path.contains("%2f") {
126 return Err("File URL path must not include encoded / characters".to_owned());
127 }
128 percent_decode(path)
129}
130
131fn windows_file_path(host: &str, path: &str) -> Result<String, String> {
134 let lower = path.to_ascii_lowercase();
135 if lower.contains("%2f") || lower.contains("%5c") {
136 return Err("File URL path must not include encoded \\ or / characters".to_owned());
137 }
138 let path = percent_decode(&path.replace('/', "\\"))?;
139 if !host.is_empty() {
140 return Ok(format!("\\\\{host}{path}"));
141 }
142 let local = path.strip_prefix('\\').unwrap_or(&path);
143 let mut characters = local.chars();
144 if characters
145 .next()
146 .is_some_and(|letter| letter.is_ascii_alphabetic())
147 && characters.next() == Some(':')
148 {
149 Ok(local.to_owned())
150 } else {
151 Err("File URL path must be absolute".to_owned())
152 }
153}
154
155fn percent_decode(path: &str) -> Result<String, String> {
156 percent_decode_str(path)
157 .decode_utf8()
158 .map(|decoded| decoded.into_owned())
159 .map_err(|error| error.to_string())
160}
161
162fn read_data_url(url: &str) -> Result<String, String> {
165 let rest = &url["data:".len()..];
166 let Some((meta, body)) = rest.split_once(',') else {
167 return Err("invalid data: URL, no comma".to_owned());
168 };
169 let body = body.split('#').next().unwrap_or("");
170 let bytes: Vec<u8> = percent_decode_str(body).collect();
171 let is_base64 = meta
172 .rsplit(';')
173 .next()
174 .is_some_and(|last| last.trim().eq_ignore_ascii_case("base64"));
175 let bytes = if is_base64 {
176 let compact: Vec<u8> = bytes
177 .into_iter()
178 .filter(|byte| !byte.is_ascii_whitespace())
179 .collect();
180 base64::engine::general_purpose::STANDARD
181 .decode(&compact)
182 .or_else(|_| base64::engine::general_purpose::STANDARD_NO_PAD.decode(&compact))
183 .map_err(|error| format!("invalid base64 in data: URL ({error})"))?
184 } else {
185 bytes
186 };
187 String::from_utf8(bytes).map_err(|error| format!("the data is not UTF-8 ({error})"))
188}
189
190fn fetch_text(url: &str, limits: ReadLimits) -> Result<String, String> {
191 let agent: ureq::Agent = ureq::Agent::config_builder()
192 .timeout_global(Some(limits.timeout))
193 .http_status_as_error(false)
194 .proxy(None)
195 .build()
196 .into();
197 let mut response = agent.get(url).call().map_err(|error| match error {
198 ureq::Error::Timeout(_) => format!("no response within {} ms", limits.timeout.as_millis()),
199 other => other.to_string(),
200 })?;
201 let status = response.status();
202 if !status.is_success() {
203 let reason = status.canonical_reason().unwrap_or("");
204 return Err(format!("HTTP {} {reason}", status.as_u16())
205 .trim_end()
206 .to_owned());
207 }
208 let declared = response
209 .headers()
210 .get("content-length")
211 .and_then(|length| length.to_str().ok())
212 .and_then(|length| length.trim().parse::<u64>().ok());
213 if let Some(declared) = declared
214 && declared > limits.max_bytes
215 {
216 return Err(format!(
217 "the response is {declared} bytes, over the {} byte limit",
218 limits.max_bytes
219 ));
220 }
221 let bytes = response
222 .body_mut()
223 .with_config()
224 .limit(limits.max_bytes)
225 .read_to_vec()
226 .map_err(|error| match error {
227 ureq::Error::Timeout(_) => format!(
228 "the body was not complete within {} ms",
229 limits.timeout.as_millis()
230 ),
231 ureq::Error::BodyExceedsLimit(_) => {
232 format!("the response is over the {} byte limit", limits.max_bytes)
233 }
234 other => other.to_string(),
235 })?;
236 String::from_utf8(bytes).map_err(|error| format!("the response is not UTF-8 ({error})"))
237}
238
239#[cfg(test)]
240mod tests {
241 use super::*;
242
243 fn absolute() -> Result<String, String> {
244 Err("File URL path must be absolute".to_owned())
245 }
246
247 #[test]
248 fn a_file_url_splits_into_host_and_path_without_query_or_fragment() {
249 assert_eq!(
250 file_url_parts("file:///tmp/a.json?x#y"),
251 ("", "/tmp/a.json")
252 );
253 assert_eq!(
254 file_url_parts("file://LOCALHOST/tmp/a.json"),
255 ("", "/tmp/a.json")
256 );
257 assert_eq!(
258 file_url_parts("file://server/share/a.json"),
259 ("server", "/share/a.json")
260 );
261 assert_eq!(file_url_parts("file:/tmp/a.json"), ("", "/tmp/a.json"));
262 }
263
264 #[test]
266 fn a_posix_path_is_the_decoded_url_path_with_no_host() {
267 let posix = |url| {
268 let (host, path) = file_url_parts(url);
269 posix_file_path(host, path)
270 };
271 assert_eq!(
272 posix("file:///tmp/a%20b.json"),
273 Ok("/tmp/a b.json".to_owned())
274 );
275 assert_eq!(
276 posix("file://localhost/tmp/a.json"),
277 Ok("/tmp/a.json".to_owned())
278 );
279 assert_eq!(
280 posix("file://example.test/x.json"),
281 Err(
282 "File URL host must be \"localhost\" or empty on this platform: example.test"
283 .to_owned()
284 )
285 );
286 assert_eq!(
287 posix("file:///tmp/a%2Fb.json"),
288 Err("File URL path must not include encoded / characters".to_owned())
289 );
290 }
291
292 #[test]
294 fn a_windows_path_needs_a_drive_letter_or_names_a_unc_share() {
295 let windows = |url| {
296 let (host, path) = file_url_parts(url);
297 windows_file_path(host, path)
298 };
299 assert_eq!(
300 windows("file:///C:/Users/a%20b/cw.json"),
301 Ok("C:\\Users\\a b\\cw.json".to_owned())
302 );
303 assert_eq!(
304 windows("file://localhost/d:/x.json"),
305 Ok("d:\\x.json".to_owned())
306 );
307 assert_eq!(
308 windows("file://server/share/x.json"),
309 Ok("\\\\server\\share\\x.json".to_owned())
310 );
311 assert_eq!(windows("file:///tmp/x.json"), absolute());
312 assert_eq!(windows("file:///1:/x.json"), absolute());
313 assert_eq!(
314 windows("file:///C:/a%5Cb.json"),
315 Err("File URL path must not include encoded \\ or / characters".to_owned())
316 );
317 assert_eq!(
318 windows("file:///C:/a%2fb.json"),
319 Err("File URL path must not include encoded \\ or / characters".to_owned())
320 );
321 }
322}