dsq_io_https/
lib.rs

1//! HTTP(S) I/O plugin for dsq
2//!
3//! This crate provides functionality for fetching files from HTTP and HTTPS URLs.
4
5use reqwest::Client;
6use std::time::Duration;
7
8/// Error type for HTTP I/O operations
9pub type Result<T> = std::result::Result<T, Error>;
10
11/// HTTP I/O error type
12#[derive(Debug, thiserror::Error)]
13pub enum Error {
14    #[error("HTTP error: {0}")]
15    Http(String),
16    #[error("Other error: {0}")]
17    Other(String),
18}
19
20impl From<reqwest::Error> for Error {
21    fn from(e: reqwest::Error) -> Self {
22        Error::Http(e.to_string())
23    }
24}
25
26/// Fetch a file from an HTTP(S) URL
27///
28/// # Examples
29///
30/// ```rust,ignore
31/// use dsq_io_https::fetch_http;
32///
33/// let data = fetch_http("https://example.com/data.csv").await.unwrap();
34/// ```
35pub async fn fetch_http(url: &str) -> Result<Vec<u8>> {
36    let client = Client::builder()
37        .timeout(Duration::from_secs(300)) // 5 minute timeout
38        .build()
39        .map_err(|e| Error::Other(format!("Failed to create HTTP client: {e}")))?;
40
41    let response = client
42        .get(url)
43        .send()
44        .await
45        .map_err(|e| Error::Http(format!("Failed to fetch URL {url}: {e}")))?;
46
47    if !response.status().is_success() {
48        return Err(Error::Http(format!(
49            "HTTP request failed with status: {}",
50            response.status()
51        )));
52    }
53
54    let bytes = response
55        .bytes()
56        .await
57        .map_err(|e| Error::Http(format!("Failed to read response body: {e}")))?;
58
59    Ok(bytes.to_vec())
60}
61
62/// Synchronous version using tokio runtime
63pub fn fetch_http_sync(url: &str) -> Result<Vec<u8>> {
64    tokio::runtime::Runtime::new()
65        .map_err(|e| Error::Other(format!("Failed to create runtime: {e}")))?
66        .block_on(fetch_http(url))
67}
68
69/// Check if a string is an HTTP(S) URL
70pub fn is_http_url(s: &str) -> bool {
71    s.starts_with("http://") || s.starts_with("https://")
72}
73
74#[cfg(test)]
75mod tests {
76    use super::*;
77
78    #[test]
79    fn test_is_http_url() {
80        assert!(is_http_url("http://example.com/data.csv"));
81        assert!(is_http_url("https://example.com/data.csv"));
82        assert!(!is_http_url("file:///data.csv"));
83        assert!(!is_http_url("/path/to/file.csv"));
84        assert!(!is_http_url("data.csv"));
85    }
86}
dsq_io_https/lib.rs

dsq_io_https/
lib.rs
