scirs2_datasets/
external.rs

1//! External data sources integration
2//!
3//! This module provides functionality for loading datasets from external sources including:
4//! - URLs and web resources
5//! - API endpoints
6//! - Popular dataset repositories
7//! - Remote file systems
8
9use std::collections::HashMap;
10use std::io::Read;
11use std::path::Path;
12use std::time::Duration;
13
14use scirs2_core::ndarray::{Array1, Array2};
15use serde::{Deserialize, Serialize};
16
17use crate::cache::DatasetCache;
18use crate::error::{DatasetsError, Result};
19use crate::loaders::{load_csv, CsvConfig};
20use crate::utils::Dataset;
21
22/// Configuration for external data source access
23#[derive(Debug, Clone, Serialize, Deserialize)]
24pub struct ExternalConfig {
25    /// Timeout for requests (in seconds)
26    pub timeout_seconds: u64,
27    /// Number of retry attempts
28    pub max_retries: u32,
29    /// User agent string for requests
30    pub user_agent: String,
31    /// Headers to include in requests
32    pub headers: HashMap<String, String>,
33    /// Whether to verify SSL certificates
34    pub verify_ssl: bool,
35    /// Cache downloaded files
36    pub use_cache: bool,
37}
38
39impl Default for ExternalConfig {
40    fn default() -> Self {
41        Self {
42            timeout_seconds: 300, // 5 minutes
43            max_retries: 3,
44            user_agent: "scirs2-datasets/0.1.0".to_string(),
45            headers: HashMap::new(),
46            verify_ssl: true,
47            use_cache: true,
48        }
49    }
50}
51
52/// Progress callback for download operations
53pub type ProgressCallback = Box<dyn Fn(u64, u64) + Send + Sync>;
54
55/// External data source client
56///
57/// # TLS crypto provider
58///
59/// When built with the `download` feature, [`ExternalClient::new`] / [`ExternalClient::with_config`]
60/// construct a [`reqwest::Client`] eagerly, which builds its rustls `ClientConfig` inside
61/// `.build()` even for a client that never issues an HTTPS request. Since the workspace's
62/// `reqwest` dependency uses `rustls-no-provider` (no aws-lc-rs/ring bundled -- see the
63/// "PURE RUST BLOCKER (reqwest)" note in the workspace root `Cargo.toml`), this crate installs
64/// the pure-Rust OxiTLS provider (`oxitls-rustcrypto-provider`, COOLJAPAN's RUSTSEC-2026-0104-fixed
65/// fork of the abandoned upstream `rustls-rustcrypto` 0.0.2-alpha, resolved under the
66/// `rustls-rustcrypto` name) as the process-default rustls `CryptoProvider` automatically,
67/// right before the first client is constructed -- unless the application has already installed
68/// a provider of its own. Applications that want a different provider (e.g. a C-linked
69/// aws-lc-rs/ring one) simply call `rustls::crypto::CryptoProvider::install_default(...)`
70/// before the first use of SciRS2 networking; an existing process default is never overridden.
71pub struct ExternalClient {
72    config: ExternalConfig,
73    cache: DatasetCache,
74    #[cfg(feature = "download")]
75    client: reqwest::Client,
76}
77
78impl ExternalClient {
79    /// Create a new external client with default configuration
80    pub fn new() -> Result<Self> {
81        Self::with_config(ExternalConfig::default())
82    }
83
84    /// Create a new external client with custom configuration
85    pub fn with_config(config: ExternalConfig) -> Result<Self> {
86        let cache = DatasetCache::new(crate::cache::get_cachedir()?);
87
88        #[cfg(feature = "download")]
89        let client = {
90            crate::tls::ensure_default_tls_provider();
91            let mut builder = reqwest::Client::builder()
92                .timeout(Duration::from_secs(config.timeout_seconds))
93                .user_agent(&config.user_agent);
94
95            if !config.verify_ssl {
96                builder = builder.danger_accept_invalid_certs(true);
97            }
98
99            builder
100                .build()
101                .map_err(|e| DatasetsError::IoError(std::io::Error::other(e)))?
102        };
103
104        Ok(Self {
105            config,
106            cache,
107            #[cfg(feature = "download")]
108            client,
109        })
110    }
111
112    /// Download a dataset from a URL
113    #[cfg(feature = "download")]
114    pub async fn download_dataset(
115        &self,
116        url: &str,
117        progress: Option<ProgressCallback>,
118    ) -> Result<Dataset> {
119        // Check cache first
120        if self.config.use_cache {
121            let cache_key = format!("external_{}", blake3::hash(url.as_bytes()).to_hex());
122            if let Ok(cached_data) = self.cache.read_cached(&cache_key) {
123                return self.parse_cached_data(&cached_data);
124            }
125        }
126
127        // Download the file
128        let response = self.make_request(url).await?;
129        let total_size = response.content_length().unwrap_or(0);
130
131        let mut downloaded = 0u64;
132        let mut buffer = Vec::new();
133        let mut stream = response.bytes_stream();
134
135        use futures_util::StreamExt;
136        while let Some(chunk) = stream.next().await {
137            let chunk = chunk.map_err(|e| DatasetsError::IoError(std::io::Error::other(e)))?;
138            downloaded += chunk.len() as u64;
139            buffer.extend_from_slice(&chunk);
140
141            if let Some(ref callback) = progress {
142                callback(downloaded, total_size);
143            }
144        }
145
146        // Cache the downloaded data
147        if self.config.use_cache {
148            let cache_key = format!("external_{}", blake3::hash(url.as_bytes()).to_hex());
149            let _ = self.cache.put(&cache_key, &buffer);
150        }
151
152        // Parse the data based on content type or URL extension
153        self.parse_downloaded_data(url, &buffer)
154    }
155
156    /// Download a dataset synchronously (blocking) - when download feature is enabled
157    #[cfg(feature = "download")]
158    pub fn download_dataset_sync(
159        &self,
160        url: &str,
161        progress: Option<ProgressCallback>,
162    ) -> Result<Dataset> {
163        // Use tokio runtime to block on the async version
164        let rt = tokio::runtime::Runtime::new()
165            .map_err(|e| DatasetsError::IoError(std::io::Error::other(e)))?;
166        rt.block_on(self.download_dataset(url, progress))
167    }
168
169    /// Download a dataset synchronously (blocking) - fallback when download feature is disabled
170    #[cfg(not(feature = "download"))]
171    #[cfg(feature = "download-sync")]
172    pub fn download_dataset_sync(
173        &self,
174        url: &str,
175        progress: Option<ProgressCallback>,
176    ) -> Result<Dataset> {
177        // Fallback implementation using ureq
178        self.download_with_ureq(url, progress)
179    }
180
181    /// Stub for download_dataset_sync when download-sync feature is disabled
182    #[cfg(not(feature = "download"))]
183    #[cfg(not(feature = "download-sync"))]
184    pub fn download_dataset_sync(
185        &self,
186        _url: &str,
187        _progress: Option<ProgressCallback>,
188    ) -> Result<Dataset> {
189        Err(DatasetsError::FormatError(
190            "Synchronous download feature is disabled. Enable 'download-sync' feature or use async download.".to_string()
191        ))
192    }
193
194    /// Download using ureq (synchronous HTTP client)
195    #[cfg(feature = "download-sync")]
196    #[allow(dead_code)]
197    fn download_with_ureq(&self, url: &str, progress: Option<ProgressCallback>) -> Result<Dataset> {
198        // Check cache first
199        if self.config.use_cache {
200            let cache_key = format!("external_{}", blake3::hash(url.as_bytes()).to_hex());
201            if let Ok(cached_data) = self.cache.read_cached(&cache_key) {
202                return self.parse_cached_data(&cached_data);
203            }
204        }
205
206        // Ensure a process-default rustls CryptoProvider exists first: ureq panics on
207        // HTTPS connections without one.
208        crate::tls::ensure_default_tls_provider();
209        let mut request = ureq::get(url).header("User-Agent", &self.config.user_agent);
210
211        // Add custom headers
212        for (key, value) in &self.config.headers {
213            request = request.header(key, value);
214        }
215
216        let response = request
217            .call()
218            .map_err(|e| DatasetsError::IoError(std::io::Error::other(e)))?;
219
220        // Get content-length header if present (case-insensitive per HTTP spec)
221        let headers = response.headers();
222        let total_size = headers
223            .get("Content-Length")
224            .and_then(|hv| hv.to_str().ok())
225            .and_then(|s| s.parse::<u64>().ok())
226            .unwrap_or(0);
227
228        // Read body via body reader (ureq 3.x)
229        let mut body = response.into_body();
230        let buffer = body
231            .read_to_vec()
232            .map_err(|e| DatasetsError::IoError(std::io::Error::other(e)))?;
233        let downloaded = buffer.len() as u64;
234        if let Some(ref callback) = progress {
235            callback(downloaded, total_size);
236        }
237
238        // Cache the downloaded data
239        if self.config.use_cache {
240            let cache_key = format!("external_{}", blake3::hash(url.as_bytes()).to_hex());
241            let _ = self.cache.put(&cache_key, &buffer);
242        }
243
244        // Parse the data
245        self.parse_downloaded_data(url, &buffer)
246    }
247
248    #[cfg(feature = "download")]
249    async fn make_request(&self, url: &str) -> Result<reqwest::Response> {
250        let mut request = self.client.get(url);
251
252        // Add custom headers
253        for (key, value) in &self.config.headers {
254            request = request.header(key, value);
255        }
256
257        let mut last_error = None;
258
259        for attempt in 0..=self.config.max_retries {
260            match request
261                .try_clone()
262                .ok_or_else(|| {
263                    DatasetsError::IoError(std::io::Error::other("Failed to clone request"))
264                })?
265                .send()
266                .await
267            {
268                Ok(response) => {
269                    if response.status().is_success() {
270                        return Ok(response);
271                    } else {
272                        last_error = Some(DatasetsError::IoError(std::io::Error::other(format!(
273                            "HTTP {}: {}",
274                            response.status(),
275                            response.status().canonical_reason().unwrap_or("Unknown")
276                        ))));
277                    }
278                }
279                Err(e) => {
280                    last_error = Some(DatasetsError::IoError(std::io::Error::other(e)));
281                }
282            }
283
284            if attempt < self.config.max_retries {
285                tokio::time::sleep(Duration::from_millis(1000 * (attempt + 1) as u64)).await;
286            }
287        }
288
289        Err(last_error.expect("Operation failed"))
290    }
291
292    fn parse_cached_data(&self, data: &[u8]) -> Result<Dataset> {
293        // Try to deserialize as JSON first (cached parsed data)
294        if let Ok(dataset) = serde_json::from_slice::<Dataset>(data) {
295            return Ok(dataset);
296        }
297
298        // Otherwise parse as raw data
299        self.parse_raw_data(data, None)
300    }
301
302    fn parse_downloaded_data(&self, url: &str, data: &[u8]) -> Result<Dataset> {
303        let extension = Path::new(url)
304            .extension()
305            .and_then(|s| s.to_str())
306            .unwrap_or("")
307            .to_lowercase();
308
309        self.parse_raw_data(data, Some(&extension))
310    }
311
312    fn parse_raw_data(&self, data: &[u8], extension: Option<&str>) -> Result<Dataset> {
313        match extension {
314            Some("csv") | None => {
315                // Try CSV parsing
316                let csv_data = String::from_utf8(data.to_vec())
317                    .map_err(|e| DatasetsError::FormatError(format!("Invalid UTF-8: {e}")))?;
318
319                // Write to temporary file for CSV parsing
320                let temp_file = tempfile::NamedTempFile::new().map_err(DatasetsError::IoError)?;
321
322                std::fs::write(temp_file.path(), &csv_data).map_err(DatasetsError::IoError)?;
323
324                load_csv(temp_file.path(), CsvConfig::default())
325            }
326            Some("json") => {
327                // Try JSON parsing
328                let json_str = String::from_utf8(data.to_vec())
329                    .map_err(|e| DatasetsError::FormatError(format!("Invalid UTF-8: {e}")))?;
330
331                serde_json::from_str(&json_str)
332                    .map_err(|e| DatasetsError::FormatError(format!("Invalid JSON: {e}")))
333            }
334            Some("arff") => {
335                // Basic ARFF parsing (simplified)
336                self.parse_arff_data(data)
337            }
338            _ => {
339                // Try to auto-detect format
340                self.auto_detect_and_parse(data)
341            }
342        }
343    }
344
345    fn parse_arff_data(&self, data: &[u8]) -> Result<Dataset> {
346        let content = String::from_utf8(data.to_vec())
347            .map_err(|e| DatasetsError::FormatError(format!("Invalid UTF-8: {e}")))?;
348
349        let lines = content.lines();
350        let mut attributes = Vec::new();
351        let mut data_section = false;
352        let mut data_lines = Vec::new();
353
354        for line in lines {
355            let line = line.trim();
356
357            if line.is_empty() || line.starts_with('%') {
358                continue;
359            }
360
361            if line.to_lowercase().starts_with("@attribute") {
362                let parts: Vec<&str> = line.split_whitespace().collect();
363                if parts.len() >= 2 {
364                    attributes.push(parts[1].to_string());
365                }
366            } else if line.to_lowercase().starts_with("@data") {
367                data_section = true;
368            } else if data_section {
369                data_lines.push(line.to_string());
370            }
371        }
372
373        // Parse data rows
374        let mut rows: Vec<Vec<f64>> = Vec::new();
375        for line in data_lines {
376            let values: Result<Vec<f64>> = line
377                .split(',')
378                .map(|s| {
379                    s.trim()
380                        .parse::<f64>()
381                        .map_err(|_| DatasetsError::FormatError(format!("Invalid number: {s}")))
382                })
383                .collect();
384
385            match values {
386                Ok(row) => rows.push(row),
387                Err(_) => continue, // Skip invalid rows
388            }
389        }
390
391        if rows.is_empty() {
392            return Err(DatasetsError::FormatError(
393                "No valid data rows found".to_string(),
394            ));
395        }
396
397        let n_features = rows[0].len();
398        let n_samples = rows.len();
399
400        // Assume last column is target if more than one column
401        let (data_cols, target_col) = if n_features > 1 {
402            (n_features - 1, Some(n_features - 1))
403        } else {
404            (n_features, None)
405        };
406
407        // Create data array
408        let mut data_vec = Vec::with_capacity(n_samples * data_cols);
409        let mut target_vec = if target_col.is_some() {
410            Some(Vec::with_capacity(n_samples))
411        } else {
412            None
413        };
414
415        for row in rows {
416            for (i, &value) in row.iter().enumerate() {
417                if i < data_cols {
418                    data_vec.push(value);
419                } else if let Some(ref mut targets) = target_vec {
420                    targets.push(value);
421                }
422            }
423        }
424
425        let data = Array2::from_shape_vec((n_samples, data_cols), data_vec)
426            .map_err(|e| DatasetsError::FormatError(e.to_string()))?;
427
428        let target = target_vec.map(Array1::from_vec);
429
430        Ok(Dataset {
431            data,
432            target,
433            featurenames: Some(attributes[..data_cols].to_vec()),
434            targetnames: None,
435            feature_descriptions: None,
436            description: Some("ARFF dataset loaded from external source".to_string()),
437            metadata: std::collections::HashMap::new(),
438        })
439    }
440
441    fn auto_detect_and_parse(&self, data: &[u8]) -> Result<Dataset> {
442        let content = String::from_utf8(data.to_vec())
443            .map_err(|e| DatasetsError::FormatError(format!("Invalid UTF-8: {e}")))?;
444
445        // Try JSON first
446        if content.trim().starts_with('{') || content.trim().starts_with('[') {
447            if let Ok(dataset) = serde_json::from_str::<Dataset>(&content) {
448                return Ok(dataset);
449            }
450        }
451
452        // Try CSV
453        if content.contains(',') || content.contains('\t') {
454            return self.parse_raw_data(data, Some("csv"));
455        }
456
457        // Try ARFF
458        if content.to_lowercase().contains("@relation") {
459            return self.parse_arff_data(data);
460        }
461
462        Err(DatasetsError::FormatError(
463            "Unable to auto-detect data format".to_string(),
464        ))
465    }
466}
467
468/// Popular dataset repository APIs
469pub mod repositories {
470    use super::*;
471
472    /// UCI Machine Learning Repository client
473    pub struct UCIRepository {
474        client: ExternalClient,
475        base_url: String,
476    }
477
478    impl UCIRepository {
479        /// Create a new UCI repository client
480        pub fn new() -> Result<Self> {
481            Ok(Self {
482                client: ExternalClient::new()?,
483                base_url: "https://archive.ics.uci.edu/ml/machine-learning-databases".to_string(),
484            })
485        }
486
487        /// Loads a dataset from the UCI Machine Learning Repository.
488        ///
489        /// # Arguments
490        /// * `name` - The name of the dataset to load
491        ///
492        /// # Returns
493        /// A `Dataset` containing the loaded data
494        #[cfg(feature = "download")]
495        pub async fn load_dataset(&self, name: &str) -> Result<Dataset> {
496            let url = match name {
497                "adult" => format!("{}/adult/adult.data", self.base_url),
498                "wine" => format!("{}/wine/wine.data", self.base_url),
499                "glass" => format!("{}/glass/glass.data", self.base_url),
500                "hepatitis" => format!("{}/hepatitis/hepatitis.data", self.base_url),
501                "heart-disease" => {
502                    format!("{}/heart-disease/processed.cleveland.data", self.base_url)
503                }
504                _ => {
505                    return Err(DatasetsError::NotFound(format!(
506                        "UCI dataset '{name}' not found"
507                    )))
508                }
509            };
510
511            self.client.download_dataset(&url, None).await
512        }
513
514        #[cfg(not(feature = "download"))]
515        /// Load a UCI dataset synchronously
516        pub fn load_dataset_sync(&self, name: &str) -> Result<Dataset> {
517            let url = match name {
518                "adult" => format!("{}/adult/adult.data", self.base_url),
519                "wine" => format!("{}/wine/wine.data", self.base_url),
520                "glass" => format!("{}/glass/glass.data", self.base_url),
521                "hepatitis" => format!("{}/hepatitis/hepatitis.data", self.base_url),
522                "heart-disease" => {
523                    format!("{}/heart-disease/processed.cleveland.data", self.base_url)
524                }
525                _ => {
526                    return Err(DatasetsError::NotFound(format!(
527                        "UCI dataset '{name}' not found"
528                    )))
529                }
530            };
531
532            self.client.download_dataset_sync(&url, None)
533        }
534
535        /// List available UCI datasets
536        pub fn list_datasets(&self) -> Vec<&'static str> {
537            vec!["adult", "wine", "glass", "hepatitis", "heart-disease"]
538        }
539    }
540
541    /// Kaggle dataset client (requires API key)
542    pub struct KaggleRepository {
543        #[allow(dead_code)]
544        client: ExternalClient,
545        #[allow(dead_code)]
546        api_key: Option<String>,
547    }
548
549    impl KaggleRepository {
550        /// Create a new Kaggle repository client
551        pub fn new(_apikey: Option<String>) -> Result<Self> {
552            let mut config = ExternalConfig::default();
553
554            if let Some(ref key) = _apikey {
555                config
556                    .headers
557                    .insert("Authorization".to_string(), format!("Bearer {key}"));
558            }
559
560            Ok(Self {
561                client: ExternalClient::with_config(config)?,
562                api_key: _apikey,
563            })
564        }
565
566        /// Loads competition data from Kaggle.
567        ///
568        /// # Arguments
569        /// * `competition` - The name of the Kaggle competition
570        ///
571        /// # Returns
572        /// A `Dataset` containing the competition data
573        #[cfg(feature = "download")]
574        pub async fn load_competition_data(&self, competition: &str) -> Result<Dataset> {
575            if self.api_key.is_none() {
576                return Err(DatasetsError::AuthenticationError(
577                    "Kaggle API key required".to_string(),
578                ));
579            }
580
581            let url = format!(
582                "https://www.kaggle.com/api/v1/competitions/{}/data/download",
583                competition
584            );
585            self.client.download_dataset(&url, None).await
586        }
587    }
588
589    /// GitHub repository client for datasets
590    pub struct GitHubRepository {
591        client: ExternalClient,
592    }
593
594    impl GitHubRepository {
595        /// Create a new GitHub repository client
596        pub fn new() -> Result<Self> {
597            Ok(Self {
598                client: ExternalClient::new()?,
599            })
600        }
601
602        /// Loads a dataset from a GitHub repository.
603        ///
604        /// # Arguments
605        /// * `user` - The GitHub username
606        /// * `repo` - The repository name
607        /// * `path` - The path to the dataset file within the repository
608        ///
609        /// # Returns
610        /// A `Dataset` containing the loaded data
611        #[cfg(feature = "download")]
612        pub async fn load_from_repo(&self, user: &str, repo: &str, path: &str) -> Result<Dataset> {
613            let url = format!("https://raw.githubusercontent.com/{user}/{repo}/main/{path}");
614            self.client.download_dataset(&url, None).await
615        }
616
617        #[cfg(not(feature = "download"))]
618        /// Load a dataset from GitHub repository synchronously
619        pub fn load_from_repo_sync(&self, user: &str, repo: &str, path: &str) -> Result<Dataset> {
620            let url = format!("https://raw.githubusercontent.com/{user}/{repo}/main/{path}");
621            self.client.download_dataset_sync(&url, None)
622        }
623    }
624}
625
626/// Convenience functions for common external data operations
627pub mod convenience {
628    use super::repositories::*;
629    use super::*;
630
631    /// Load a dataset from a URL with progress tracking
632    #[cfg(feature = "download")]
633    pub async fn load_from_url(url: &str, config: Option<ExternalConfig>) -> Result<Dataset> {
634        let client = match config {
635            Some(cfg) => ExternalClient::with_config(cfg)?,
636            None => ExternalClient::new()?,
637        };
638
639        client
640            .download_dataset(
641                url,
642                Some(Box::new(|downloaded, total| {
643                    if let Some(percent) = (downloaded * 100).checked_div(total) {
644                        eprintln!("Downloaded: {percent:.1}% ({downloaded}/{total})");
645                    } else {
646                        eprintln!("Downloaded: {downloaded} bytes");
647                    }
648                })),
649            )
650            .await
651    }
652
653    /// Load a dataset from a URL synchronously
654    pub fn load_from_url_sync(url: &str, config: Option<ExternalConfig>) -> Result<Dataset> {
655        let client = match config {
656            Some(cfg) => ExternalClient::with_config(cfg)?,
657            None => ExternalClient::new()?,
658        };
659
660        client.download_dataset_sync(
661            url,
662            Some(Box::new(|downloaded, total| {
663                if let Some(percent) = (downloaded * 100).checked_div(total) {
664                    eprintln!("Downloaded: {percent:.1}% ({downloaded}/{total})");
665                } else {
666                    eprintln!("Downloaded: {downloaded} bytes");
667                }
668            })),
669        )
670    }
671
672    /// Load a UCI dataset by name
673    #[cfg(feature = "download")]
674    pub async fn load_uci_dataset(name: &str) -> Result<Dataset> {
675        let repo = UCIRepository::new()?;
676        repo.load_dataset(name).await
677    }
678
679    /// Load a UCI dataset by name synchronously
680    #[cfg(not(feature = "download"))]
681    pub fn load_uci_dataset_sync(name: &str) -> Result<Dataset> {
682        let repo = UCIRepository::new()?;
683        repo.load_dataset_sync(name)
684    }
685
686    /// Load a dataset from GitHub repository
687    #[cfg(feature = "download")]
688    pub async fn load_github_dataset(user: &str, repo: &str, path: &str) -> Result<Dataset> {
689        let github = GitHubRepository::new()?;
690        github.load_from_repo(user, repo, path).await
691    }
692
693    /// Load a dataset from GitHub repository synchronously
694    #[cfg(not(feature = "download"))]
695    pub fn load_github_dataset_sync(user: &str, repo: &str, path: &str) -> Result<Dataset> {
696        let github = GitHubRepository::new()?;
697        github.load_from_repo_sync(user, repo, path)
698    }
699
700    /// List available UCI datasets
701    pub fn list_uci_datasets() -> Result<Vec<&'static str>> {
702        let repo = UCIRepository::new()?;
703        Ok(repo.list_datasets())
704    }
705}
706
707#[cfg(test)]
708mod tests {
709    use super::convenience::*;
710    use super::*;
711
712    #[test]
713    fn test_external_config_default() {
714        let config = ExternalConfig::default();
715        assert_eq!(config.timeout_seconds, 300);
716        assert_eq!(config.max_retries, 3);
717        assert!(config.verify_ssl);
718        assert!(config.use_cache);
719    }
720
721    #[test]
722    fn test_uci_repository_list_datasets() {
723        let datasets = list_uci_datasets().expect("Operation failed");
724        assert!(!datasets.is_empty());
725        assert!(datasets.contains(&"wine"));
726        assert!(datasets.contains(&"adult"));
727    }
728
729    #[test]
730    fn test_parse_arff_data() {
731        let arff_content = r#"
732@relation test
733@attribute feature1 numeric
734@attribute feature2 numeric
735@attribute class {0,1}
736@data
7371.0,2.0,0
7383.0,4.0,1
7395.0,6.0,0
740"#;
741
742        let client = ExternalClient::new().expect("Operation failed");
743        let dataset = client
744            .parse_arff_data(arff_content.as_bytes())
745            .expect("Operation failed");
746
747        assert_eq!(dataset.n_samples(), 3);
748        assert_eq!(dataset.n_features(), 2);
749        assert!(dataset.target.is_some());
750    }
751
752    #[tokio::test]
753    #[cfg(feature = "download")]
754    async fn test_download_small_csv() {
755        // Test with a small public CSV dataset
756        let url = "https://raw.githubusercontent.com/mwaskom/seaborn-data/master/iris.csv";
757
758        let result = load_from_url(url, None).await;
759        match result {
760            Ok(dataset) => {
761                assert!(dataset.n_samples() > 0);
762                assert!(dataset.n_features() > 0);
763            }
764            Err(e) => {
765                // Network tests may fail in CI, so we just log the error
766                eprintln!("Network test failed (expected in CI): {}", e);
767            }
768        }
769    }
770}
scirs2_datasets/external.rs

scirs2_datasets/
external.rs
