mocra 0.3.0

A distributed, event-driven crawling and data collection framework
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

use super::ConfigProvider;
use crate::common::model::config::Config;
use async_trait::async_trait;
use log::{error, info};
use std::path::PathBuf;
use std::time::Duration;
use tokio::sync::watch;

/// File-based configuration provider.
///
/// Loads config from local filesystem and supports file-change watching (hot reload).
pub struct FileConfigProvider {
    path: PathBuf,
}

impl FileConfigProvider {
    /// Creates a new file config provider.
    ///
    /// # Arguments
    /// * `path` - Configuration file path.
    pub fn new(path: &str) -> Self {
        Self {
            path: PathBuf::from(path),
        }
    }
}

#[async_trait]
impl ConfigProvider for FileConfigProvider {
    async fn load_config(&self) -> Result<Config, String> {
        let path_str = self.path.to_str().ok_or("Invalid path")?;
        Config::load(path_str)
    }

    async fn watch(&self) -> Result<watch::Receiver<Config>, String> {
        let initial_config = self.load_config().await?;
        let (tx, rx) = watch::channel(initial_config);
        let path = self.path.clone();

        tokio::spawn(async move {
            let mut last_modified = std::fs::metadata(&path).and_then(|m| m.modified()).ok();
            loop {
                tokio::time::sleep(Duration::from_secs(5)).await;
                if let Ok(metadata) = std::fs::metadata(&path)
                    && let Ok(modified) = metadata.modified()
                    && last_modified != Some(modified)
                {
                    last_modified = Some(modified);
                    info!("Config file changed, reloading...");
                    match Config::load(path.to_str().unwrap_or_default()) {
                        Ok(config) => {
                            if let Err(e) = tx.send(config) {
                                error!("Failed to send config update: {}", e);
                                break;
                            }
                        }
                        Err(e) => error!("Failed to reload config: {}", e),
                    }
                }
            }
        });

        Ok(rx)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::Write;
    use tempfile::NamedTempFile;

    #[tokio::test]
    async fn test_load_config_success() {
        // Create a temp file with valid config
        let mut file = NamedTempFile::new().unwrap();
        let config_content = r#"
            name = "test_app"
            
            [db]
            url = "postgres://user:password@localhost:5432/db"
            database_schema = "public"
            
            [download_config]
            downloader_expire = 3600
            timeout = 30
            rate_limit = 10.0
            enable_session = true
            enable_locker = false
            enable_rate_limit = true
            cache_ttl = 600
            wss_timeout = 60
            
            [cache]
            ttl = 3600
            
            [crawler]
            request_max_retries = 3
            task_max_errors = 5
            module_max_errors = 10
            module_locker_ttl = 60
            
            [channel_config]
            minid_time = 0
            capacity = 1000
            compression_threshold = 1024
        "#;
        write!(file, "{}", config_content).unwrap();

        let provider = FileConfigProvider::new(file.path().to_str().unwrap());
        let config = provider.load_config().await;
        assert!(config.is_ok(), "Config load failed: {:?}", config.err());
    }

    #[tokio::test]
    async fn test_load_config_not_found() {
        let provider = FileConfigProvider::new("non_existent_file.toml");
        let result = provider.load_config().await;
        assert!(result.is_err());
    }
}