mocra-core 0.4.0

The mocra crawler framework runtime: errors, cache, utilities, domain models, downloader, data-plane queue, coordination, scheduler and engine.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

use super::ConfigProvider;
use crate::common::model::config::Config;
use async_trait::async_trait;
use log::{error, info};
use redis::AsyncCommands;
use std::time::Duration;
use tokio::sync::watch;

/// Redis-backed configuration provider.
///
/// Loads configuration JSON from a Redis key and supports change watching via polling.
pub struct RedisConfigProvider {
    client: redis::Client,
    key: String,
    // channel: String, // Not used for polling
}

impl RedisConfigProvider {
    /// Creates a new Redis config provider.
    ///
    /// # Arguments
    /// * `redis_url` - Redis connection string.
    /// * `key` - Key where config is stored.
    /// * `_channel` - (unused) Pub/Sub channel.
    pub fn new(redis_url: &str, key: &str, _channel: &str) -> Result<Self, String> {
        let client = redis::Client::open(redis_url).map_err(|e| e.to_string())?;
        Ok(Self {
            client,
            key: key.to_string(),
        })
    }

    /// Fetches config from Redis.
    async fn fetch_config(&self) -> Result<Config, String> {
        let mut con = self
            .client
            .get_multiplexed_async_connection()
            .await
            .map_err(|e| e.to_string())?;
        let config_str: String = con.get(&self.key).await.map_err(|e| e.to_string())?;
        serde_json::from_str(&config_str).map_err(|e| e.to_string())
    }
}

#[async_trait]
impl ConfigProvider for RedisConfigProvider {
    async fn load_config(&self) -> Result<Config, String> {
        self.fetch_config().await
    }

    async fn watch(&self) -> Result<watch::Receiver<Config>, String> {
        let initial_config = self.load_config().await?;
        let (tx, rx) = watch::channel(initial_config);

        let client = self.client.clone();
        let key = self.key.clone();

        tokio::spawn(async move {
            let mut last_config_str = String::new();

            loop {
                tokio::time::sleep(Duration::from_secs(10)).await;

                let mut con = match client.get_multiplexed_async_connection().await {
                    Ok(c) => c,
                    Err(e) => {
                        error!("Failed to get redis connection: {}", e);
                        continue;
                    }
                };

                let config_str: String = match con.get(&key).await {
                    Ok(s) => s,
                    Err(e) => {
                        error!("Failed to fetch config from Redis: {}", e);
                        continue;
                    }
                };

                if config_str != last_config_str {
                    match serde_json::from_str::<Config>(&config_str) {
                        Ok(config) => {
                            info!("Config changed in Redis, reloading...");
                            if let Err(e) = tx.send(config) {
                                error!("Failed to send config update: {}", e);
                                break;
                            }
                            last_config_str = config_str;
                        }
                        Err(e) => error!("Failed to parse config: {}", e),
                    }
                }
            }
        });

        Ok(rx)
    }
}