dexter_daemon/
lib.rs

1/// Worker allocation plan for staged daemon pipelines where fetch and persistence
2/// run concurrently.
3#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4pub struct WorkerRepurposePlan {
5    /// Number of fetch workers actively scraping upstream sources.
6    pub fetch_workers: usize,
7    /// Number of dedicated persistence workers (CSV + inserts + post-fetch side effects).
8    pub persist_workers: usize,
9    /// Capacity of the fetch queue feeding fetch workers.
10    pub fetch_queue_capacity: usize,
11    /// Capacity of the persistence queue feeding persist workers.
12    pub persist_queue_capacity: usize,
13    /// Number of randomized fetch jobs to keep in flight.
14    pub dispatch_target: usize,
15}
16
17impl WorkerRepurposePlan {
18    /// Build a plan for randomized registry ingestion.
19    ///
20    /// `dispatch_target` should already include provider-specific prefetch shaping.
21    pub fn for_randomized_registry(
22        fetch_workers: usize,
23        fetch_queue_capacity: usize,
24        dispatch_target: usize,
25    ) -> Self {
26        let fetch_workers = fetch_workers.max(1);
27        let fetch_queue_capacity = fetch_queue_capacity.max(fetch_workers * 2).max(4);
28        let dispatch_target = dispatch_target.max(1);
29        let persist_workers = 1;
30        let persist_queue_capacity = fetch_queue_capacity.max(dispatch_target).max(4);
31        Self {
32            fetch_workers,
33            persist_workers,
34            fetch_queue_capacity,
35            persist_queue_capacity,
36            dispatch_target,
37        }
38    }
39}
40
41#[cfg(test)]
42mod tests {
43    use super::WorkerRepurposePlan;
44
45    #[test]
46    fn randomized_registry_plan_keeps_minimum_bounds() {
47        let plan = WorkerRepurposePlan::for_randomized_registry(0, 0, 0);
48        assert_eq!(plan.fetch_workers, 1);
49        assert_eq!(plan.persist_workers, 1);
50        assert_eq!(plan.fetch_queue_capacity, 4);
51        assert_eq!(plan.persist_queue_capacity, 4);
52        assert_eq!(plan.dispatch_target, 1);
53    }
54
55    #[test]
56    fn randomized_registry_plan_scales_persist_queue_to_dispatch_target() {
57        let plan = WorkerRepurposePlan::for_randomized_registry(4, 8, 64);
58        assert_eq!(plan.fetch_workers, 4);
59        assert_eq!(plan.fetch_queue_capacity, 8);
60        assert_eq!(plan.persist_workers, 1);
61        assert_eq!(plan.persist_queue_capacity, 64);
62        assert_eq!(plan.dispatch_target, 64);
63    }
64}
dexter_daemon/lib.rs

dexter_daemon/
lib.rs
