sync_engine/
config.rs

1//! Configuration for the sync engine.
2//!
3//! # Example
4//!
5//! ```
6//! use sync_engine::SyncEngineConfig;
7//!
8//! // Minimal config (uses defaults)
9//! let config = SyncEngineConfig::default();
10//! assert_eq!(config.l1_max_bytes, 256 * 1024 * 1024); // 256 MB
11//!
12//! // Full config
13//! let config = SyncEngineConfig {
14//!     redis_url: Some("redis://localhost:6379".into()),
15//!     sql_url: Some("mysql://user:pass@localhost/db".into()),
16//!     l1_max_bytes: 128 * 1024 * 1024, // 128 MB
17//!     batch_flush_count: 100,
18//!     batch_flush_ms: 50,
19//!     ..Default::default()
20//! };
21//! ```
22
23use serde::Deserialize;
24
25/// Configuration for the sync engine.
26///
27/// All fields have sensible defaults. At minimum, you should configure
28/// `redis_url` and `sql_url` for production use.
29#[derive(Debug, Clone, Deserialize)]
30pub struct SyncEngineConfig {
31    /// Redis connection string (e.g., "redis://localhost:6379")
32    #[serde(default)]
33    pub redis_url: Option<String>,
34    
35    /// Redis key prefix for namespacing (e.g., "myapp:" → keys become "myapp:user.alice")
36    /// Allows sync-engine to coexist with other data in the same Redis instance.
37    #[serde(default)]
38    pub redis_prefix: Option<String>,
39    
40    /// SQL connection string (e.g., "sqlite:sync.db" or "mysql://user:pass@host/db")
41    #[serde(default)]
42    pub sql_url: Option<String>,
43    
44    /// L1 cache max size in bytes (default: 256 MB)
45    #[serde(default = "default_l1_max_bytes")]
46    pub l1_max_bytes: usize,
47    
48    /// Maximum payload size in bytes (default: 16 MB)
49    /// 
50    /// Payloads larger than this will be rejected with an error.
51    /// This prevents a single large item (e.g., 1TB file) from exhausting
52    /// the L1 cache. Set to 0 for unlimited (not recommended).
53    /// 
54    /// **Important**: This is a safety limit. Developers should choose a value
55    /// appropriate for their use case. For binary blobs, consider using
56    /// external object storage (S3, GCS) and storing only references here.
57    #[serde(default = "default_max_payload_bytes")]
58    pub max_payload_bytes: usize,
59    
60    /// Backpressure thresholds
61    #[serde(default = "default_backpressure_warn")]
62    pub backpressure_warn: f64,
63    #[serde(default = "default_backpressure_critical")]
64    pub backpressure_critical: f64,
65    
66    /// Batch flush settings
67    #[serde(default = "default_batch_flush_ms")]
68    pub batch_flush_ms: u64,
69    #[serde(default = "default_batch_flush_count")]
70    pub batch_flush_count: usize,
71    #[serde(default = "default_batch_flush_bytes")]
72    pub batch_flush_bytes: usize,
73    
74    /// Cuckoo filter warmup
75    #[serde(default = "default_cuckoo_warmup_batch_size")]
76    pub cuckoo_warmup_batch_size: usize,
77    
78    /// WAL path (SQLite file for durability during MySQL outages)
79    #[serde(default)]
80    pub wal_path: Option<String>,
81    
82    /// WAL max items before backpressure
83    #[serde(default)]
84    pub wal_max_items: Option<u64>,
85    
86    /// WAL drain batch size
87    #[serde(default = "default_wal_drain_batch_size")]
88    pub wal_drain_batch_size: usize,
89    
90    /// CF snapshot interval in seconds (0 = disabled)
91    #[serde(default = "default_cf_snapshot_interval_secs")]
92    pub cf_snapshot_interval_secs: u64,
93    
94    /// CF snapshot after N inserts (0 = disabled)
95    #[serde(default = "default_cf_snapshot_insert_threshold")]
96    pub cf_snapshot_insert_threshold: u64,
97    
98    /// Redis eviction: enable proactive eviction before Redis LRU kicks in
99    #[serde(default = "default_redis_eviction_enabled")]
100    pub redis_eviction_enabled: bool,
101    
102    /// Redis eviction: pressure threshold to start evicting (0.0-1.0, default: 0.75)
103    #[serde(default = "default_redis_eviction_start")]
104    pub redis_eviction_start: f64,
105    
106    /// Redis eviction: target pressure after eviction (0.0-1.0, default: 0.60)
107    #[serde(default = "default_redis_eviction_target")]
108    pub redis_eviction_target: f64,
109    
110    /// Merkle calculation: enable merkle tree updates on this instance.
111    /// 
112    /// In a multi-instance deployment with shared SQL, only a few nodes need to
113    /// run merkle calculations for resilience. Set to false on most nodes.
114    /// Default: true (single-instance default)
115    #[serde(default = "default_merkle_calc_enabled")]
116    pub merkle_calc_enabled: bool,
117    
118    /// Merkle calculation: jitter range in milliseconds.
119    /// 
120    /// Adds random delay (0 to N ms) before merkle batch calculation to reduce
121    /// contention when multiple instances are calculating. Default: 0 (no jitter)
122    #[serde(default)]
123    pub merkle_calc_jitter_ms: u64,
124    
125    /// CDC Stream: Enable Change Data Capture output to Redis Stream.
126    /// 
127    /// When enabled, every Put/Delete writes to `{redis_prefix}__local__:cdc`.
128    /// This enables external replication agents to tail changes.
129    /// Default: false (opt-in feature)
130    #[serde(default)]
131    pub enable_cdc_stream: bool,
132    
133    /// CDC Stream: Maximum entries before approximate trimming (MAXLEN ~).
134    /// 
135    /// Consumers that fall behind this limit rely on Merkle repair.
136    /// Default: 100,000 entries
137    #[serde(default = "default_cdc_stream_maxlen")]
138    pub cdc_stream_maxlen: u64,
139}
140
141fn default_l1_max_bytes() -> usize { 256 * 1024 * 1024 } // 256 MB
142fn default_max_payload_bytes() -> usize { 16 * 1024 * 1024 } // 16 MB
143fn default_backpressure_warn() -> f64 { 0.7 }
144fn default_backpressure_critical() -> f64 { 0.9 }
145fn default_batch_flush_ms() -> u64 { 100 }
146fn default_batch_flush_count() -> usize { 1000 }
147fn default_batch_flush_bytes() -> usize { 1024 * 1024 } // 1 MB
148fn default_cuckoo_warmup_batch_size() -> usize { 10000 }
149fn default_wal_drain_batch_size() -> usize { 100 }
150fn default_cf_snapshot_interval_secs() -> u64 { 30 }
151fn default_cf_snapshot_insert_threshold() -> u64 { 10_000 }
152fn default_redis_eviction_enabled() -> bool { true }
153fn default_redis_eviction_start() -> f64 { 0.75 }
154fn default_redis_eviction_target() -> f64 { 0.60 }
155fn default_merkle_calc_enabled() -> bool { true }
156fn default_cdc_stream_maxlen() -> u64 { 100_000 }
157
158impl Default for SyncEngineConfig {
159    fn default() -> Self {
160        Self {
161            redis_url: None,
162            sql_url: None,
163            redis_prefix: None,
164            l1_max_bytes: default_l1_max_bytes(),
165            max_payload_bytes: default_max_payload_bytes(),
166            backpressure_warn: default_backpressure_warn(),
167            backpressure_critical: default_backpressure_critical(),
168            batch_flush_ms: default_batch_flush_ms(),
169            batch_flush_count: default_batch_flush_count(),
170            batch_flush_bytes: default_batch_flush_bytes(),
171            cuckoo_warmup_batch_size: default_cuckoo_warmup_batch_size(),
172            wal_path: None,
173            wal_max_items: None,
174            wal_drain_batch_size: default_wal_drain_batch_size(),
175            cf_snapshot_interval_secs: default_cf_snapshot_interval_secs(),
176            cf_snapshot_insert_threshold: default_cf_snapshot_insert_threshold(),
177            redis_eviction_enabled: default_redis_eviction_enabled(),
178            redis_eviction_start: default_redis_eviction_start(),
179            redis_eviction_target: default_redis_eviction_target(),
180            merkle_calc_enabled: default_merkle_calc_enabled(),
181            merkle_calc_jitter_ms: 0,
182            enable_cdc_stream: false,
183            cdc_stream_maxlen: default_cdc_stream_maxlen(),
184        }
185    }
186}
sync_engine/config.rs

sync_engine/
config.rs
