edgestore_repl/replicated_engine.rs
1//! `ReplicatedEngine` — convenience wrapper for the common primary + replica pattern.
2//!
3//! Eliminates the three-step boilerplate of wiring an `Engine` with
4//! `HttpReplicationServer` and `AntiEntropyLoop`.
5//!
6//! ## Primary
7//!
8//! ```rust,no_run
9//! use edgestore::EdgestoreConfig;
10//! use edgestore_repl::ReplicatedEngine;
11//!
12//! let primary = ReplicatedEngine::open_primary(
13//! EdgestoreConfig::new("/var/db/primary"),
14//! "0.0.0.0:8900",
15//! ).unwrap();
16//!
17//! primary.engine().lock().unwrap().put(b"products", b"p1", b"data").unwrap();
18//! ```
19//!
20//! ## Replica
21//!
22//! ```rust,no_run
23//! use edgestore::EdgestoreConfig;
24//! use edgestore_repl::ReplicatedEngine;
25//!
26//! let replica = ReplicatedEngine::open_replica(
27//! EdgestoreConfig::new("/var/db/replica"),
28//! "http://primary-host:8900",
29//! ).unwrap();
30//!
31//! // Engine is read-only — write attempts return Err(EdgestoreError::ReadOnly).
32//! let val = replica.engine().lock().unwrap().get(b"products", b"p1").unwrap();
33//! ```
34//!
35//! ## Wake-on-flush (reduced replication lag)
36//!
37//! Primary registers an `on_segment_flushed` callback via `Engine::with_on_segment_flushed`
38//! before passing the engine to this wrapper. The callback wakes the anti-entropy loop
39//! via a channel instead of waiting for the poll interval.
40
41use std::path::PathBuf;
42use std::sync::{Arc, Mutex};
43
44use edgestore::{EdgestoreConfig, EdgestoreError, Engine};
45
46use crate::anti_entropy::AntiEntropyLoop;
47use crate::http_client::HttpReplicationClient;
48use crate::http_server::HttpReplicationServer;
49use edgestore::replication::ReplicationProtocol;
50
51/// Wraps `Engine` + `HttpReplicationServer` (primary) or `Engine` + `AntiEntropyLoop` (replica).
52///
53/// Background threads are owned by this struct. When dropped, threads continue
54/// running until the process exits — same behavior as calling `start()` manually.
55pub struct ReplicatedEngine {
56 engine: Arc<Mutex<Engine>>,
57 // JoinHandles held to prevent accidental detach; threads are daemon-like (run until exit).
58 #[allow(dead_code)]
59 handles: Vec<std::thread::JoinHandle<()>>,
60 bound_port: Option<u16>,
61}
62
63impl ReplicatedEngine {
64 /// Open a primary replication node.
65 ///
66 /// Starts `HttpReplicationServer` on `bind_addr` in a background thread.
67 /// The engine is writable. Other nodes point their `AntiEntropyLoop` at this address.
68 pub fn open_primary(
69 config: EdgestoreConfig,
70 bind_addr: &str,
71 ) -> Result<ReplicatedEngine, EdgestoreError> {
72 let engine = Arc::new(Mutex::new(Engine::open(config)?));
73 let server = HttpReplicationServer::new(Arc::clone(&engine));
74 let (handle, port) = server.start(bind_addr)?;
75 Ok(ReplicatedEngine {
76 engine,
77 handles: vec![handle],
78 bound_port: Some(port),
79 })
80 }
81
82 /// Open a replica node.
83 ///
84 /// Opens the engine in read-only mode (`Engine::open_readonly`) and starts an
85 /// `AntiEntropyLoop` that pulls from `primary_url`. The engine rejects all write
86 /// calls with `Err(EdgestoreError::ReadOnly)`.
87 ///
88 /// `primary_url` must be the base URL of the primary's `HttpReplicationServer`
89 /// (e.g. `"http://host:8900"`).
90 pub fn open_replica(
91 config: EdgestoreConfig,
92 primary_url: &str,
93 ) -> Result<ReplicatedEngine, EdgestoreError> {
94 let db_path: PathBuf = config.path.clone();
95 let engine = Arc::new(Mutex::new(Engine::open_readonly(config)?));
96 let peer_id = "primary".to_string();
97 let loop_ = AntiEntropyLoop::new(
98 Arc::clone(&engine),
99 primary_url.to_string(),
100 peer_id,
101 db_path,
102 );
103 let handle = loop_.start();
104 Ok(ReplicatedEngine {
105 engine,
106 handles: vec![handle],
107 bound_port: None,
108 })
109 }
110
111 /// Promote a replica engine to primary with write-token fencing (BtrLog §4.2).
112 ///
113 /// # Fencing protocol
114 ///
115 /// 1. If `old_primary_url` is provided, fetch its current `wtoken` via `GET /watermark`.
116 /// Failure (network or 404) is non-fatal — the local token is used as the floor.
117 /// 2. Compute `new_token = max(fetched_wtoken, local_wtoken) + 1`.
118 /// 3. Persist `new_token` to `{db_path}/WTOKEN` via `engine.set_write_token`.
119 /// 4. Start `HttpReplicationServer` on `bind_addr`.
120 ///
121 /// Anti-entropy loops on other replicas that have already seen the old primary's
122 /// `wtoken` will observe the higher token and can log or refuse stale segments.
123 ///
124 /// # Arguments
125 ///
126 /// * `config` — database configuration for the new primary.
127 /// * `bind_addr` — address to bind the HTTP replication server (e.g. `"0.0.0.0:8900"`).
128 /// * `old_primary_url` — base URL of the old primary, used to fetch its `wtoken` before
129 /// fencing. Pass `None` when the old primary is already down.
130 pub fn promote_to_primary(
131 config: EdgestoreConfig,
132 bind_addr: &str,
133 old_primary_url: Option<&str>,
134 ) -> Result<ReplicatedEngine, EdgestoreError> {
135 let mut engine = Engine::open(config)?;
136
137 let fetched_wtoken = old_primary_url.and_then(|url| {
138 let client = HttpReplicationClient::new(url);
139 match client.watermark() {
140 Ok(wm) => Some(wm.wtoken),
141 Err(e) => {
142 eprintln!(
143 "[edgestore] promote_to_primary: could not fetch watermark from {}: {}",
144 url, e
145 );
146 None
147 }
148 }
149 });
150
151 let new_token = fetched_wtoken.unwrap_or(0).max(engine.write_token()) + 1;
152
153 engine.set_write_token(new_token)?;
154
155 eprintln!(
156 "[edgestore] promote_to_primary: write token advanced to {}",
157 new_token
158 );
159
160 let engine = Arc::new(Mutex::new(engine));
161 let server = HttpReplicationServer::new(Arc::clone(&engine));
162 let (handle, port) = server.start(bind_addr)?;
163 Ok(ReplicatedEngine {
164 engine,
165 handles: vec![handle],
166 bound_port: Some(port),
167 })
168 }
169
170 /// Access the underlying `Arc<Mutex<Engine>>` for reads (replica) or reads+writes (primary).
171 pub fn engine(&self) -> &Arc<Mutex<Engine>> {
172 &self.engine
173 }
174
175 /// Port the `HttpReplicationServer` is listening on, or `None` for a replica.
176 pub fn bound_port(&self) -> Option<u16> {
177 self.bound_port
178 }
179}