Skip to main content

edgestore_repl/
replicated_engine.rs

1//! `ReplicatedEngine` — convenience wrapper for the common primary + replica pattern.
2//!
3//! Eliminates the three-step boilerplate of wiring an `Engine` with
4//! `HttpReplicationServer` and `AntiEntropyLoop`.
5//!
6//! ## Primary
7//!
8//! ```rust,no_run
9//! use edgestore::EdgestoreConfig;
10//! use edgestore_repl::ReplicatedEngine;
11//!
12//! let primary = ReplicatedEngine::open_primary(
13//!     EdgestoreConfig::new("/var/db/primary"),
14//!     "0.0.0.0:8900",
15//! ).unwrap();
16//!
17//! primary.engine().lock().unwrap().put(b"products", b"p1", b"data").unwrap();
18//! ```
19//!
20//! ## Replica
21//!
22//! ```rust,no_run
23//! use edgestore::EdgestoreConfig;
24//! use edgestore_repl::ReplicatedEngine;
25//!
26//! let replica = ReplicatedEngine::open_replica(
27//!     EdgestoreConfig::new("/var/db/replica"),
28//!     "http://primary-host:8900",
29//! ).unwrap();
30//!
31//! // Engine is read-only — write attempts return Err(EdgestoreError::ReadOnly).
32//! let val = replica.engine().lock().unwrap().get(b"products", b"p1").unwrap();
33//! ```
34//!
35//! ## Wake-on-flush (reduced replication lag)
36//!
37//! Primary registers an `on_segment_flushed` callback via `Engine::with_on_segment_flushed`
38//! before passing the engine to this wrapper. The callback wakes the anti-entropy loop
39//! via a channel instead of waiting for the poll interval.
40
41use std::path::PathBuf;
42use std::sync::{Arc, Mutex};
43
44use edgestore::{EdgestoreConfig, EdgestoreError, Engine};
45
46use crate::anti_entropy::AntiEntropyLoop;
47use crate::http_client::HttpReplicationClient;
48use crate::http_server::HttpReplicationServer;
49use edgestore::replication::ReplicationProtocol;
50
51/// Wraps `Engine` + `HttpReplicationServer` (primary) or `Engine` + `AntiEntropyLoop` (replica).
52///
53/// Background threads are owned by this struct. When dropped, threads continue
54/// running until the process exits — same behavior as calling `start()` manually.
55pub struct ReplicatedEngine {
56    engine: Arc<Mutex<Engine>>,
57    // JoinHandles held to prevent accidental detach; threads are daemon-like (run until exit).
58    #[allow(dead_code)]
59    handles: Vec<std::thread::JoinHandle<()>>,
60    bound_port: Option<u16>,
61}
62
63impl ReplicatedEngine {
64    /// Open a primary replication node.
65    ///
66    /// Starts `HttpReplicationServer` on `bind_addr` in a background thread.
67    /// The engine is writable. Other nodes point their `AntiEntropyLoop` at this address.
68    pub fn open_primary(
69        config: EdgestoreConfig,
70        bind_addr: &str,
71    ) -> Result<ReplicatedEngine, EdgestoreError> {
72        let engine = Arc::new(Mutex::new(Engine::open(config)?));
73        let server = HttpReplicationServer::new(Arc::clone(&engine));
74        let (handle, port) = server.start(bind_addr)?;
75        Ok(ReplicatedEngine {
76            engine,
77            handles: vec![handle],
78            bound_port: Some(port),
79        })
80    }
81
82    /// Open a replica node.
83    ///
84    /// Opens the engine in read-only mode (`Engine::open_readonly`) and starts an
85    /// `AntiEntropyLoop` that pulls from `primary_url`. The engine rejects all write
86    /// calls with `Err(EdgestoreError::ReadOnly)`.
87    ///
88    /// `primary_url` must be the base URL of the primary's `HttpReplicationServer`
89    /// (e.g. `"http://host:8900"`).
90    pub fn open_replica(
91        config: EdgestoreConfig,
92        primary_url: &str,
93    ) -> Result<ReplicatedEngine, EdgestoreError> {
94        let db_path: PathBuf = config.path.clone();
95        let engine = Arc::new(Mutex::new(Engine::open_readonly(config)?));
96        let peer_id = "primary".to_string();
97        let loop_ = AntiEntropyLoop::new(
98            Arc::clone(&engine),
99            primary_url.to_string(),
100            peer_id,
101            db_path,
102        );
103        let handle = loop_.start();
104        Ok(ReplicatedEngine {
105            engine,
106            handles: vec![handle],
107            bound_port: None,
108        })
109    }
110
111    /// Promote a replica engine to primary with write-token fencing (BtrLog §4.2).
112    ///
113    /// # Fencing protocol
114    ///
115    /// 1. If `old_primary_url` is provided, fetch its current `wtoken` via `GET /watermark`.
116    ///    Failure (network or 404) is non-fatal — the local token is used as the floor.
117    /// 2. Compute `new_token = max(fetched_wtoken, local_wtoken) + 1`.
118    /// 3. Persist `new_token` to `{db_path}/WTOKEN` via `engine.set_write_token`.
119    /// 4. Start `HttpReplicationServer` on `bind_addr`.
120    ///
121    /// Anti-entropy loops on other replicas that have already seen the old primary's
122    /// `wtoken` will observe the higher token and can log or refuse stale segments.
123    ///
124    /// # Arguments
125    ///
126    /// * `config`          — database configuration for the new primary.
127    /// * `bind_addr`       — address to bind the HTTP replication server (e.g. `"0.0.0.0:8900"`).
128    /// * `old_primary_url` — base URL of the old primary, used to fetch its `wtoken` before
129    ///   fencing.  Pass `None` when the old primary is already down.
130    pub fn promote_to_primary(
131        config: EdgestoreConfig,
132        bind_addr: &str,
133        old_primary_url: Option<&str>,
134    ) -> Result<ReplicatedEngine, EdgestoreError> {
135        let mut engine = Engine::open(config)?;
136
137        let fetched_wtoken = old_primary_url.and_then(|url| {
138            let client = HttpReplicationClient::new(url);
139            match client.watermark() {
140                Ok(wm) => Some(wm.wtoken),
141                Err(e) => {
142                    eprintln!(
143                        "[edgestore] promote_to_primary: could not fetch watermark from {}: {}",
144                        url, e
145                    );
146                    None
147                }
148            }
149        });
150
151        let new_token = fetched_wtoken.unwrap_or(0).max(engine.write_token()) + 1;
152
153        engine.set_write_token(new_token)?;
154
155        eprintln!(
156            "[edgestore] promote_to_primary: write token advanced to {}",
157            new_token
158        );
159
160        let engine = Arc::new(Mutex::new(engine));
161        let server = HttpReplicationServer::new(Arc::clone(&engine));
162        let (handle, port) = server.start(bind_addr)?;
163        Ok(ReplicatedEngine {
164            engine,
165            handles: vec![handle],
166            bound_port: Some(port),
167        })
168    }
169
170    /// Access the underlying `Arc<Mutex<Engine>>` for reads (replica) or reads+writes (primary).
171    pub fn engine(&self) -> &Arc<Mutex<Engine>> {
172        &self.engine
173    }
174
175    /// Port the `HttpReplicationServer` is listening on, or `None` for a replica.
176    pub fn bound_port(&self) -> Option<u16> {
177        self.bound_port
178    }
179}