1use std::collections::BTreeMap;
2use std::io::{BufRead, BufReader, BufWriter, Read, Write};
3use std::path::Path;
4
5use anyhow::{Context, Result};
6use second_brain_core::store::SkipReason;
7use serde_json::Value;
8
9use crate::crypto::{SyncEncryptor, open_line, seal_line};
10use crate::protocol::{SyncHeader, read_header};
11use crate::ssh::untrusted_host_hint;
12
13const IMPORT_CHUNK: usize = 1000;
16const EXPORT_PAGE: usize = 2000;
17
18pub trait Local {
21 fn cursors(&self) -> Result<Vec<(String, u64)>>;
22 fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()>;
23 fn import(&self, header: &SyncHeader, records: &[Value]) -> Result<Imported>;
25 fn export(&self, after_seq: u64, limit: usize) -> Result<Exported>;
26}
27
28#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
29pub struct Imported {
30 pub created: u64,
31 pub updated: u64,
32 pub deleted: u64,
33 pub skipped: u64,
34 pub dangling: u64,
35 pub max_seq: u64,
36}
37
38impl Imported {
39 pub fn dangling_in(skipped_by_reason: &BTreeMap<String, u64>) -> u64 {
40 skipped_by_reason
41 .get(SkipReason::DanglingEndpoint.as_str())
42 .copied()
43 .unwrap_or(0)
44 }
45}
46
47pub struct Exported {
48 pub records: Vec<Value>,
49 pub max_seq: u64,
50}
51
52pub trait Transport {
53 fn host(&self) -> &str;
54 fn whoami(&self) -> Result<String>;
55 fn export(&self, after_seq: u64) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)>;
56 fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Remote>)>;
57}
58
59pub trait Remote {
61 fn wait(&mut self) -> Result<Exit>;
62 fn stop(&mut self) -> Result<()>;
63}
64
65pub struct Exit {
66 pub success: bool,
67 pub status: String,
68}
69
70pub fn sync(
73 local: &dyn Local,
74 transport: &dyn Transport,
75 encryptor: &dyn SyncEncryptor,
76 data_dir: &Path,
77 report: &dyn Fn(&str),
78) -> Result<()> {
79 let peer_host = transport.host();
80 let peer_machine_id = transport
81 .whoami()
82 .context("resolving peer machine id (is the peer's daemon running and reachable?)")?;
83
84 report(&format!("pulling from {peer_host}..."));
85 let pull_stats = pull(local, transport, &peer_machine_id, encryptor)?;
86 report(&format!(
87 " {} created, {} updated, {} deleted, {} skipped",
88 pull_stats.created, pull_stats.updated, pull_stats.deleted, pull_stats.skipped
89 ));
90 if pull_stats.dangling > 0 {
91 report(&format!(
92 " warning: {} records dropped with unresolvable endpoints; \
93 run `sb prune-relations` on the peer if this persists",
94 pull_stats.dangling
95 ));
96 }
97
98 report(&format!("pushing to {peer_host}..."));
99 let max_seq = push(local, transport, &peer_machine_id, encryptor, report)?;
100 report(&format!(" pushed through seq {max_seq}"));
101
102 crate::peers::record_success(data_dir, chrono::Utc::now())
103 .context("recording that the sync succeeded")?;
104 report("sync complete.");
105 Ok(())
106}
107
108fn cursor(local: &dyn Local, key: &str, legacy_key: &str) -> Result<u64> {
112 let cursors = local.cursors().context("getting sync status")?;
113 Ok(cursors
114 .iter()
115 .find(|(peer_id, _)| peer_id == key)
116 .or_else(|| cursors.iter().find(|(peer_id, _)| peer_id == legacy_key))
117 .map(|(_, last_seq)| *last_seq)
118 .unwrap_or(0))
119}
120
121pub fn pull(
122 local: &dyn Local,
123 transport: &dyn Transport,
124 peer_machine_id: &str,
125 encryptor: &dyn SyncEncryptor,
126) -> Result<Imported> {
127 let peer_host = transport.host();
128 let after_seq = cursor(local, peer_machine_id, peer_host)?;
129
130 let (output, mut remote) = transport.export(after_seq)?;
131 let mut reader = BufReader::new(output);
132 let peer_header = match read_checked_header(&mut reader, peer_host) {
133 Ok(header) => header,
134 Err(e) => {
135 if matches!(reader.fill_buf(), Ok(rest) if rest.is_empty()) {
138 let exit = remote.wait()?;
139 if !exit.success {
140 anyhow::bail!(
141 "sync-export on {peer_host} exited with {}; its error is printed above.\n hint: {}",
142 exit.status,
143 untrusted_host_hint(peer_host)
144 );
145 }
146 } else {
147 remote.stop()?;
148 }
149 return Err(e);
150 }
151 };
152
153 let mut totals = Imported::default();
154 let mut chunk = Vec::with_capacity(IMPORT_CHUNK);
155 let mut post_chunk = |chunk: &[Value]| -> Result<()> {
156 if chunk.is_empty() {
157 return Ok(());
158 }
159 let resp = local.import(&peer_header, chunk)?;
160 totals.created += resp.created;
161 totals.updated += resp.updated;
162 totals.deleted += resp.deleted;
163 totals.skipped += resp.skipped;
164 totals.dangling += resp.dangling;
165 totals.max_seq = totals.max_seq.max(resp.max_seq);
166 Ok(())
167 };
168 let mut import_stream = || -> Result<()> {
169 for line in (&mut reader).lines() {
170 let line = line?;
171 if line.trim().is_empty() {
172 continue;
173 }
174 let plaintext = open_line(encryptor, &line)?;
175 chunk.push(serde_json::from_slice(&plaintext)?);
176 if chunk.len() >= IMPORT_CHUNK {
177 post_chunk(&chunk)?;
178 chunk.clear();
179 }
180 }
181 post_chunk(&chunk)?;
182 chunk.clear();
183 Ok(())
184 };
185 if let Err(e) = import_stream() {
186 remote.stop()?;
187 return Err(e);
188 }
189
190 let exit = remote.wait()?;
191 if !exit.success {
192 anyhow::bail!(
193 "ssh pull from {peer_host} exited with {}.\n hint: {}",
194 exit.status,
195 untrusted_host_hint(peer_host)
196 );
197 }
198
199 if totals.max_seq > after_seq {
200 local
201 .set_cursor(peer_machine_id, totals.max_seq)
202 .context("updating sync state")?;
203 }
204
205 Ok(totals)
206}
207
208pub fn push(
209 local: &dyn Local,
210 transport: &dyn Transport,
211 peer_machine_id: &str,
212 encryptor: &dyn SyncEncryptor,
213 report: &dyn Fn(&str),
214) -> Result<u64> {
215 let peer_host = transport.host();
216 let push_peer = format!("push:{peer_machine_id}");
217 let mut cursor = cursor(local, &push_peer, &format!("push:{peer_host}"))?;
218
219 let (input, mut remote) = transport.import()?;
220 let mut total_pushed = 0u64;
221 let stream = || -> Result<()> {
222 let mut writer = BufWriter::new(input);
223 writeln!(writer, "{}", SyncHeader::local().to_line())?;
224
225 loop {
226 let export = local.export(cursor, EXPORT_PAGE)?;
227 if export.records.is_empty() {
228 break;
229 }
230 let batch_count = export.records.len();
231 for record in &export.records {
232 let plaintext = serde_json::to_vec(record)?;
233 let line = seal_line(encryptor, &plaintext)?;
234 writer.write_all(line.as_bytes())?;
235 writer.write_all(b"\n")?;
236 }
237 writer.flush()?;
238
239 total_pushed += batch_count as u64;
240 cursor = export.max_seq;
241 report(&format!(
242 " batch: {batch_count} records (through seq {cursor})"
243 ));
244
245 if batch_count < EXPORT_PAGE {
246 break;
247 }
248 }
249 writer.flush()?;
250 Ok(())
251 };
252 let streamed = stream();
253
254 let exit = remote.wait()?;
257 if !exit.success {
258 anyhow::bail!(
259 "ssh push to {peer_host} exited with {}.\n hint: {}",
260 exit.status,
261 untrusted_host_hint(peer_host)
262 );
263 }
264 streamed?;
265
266 if total_pushed > 0 {
267 local
268 .set_cursor(&push_peer, cursor)
269 .context("updating push sync state")?;
270 }
271
272 report(&format!(" total: {total_pushed} records pushed"));
273 Ok(cursor)
274}
275
276pub fn read_checked_header<R: BufRead>(reader: &mut R, sender: &str) -> Result<SyncHeader> {
277 let header =
278 read_header(reader).with_context(|| format!("reading the sync header from {sender}"))?;
279 SyncHeader::local().ensure_matches("this machine", &header, sender)?;
280 Ok(header)
281}