use std::collections::BTreeMap;
use std::io::{BufRead, BufReader, BufWriter, Read, Write};
use std::path::Path;
use anyhow::{Context, Result};
use second_brain_core::store::SkipReason;
use serde_json::Value;
use crate::crypto::{SyncEncryptor, open_line, seal_line};
use crate::protocol::{SyncHeader, read_header};
use crate::ssh::untrusted_host_hint;
const IMPORT_CHUNK: usize = 1000;
const EXPORT_PAGE: usize = 2000;
pub trait Local {
fn cursors(&self) -> Result<Vec<(String, u64)>>;
fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()>;
fn import(&self, header: &SyncHeader, records: &[Value]) -> Result<Imported>;
fn export(&self, after_seq: u64, limit: usize) -> Result<Exported>;
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct Imported {
pub created: u64,
pub updated: u64,
pub deleted: u64,
pub skipped: u64,
pub dangling: u64,
pub max_seq: u64,
}
impl Imported {
pub fn dangling_in(skipped_by_reason: &BTreeMap<String, u64>) -> u64 {
skipped_by_reason
.get(SkipReason::DanglingEndpoint.as_str())
.copied()
.unwrap_or(0)
}
}
pub struct Exported {
pub records: Vec<Value>,
pub max_seq: u64,
}
pub trait Transport {
fn host(&self) -> &str;
fn whoami(&self) -> Result<String>;
fn export(&self, after_seq: u64) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)>;
fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Remote>)>;
}
pub trait Remote {
fn wait(&mut self) -> Result<Exit>;
fn stop(&mut self) -> Result<()>;
}
pub struct Exit {
pub success: bool,
pub status: String,
}
pub fn sync(
local: &dyn Local,
transport: &dyn Transport,
encryptor: &dyn SyncEncryptor,
data_dir: &Path,
report: &dyn Fn(&str),
) -> Result<()> {
let peer_host = transport.host();
let peer_machine_id = transport
.whoami()
.context("resolving peer machine id (is the peer's daemon running and reachable?)")?;
report(&format!("pulling from {peer_host}..."));
let pull_stats = pull(local, transport, &peer_machine_id, encryptor)?;
report(&format!(
" {} created, {} updated, {} deleted, {} skipped",
pull_stats.created, pull_stats.updated, pull_stats.deleted, pull_stats.skipped
));
if pull_stats.dangling > 0 {
report(&format!(
" warning: {} records dropped with unresolvable endpoints; \
run `sb prune-relations` on the peer if this persists",
pull_stats.dangling
));
}
report(&format!("pushing to {peer_host}..."));
let max_seq = push(local, transport, &peer_machine_id, encryptor, report)?;
report(&format!(" pushed through seq {max_seq}"));
crate::peers::record_success(data_dir, chrono::Utc::now())
.context("recording that the sync succeeded")?;
report("sync complete.");
Ok(())
}
fn cursor(local: &dyn Local, key: &str, legacy_key: &str) -> Result<u64> {
let cursors = local.cursors().context("getting sync status")?;
Ok(cursors
.iter()
.find(|(peer_id, _)| peer_id == key)
.or_else(|| cursors.iter().find(|(peer_id, _)| peer_id == legacy_key))
.map(|(_, last_seq)| *last_seq)
.unwrap_or(0))
}
pub fn pull(
local: &dyn Local,
transport: &dyn Transport,
peer_machine_id: &str,
encryptor: &dyn SyncEncryptor,
) -> Result<Imported> {
let peer_host = transport.host();
let after_seq = cursor(local, peer_machine_id, peer_host)?;
let (output, mut remote) = transport.export(after_seq)?;
let mut reader = BufReader::new(output);
let peer_header = match read_checked_header(&mut reader, peer_host) {
Ok(header) => header,
Err(e) => {
if matches!(reader.fill_buf(), Ok(rest) if rest.is_empty()) {
let exit = remote.wait()?;
if !exit.success {
anyhow::bail!(
"sync-export on {peer_host} exited with {}; its error is printed above.\n hint: {}",
exit.status,
untrusted_host_hint(peer_host)
);
}
} else {
remote.stop()?;
}
return Err(e);
}
};
let mut totals = Imported::default();
let mut chunk = Vec::with_capacity(IMPORT_CHUNK);
let mut post_chunk = |chunk: &[Value]| -> Result<()> {
if chunk.is_empty() {
return Ok(());
}
let resp = local.import(&peer_header, chunk)?;
totals.created += resp.created;
totals.updated += resp.updated;
totals.deleted += resp.deleted;
totals.skipped += resp.skipped;
totals.dangling += resp.dangling;
totals.max_seq = totals.max_seq.max(resp.max_seq);
Ok(())
};
let mut import_stream = || -> Result<()> {
for line in (&mut reader).lines() {
let line = line?;
if line.trim().is_empty() {
continue;
}
let plaintext = open_line(encryptor, &line)?;
chunk.push(serde_json::from_slice(&plaintext)?);
if chunk.len() >= IMPORT_CHUNK {
post_chunk(&chunk)?;
chunk.clear();
}
}
post_chunk(&chunk)?;
chunk.clear();
Ok(())
};
if let Err(e) = import_stream() {
remote.stop()?;
return Err(e);
}
let exit = remote.wait()?;
if !exit.success {
anyhow::bail!(
"ssh pull from {peer_host} exited with {}.\n hint: {}",
exit.status,
untrusted_host_hint(peer_host)
);
}
if totals.max_seq > after_seq {
local
.set_cursor(peer_machine_id, totals.max_seq)
.context("updating sync state")?;
}
Ok(totals)
}
pub fn push(
local: &dyn Local,
transport: &dyn Transport,
peer_machine_id: &str,
encryptor: &dyn SyncEncryptor,
report: &dyn Fn(&str),
) -> Result<u64> {
let peer_host = transport.host();
let push_peer = format!("push:{peer_machine_id}");
let mut cursor = cursor(local, &push_peer, &format!("push:{peer_host}"))?;
let (input, mut remote) = transport.import()?;
let mut total_pushed = 0u64;
let stream = || -> Result<()> {
let mut writer = BufWriter::new(input);
writeln!(writer, "{}", SyncHeader::local().to_line())?;
loop {
let export = local.export(cursor, EXPORT_PAGE)?;
if export.records.is_empty() {
break;
}
let batch_count = export.records.len();
for record in &export.records {
let plaintext = serde_json::to_vec(record)?;
let line = seal_line(encryptor, &plaintext)?;
writer.write_all(line.as_bytes())?;
writer.write_all(b"\n")?;
}
writer.flush()?;
total_pushed += batch_count as u64;
cursor = export.max_seq;
report(&format!(
" batch: {batch_count} records (through seq {cursor})"
));
if batch_count < EXPORT_PAGE {
break;
}
}
writer.flush()?;
Ok(())
};
let streamed = stream();
let exit = remote.wait()?;
if !exit.success {
anyhow::bail!(
"ssh push to {peer_host} exited with {}.\n hint: {}",
exit.status,
untrusted_host_hint(peer_host)
);
}
streamed?;
if total_pushed > 0 {
local
.set_cursor(&push_peer, cursor)
.context("updating push sync state")?;
}
report(&format!(" total: {total_pushed} records pushed"));
Ok(cursor)
}
pub fn read_checked_header<R: BufRead>(reader: &mut R, sender: &str) -> Result<SyncHeader> {
let header =
read_header(reader).with_context(|| format!("reading the sync header from {sender}"))?;
SyncHeader::local().ensure_matches("this machine", &header, sender)?;
Ok(header)
}