use std::io::Read;
use anyhow::{Context, Result};
use second_brain_sync::keyring::IdentityMismatch;
use second_brain_sync::peers;
use second_brain_sync::protocol::{BelowFloorLine, Hello, Merging, RefusedLine, SyncHeader};
use second_brain_sync::session::{self, Exported, Imported, Local, Merged, read_checked_header};
use second_brain_sync::ssh::SshTransport;
use crate::api;
pub(crate) fn cmd_sync_trust(host: &str) -> Result<()> {
use std::io::Write;
let scan = std::process::Command::new("ssh-keyscan")
.arg("-T")
.arg("10")
.arg(host)
.output()
.context("running ssh-keyscan (is openssh-client installed?)")?;
if !scan.status.success() {
let stderr = String::from_utf8_lossy(&scan.stderr);
anyhow::bail!(
"ssh-keyscan {host} failed (exit {}): {}",
scan.status,
stderr.trim()
);
}
let raw = String::from_utf8(scan.stdout).context("ssh-keyscan returned non-UTF8 output")?;
let lines = second_brain_sync::ssh::parse_keyscan_output(&raw);
if lines.is_empty() {
anyhow::bail!("ssh-keyscan {host} returned no host keys — host unreachable or filtered");
}
let canonical: String = lines
.iter()
.map(second_brain_sync::ssh::compose_known_host_entry)
.collect();
let mut fp_child = std::process::Command::new("ssh-keygen")
.arg("-lf")
.arg("-")
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::inherit())
.spawn()
.context("running ssh-keygen (is openssh-client installed?)")?;
{
let stdin = fp_child.stdin.take().unwrap();
let mut writer = std::io::BufWriter::new(stdin);
writer.write_all(canonical.as_bytes())?;
}
let fp_out = fp_child.wait_with_output()?;
if !fp_out.status.success() {
anyhow::bail!("ssh-keygen -lf failed: exit {}", fp_out.status);
}
let fingerprints = String::from_utf8_lossy(&fp_out.stdout);
eprintln!("Host: {host}");
eprintln!("Fetched {} key(s). Fingerprints:", lines.len());
for fp_line in fingerprints.lines() {
eprintln!(" {fp_line}");
}
eprintln!();
eprintln!("Verify these fingerprints out-of-band before trusting them.");
eprint!("Pin to ~/.second-brain/known_hosts? [y/N] ");
std::io::stderr().flush().ok();
let mut answer = String::new();
std::io::stdin()
.read_line(&mut answer)
.context("reading confirmation")?;
let trimmed = answer.trim();
if !trimmed.eq_ignore_ascii_case("y") && !trimmed.eq_ignore_ascii_case("yes") {
eprintln!("aborted; nothing written.");
return Ok(());
}
let path = second_brain_sync::ssh::known_hosts_path()?;
second_brain_sync::ssh::append_known_host(&path, &lines)?;
eprintln!(
"pinned {} key(s) for {host} -> {}",
lines.len(),
path.display()
);
Ok(())
}
const SNAPSHOT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(6 * 3600);
const MERGE_POLL: std::time::Duration = std::time::Duration::from_secs(2);
const MERGE_WAIT: std::time::Duration = std::time::Duration::from_secs(600);
enum ExportPage {
Lines {
lines: Vec<String>,
max_seq: u64,
hello: Option<Hello>,
},
BelowFloor(BelowFloorLine),
Refused(RefusedLine),
}
fn below_floor_line(body: &str) -> Option<BelowFloorLine> {
let value: serde_json::Value = serde_json::from_str(body).ok()?;
if value["below_floor"] != true {
return None;
}
Some(BelowFloorLine {
after_seq: value["after_seq"].as_u64()?,
log_floor: value["log_floor"].as_u64()?,
})
}
fn fetch_export_page(
client: &reqwest::blocking::Client,
request: &serde_json::Value,
) -> Result<ExportPage> {
#[derive(serde::Deserialize)]
struct ExportResp {
lines: Vec<String>,
max_seq: u64,
#[serde(default)]
hello: Option<Hello>,
}
let resp = client
.post(format!("{}/v1/sync/export", api::base_url()))
.json(request)
.send()
.context("calling sync/export on daemon")?;
if resp.status() == reqwest::StatusCode::CONFLICT {
let body = resp.text().context("reading the daemon's sync refusal")?;
if let Some(line) = below_floor_line(&body) {
return Ok(ExportPage::BelowFloor(line));
}
if let Some(refused) = RefusedLine::from_refusal_body(&body) {
return Ok(ExportPage::Refused(refused));
}
anyhow::bail!("sync/export returned error: {}", api::error_message(&body));
}
let export: ExportResp = api::daemon_ok(resp)
.context("sync/export returned error")?
.json()?;
Ok(ExportPage::Lines {
lines: export.lines,
max_seq: export.max_seq,
hello: export.hello,
})
}
pub(crate) fn cmd_sync_snapshot(hello_arg: Option<&str>, apply: bool) -> Result<()> {
snapshot(hello_arg, apply, None)
}
pub(crate) fn cmd_sync_snapshot_pinned(
hello_arg: Option<&str>,
apply: bool,
exporter: &str,
) -> Result<()> {
snapshot(hello_arg, apply, Some(exporter))
}
fn snapshot(hello_arg: Option<&str>, apply: bool, exporter: Option<&str>) -> Result<()> {
let client = api::sync_client(SNAPSHOT_TIMEOUT)?;
let base = api::base_url();
if apply {
let request = match exporter {
Some(exporter) => apply_request(&client, &base, exporter, Box::new(std::io::stdin())),
None => client
.post(format!("{base}/v1/sync/snapshot/apply"))
.header(reqwest::header::CONTENT_TYPE, "application/octet-stream")
.body(reqwest::blocking::Body::new(std::io::stdin())),
};
let resp = api::daemon_ok(
request
.send()
.context("streaming the snapshot to the daemon")?,
)
.context("sync/snapshot/apply returned error")?;
let merging: Merging = resp.json().context("reading the daemon's merge answer")?;
println!("{}", serde_json::to_string(&merging)?);
return Ok(());
}
let arg = hello_arg.context("usage: sb sync-snapshot --hello <hello> | --apply")?;
let hello = Hello::from_arg(arg).context("reading --hello")?;
let mut resp = api::daemon_ok(
client
.post(format!("{base}/v1/sync/snapshot"))
.json(&serde_json::json!({"hello": hello, "with_chunks": true}))
.send()
.context("calling sync/snapshot on daemon")?,
)
.context("sync/snapshot returned error")?;
let stdout = std::io::stdout();
let mut out = stdout.lock();
std::io::copy(&mut resp, &mut out).context("streaming the snapshot to stdout")?;
std::io::Write::flush(&mut out)?;
Ok(())
}
pub(crate) fn cmd_sync_export(after_seq: u64, hello_arg: Option<&str>) -> Result<()> {
let hello = hello_arg
.map(Hello::from_arg)
.transpose()
.context("reading --hello")?;
let client = api::sync_client(std::time::Duration::from_secs(120))?;
const BATCH_SIZE: usize = 2000;
let stdout = std::io::stdout();
let mut writer = std::io::BufWriter::new(stdout.lock());
let mut cursor = after_seq;
let mut total = 0u64;
let mut header_written = false;
loop {
use std::io::Write;
let page = fetch_export_page(&client, &export_request(cursor, BATCH_SIZE, hello.as_ref()))?;
let (lines, max_seq, resp_hello) = match page {
ExportPage::Lines {
lines,
max_seq,
hello,
} => (lines, max_seq, hello),
ExportPage::BelowFloor(line) if !header_written => {
writeln!(writer, "{}", line.to_line())?;
writer.flush()?;
eprintln!(
"the log no longer holds records after seq {}: its floor is {}; the puller merges a snapshot",
line.after_seq, line.log_floor
);
return Ok(());
}
ExportPage::BelowFloor(line) => anyhow::bail!(
"the log floor moved to {} while exporting after seq {}",
line.log_floor,
line.after_seq
),
ExportPage::Refused(refused) => {
if !header_written {
writeln!(writer, "{}", refused.to_line())?;
writer.flush()?;
}
anyhow::bail!("sync/export returned error: {}", refused.error);
}
};
if !header_written {
let header = SyncHeader {
hello: resp_hello,
..SyncHeader::local()
};
writeln!(writer, "{}", header.to_line())?;
writer.flush()?;
header_written = true;
}
if lines.is_empty() {
break;
}
let batch_count = lines.len();
for line in &lines {
writer.write_all(line.as_bytes())?;
writer.write_all(b"\n")?;
}
writer.flush()?;
total += batch_count as u64;
cursor = max_seq;
if batch_count < BATCH_SIZE {
break;
}
}
eprintln!("exported {total} records through seq {cursor}");
Ok(())
}
pub(crate) fn cmd_sync_key() -> Result<()> {
anyhow::bail!(
"sync keys no longer leave the daemon: it turns an existing sync.key into its own \
wrapped identity key on start and deletes the file; add a device with `sb pair`"
)
}
pub(crate) fn ensure_pinned(named: Option<&str>, pinned: &str) -> Result<()> {
match named {
Some(id) if id == pinned => Ok(()),
Some(id) => anyhow::bail!("sync-serve refused: this key belongs to {pinned}, not {id}"),
None => anyhow::bail!("sync-serve refused: the stream names no machine, expected {pinned}"),
}
}
pub(crate) fn cmd_sync_import() -> Result<()> {
import(None)
}
pub(crate) fn cmd_sync_import_pinned(expected: &str) -> Result<()> {
import(Some(expected))
}
fn import(expected: Option<&str>) -> Result<()> {
let stdin = std::io::stdin();
let mut input = std::io::BufReader::new(stdin.lock());
let header = read_checked_header(&mut input, "the sender")?;
if let Some(expected) = expected {
ensure_pinned(
header.hello.as_ref().map(|h| h.machine_id.as_str()),
expected,
)?;
}
let local = DaemonApi {
client: api::sync_client(std::time::Duration::from_secs(600))?,
base: api::base_url(),
};
let stdout = std::io::stdout();
let totals = session::serve_push(
&local,
&header,
input,
std::io::BufWriter::new(stdout.lock()),
)?;
eprintln!(
"imported: {} created, {} updated, {} deleted, {} skipped, {} errors (through seq {})",
totals.created,
totals.updated,
totals.deleted,
totals.skipped,
totals.errors,
totals.max_seq
);
Ok(())
}
pub(crate) fn cmd_sync_status() -> Result<()> {
let api_base = api::base_url();
let client = api::client(std::time::Duration::from_secs(30))?;
#[derive(serde::Deserialize)]
struct PeerInfo {
peers: Vec<PeerState>,
}
#[derive(serde::Deserialize)]
struct PeerState {
peer_id: String,
last_seq: u64,
last_sync_at: String,
}
let resp: PeerInfo = api::daemon_ok(
client
.get(format!("{api_base}/v1/sync/status"))
.send()
.context("getting sync status from daemon")?,
)?
.json()?;
if resp.peers.is_empty() {
println!("no sync peers configured");
} else {
for state in &resp.peers {
println!(
"peer: {}, last_seq: {}, last_sync: {}",
state.peer_id, state.last_seq, state.last_sync_at
);
}
}
Ok(())
}
pub(crate) fn cmd_sync_compact(ops: &[String], dry_run: bool) -> Result<()> {
let api_base = api::base_url();
let client = api::client(std::time::Duration::from_secs(30))?;
#[derive(serde::Deserialize)]
struct CompactResp {
groups_with_duplicates: u64,
rows_deleted: u64,
}
let resp: CompactResp = api::daemon_ok(
client
.post(format!("{api_base}/v1/sync/compact"))
.json(&serde_json::json!({"ops": ops, "dry_run": dry_run}))
.send()
.context("calling compact on daemon")?,
)?
.json()?;
if dry_run {
eprintln!(
"dry run: {} groups have duplicates, {} rows would be deleted",
resp.groups_with_duplicates, resp.rows_deleted
);
} else {
eprintln!(
"compacted {} groups, deleted {} duplicate rows",
resp.groups_with_duplicates, resp.rows_deleted
);
}
Ok(())
}
pub(crate) fn cmd_sync(peer_host: &str, args: &[String]) -> Result<()> {
let data_dir = second_brain_core::machine::data_dir()?;
let sb_binary = peer_sb(args, &data_dir, peer_host)?;
let local = DaemonApi {
client: api::sync_client(std::time::Duration::from_secs(600))?,
base: api::base_url(),
};
let transport = SshTransport::new(peer_host, &sb_binary)?;
session::sync(&local, &transport, &data_dir, &|line| eprintln!("{line}"))
}
fn peer_sb(args: &[String], data_dir: &std::path::Path, peer_host: &str) -> Result<String> {
let flag = args
.iter()
.position(|arg| arg == "--sb-path")
.and_then(|i| args.get(i + 1));
if let Some(sb) = flag {
return Ok(sb.clone());
}
Ok(peers::configured_peer(data_dir, peer_host)
.context("reading sync-peers")?
.map_or_else(|| "sb".to_string(), |peer| peer.sb))
}
fn apply_request(
client: &reqwest::blocking::Client,
base: &str,
exporter: &str,
stream: Box<dyn Read + Send>,
) -> reqwest::blocking::RequestBuilder {
client
.post(format!("{base}/v1/sync/snapshot/apply"))
.query(&[("exporter", exporter)])
.header(reqwest::header::CONTENT_TYPE, "application/octet-stream")
.body(reqwest::blocking::Body::new(stream))
}
fn merged_unless_dropped(started: &Merging, cursors: &[(String, u64)]) -> Result<Merged> {
let cursor = cursors
.iter()
.find(|(peer, _)| *peer == started.exporter)
.map_or(0, |(_, seq)| *seq);
if cursor < started.seq {
anyhow::bail!(
"the daemon dropped the snapshot from {} without merging it (see its log); the next \
sync sends another",
started.exporter
);
}
Ok(Merged::Done { seq: started.seq })
}
fn refusal_error(body: &str) -> anyhow::Error {
match RefusedLine::from_refusal_body(body) {
Some(refused) if refused.identity => IdentityMismatch(refused.error).into(),
_ => anyhow::anyhow!("{}", api::error_message(body)),
}
}
struct DaemonApi {
client: reqwest::blocking::Client,
base: String,
}
impl Local for DaemonApi {
fn cursors(&self) -> Result<Vec<(String, u64)>> {
#[derive(serde::Deserialize)]
struct PeerInfo {
peers: Vec<PeerState>,
}
#[derive(serde::Deserialize)]
struct PeerState {
peer_id: String,
last_seq: u64,
}
let status: PeerInfo = self
.client
.get(format!("{}/v1/sync/status", self.base))
.send()?
.json()?;
Ok(status
.peers
.into_iter()
.map(|p| (p.peer_id, p.last_seq))
.collect())
}
fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()> {
self.client
.post(format!("{}/v1/sync/state", self.base))
.json(&serde_json::json!({"peer_id": peer_id, "last_seq": last_seq}))
.send()?;
Ok(())
}
fn import(&self, header: &SyncHeader, lines: &[String]) -> Result<Imported> {
#[derive(serde::Deserialize)]
struct ImportResp {
created: u64,
updated: u64,
deleted: u64,
skipped: u64,
#[serde(default)]
errors: u64,
max_seq: u64,
#[serde(default)]
reached: Option<u64>,
#[serde(default)]
skipped_by_reason: std::collections::BTreeMap<String, u64>,
}
let resp: ImportResp = api::daemon_ok(
self.client
.post(format!("{}/v1/sync/import", self.base))
.json(&serde_json::json!({"header": header, "lines": lines}))
.send()
.context("posting import to daemon")?,
)
.context("import returned error")?
.json()?;
Ok(Imported {
created: resp.created,
updated: resp.updated,
deleted: resp.deleted,
skipped: resp.skipped,
dangling: Imported::dangling_in(&resp.skipped_by_reason),
errors: resp.errors,
max_seq: resp.max_seq,
reached: resp.reached.unwrap_or(resp.max_seq),
})
}
fn export(&self, peer_id: &str, after_seq: u64, limit: usize) -> Result<Exported> {
#[derive(serde::Deserialize)]
struct ExportResp {
lines: Vec<String>,
max_seq: u64,
}
let mut request = export_request(after_seq, limit, None);
request["peer_id"] = serde_json::json!(peer_id);
let export: ExportResp = api::daemon_ok(
self.client
.post(format!("{}/v1/sync/export", self.base))
.json(&request)
.send()
.context("getting export from daemon")?,
)
.context("export returned error")?
.json()?;
Ok(Exported {
lines: export.lines,
max_seq: export.max_seq,
})
}
fn hello_for(&self, peer_id: &str) -> Result<Hello> {
Ok(api::daemon_ok(
self.client
.post(format!("{}/v1/sync/hello", self.base))
.json(&serde_json::json!({"peer_id": peer_id}))
.send()
.context("getting the sync hello from daemon")?,
)
.context("sync/hello returned error")?
.json()?)
}
fn accept_hello(&self, hello: &Hello) -> Result<()> {
let resp = self
.client
.post(format!("{}/v1/sync/accept-hello", self.base))
.json(&serde_json::json!({"hello": hello}))
.send()
.context("passing the peer's sync hello to daemon")?;
if resp.status() == reqwest::StatusCode::CONFLICT {
let body = resp.text().context("reading the daemon's sync refusal")?;
return Err(refusal_error(&body).context("sync/accept-hello returned error"));
}
api::daemon_ok(resp).context("sync/accept-hello returned error")?;
Ok(())
}
fn record_ack(&self, peer_id: &str, acked_seq: u64) -> Result<()> {
api::daemon_ok(
self.client
.post(format!("{}/v1/sync/ack", self.base))
.json(&serde_json::json!({"peer_id": peer_id, "acked_seq": acked_seq}))
.send()
.context("recording the peer's ack on daemon")?,
)
.context("sync/ack returned error")?;
Ok(())
}
fn record_peer_in(&self, peer_id: &str, seq: u64) -> Result<()> {
api::daemon_ok(
self.client
.post(format!("{}/v1/sync/peer-in", self.base))
.json(&serde_json::json!({"peer_id": peer_id, "seq": seq}))
.send()
.context("recording what the push applied on daemon")?,
)
.context("sync/peer-in returned error")?;
Ok(())
}
fn merging(&self) -> Result<Option<Merging>> {
#[derive(serde::Deserialize)]
struct Status {
merging: Option<Merging>,
}
let status: Status = api::daemon_ok(
self.client
.get(format!("{}/v1/sync/snapshot/status", self.base))
.send()
.context("reading the snapshot merge status from daemon")?,
)
.context("sync/snapshot/status returned error")?
.json()?;
Ok(status.merging)
}
fn export_snapshot(&self, peer: &Hello) -> Result<Box<dyn Read + Send>> {
let resp = api::daemon_ok(
api::client(SNAPSHOT_TIMEOUT)?
.post(format!("{}/v1/sync/snapshot", self.base))
.json(&serde_json::json!({"hello": peer, "with_chunks": true}))
.send()
.context("asking the daemon for a snapshot")?,
)
.context("sync/snapshot returned error")?;
Ok(Box::new(resp))
}
fn merge_snapshot(&self, exporter: &str, stream: Box<dyn Read + Send>) -> Result<Merged> {
let started: Merging = api::daemon_ok(
apply_request(
&api::client(SNAPSHOT_TIMEOUT)?,
&self.base,
exporter,
stream,
)
.send()
.context("streaming the snapshot to the daemon")?,
)
.context("sync/snapshot/apply returned error")?
.json()
.context("reading the daemon's merge answer")?;
let deadline = std::time::Instant::now() + MERGE_WAIT;
loop {
match self.merging()? {
Some(running) if running.snapshot_id == started.snapshot_id => {
if std::time::Instant::now() >= deadline {
return Ok(Merged::Running(running));
}
std::thread::sleep(MERGE_POLL);
}
_ => break,
}
}
merged_unless_dropped(&started, &self.cursors()?)
}
fn push_report(&self, peer_id: &str) -> Result<Option<u64>> {
#[derive(serde::Deserialize)]
struct Report {
reported: Option<u64>,
}
let report: Report = api::daemon_ok(
self.client
.get(format!("{}/v1/sync/push-report", self.base))
.query(&[("peer_id", peer_id)])
.send()
.context("reading the peer's last push report from daemon")?,
)
.context("sync/push-report returned error")?
.json()?;
Ok(report.reported)
}
fn record_push_report(&self, peer_id: &str, reported: Option<u64>) -> Result<()> {
api::daemon_ok(
self.client
.post(format!("{}/v1/sync/push-report", self.base))
.json(&serde_json::json!({"peer_id": peer_id, "reported": reported}))
.send()
.context("recording the peer's push report on daemon")?,
)
.context("sync/push-report returned error")?;
Ok(())
}
}
fn export_request(after_seq: u64, limit: usize, hello: Option<&Hello>) -> serde_json::Value {
serde_json::json!({
"after_seq": after_seq,
"limit": limit,
"header": SyncHeader::local(),
"hello": hello,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_merge_that_never_raised_the_exporters_cursor_is_reported_dropped() {
let started = Merging {
snapshot_id: "s1".to_string(),
exporter: "user-pi".to_string(),
seq: 40,
};
let below = vec![("user-pi".to_string(), 39), ("other".to_string(), 99)];
let error = merged_unless_dropped(&started, &below).unwrap_err();
assert!(format!("{error:#}").contains("dropped"), "{error:#}");
assert!(merged_unless_dropped(&started, &[]).is_err());
let at = vec![("user-pi".to_string(), 40)];
assert!(matches!(
merged_unless_dropped(&started, &at).unwrap(),
Merged::Done { seq: 40 }
));
}
#[test]
fn a_pulled_snapshot_is_applied_naming_its_exporter() {
let request = apply_request(
&reqwest::blocking::Client::new(),
"http://127.0.0.1:9",
"user pi",
Box::new(std::io::empty()),
)
.build()
.unwrap();
assert_eq!(request.url().path(), "/v1/sync/snapshot/apply");
assert_eq!(request.url().query(), Some("exporter=user+pi"));
}
#[test]
fn a_refusal_the_daemon_flags_as_identity_is_an_identity_mismatch() {
let flagged = refusal_error(r#"{"error":"run `sb pair`","identity":true}"#);
assert_eq!(
flagged.downcast_ref::<IdentityMismatch>(),
Some(&IdentityMismatch("run `sb pair`".to_string()))
);
let other = refusal_error(r#"{"error":"sync hello rejected: from a: wall clock"}"#);
assert!(other.downcast_ref::<IdentityMismatch>().is_none());
assert_eq!(other.to_string(), "sync hello rejected: from a: wall clock");
}
#[test]
fn sb_sync_finds_the_peers_sb_like_the_daemon_unless_told() {
let dir = tempfile::tempdir().unwrap();
let none: Vec<String> = Vec::new();
assert_eq!(peer_sb(&none, dir.path(), "user@pi").unwrap(), "sb");
std::fs::write(dir.path().join(peers::PEERS_FILE), "user@pi /opt/sb\n").unwrap();
assert_eq!(peer_sb(&none, dir.path(), "user@pi").unwrap(), "/opt/sb");
let flag = vec!["--sb-path".to_string(), "/usr/bin/sb".to_string()];
assert_eq!(
peer_sb(&flag, dir.path(), "user@pi").unwrap(),
"/usr/bin/sb"
);
}
}