use std::collections::{HashMap, HashSet};
use grain_id::GrainId;
use sapphire_sync::{
Content, ContentHash, ContentSource, PathUpdate, Replica, Report, VersionVector,
};
use tokio::io::{AsyncRead, AsyncWrite, ReadHalf};
use tokio::sync::{Mutex, mpsc};
use tokio::task::JoinHandle;
use crate::error::{Error, Result};
use crate::frame::{Frame, read_frame, write_blob, write_control};
use crate::message::Message;
use crate::{INLINE_LIMIT, SESSION_FORMAT_VERSION};
const PAGE: usize = 256;
const QUEUE: usize = 64;
pub const HELLO_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(90);
#[derive(Clone, Debug, Default)]
pub struct SessionOutcome {
pub sent: usize,
pub received: usize,
pub report: Report,
}
pub(crate) enum Out {
Control(Message),
Blob(ContentHash, Vec<u8>),
}
#[derive(Default)]
pub(crate) struct Received(pub(crate) HashMap<ContentHash, Vec<u8>>);
impl ContentSource for Received {
fn fetch(&self, hash: &ContentHash) -> Option<Vec<u8>> {
self.0.get(hash).cloned()
}
}
pub(crate) enum Stop {
Complete,
Gone,
Refused(String),
Protocol(String),
}
pub(crate) struct Exchange<S> {
pub(crate) reader: ReadHalf<S>,
pub(crate) out: mpsc::Sender<Out>,
pub(crate) writer: JoinHandle<()>,
pub(crate) queued: JoinHandle<()>,
pub(crate) peer_vv: VersionVector,
pub(crate) sent_vv: VersionVector,
pub(crate) received: Received,
pub(crate) outcome: SessionOutcome,
pub(crate) stop: Stop,
}
pub async fn run_session<S>(
stream: S,
replica: &Mutex<Replica>,
workspace_id: GrainId,
) -> Result<SessionOutcome>
where
S: AsyncRead + AsyncWrite + Send + Unpin + 'static,
{
let Exchange {
out,
writer,
queued,
peer_vv,
received,
outcome,
stop,
..
} = initial_exchange(stream, replica, workspace_id).await?;
match stop {
Stop::Complete => {
let _ = queued.await;
let result = {
let mut locked = replica.lock().await;
materialise(&mut locked, &received, &peer_vv, outcome)
};
close(out, writer, result).await
}
Stop::Gone => {
queued.abort();
abort(out, writer);
Ok(outcome)
}
Stop::Refused(why) => {
queued.abort();
abort(out, writer);
Err(Error::Refused(why))
}
Stop::Protocol(why) => {
queued.abort();
abort(out, writer);
Err(Error::Protocol(why))
}
}
}
pub(crate) async fn initial_exchange<S>(
stream: S,
replica: &Mutex<Replica>,
workspace_id: GrainId,
) -> Result<Exchange<S>>
where
S: AsyncRead + AsyncWrite + Send + Unpin + 'static,
{
let (mut reader, mut writer) = tokio::io::split(stream);
let (out_tx, mut out_rx) = mpsc::channel::<Out>(QUEUE);
let writer_task = tokio::spawn(async move {
while let Some(item) = out_rx.recv().await {
let result = match item {
Out::Control(msg) => write_control(&mut writer, &msg).await,
Out::Blob(hash, bytes) => write_blob(&mut writer, &hash, &bytes).await,
};
if result.is_err() {
break;
}
}
});
let (replica_id, vv) = {
let locked = replica.lock().await;
(locked.replica_id(), locked.vv().clone())
};
send(
&out_tx,
Out::Control(Message::Hello {
format: SESSION_FORMAT_VERSION,
workspace_id,
replica_id,
vv,
}),
)
.await?;
let hello = match tokio::time::timeout(HELLO_TIMEOUT, read_frame(&mut reader)).await {
Ok(frame) => frame,
Err(_) => {
abort(out_tx, writer_task);
return Err(Error::Protocol(format!(
"the peer sent no Hello within {HELLO_TIMEOUT:?}"
)));
}
};
let peer_vv = match hello? {
Some(Frame::Control(Message::Hello {
format,
workspace_id: theirs,
vv,
..
})) => {
if format != SESSION_FORMAT_VERSION {
let _ = send(
&out_tx,
Out::Control(Message::Refused(format!(
"session format {format}, expected {SESSION_FORMAT_VERSION}"
))),
)
.await;
return refuse(
out_tx,
writer_task,
Error::VersionMismatch {
ours: SESSION_FORMAT_VERSION,
theirs: format,
},
)
.await;
}
if theirs != workspace_id {
let _ = send(
&out_tx,
Out::Control(Message::Refused(format!(
"this stream is for workspace {workspace_id}, not {theirs}"
))),
)
.await;
return refuse(
out_tx,
writer_task,
Error::Protocol(format!(
"workspace mismatch: the peer is syncing {theirs}, this session is for \
{workspace_id}"
)),
)
.await;
}
vv
}
Some(Frame::Control(Message::Refused(why))) => {
return refuse(out_tx, writer_task, Error::Refused(why)).await;
}
other => {
return refuse(
out_tx,
writer_task,
Error::Protocol(format!("expected Hello, got {other:?}")),
)
.await;
}
};
let mut replica = replica.lock().await;
let delta = replica
.delta_for(&peer_vv)
.map_err(|e| Error::Sync(e.to_string()))?;
let sent_vv = replica.vv().clone();
let sent = delta.len();
let mut outgoing: Vec<Out> = Vec::new();
for page in delta.chunks(PAGE) {
for update in page {
for entry in &update.versions {
let Content::File { hash, len } = entry.content else {
continue;
};
if len as usize > INLINE_LIMIT {
continue;
}
if let Ok(Some(bytes)) = replica.read_content(&hash) {
outgoing.push(Out::Blob(hash, bytes));
}
}
}
outgoing.push(Out::Control(Message::Updates(page.to_vec())));
}
outgoing.push(Out::Control(Message::Done));
let sender = out_tx.clone();
let send_task = tokio::spawn(async move {
for item in outgoing {
if sender.send(item).await.is_err() {
break;
}
}
});
let mut received = Received::default();
let mut wanted: HashSet<ContentHash> = HashSet::new();
let mut report = Report::default();
let mut received_count = 0usize;
let mut peer_done = false;
let mut peer_settled = false;
let mut settled = false;
let stop = loop {
if peer_done && peer_settled && wanted.is_empty() {
break Stop::Complete;
}
let Some(frame) = read_frame(&mut reader).await? else {
break Stop::Gone;
};
match frame {
Frame::Control(Message::Updates(updates)) => {
received_count += updates.len();
let page = replica
.apply(&updates, &received)
.map_err(|e| Error::Sync(e.to_string()))?;
merge_report(&mut report, page);
for hash in needed(&replica, &updates, &received)? {
if wanted.insert(hash) {
send(&out_tx, Out::Control(Message::Want(hash))).await?;
}
}
}
Frame::Control(Message::Done) => peer_done = true,
Frame::Control(Message::Settled) => peer_settled = true,
Frame::Control(Message::Want(hash)) => {
let answer = match replica.read_content(&hash) {
Ok(Some(bytes)) => Out::Blob(hash, bytes),
_ => Out::Control(Message::Missing(hash)),
};
send(&out_tx, answer).await?;
}
Frame::Control(Message::Missing(hash)) => {
wanted.remove(&hash);
}
Frame::Control(Message::Live(_)) => {
break Stop::Protocol(
"a Live push before the initial exchange finished".to_owned(),
);
}
Frame::Control(Message::Refused(why)) => {
break Stop::Refused(why);
}
Frame::Control(Message::Hello { .. }) => {
break Stop::Protocol("a second Hello".to_owned());
}
Frame::Blob { hash, bytes } => {
if ContentHash::of_bytes(&bytes) != hash {
break Stop::Protocol(format!("content does not match {hash}"));
}
wanted.remove(&hash);
received.0.insert(hash, bytes);
}
}
if peer_done && wanted.is_empty() && !settled {
settled = true;
send(&out_tx, Out::Control(Message::Settled)).await?;
}
};
Ok(Exchange {
sent_vv,
reader,
out: out_tx,
writer: writer_task,
queued: send_task,
peer_vv,
received,
outcome: SessionOutcome {
sent,
received: received_count,
report,
},
stop,
})
}
pub(crate) fn materialise(
replica: &mut Replica,
received: &Received,
peer_vv: &VersionVector,
mut outcome: SessionOutcome,
) -> Result<SessionOutcome> {
let fetched = replica
.fetch_missing(received)
.map_err(|e| Error::Sync(e.to_string()))?;
merge_report(&mut outcome.report, fetched);
replica
.commit_session(peer_vv)
.map_err(|e| Error::Sync(e.to_string()))?;
tracing::debug!(
sent = outcome.sent,
received = outcome.received,
changed = outcome.report.changed,
"session complete"
);
Ok(outcome)
}
pub(crate) async fn send(tx: &mpsc::Sender<Out>, item: Out) -> Result<()> {
tx.send(item)
.await
.map_err(|_| Error::Protocol("the stream closed".to_owned()))
}
async fn close(
out_tx: mpsc::Sender<Out>,
writer: JoinHandle<()>,
result: Result<SessionOutcome>,
) -> Result<SessionOutcome> {
drop(out_tx);
let _ = writer.await;
result
}
async fn refuse<S>(
out_tx: mpsc::Sender<Out>,
writer: JoinHandle<()>,
err: Error,
) -> Result<Exchange<S>> {
drop(out_tx);
let _ = writer.await;
Err(err)
}
pub(crate) fn abort(out_tx: mpsc::Sender<Out>, writer: JoinHandle<()>) {
drop(out_tx);
writer.abort();
}
pub(crate) fn needed(
replica: &Replica,
updates: &[PathUpdate],
received: &Received,
) -> Result<Vec<ContentHash>> {
let mut out = Vec::new();
for update in updates {
for entry in &update.versions {
let Some(hash) = entry.content.hash() else {
continue;
};
if received.0.contains_key(&hash) {
continue;
}
let have = replica
.read_content(&hash)
.map_err(|e| Error::Sync(e.to_string()))?
.is_some();
if !have {
out.push(hash);
}
}
}
Ok(out)
}
pub(crate) fn merge_report(into: &mut Report, from: Report) {
into.changed += from.changed;
into.recorded.extend(from.recorded);
into.conflicts.extend(from.conflicts);
into.skipped.extend(from.skipped);
}