use std::collections::HashSet;
use grain_id::GrainId;
use sapphire_sync::{
Content, ContentHash, ContentSource, PathUpdate, Replica, Report, VersionVector,
};
use tokio::io::{AsyncRead, AsyncWrite, ReadHalf};
use tokio::sync::Mutex;
use tokio::task::JoinHandle;
use tokio::time::Instant;
use crate::error::{Error, Result};
use crate::frame::{Frame, read_frame};
use crate::incoming::Incoming;
use crate::message::Message;
use crate::progress::Progress;
use crate::writer::{Out, Outbox, spawn_writer};
use crate::{IDLE_TIMEOUT, INLINE_LIMIT, SESSION_FORMAT_VERSION};
const PAGE: usize = 256;
pub const HELLO_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(90);
#[derive(Clone, Debug, Default)]
pub struct SessionOutcome {
pub sent: usize,
pub received: usize,
pub report: Report,
}
pub(crate) struct NoContent;
impl ContentSource for NoContent {
fn locate(&self, _: &ContentHash) -> Option<std::path::PathBuf> {
None
}
}
pub(crate) enum Stop {
Complete,
Gone,
Refused(String),
Protocol(String),
}
pub(crate) struct Exchange<S> {
pub(crate) reader: ReadHalf<S>,
pub(crate) out: Outbox,
pub(crate) writer: JoinHandle<()>,
pub(crate) progress: Progress,
pub(crate) peer_vv: VersionVector,
pub(crate) sent_vv: VersionVector,
pub(crate) incoming: Incoming,
pub(crate) outcome: SessionOutcome,
pub(crate) stop: Stop,
}
pub async fn run_session<S>(
stream: S,
replica: &Mutex<Replica>,
workspace_id: GrainId,
) -> Result<SessionOutcome>
where
S: AsyncRead + AsyncWrite + Send + Unpin + 'static,
{
let Exchange {
out,
writer,
progress,
peer_vv,
outcome,
stop,
..
} = initial_exchange(stream, replica, workspace_id).await?;
match stop {
Stop::Complete => {
let result = {
let mut locked = replica.lock().await;
materialise(&mut locked, &peer_vv, outcome)
};
close(out, writer, &progress, result).await
}
Stop::Gone => {
abort(out, writer);
Ok(outcome)
}
Stop::Refused(why) => {
abort(out, writer);
Err(Error::Refused(why))
}
Stop::Protocol(why) => {
abort(out, writer);
Err(Error::Protocol(why))
}
}
}
pub(crate) async fn initial_exchange<S>(
stream: S,
replica: &Mutex<Replica>,
workspace_id: GrainId,
) -> Result<Exchange<S>>
where
S: AsyncRead + AsyncWrite + Send + Unpin + 'static,
{
let (mut reader, writer) = tokio::io::split(stream);
let progress = Progress::new();
let (out_tx, writer_task) = spawn_writer(writer, progress.clone());
let (replica_id, vv, max_file_size) = {
let locked = replica.lock().await;
(
locked.replica_id(),
locked.vv().clone(),
locked.config().max_file_size,
)
};
send(
&out_tx,
Out::Control(Message::Hello {
format: SESSION_FORMAT_VERSION,
workspace_id,
replica_id,
vv,
max_file_size,
}),
)?;
let hello = match tokio::time::timeout(HELLO_TIMEOUT, read_frame(&mut reader)).await {
Ok(frame) => {
progress.touch();
frame
}
Err(_) => {
abort(out_tx, writer_task);
return Err(Error::Protocol(format!(
"the peer sent no Hello within {HELLO_TIMEOUT:?}"
)));
}
};
let (peer_vv, peer_max) = match hello? {
Some(Frame::Control(Message::Hello {
format,
workspace_id: theirs,
vv,
max_file_size,
..
})) => {
if format != SESSION_FORMAT_VERSION {
let _ = send(
&out_tx,
Out::Control(Message::Refused(format!(
"session format {format}, expected {SESSION_FORMAT_VERSION}"
))),
);
return refuse(
out_tx,
writer_task,
Error::VersionMismatch {
ours: SESSION_FORMAT_VERSION,
theirs: format,
},
)
.await;
}
if theirs != workspace_id {
let _ = send(
&out_tx,
Out::Control(Message::Refused(format!(
"this stream is for workspace {workspace_id}, not {theirs}"
))),
);
return refuse(
out_tx,
writer_task,
Error::Protocol(format!(
"workspace mismatch: the peer is syncing {theirs}, this session is for \
{workspace_id}"
)),
)
.await;
}
(vv, max_file_size)
}
Some(Frame::Control(Message::Refused(why))) => {
return refuse(out_tx, writer_task, Error::Refused(why)).await;
}
other => {
return refuse(
out_tx,
writer_task,
Error::Protocol(format!("expected Hello, got {other:?}")),
)
.await;
}
};
let (mut incoming, sent_vv, sent, outgoing) = {
let replica = replica.lock().await;
let incoming = Incoming::new(replica.stager());
let delta = replica
.delta_for(&peer_vv)
.map_err(|e| Error::Sync(e.to_string()))?;
let inline_max = (INLINE_LIMIT as u64).min(peer_max);
let inline = replica
.content_paths(&inline_hashes(&delta, inline_max))
.unwrap_or_default();
let mut outgoing: Vec<Out> = Vec::new();
for page in delta.chunks(PAGE) {
for hash in inline_hashes(page, inline_max) {
if let Some((path, len)) = inline.get(&hash) {
outgoing.push(Out::Blob {
hash,
path: path.clone(),
len: *len,
});
}
}
outgoing.push(Out::Control(Message::Updates(page.to_vec())));
}
outgoing.push(Out::Control(Message::Done));
(incoming, replica.vv().clone(), delta.len(), outgoing)
};
let mut wanted: HashSet<ContentHash> = HashSet::new();
let mut lacked: HashSet<ContentHash> = HashSet::new();
let mut report = Report::default();
let mut received_count = 0usize;
let mut peer_done = false;
let mut peer_settled = false;
let mut settled = false;
let looped: Result<Stop> = async {
for item in outgoing {
send(&out_tx, item)?;
}
Ok(loop {
if peer_done && peer_settled && wanted.is_empty() {
break Stop::Complete;
}
let Some(frame) = read_or_idle(&mut reader, &progress).await? else {
break Stop::Gone;
};
match frame {
Frame::Control(Message::Updates(updates)) => {
received_count += updates.len();
let need = {
let mut replica = replica.lock().await;
let page = replica
.apply(&updates, &NoContent)
.map_err(|e| Error::Sync(e.to_string()))?;
merge_report(&mut report, page);
needed(&replica, &updates)?
};
for hash in need {
if wanted.insert(hash) {
send(&out_tx, Out::Control(Message::Want(hash)))?;
}
}
}
Frame::Control(Message::Done) => {
peer_done = true;
let again = replica
.lock()
.await
.unavailable()
.map_err(|e| Error::Sync(e.to_string()))?;
for hash in again {
if !lacked.contains(&hash) && wanted.insert(hash) {
send(&out_tx, Out::Control(Message::Want(hash)))?;
}
}
}
Frame::Control(Message::Settled) => peer_settled = true,
Frame::Control(Message::Want(hash)) => {
let reply = answer(&*replica.lock().await, hash);
send(&out_tx, reply)?;
}
Frame::Control(Message::Missing(hash)) => {
wanted.remove(&hash);
lacked.insert(hash);
}
Frame::Control(Message::Aborted(hash)) => {
if let Err(e) = incoming.aborted(hash) {
break Stop::Protocol(reason(e));
}
wanted.remove(&hash);
lacked.insert(hash);
}
Frame::Control(Message::Live(_)) => {
break Stop::Protocol(
"a Live push before the initial exchange finished".to_owned(),
);
}
Frame::Control(Message::Refused(why)) => {
break Stop::Refused(why);
}
Frame::Control(Message::Hello { .. }) => {
break Stop::Protocol("a second Hello".to_owned());
}
Frame::BlobStart { hash, len } => {
if let Err(e) = incoming.start(hash, len) {
break Stop::Protocol(reason(e));
}
}
Frame::BlobChunk(bytes) => {
if let Err(e) = incoming.chunk(&bytes) {
break Stop::Protocol(reason(e));
}
}
Frame::BlobEnd => match incoming.end() {
Ok(hash) => {
wanted.remove(&hash);
}
Err(e) => break Stop::Protocol(reason(e)),
},
}
if peer_done && wanted.is_empty() && !settled {
settled = true;
send(&out_tx, Out::Control(Message::Settled))?;
}
})
}
.await;
let stop = match looped {
Ok(stop) => stop,
Err(Error::Closed) => Stop::Gone,
Err(e) => {
abort(out_tx, writer_task);
return Err(e);
}
};
Ok(Exchange {
sent_vv,
reader,
out: out_tx,
writer: writer_task,
progress,
peer_vv,
incoming,
outcome: SessionOutcome {
sent,
received: received_count,
report,
},
stop,
})
}
pub(crate) async fn read_or_idle<R: AsyncRead + Unpin>(
r: &mut R,
progress: &Progress,
) -> Result<Option<Frame>> {
let read = read_frame(r);
tokio::pin!(read);
loop {
let deadline = Instant::now() + IDLE_TIMEOUT.saturating_sub(progress.idle_for());
tokio::select! {
frame = &mut read => {
progress.touch();
return frame;
}
_ = tokio::time::sleep_until(deadline) => {
if progress.idle_for() >= IDLE_TIMEOUT {
return Err(Error::Protocol(format!("no progress for {IDLE_TIMEOUT:?}")));
}
}
}
}
}
pub(crate) fn materialise(
replica: &mut Replica,
peer_vv: &VersionVector,
mut outcome: SessionOutcome,
) -> Result<SessionOutcome> {
let fetched = replica
.fetch_missing(&NoContent)
.map_err(|e| Error::Sync(e.to_string()))?;
merge_report(&mut outcome.report, fetched);
replica
.commit_session(peer_vv)
.map_err(|e| Error::Sync(e.to_string()))?;
tracing::debug!(
sent = outcome.sent,
received = outcome.received,
changed = outcome.report.changed,
"session complete"
);
Ok(outcome)
}
pub(crate) fn send(out: &Outbox, item: Out) -> Result<()> {
out.send(item)
}
pub(crate) fn answer(replica: &Replica, hash: ContentHash) -> Out {
match replica.content_path(&hash) {
Ok(Some((path, len))) => Out::Blob { hash, path, len },
_ => Out::Control(Message::Missing(hash)),
}
}
pub(crate) fn reason(err: Error) -> String {
match err {
Error::Protocol(why) => why,
other => other.to_string(),
}
}
async fn close(
out_tx: Outbox,
mut writer: JoinHandle<()>,
progress: &Progress,
result: Result<SessionOutcome>,
) -> Result<SessionOutcome> {
drop(out_tx);
loop {
let deadline = Instant::now() + IDLE_TIMEOUT.saturating_sub(progress.idle_for());
tokio::select! {
_ = &mut writer => break,
_ = tokio::time::sleep_until(deadline) => {
if progress.idle_for() >= IDLE_TIMEOUT {
tracing::debug!("the peer stopped reading; dropping what was still queued");
writer.abort();
break;
}
}
}
}
result
}
async fn refuse<S>(out_tx: Outbox, writer: JoinHandle<()>, err: Error) -> Result<Exchange<S>> {
drop(out_tx);
let _ = writer.await;
Err(err)
}
pub(crate) fn abort(out_tx: Outbox, writer: JoinHandle<()>) {
drop(out_tx);
writer.abort();
}
fn inline_hashes(updates: &[PathUpdate], max: u64) -> Vec<ContentHash> {
updates
.iter()
.flat_map(|update| &update.versions)
.filter_map(|entry| match entry.content {
Content::File { hash, len } if len <= max => Some(hash),
_ => None,
})
.collect()
}
pub(crate) fn needed(replica: &Replica, updates: &[PathUpdate]) -> Result<Vec<ContentHash>> {
let max = replica.config().max_file_size;
let hashes: Vec<ContentHash> = updates
.iter()
.flat_map(|update| &update.versions)
.filter_map(|entry| match entry.content {
Content::File { hash, len } if len <= max => Some(hash),
_ => None,
})
.collect();
let have = replica
.content_paths(&hashes)
.map_err(|e| Error::Sync(e.to_string()))?;
Ok(hashes
.into_iter()
.filter(|hash| !have.contains_key(hash))
.collect())
}
pub(crate) fn merge_report(into: &mut Report, from: Report) {
into.changed += from.changed;
into.recorded.extend(from.recorded);
into.conflicts.extend(from.conflicts);
into.skipped.extend(from.skipped);
}