1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
//! Inbound frame receive loop and dispatch table.
//!
//! `receive_loop` reads `Message::Binary` frames off the WebSocket stream,
//! decodes the `SyncFrame` envelope, and hands each frame to `dispatch_frame`,
//! which fans out to per-message-type handlers on `SyncClient` and
//! `SyncDelegate`. Pulled out of the main transport module so the giant
//! `match` over message types lives in one self-contained file instead of
//! being interleaved with the push loop.
//!
//! Engine-level ack dispatch (with AckStatus handling) lives in
//! `dispatch_acks`. Array reject and push-window pong handling lives in
//! `dispatch_array`.
use std::sync::Arc;
use futures::StreamExt;
use tokio_tungstenite::tungstenite::Message;
use nodedb_types::sync::wire::{AckStatus, SyncFrame, SyncMessageType};
use super::dispatch_acks;
use crate::error::LiteError;
use crate::sync::client::SyncClient;
use crate::sync::delegate::SyncDelegate;
/// Receive and dispatch incoming frames from Origin.
pub(super) async fn receive_loop<S>(
client: &Arc<SyncClient>,
delegate: &Arc<dyn SyncDelegate>,
stream: &mut S,
) -> Result<(), LiteError>
where
S: StreamExt<Item = Result<Message, tokio_tungstenite::tungstenite::Error>> + Unpin,
{
while let Some(msg_result) = stream.next().await {
let msg = msg_result.map_err(|e| LiteError::Sync {
detail: format!("WebSocket read error: {e}"),
})?;
let bytes = match &msg {
Message::Binary(b) => b.as_ref(),
Message::Close(_) => return Ok(()),
// Sync frames are binary. Control frames and text carry none.
Message::Ping(_) | Message::Pong(_) | Message::Text(_) | Message::Frame(_) => continue,
};
let Some(frame) = SyncFrame::from_bytes(bytes) else {
tracing::warn!("received malformed frame, skipping");
continue;
};
dispatch_frame(client, delegate, &frame).await;
}
Ok(())
}
/// Dispatch a single incoming frame to the appropriate handler.
pub(super) async fn dispatch_frame(
client: &Arc<SyncClient>,
delegate: &Arc<dyn SyncDelegate>,
frame: &SyncFrame,
) {
match frame.msg_type {
SyncMessageType::DeltaAck => {
if let Some(ack) = frame.decode_body::<nodedb_types::sync::wire::DeltaAckMsg>() {
match &ack.status {
AckStatus::Applied | AckStatus::Duplicate => {
// DeltaAckMsg does not carry a collection field so we
// cannot derive a stream_id to advance the frontier here.
// The durable last_assigned from StreamSeqTracker already
// prevents re-sending un-acked seqs; frontier advancement
// for CRDT deltas is deferred until DeltaAckMsg gains a
// collection field.
delegate.acknowledge(ack.mutation_id);
}
AckStatus::Accepted => {
// Provisional: Origin admitted the delta but has not
// applied it. Keep it queued until a terminal status
// arrives.
tracing::trace!(
mutation_id = ack.mutation_id,
"DeltaAck: accepted, awaiting apply outcome"
);
}
AckStatus::Fenced => {
// Origin did NOT apply this delta — the producer epoch
// was rejected. Retiring it here would drop the write;
// it stays queued for re-push after the producer
// identity is re-established.
tracing::error!(
mutation_id = ack.mutation_id,
"DeltaAck: producer fenced by Origin; halting push, \
delta retained for re-send"
);
client.set_fenced();
}
AckStatus::Gap { expected } => {
// Origin did NOT apply this delta — it expected a
// different sequence number. The delta stays queued so
// the push loop re-sends it at its stable seq.
tracing::warn!(
mutation_id = ack.mutation_id,
expected,
"DeltaAck: sequence gap detected by Origin; delta \
retained for re-send"
);
}
AckStatus::Rejected { reason } => {
// Terminal failure: per the wire contract this delta
// "will never apply". Unlike `Fenced` (recoverable once
// the producer identity is re-established) and `Gap`
// (recoverable by re-sending at the right seq), there is
// nothing to retry — retaining it would stall the queue
// forever behind a message Origin has permanently
// refused. So it is retired, which DOES drop the write:
// logged at ERROR with Origin's reason so the loss is
// never silent.
tracing::error!(
mutation_id = ack.mutation_id,
reason = %reason,
"DeltaAck: Origin permanently rejected this delta; \
dropping it — the write is LOST and will not retry"
);
delegate.acknowledge(ack.mutation_id);
}
}
client.handle_delta_ack(&ack).await;
} else {
client.metrics().record_stale_timeouts(1);
tracing::warn!(
frame_len = frame.body.len(),
"DeltaAck frame body failed to decode; \
in-flight entry will be evicted by the stale-timeout pass"
);
}
}
SyncMessageType::RowPush => {
// Server-originated row post-image (SQL DML on Origin, or a
// DDL-managed system row). Applied locally; never echoed back.
if let Some(msg) = frame.decode_body::<nodedb_types::sync::wire::RowPushMsg>() {
// Origin's client→server writes are protected by sync_admit
// (producer/epoch/seq); this is the symmetric gate for the
// server→client direction, without which a re-delivered or
// out-of-order RowPush could resurrect a deleted row.
if client
.admit_row_push(msg.peer_id, &msg.collection, msg.sequence)
.await
&& let Err(e) = delegate.apply_remote_row(&msg).await
{
// The refusal goes back to Origin as a `RowPushReject`
// on the next push tick.
tracing::error!(
collection = %msg.collection,
doc = %msg.document_id,
sequence = msg.sequence,
error = %e,
"RowPush refused; row not applied, refusal queued for Origin"
);
client.queue_row_push_reject(&msg, &e).await;
}
} else {
tracing::warn!(
frame_len = frame.body.len(),
"RowPush frame body failed to decode; row not applied"
);
}
}
SyncMessageType::ResyncRequest => {
// Origin is requesting us to re-sync. Log; the push loop re-sends
// from the requested mutation ID on the next tick.
if let Some(msg) = frame.decode_body::<nodedb_types::sync::wire::ResyncRequestMsg>() {
tracing::warn!(
reason = ?msg.reason,
from_mutation_id = msg.from_mutation_id,
collection = %msg.collection,
"Origin requested re-sync"
);
}
}
SyncMessageType::DeltaReject => {
if let Some(reject) = frame.decode_body::<nodedb_types::sync::wire::DeltaRejectMsg>() {
// Detect auth-related rejection → pause push, trigger token refresh.
if matches!(
&reject.compensation,
Some(nodedb_types::sync::compensation::CompensationHint::PermissionDenied)
) && client.config().token_provider.is_some()
{
client.pause_for_auth().await;
}
// A peer-id collision is the one refusal the client can act on
// by itself, and the one it must: every later write carries the
// same refused id, so a replica that only reports it is
// permanently unable to sync. Rotating re-authors the local
// documents under a new identity and re-queues this delta with
// them, so it replaces the rollback rather than following it.
let collision = reject
.compensation
.as_ref()
.is_some_and(|hint| hint.is_peer_id_collision());
if collision {
delegate.rotate_peer_id().await;
} else if let Some(hint) = &reject.compensation {
delegate.reject_with_policy(reject.mutation_id, hint);
} else {
delegate.reject(reject.mutation_id);
}
client.handle_delta_reject(&reject).await;
}
}
SyncMessageType::TokenRefreshAck => {
if let Some(ack) = frame.decode_body::<nodedb_types::sync::wire::TokenRefreshAckMsg>() {
client.handle_token_refresh_ack(&ack).await;
}
}
SyncMessageType::ShapeSnapshot => {
if let Some(snapshot) =
frame.decode_body::<nodedb_types::sync::wire::ShapeSnapshotMsg>()
{
// `ShapeSnapshotMsg` carries no collection of its own, so it is
// resolved from the subscription the snapshot answers. A shape
// with no collection (graph, array) or an unknown shape_id
// cannot be routed to a document — importing it anywhere would
// merge foreign state into an unrelated collection.
if !snapshot.data.is_empty() {
let collection = {
let shapes = client.shapes().lock().await;
shapes
.get(&snapshot.shape_id)
.and_then(|sub| sub.definition.collection().map(str::to_string))
};
match collection {
Some(collection) => delegate.import_remote(&collection, &snapshot.data),
None => tracing::error!(
shape_id = %snapshot.shape_id,
"ShapeSnapshot has no resolvable collection — discarding payload \
rather than importing it into an unrelated document"
),
}
}
client.handle_shape_snapshot(&snapshot).await;
}
}
SyncMessageType::ShapeDelta => {
if let Some(delta) = frame.decode_body::<nodedb_types::sync::wire::ShapeDeltaMsg>() {
client.metrics().record_received();
if let Some(resync) = client.check_sequence_gap(&delta.shape_id, delta.lsn).await {
tracing::warn!(
shape_id = %delta.shape_id,
"requesting re-sync due to sequence gap"
);
// Stash for the push loop to send on its next tick — the
// dispatch path does not own the sink.
client.set_pending_resync(resync).await;
}
if !delta.delta.is_empty() {
delegate.import_remote(&delta.collection, &delta.delta);
}
client.handle_shape_delta(&delta).await;
}
}
SyncMessageType::VectorClockSync => {
if let Some(clock_msg) =
frame.decode_body::<nodedb_types::sync::wire::VectorClockSyncMsg>()
{
client.handle_clock_sync(&clock_msg).await;
}
}
SyncMessageType::DefinitionSync => {
if let Some(msg) = frame.decode_body::<nodedb_types::sync::wire::DefinitionSyncMsg>() {
delegate.import_definition(&msg).await;
}
}
SyncMessageType::CollectionSchema => {
if let Some(msg) =
frame.decode_body::<nodedb_types::sync::wire::CollectionSchemaSyncMsg>()
{
delegate.import_collection_schema(&msg).await;
} else {
tracing::warn!("CollectionSchema: failed to decode frame body");
}
}
SyncMessageType::ArrayDelta => {
if let Some(msg) = frame.decode_body::<nodedb_types::sync::wire::ArrayDeltaMsg>() {
if let Some(ack) = delegate.handle_array_delta(&msg).await {
client.set_pending_array_ack(ack).await;
}
} else {
tracing::warn!("ArrayDelta: failed to decode frame body");
}
}
SyncMessageType::ArrayDeltaBatch => {
if let Some(msg) = frame.decode_body::<nodedb_types::sync::wire::ArrayDeltaBatchMsg>() {
if let Some(ack) = delegate.handle_array_delta_batch(&msg).await {
client.set_pending_array_ack(ack).await;
}
} else {
tracing::warn!("ArrayDeltaBatch: failed to decode frame body");
}
}
SyncMessageType::ArraySnapshot => {
// Origin's catch-up server sends the header first, then the
// chunks, on this ordered stream.
if let Some(msg) = frame.decode_body::<nodedb_types::sync::wire::ArraySnapshotMsg>() {
delegate.handle_array_snapshot(&msg).await;
} else {
tracing::warn!("ArraySnapshot: failed to decode frame body");
}
}
SyncMessageType::ArraySnapshotChunk => {
if let Some(msg) =
frame.decode_body::<nodedb_types::sync::wire::ArraySnapshotChunkMsg>()
{
// The ack for the last chunk carries the snapshot HLC, which
// advances Origin's GC frontier for this replica.
if let Some(ack) = delegate.handle_array_snapshot_chunk(&msg).await {
client.set_pending_array_ack(ack).await;
}
} else {
tracing::warn!("ArraySnapshotChunk: failed to decode frame body");
}
}
SyncMessageType::ArrayReject => {
super::dispatch_array::handle_array_reject(client, delegate, frame).await;
}
SyncMessageType::ArrayAdvisoryRefused => {
super::dispatch_array::handle_array_advisory_refused(frame);
}
SyncMessageType::ColumnarInsertAck => {
dispatch_acks::handle_columnar_insert_ack(client, delegate, frame).await;
}
SyncMessageType::VectorInsertAck => {
dispatch_acks::handle_vector_insert_ack(client, delegate, frame).await;
}
SyncMessageType::VectorDeleteAck => {
dispatch_acks::handle_vector_delete_ack(client, delegate, frame).await;
}
SyncMessageType::FtsIndexAck => {
dispatch_acks::handle_fts_index_ack(client, delegate, frame).await;
}
SyncMessageType::FtsDeleteAck => {
dispatch_acks::handle_fts_delete_ack(client, delegate, frame).await;
}
SyncMessageType::SpatialInsertAck => {
dispatch_acks::handle_spatial_insert_ack(client, delegate, frame).await;
}
SyncMessageType::SpatialDeleteAck => {
dispatch_acks::handle_spatial_delete_ack(client, delegate, frame).await;
}
SyncMessageType::TimeseriesAck => {
dispatch_acks::handle_timeseries_ack(client, delegate, frame).await;
}
SyncMessageType::KvPushAck => {
super::dispatch_kv::handle_kv_push_ack(client, delegate, frame).await;
}
SyncMessageType::PingPong => {
super::dispatch_array::handle_ping_pong(client, delegate, frame).await;
}
// Types Lite has no handler for: client-to-server frames, a second
// handshake ack, and server frames Lite does not act on. Origin logs
// an unhandled type at warn and keeps the session, so Lite does too.
SyncMessageType::Handshake
| SyncMessageType::HandshakeAck
| SyncMessageType::DeltaPush
| SyncMessageType::CollectionPurged
| SyncMessageType::RowPushReject
| SyncMessageType::ShapeSubscribe
| SyncMessageType::ShapeUnsubscribe
| SyncMessageType::TimeseriesPush
| SyncMessageType::Throttle
| SyncMessageType::TokenRefresh
| SyncMessageType::PresenceUpdate
| SyncMessageType::PresenceBroadcast
| SyncMessageType::PresenceLeave
| SyncMessageType::ArraySchema
| SyncMessageType::ArrayAck
| SyncMessageType::ArrayCatchupRequest
| SyncMessageType::ArrayOpResults
| SyncMessageType::ColumnarInsert
| SyncMessageType::VectorInsert
| SyncMessageType::VectorDelete
| SyncMessageType::FtsIndex
| SyncMessageType::FtsDelete
| SyncMessageType::SpatialInsert
| SyncMessageType::SpatialDelete
| SyncMessageType::KvPush => warn_unhandled(frame),
// `SyncMessageType` is `#[non_exhaustive]`, so a match outside its
// crate needs this arm. A type added later is unhandled the same way.
_ => warn_unhandled(frame),
}
}
/// Log a frame of a type Lite does not handle. The session stays open.
fn warn_unhandled(frame: &SyncFrame) {
tracing::warn!(
msg_type = ?frame.msg_type,
msg_type_code = frame.msg_type as u8,
frame_len = frame.body.len(),
"unhandled sync message type from Origin; frame ignored"
);
}