use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::str::FromStr;
use std::sync::Arc;
use std::sync::Mutex;
use async_trait::async_trait;
use bytes::Bytes;
use chrono::Utc;
use rings_transport::connection_ref::ConnectionRef;
#[cfg(feature = "dummy")]
pub use rings_transport::connections::DummyConnection as ConnectionOwner;
#[cfg(feature = "dummy")]
pub use rings_transport::connections::DummyTransport as Transport;
#[cfg(feature = "wasm")]
pub use rings_transport::connections::WebSysWebrtcConnection as ConnectionOwner;
#[cfg(feature = "wasm")]
pub use rings_transport::connections::WebSysWebrtcTransport as Transport;
#[cfg(all(not(feature = "wasm"), not(feature = "dummy")))]
use rings_transport::connections::WebrtcConnection as ConnectionOwner;
#[cfg(all(not(feature = "wasm"), not(feature = "dummy")))]
use rings_transport::connections::WebrtcTransport as Transport;
use rings_transport::core::transport::ConnectionInterface;
use rings_transport::core::transport::TransportInterface;
use rings_transport::core::transport::TransportMessage;
use rings_transport::core::transport::WebrtcConnectionState;
use rings_transport::delivery::DeliveryFuture;
use rings_transport::webrtc_config::WebrtcUdpPortRange;
use crate::chunk::Chunk;
use crate::chunk::ChunkList;
use crate::chunk::Framing;
use crate::chunk::ReassemblyLimits;
use crate::chunk::WireReserves;
use crate::consts::TRANSPORT_MAX_SIZE;
use crate::dht::entry::PlacementMiss;
use crate::dht::Did;
use crate::dht::LiveDid;
use crate::dht::PeerRing;
use crate::error::Error;
use crate::error::Result;
use crate::measure::order_peers_by_quality;
use crate::measure::MeasureCounter;
use crate::measure::MeasureImpl;
use crate::measure::PeerQuality;
use crate::message::ConnectNodeReport;
use crate::message::ConnectNodeSend;
use crate::message::Message;
use crate::message::MessagePayload;
use crate::message::PayloadSender;
use crate::session::SessionSk;
use crate::swarm::callback::InnerSwarmCallback;
const STORAGE_LOOKUP_OBSERVATION_TTL_MS: i64 = 30_000;
pub(crate) const STORAGE_LOOKUP_OBSERVATION_CAPACITY: usize = 1024;
type StorageLookupObservationMap = BTreeMap<StorageLookupObservationKey, StorageLookupObservation>;
pub struct SwarmTransport {
pub(crate) network_id: u32,
transport: Transport,
session_sk: SessionSk,
pub(crate) dht: Arc<PeerRing>,
storage_redundancy: u16,
reassembly_limits: ReassemblyLimits,
storage_lookup_observations: Mutex<StorageLookupObservationMap>,
measured_disconnects: Mutex<BTreeSet<Did>>,
measure: Option<MeasureImpl>,
}
#[derive(Clone, Copy)]
pub(crate) struct SwarmTransportSettings {
storage_redundancy: u16,
reassembly_limits: ReassemblyLimits,
}
impl SwarmTransportSettings {
pub(crate) fn new(storage_redundancy: u16, reassembly_limits: ReassemblyLimits) -> Self {
Self {
storage_redundancy,
reassembly_limits,
}
}
}
pub(crate) struct SwarmWebrtcConfig {
ice_servers: String,
external_address: Option<String>,
udp_port_range: Option<WebrtcUdpPortRange>,
}
impl SwarmWebrtcConfig {
pub(crate) fn new(
ice_servers: String,
external_address: Option<String>,
udp_port_range: Option<WebrtcUdpPortRange>,
) -> Self {
Self {
ice_servers,
external_address,
udp_port_range,
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
struct StorageLookupObservationKey {
resource: Did,
redundancy: u16,
}
struct StorageLookupObservation {
observed_at_ms: i64,
misses: BTreeSet<PlacementMiss>,
}
fn storage_lookup_observation_now_ms() -> i64 {
Utc::now().timestamp_millis()
}
fn evict_storage_lookup_observations(observations: &mut StorageLookupObservationMap, now_ms: i64) {
observations.retain(|_, observation| {
now_ms.saturating_sub(observation.observed_at_ms) <= STORAGE_LOOKUP_OBSERVATION_TTL_MS
});
while observations.len() > STORAGE_LOOKUP_OBSERVATION_CAPACITY {
let Some(stale_key) = observations
.iter()
.min_by_key(|(_, observation)| observation.observed_at_ms)
.map(|(key, _)| *key)
else {
break;
};
observations.remove(&stale_key);
}
}
#[derive(Clone)]
pub struct SwarmConnection {
peer: Did,
pub connection: ConnectionRef<ConnectionOwner>,
}
async fn record_measurement(measure: Option<MeasureImpl>, did: Did, counter: MeasureCounter) {
if let Some(measure) = measure {
measure.incr(did, counter).await;
}
}
#[cfg(feature = "wasm")]
fn spawn_delivery(fut: DeliveryFuture, did: Did, measure: Option<MeasureImpl>) {
wasm_bindgen_futures::spawn_local(async move {
match fut.await {
Ok(()) => record_measurement(measure, did, MeasureCounter::Sent).await,
Err(e) => {
tracing::warn!("Message to {did} was not delivered: {e}");
record_measurement(measure, did, MeasureCounter::FailedToSend).await;
}
}
});
}
#[cfg(not(feature = "wasm"))]
fn spawn_delivery(fut: DeliveryFuture, did: Did, measure: Option<MeasureImpl>) {
tokio::spawn(async move {
match fut.await {
Ok(()) => record_measurement(measure, did, MeasureCounter::Sent).await,
Err(e) => {
tracing::warn!("Message to {did} was not delivered: {e}");
record_measurement(measure, did, MeasureCounter::FailedToSend).await;
}
}
});
}
fn frame_chunk(session_sk: &SessionSk, did: Did, chunk: Chunk) -> Result<Bytes> {
MessagePayload::new_send(Message::Chunk(chunk), session_sk, did, did)?.to_bincode()
}
#[cfg(not(feature = "wasm"))]
type ChunkTail = Box<dyn Iterator<Item = Chunk> + Send>;
#[cfg(feature = "wasm")]
type ChunkTail = Box<dyn Iterator<Item = Chunk>>;
async fn run_chunked_send(
conn: SwarmConnection,
tail: ChunkTail,
first_delivery: DeliveryFuture,
session_sk: SessionSk,
did: Did,
measure: Option<MeasureImpl>,
) {
if let Err(e) = first_delivery.await {
tracing::warn!("Chunked send to {did} stopped before the first chunk flushed: {e}");
record_measurement(measure, did, MeasureCounter::FailedToSend).await;
return;
}
for chunk in tail {
let bytes = match frame_chunk(&session_sk, did, chunk) {
Ok(bytes) => bytes,
Err(e) => {
tracing::warn!("Chunked send to {did} aborted while framing a chunk: {e}");
return;
}
};
match conn.send_data(bytes).await {
Ok(delivery) => {
if let Err(e) = delivery.await {
tracing::warn!("Chunked send to {did} stopped before flush: {e}");
record_measurement(measure, did, MeasureCounter::FailedToSend).await;
return;
}
}
Err(e) => {
tracing::warn!("Chunked send to {did} stopped: {e}");
record_measurement(measure, did, MeasureCounter::FailedToSend).await;
return;
}
}
}
record_measurement(measure, did, MeasureCounter::Sent).await;
}
#[cfg(feature = "wasm")]
fn spawn_chunked_send(
conn: SwarmConnection,
tail: ChunkTail,
first_delivery: DeliveryFuture,
session_sk: SessionSk,
did: Did,
measure: Option<MeasureImpl>,
) {
wasm_bindgen_futures::spawn_local(run_chunked_send(
conn,
tail,
first_delivery,
session_sk,
did,
measure,
));
}
#[cfg(not(feature = "wasm"))]
fn spawn_chunked_send(
conn: SwarmConnection,
tail: ChunkTail,
first_delivery: DeliveryFuture,
session_sk: SessionSk,
did: Did,
measure: Option<MeasureImpl>,
) {
tokio::spawn(run_chunked_send(
conn,
tail,
first_delivery,
session_sk,
did,
measure,
));
}
impl SwarmTransport {
pub(crate) fn new(
network_id: u32,
webrtc: SwarmWebrtcConfig,
session_sk: SessionSk,
dht: Arc<PeerRing>,
measure: Option<MeasureImpl>,
settings: SwarmTransportSettings,
) -> Self {
Self {
network_id,
transport: Transport::new(
&webrtc.ice_servers,
webrtc.external_address,
webrtc.udp_port_range,
),
session_sk,
dht,
storage_redundancy: settings.storage_redundancy,
reassembly_limits: settings.reassembly_limits,
storage_lookup_observations: Mutex::new(BTreeMap::new()),
measured_disconnects: Mutex::new(BTreeSet::new()),
measure,
}
}
pub(crate) fn storage_redundancy(&self) -> u16 {
self.storage_redundancy
}
pub(crate) fn reassembly_limits(&self) -> ReassemblyLimits {
self.reassembly_limits
}
async fn record_peer_measurement(&self, peer: Did, counter: MeasureCounter) {
record_measurement(self.measure.clone(), peer, counter).await;
}
pub(crate) async fn record_peer_connected(&self, peer: Did) {
match self.measured_disconnects.lock() {
Ok(mut measured) => {
measured.remove(&peer);
}
Err(_) => {
tracing::warn!("Failed to update disconnect epoch for connected peer {peer}");
}
}
self.record_peer_measurement(peer, MeasureCounter::Connect)
.await;
}
pub(crate) async fn record_peer_disconnected(&self, peer: Did) {
let should_record = match self.measured_disconnects.lock() {
Ok(mut measured) => measured.insert(peer),
Err(_) => {
tracing::warn!("Failed to update disconnect epoch for disconnected peer {peer}");
true
}
};
if should_record {
self.record_peer_measurement(peer, MeasureCounter::Disconnected)
.await;
}
}
pub(crate) async fn record_peer_message_received(&self, peer: Did) {
self.record_peer_measurement(peer, MeasureCounter::Received)
.await;
}
pub(crate) async fn record_peer_message_receive_failed(&self, peer: Did) {
self.record_peer_measurement(peer, MeasureCounter::FailedToReceive)
.await;
}
pub(crate) async fn record_peer_message_send_failed(&self, peer: Did) {
self.record_peer_measurement(peer, MeasureCounter::FailedToSend)
.await;
}
pub(crate) async fn peer_quality(&self, peer: Did) -> PeerQuality {
match &self.measure {
Some(measure) => measure.quality(peer).await,
None => PeerQuality::Unknown,
}
}
pub(crate) async fn order_dht_candidates_by_quality(
&self,
candidates: impl IntoIterator<Item = Did>,
) -> Vec<Did> {
let mut measured = Vec::new();
for did in candidates {
measured.push((did, self.peer_quality(did).await));
}
order_peers_by_quality(measured)
}
pub(crate) fn ensure_storage_redundancy<const REDUNDANT: u16>(&self) -> Result<()> {
if self.storage_redundancy == REDUNDANT {
Ok(())
} else {
Err(Error::StorageRedundancyMismatch {
configured: self.storage_redundancy,
requested: REDUNDANT,
})
}
}
pub(crate) fn start_storage_lookup(&self, resource: Did, redundancy: u16) -> Result<()> {
let mut observations = self
.storage_lookup_observations
.lock()
.map_err(|_| Error::DHTSyncLockError)?;
let now = storage_lookup_observation_now_ms();
evict_storage_lookup_observations(&mut observations, now);
observations.remove(&StorageLookupObservationKey {
resource,
redundancy,
});
Ok(())
}
pub(crate) fn observe_storage_misses(
&self,
resource: Did,
redundancy: u16,
misses: impl IntoIterator<Item = PlacementMiss>,
) -> Result<()> {
let mut misses = misses.into_iter().peekable();
if misses.peek().is_none() {
return Ok(());
}
let mut observations = self
.storage_lookup_observations
.lock()
.map_err(|_| Error::DHTSyncLockError)?;
let now = storage_lookup_observation_now_ms();
evict_storage_lookup_observations(&mut observations, now);
let key = StorageLookupObservationKey {
resource,
redundancy,
};
let observation = observations
.entry(key)
.or_insert_with(|| StorageLookupObservation {
observed_at_ms: now,
misses: BTreeSet::new(),
});
observation.observed_at_ms = now;
observation.misses.extend(misses);
evict_storage_lookup_observations(&mut observations, now);
Ok(())
}
pub(crate) fn take_storage_misses(
&self,
resource: Did,
redundancy: u16,
) -> Result<Vec<PlacementMiss>> {
let mut observations = self
.storage_lookup_observations
.lock()
.map_err(|_| Error::DHTSyncLockError)?;
let now = storage_lookup_observation_now_ms();
evict_storage_lookup_observations(&mut observations, now);
let key = StorageLookupObservationKey {
resource,
redundancy,
};
Ok(observations
.remove(&key)
.map(|observation| observation.misses.into_iter().collect())
.unwrap_or_default())
}
#[cfg(all(test, not(feature = "wasm")))]
pub(crate) fn expire_storage_lookup_observation(
&self,
resource: Did,
redundancy: u16,
) -> Result<()> {
let mut observations = self
.storage_lookup_observations
.lock()
.map_err(|_| Error::DHTSyncLockError)?;
if let Some(observation) = observations.get_mut(&StorageLookupObservationKey {
resource,
redundancy,
}) {
observation.observed_at_ms = storage_lookup_observation_now_ms()
.saturating_sub(STORAGE_LOOKUP_OBSERVATION_TTL_MS + 1);
}
Ok(())
}
#[cfg(all(test, not(feature = "wasm")))]
pub(crate) fn storage_lookup_observation_count(&self) -> Result<usize> {
let observations = self
.storage_lookup_observations
.lock()
.map_err(|_| Error::DHTSyncLockError)?;
Ok(observations.len())
}
pub async fn new_connection(&self, peer: Did, callback: InnerSwarmCallback) -> Result<()> {
if peer == self.dht.did {
return Ok(());
}
let cid = peer.to_string();
self.transport
.new_connection(&cid, Box::new(callback))
.await
.map_err(Error::Transport)
}
pub fn get_connection(&self, peer: Did) -> Option<SwarmConnection> {
self.transport
.connection(&peer.to_string())
.map(|conn| SwarmConnection {
peer,
connection: conn,
})
.ok()
}
pub fn get_connections(&self) -> Vec<(Did, SwarmConnection)> {
self.transport
.connections()
.into_iter()
.filter_map(|(k, v)| {
Did::from_str(&k).ok().map(|did| {
(did, SwarmConnection {
peer: did,
connection: v,
})
})
})
.collect()
}
pub fn get_connection_ids(&self) -> Vec<Did> {
self.transport
.connection_ids()
.into_iter()
.filter_map(|k| Did::from_str(&k).ok())
.collect()
}
pub async fn disconnect(&self, peer: Did) -> Result<()> {
tracing::info!("removing {peer} from DHT");
self.dht.remove(peer)?;
self.transport
.close_connection(&peer.to_string())
.await
.map_err(|e| e.into())
}
pub async fn connect(&self, peer: Did, callback: InnerSwarmCallback) -> Result<()> {
let offer_msg = match self.prepare_connection_offer(peer, callback).await {
Ok(offer_msg) => offer_msg,
Err(Error::AlreadyConnected) => return Err(Error::AlreadyConnected),
Err(e) => {
self.record_peer_message_send_failed(peer).await;
return Err(e);
}
};
self.send_message(Message::ConnectNodeSend(offer_msg), peer)
.await?;
Ok(())
}
pub async fn get_and_check_connection(&self, peer: Did) -> Option<SwarmConnection> {
let conn = self.get_connection(peer)?;
if let Err(e) = conn.connection.webrtc_wait_for_data_channel_open().await {
tracing::warn!(
"[get_and_check_connection] connection {peer} data channel not open, will be dropped, reason: {e:?}"
);
if let Err(e) = self.disconnect(peer).await {
tracing::error!("Failed on close connection {peer}: {e:?}");
}
return None;
};
Some(conn)
}
pub async fn prepare_connection_offer(
&self,
peer: Did,
callback: InnerSwarmCallback,
) -> Result<ConnectNodeSend> {
if self.get_and_check_connection(peer).await.is_some() {
return Err(Error::AlreadyConnected);
};
self.new_connection(peer, callback).await?;
let conn = self
.transport
.connection(&peer.to_string())
.map_err(Error::Transport)?;
let offer = conn.webrtc_create_offer().await.map_err(Error::Transport)?;
let offer_str = serde_json::to_string(&offer).map_err(|_| Error::SerializeToString)?;
let offer_msg = ConnectNodeSend {
sdp: offer_str,
network_id: self.network_id,
};
Ok(offer_msg)
}
pub async fn answer_remote_connection(
&self,
peer: Did,
callback: InnerSwarmCallback,
offer_msg: &ConnectNodeSend,
) -> Result<ConnectNodeReport> {
let offer = serde_json::from_str(&offer_msg.sdp).map_err(Error::Deserialize)?;
if let Some(swarm_conn) = self.get_connection(peer) {
if swarm_conn.connection.webrtc_connection_state() == WebrtcConnectionState::New {
if self.dht.did > peer {
self.disconnect(peer).await?;
} else {
return Err(Error::AlreadyConnected);
}
} else if self.get_and_check_connection(peer).await.is_some() {
return Err(Error::AlreadyConnected);
};
};
self.new_connection(peer, callback).await?;
let conn = self
.transport
.connection(&peer.to_string())
.map_err(Error::Transport)?;
let answer = conn
.webrtc_answer_offer(offer)
.await
.map_err(Error::Transport)?;
let answer_str = serde_json::to_string(&answer).map_err(|_| Error::SerializeToString)?;
let answer_msg = ConnectNodeReport { sdp: answer_str };
Ok(answer_msg)
}
pub async fn accept_remote_connection(
&self,
peer: Did,
answer_msg: &ConnectNodeReport,
) -> Result<()> {
let answer = serde_json::from_str(&answer_msg.sdp).map_err(Error::Deserialize)?;
let conn = self
.transport
.connection(&peer.to_string())
.map_err(Error::Transport)?;
conn.webrtc_accept_answer(answer)
.await
.map_err(Error::Transport)?;
Ok(())
}
}
impl SwarmConnection {
pub async fn send_data(&self, data: Bytes) -> Result<DeliveryFuture> {
self.connection
.send_message(TransportMessage::Custom(data.to_vec()))
.await
.map_err(|e| e.into())
}
pub fn webrtc_connection_state(&self) -> WebrtcConnectionState {
self.connection.webrtc_connection_state()
}
pub fn max_message_size(&self) -> usize {
self.connection.max_message_size()
}
}
#[cfg_attr(feature = "wasm", async_trait(?Send))]
#[cfg_attr(not(feature = "wasm"), async_trait)]
impl PayloadSender for SwarmTransport {
fn session_sk(&self) -> &SessionSk {
&self.session_sk
}
fn dht(&self) -> Arc<PeerRing> {
self.dht.clone()
}
fn is_connected(&self, did: Did) -> bool {
let Some(conn) = self.get_connection(did) else {
return false;
};
conn.webrtc_connection_state() == WebrtcConnectionState::Connected
}
async fn do_send_payload(&self, did: Did, payload: MessagePayload) -> Result<()> {
let Some(conn) = self.get_and_check_connection(did).await else {
self.record_peer_message_send_failed(did).await;
return Err(Error::SwarmMissDidInTable(did));
};
tracing::debug!(
"Try send {:?}, to node {:?}",
payload.clone(),
payload.relay.next_hop,
);
let data = payload.to_bincode()?;
if data.len() > TRANSPORT_MAX_SIZE {
tracing::error!("Message is too large: {:?}", payload);
return Err(Error::MessageTooLarge(data.len()));
}
let Some(plan) = WireReserves::PRODUCTION.plan(data.len(), conn.max_message_size()) else {
self.record_peer_message_send_failed(did).await;
return Err(Error::PeerMaxMessageSizeTooSmall(conn.max_message_size()));
};
match plan {
Framing::Whole => match conn.send_data(data).await {
Ok(delivery) => spawn_delivery(delivery, did, self.measure.clone()),
Err(e) => {
self.record_peer_message_send_failed(did).await;
return Err(e);
}
},
Framing::Chunked { chunk_size } => {
let mut chunks = ChunkList::stream(data, chunk_size);
if let Some(first) = chunks.next() {
let first = frame_chunk(&self.session_sk, did, first)?;
match conn.send_data(first).await {
Ok(first_delivery) => {
spawn_chunked_send(
conn,
Box::new(chunks),
first_delivery,
self.session_sk.clone(),
did,
self.measure.clone(),
);
}
Err(e) => {
self.record_peer_message_send_failed(did).await;
return Err(e);
}
}
}
}
}
tracing::debug!(
"Sent {:?}, to node {:?}",
payload.clone(),
payload.relay.next_hop,
);
Ok(())
}
}
#[cfg_attr(feature = "wasm", async_trait(?Send))]
#[cfg_attr(not(feature = "wasm"), async_trait)]
impl LiveDid for SwarmConnection {
async fn live(&self) -> bool {
self.webrtc_connection_state() == WebrtcConnectionState::Connected
}
}
impl From<SwarmConnection> for Did {
fn from(conn: SwarmConnection) -> Self {
conn.peer
}
}
#[cfg(all(test, not(feature = "wasm")))]
mod tests;