use bmrng::unbounded::{UnboundedRequestReceiver, UnboundedRequestSender};
use chrono::Utc;
use livekit_common::{
ClientCapability, ParticipantIdentity, RemoteParticipantRegistry,
CLIENT_PROTOCOL_DATA_STREAM_V2,
};
use livekit_protocol as proto;
use std::{path::Path, sync::Arc};
use tokio::sync::Mutex;
use crate::{
info::{ByteStreamInfo, TextStreamInfo},
types::{ByteHeader, CompressionType, ContentHeader, Header, StreamId, TextHeader},
utf8_chunk::Utf8AwareChunkExt,
utils::{SendError, StreamError, StreamResult},
};
use super::{
constants,
raw_stream::{RawStream, RawStreamOpenOptions},
stream_writer::{ByteStreamWriter, TextStreamWriter},
StreamByteOptions, StreamTextOptions,
};
fn create_random_uuid() -> String {
uuid::Uuid::new_v4().to_string()
}
#[derive(Clone)]
pub struct Manager {
packet_tx: UnboundedRequestSender<proto::DataPacket, Result<(), SendError>>,
}
impl Manager {
pub fn new() -> (Self, UnboundedRequestReceiver<proto::DataPacket, Result<(), SendError>>) {
let (packet_tx, packet_rx) = bmrng::unbounded_channel();
let manager = Self { packet_tx };
(manager, packet_rx)
}
pub async fn stream_text(&self, options: StreamTextOptions) -> StreamResult<TextStreamWriter> {
let stream_id: StreamId = options.id.clone().unwrap_or_else(create_random_uuid).into();
let dests = options.destination_identities.clone();
let (header, text_header) =
build_text_header(&options, stream_id, None, None, CompressionType::None);
enforce_header_size(&header, &dests)?;
let open_options = RawStreamOpenOptions {
header: header.clone(),
destination_identities: dests,
sender_identity: options.sender_identity.clone(),
packet_tx: self.packet_tx.clone(),
};
let writer = TextStreamWriter::new(
Arc::new(TextStreamInfo::from_headers(header, text_header)),
Arc::new(Mutex::new(RawStream::open(open_options).await?)),
);
Ok(writer)
}
pub async fn stream_bytes(&self, options: StreamByteOptions) -> StreamResult<ByteStreamWriter> {
let stream_id: StreamId = options.id.clone().unwrap_or_else(create_random_uuid).into();
let name = options.name.clone().unwrap_or_default();
let dests = options.destination_identities.clone();
let (header, byte_header) = build_byte_header(
&options,
stream_id,
name,
options.total_length,
None,
CompressionType::None,
);
enforce_header_size(&header, &dests)?;
let open_options = RawStreamOpenOptions {
header: header.clone(),
destination_identities: dests,
sender_identity: options.sender_identity.clone(),
packet_tx: self.packet_tx.clone(),
};
let writer = ByteStreamWriter::new(
Arc::new(ByteStreamInfo::from_headers(header, byte_header)),
Arc::new(Mutex::new(RawStream::open(open_options).await?)),
);
Ok(writer)
}
pub async fn send_text(
&self,
text: &str,
options: StreamTextOptions,
remote_participant_registry: &dyn RemoteParticipantRegistry,
) -> StreamResult<TextStreamInfo> {
let stream_id: StreamId = options.id.clone().unwrap_or_else(create_random_uuid).into();
let total_length = text.len() as u64;
let eligibility =
evaluate_eligibility(remote_participant_registry, &options.destination_identities);
let can_compress = options.compress.unwrap_or(true) && eligibility.compression;
let text_bytes = text.as_bytes();
let mut maybe_compressed = MaybeCollectedAsyncReader::from_async_reader(
async_compression::futures::bufread::DeflateEncoder::new(
futures_util::io::Cursor::new(text_bytes.to_vec()),
),
);
let use_compression = can_compress
&& maybe_compressed.as_bytes().await.is_ok_and(|c| c.len() < text_bytes.len());
let (mut header, text_header) = if use_compression {
build_text_header(
&options,
stream_id.clone(),
Some(total_length),
Some(maybe_compressed.as_bytes().await?.to_owned()),
CompressionType::DeflateRaw,
)
} else {
build_text_header(
&options,
stream_id.clone(),
Some(total_length),
Some(text_bytes.to_vec()),
CompressionType::None,
)
};
let proto_header = header.clone().into();
if eligibility.inline
&& options.attached_stream_ids.is_empty()
&& header_packet_fits(&proto_header, &options.destination_identities)
{
let mut packet =
RawStream::create_header_packet(proto_header, options.destination_identities);
packet.participant_identity =
options.sender_identity.map(|id| id.into()).unwrap_or_default();
RawStream::send_packet(&self.packet_tx, packet).await?;
return Ok(TextStreamInfo::from_headers(header, text_header));
}
header.inline_content = None;
enforce_header_size(&header, &options.destination_identities)?;
let open_options = RawStreamOpenOptions {
header: header.clone(),
destination_identities: options.destination_identities,
sender_identity: options.sender_identity,
packet_tx: self.packet_tx.clone(),
};
let info = TextStreamInfo::from_headers(header, text_header);
let mut stream = RawStream::open(open_options).await?;
if use_compression {
let compressed_bytes = maybe_compressed.as_bytes().await?;
stream.write_raw_chunks(compressed_bytes).await?;
} else {
for chunk in text_bytes.utf8_aware_chunks(constants::STREAM_CHUNK_SIZE_BYTES) {
stream.write_chunk(chunk).await?;
}
}
stream.close(None, None).await?;
Ok(info)
}
pub async fn send_bytes(
&self,
data: impl AsRef<[u8]>,
options: StreamByteOptions,
remote_participant_registry: &dyn RemoteParticipantRegistry,
) -> StreamResult<ByteStreamInfo> {
if options.total_length.is_some() {
log::warn!("Ignoring total_length option specified for send_bytes");
}
let bytes = data.as_ref();
let stream_id: StreamId = options.id.clone().unwrap_or_else(create_random_uuid).into();
let name = options.name.clone().unwrap_or_else(|| constants::BYTE_DEFAULT_NAME.to_owned());
let total_length = bytes.len() as u64;
let eligibility =
evaluate_eligibility(remote_participant_registry, &options.destination_identities);
let can_compress = options.compress.unwrap_or(true) && eligibility.compression;
let mut maybe_compressed = MaybeCollectedAsyncReader::from_async_reader(
async_compression::futures::bufread::DeflateEncoder::new(
futures_util::io::Cursor::new(bytes.to_vec()),
),
);
let use_compression =
can_compress && maybe_compressed.as_bytes().await.is_ok_and(|c| c.len() < bytes.len());
let (mut header, byte_header) = if use_compression {
build_byte_header(
&options,
stream_id.clone(),
name.clone(),
Some(total_length), Some(maybe_compressed.as_bytes().await?.to_owned()),
CompressionType::DeflateRaw,
)
} else {
build_byte_header(
&options,
stream_id.clone(),
name.clone(),
Some(total_length), Some(bytes.to_vec()),
CompressionType::None,
)
};
let proto_header = header.clone().into();
if eligibility.inline && header_packet_fits(&proto_header, &options.destination_identities)
{
let mut packet =
RawStream::create_header_packet(proto_header, options.destination_identities);
packet.participant_identity =
options.sender_identity.map(|id| id.into()).unwrap_or_default();
RawStream::send_packet(&self.packet_tx, packet).await?;
return Ok(ByteStreamInfo::from_headers(header, byte_header));
}
header.inline_content = None;
enforce_header_size(&header, &options.destination_identities)?;
let open_options = RawStreamOpenOptions {
header: header.clone(),
destination_identities: options.destination_identities,
sender_identity: options.sender_identity,
packet_tx: self.packet_tx.clone(),
};
let info = ByteStreamInfo::from_headers(header, byte_header);
let mut stream = RawStream::open(open_options).await?;
if use_compression {
let compressed_bytes = maybe_compressed.as_bytes().await?;
stream.write_raw_chunks(compressed_bytes).await?;
} else {
stream.write_raw_chunks(bytes).await?;
}
stream.close(None, None).await?;
Ok(info)
}
pub async fn send_file(
&self,
path: impl AsRef<Path>,
options: StreamByteOptions,
remote_participant_registry: &dyn RemoteParticipantRegistry,
) -> StreamResult<ByteStreamInfo> {
let path = path.as_ref();
let file_size = tokio::fs::metadata(path)
.await
.map(|metadata| metadata.len())
.map_err(StreamError::from)?;
let name = options.name.clone().unwrap_or_else(|| {
path.file_name().and_then(|n| n.to_str()).unwrap_or_default().to_owned()
});
let stream_id: StreamId = options.id.clone().unwrap_or_else(create_random_uuid).into();
let dests = options.destination_identities.clone();
let eligibility = evaluate_eligibility(remote_participant_registry, &dests);
let should_compress = options.compress.unwrap_or(true) && eligibility.compression;
let compression =
if should_compress { CompressionType::DeflateRaw } else { CompressionType::None };
let (header, byte_header) =
build_byte_header(&options, stream_id, name, Some(file_size), None, compression);
enforce_header_size(&header, &dests)?;
let open_options = RawStreamOpenOptions {
header: header.clone(),
destination_identities: dests,
sender_identity: options.sender_identity.clone(),
packet_tx: self.packet_tx.clone(),
};
let info = ByteStreamInfo::from_headers(header, byte_header);
let mut stream = RawStream::open(open_options).await?;
stream.write_file(path, should_compress).await?;
stream.close(None, None).await?;
Ok(info)
}
}
struct SendEligibility {
inline: bool,
compression: bool,
}
fn evaluate_eligibility(
registry: &dyn RemoteParticipantRegistry,
destinations: &[ParticipantIdentity],
) -> SendEligibility {
let recipients: Vec<ParticipantIdentity> =
if destinations.is_empty() { registry.remote_identities() } else { destinations.to_vec() };
let inline = recipients
.iter()
.all(|id| registry.remote_client_protocol(id) >= CLIENT_PROTOCOL_DATA_STREAM_V2);
let compression = inline
&& recipients.iter().all(|id| {
registry.remote_capabilities(id).contains(&ClientCapability::CompressionDeflateRaw)
});
SendEligibility { inline, compression }
}
enum MaybeCollectedAsyncReader<Reader: futures_util::io::AsyncRead + Unpin> {
Reader(Reader),
Collected(Vec<u8>),
}
impl<Reader: futures_util::io::AsyncRead + Unpin> MaybeCollectedAsyncReader<Reader> {
fn from_async_reader(reader: Reader) -> Self {
Self::Reader(reader)
}
async fn as_bytes(&mut self) -> Result<&[u8], std::io::Error> {
use futures_util::io::AsyncReadExt;
match self {
Self::Collected(_) => { }
Self::Reader(reader) => {
let mut buf = Vec::new();
reader.read_to_end(&mut buf).await?;
*self = Self::Collected(buf);
}
}
let Self::Collected(bytes) = self else { unreachable!("just set to Collected") };
Ok(bytes)
}
}
fn header_packet_fits(
header: &proto::data_stream::Header,
destinations: &[ParticipantIdentity],
) -> bool {
use prost::Message;
let packet = RawStream::create_header_packet(header.clone(), destinations.to_vec());
packet.encoded_len() <= constants::STREAM_CHUNK_SIZE_BYTES
}
fn enforce_header_size(header: &Header, destinations: &[ParticipantIdentity]) -> StreamResult<()> {
let proto_header: proto::data_stream::Header = header.clone().into();
if header_packet_fits(&proto_header, destinations) {
Ok(())
} else {
Err(StreamError::HeaderTooLarge)
}
}
fn build_text_header(
options: &StreamTextOptions,
stream_id: StreamId,
total_length: Option<u64>,
inline_content: Option<Vec<u8>>,
compression: CompressionType,
) -> (Header, TextHeader) {
let text_header = TextHeader {
operation_type: options.operation_type.unwrap_or_default(),
version: options.version.unwrap_or_default(),
reply_to_stream_id: options.reply_to_stream_id.clone().map(StreamId::from),
attached_stream_ids: options
.attached_stream_ids
.clone()
.into_iter()
.map(StreamId::from)
.collect(),
generated: options.generated.unwrap_or_default(),
};
let header = Header {
stream_id,
timestamp: Utc::now().timestamp_millis(),
topic: options.topic.clone(),
mime_type: constants::TEXT_MIME_TYPE.to_owned(),
total_length,
attributes: options.attributes.clone(),
content_header: Some(ContentHeader::TextHeader(text_header.clone().into())),
inline_content,
compression,
};
(header, text_header)
}
fn build_byte_header(
options: &StreamByteOptions,
stream_id: StreamId,
name: String,
total_length: Option<u64>,
inline_content: Option<Vec<u8>>,
compression: CompressionType,
) -> (Header, ByteHeader) {
let byte_header = ByteHeader { name };
let header = Header {
stream_id,
timestamp: Utc::now().timestamp_millis(),
topic: options.topic.clone(),
mime_type: options
.mime_type
.clone()
.unwrap_or_else(|| constants::BYTE_MIME_TYPE.to_owned()),
total_length,
attributes: options.attributes.clone(),
content_header: Some(byte_header.clone().into()),
inline_content,
compression,
};
(header, byte_header)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{backend::somewhat_compressible, outgoing::StreamWriter};
use livekit_common::{CLIENT_PROTOCOL_DATA_STREAM_RPC, CLIENT_PROTOCOL_DEFAULT};
use std::{collections::HashMap, sync::Mutex as StdMutex};
struct FakeRegistry {
remotes: HashMap<String, (i32, Vec<ClientCapability>)>,
}
impl FakeRegistry {
fn new() -> Self {
Self { remotes: HashMap::new() }
}
fn add(mut self, id: &str, client_protocol: i32, caps: &[ClientCapability]) -> Self {
self.remotes.insert(id.to_string(), (client_protocol, caps.to_vec()));
self
}
}
impl RemoteParticipantRegistry for FakeRegistry {
fn remote_client_protocol(&self, identity: &ParticipantIdentity) -> i32 {
self.remotes.get(&identity.0).map(|(p, _)| *p).unwrap_or(0)
}
fn remote_capabilities(&self, identity: &ParticipantIdentity) -> Vec<ClientCapability> {
self.remotes.get(&identity.0).map(|(_, c)| c.clone()).unwrap_or_default()
}
fn remote_identities(&self) -> Vec<ParticipantIdentity> {
self.remotes.keys().map(|k| ParticipantIdentity(k.clone())).collect()
}
}
fn pre_v2_room() -> FakeRegistry {
FakeRegistry::new()
.add("alice", CLIENT_PROTOCOL_DEFAULT, &[])
.add("bob", CLIENT_PROTOCOL_DEFAULT, &[])
.add("jim", CLIENT_PROTOCOL_DATA_STREAM_RPC, &[])
}
fn all_v2_room() -> FakeRegistry {
FakeRegistry::new()
.add(
"alice",
CLIENT_PROTOCOL_DATA_STREAM_V2,
&[ClientCapability::CompressionDeflateRaw],
)
.add("bob", CLIENT_PROTOCOL_DATA_STREAM_V2, &[ClientCapability::CompressionDeflateRaw])
.add("noCompression", CLIENT_PROTOCOL_DATA_STREAM_V2, &[])
}
fn mixed_room() -> FakeRegistry {
FakeRegistry::new()
.add("alice", CLIENT_PROTOCOL_DEFAULT, &[])
.add("bob", CLIENT_PROTOCOL_DATA_STREAM_V2, &[ClientCapability::CompressionDeflateRaw])
.add("jim", CLIENT_PROTOCOL_DATA_STREAM_V2, &[ClientCapability::CompressionDeflateRaw])
.add("mallory", CLIENT_PROTOCOL_DEFAULT, &[])
.add("noCompression", CLIENT_PROTOCOL_DATA_STREAM_V2, &[])
}
fn all_v2_capable_room() -> FakeRegistry {
FakeRegistry::new()
.add(
"alice",
CLIENT_PROTOCOL_DATA_STREAM_V2,
&[ClientCapability::CompressionDeflateRaw],
)
.add("bob", CLIENT_PROTOCOL_DATA_STREAM_V2, &[ClientCapability::CompressionDeflateRaw])
}
type Sent = Arc<StdMutex<Vec<proto::DataPacket>>>;
fn setup() -> (Manager, Sent) {
let (manager, mut packet_rx) = Manager::new();
let sent: Sent = Arc::new(StdMutex::new(Vec::new()));
let sink = sent.clone();
tokio::spawn(async move {
while let Ok((packet, responder)) = packet_rx.recv().await {
sink.lock().unwrap().push(packet);
let _ = responder.respond(Ok(()));
}
});
(manager, sent)
}
fn ids(list: &[&str]) -> Vec<ParticipantIdentity> {
list.iter().map(|s| ParticipantIdentity(s.to_string())).collect()
}
fn text_opts(topic: &str, dests: &[&str]) -> StreamTextOptions {
StreamTextOptions::new_with_topic(topic).with_destination_identities(ids(dests))
}
fn byte_opts(topic: &str, dests: &[&str]) -> StreamByteOptions {
StreamByteOptions::new_with_topic(topic).with_destination_identities(ids(dests))
}
fn header(p: &proto::DataPacket) -> &proto::data_stream::Header {
match p.value.as_ref().unwrap() {
proto::data_packet::Value::StreamHeader(h) => h,
_ => panic!("expected stream header"),
}
}
fn chunk(p: &proto::DataPacket) -> &proto::data_stream::Chunk {
match p.value.as_ref().unwrap() {
proto::data_packet::Value::StreamChunk(c) => c,
_ => panic!("expected stream chunk"),
}
}
fn is_text_header(h: &proto::data_stream::Header) -> bool {
matches!(h.content_header, Some(proto::data_stream::header::ContentHeader::TextHeader(_)))
}
fn is_byte_header(h: &proto::data_stream::Header) -> bool {
matches!(h.content_header, Some(proto::data_stream::header::ContentHeader::ByteHeader(_)))
}
fn assert_trailer(p: &proto::DataPacket) {
match p.value.as_ref().unwrap() {
proto::data_packet::Value::StreamTrailer(t) => assert_eq!(t.reason, ""),
_ => panic!("expected stream trailer"),
}
}
fn random_bytes(len: usize) -> Vec<u8> {
use rand::{rngs::StdRng, Rng, SeedableRng};
let mut rng = StdRng::seed_from_u64(0xdead_beef);
(0..len).map(|_| rng.random::<u8>()).collect()
}
fn text_content_header(h: &proto::data_stream::Header) -> &proto::data_stream::TextHeader {
match h.content_header.as_ref().unwrap() {
proto::data_stream::header::ContentHeader::TextHeader(t) => t,
_ => panic!("expected text header"),
}
}
mod room_with_pre_data_streams_v2_participants {
use super::*;
#[tokio::test]
async fn pre_v2_short_text_is_legacy_multipacket() {
let (m, sent) = setup();
m.send_text("hello world", text_opts("chat", &[]), &pre_v2_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 3);
let h = header(&p[0]);
assert!(is_text_header(h));
assert_eq!(h.topic, "chat");
assert_eq!(h.compression(), proto::data_stream::CompressionType::None);
assert!(h.inline_content.is_none());
let c = chunk(&p[1]);
assert_eq!(c.chunk_index, 0);
assert_eq!(c.content, b"hello world");
assert_trailer(&p[2]);
}
#[tokio::test]
async fn pre_v2_long_text_splits_at_mtu() {
let (m, sent) = setup();
let text = "A".repeat(40_000);
m.send_text(&text, text_opts("chat", &[]), &pre_v2_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 5); assert_eq!(header(&p[0]).compression(), proto::data_stream::CompressionType::None);
assert_eq!(chunk(&p[1]).content.len(), 15_000);
assert_eq!(chunk(&p[2]).content.len(), 15_000);
assert_eq!(chunk(&p[3]).content.len(), 10_000);
assert_eq!(chunk(&p[1]).chunk_index, 0);
assert_eq!(chunk(&p[3]).chunk_index, 2);
assert_trailer(&p[4]);
}
#[tokio::test]
async fn pre_v2_bytes_is_legacy_multipacket() {
let (m, sent) = setup();
m.send_bytes([0u8, 1, 2, 3], byte_opts("blob", &[]), &pre_v2_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 3);
let h = header(&p[0]);
assert!(is_byte_header(h));
assert_eq!(h.compression(), proto::data_stream::CompressionType::None);
assert!(h.inline_content.is_none());
assert_eq!(chunk(&p[1]).content, vec![0, 1, 2, 3]);
assert_trailer(&p[2]);
}
#[tokio::test]
async fn pre_v2_empty_text_sends_header_and_trailer() {
let (m, sent) = setup();
m.send_text("", text_opts("chat", &[]), &pre_v2_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 2);
assert_eq!(header(&p[0]).total_length, Some(0));
assert_trailer(&p[1]);
}
}
mod room_with_all_data_streams_v2_participants {
use super::*;
mod send_text {
use super::*;
#[tokio::test]
async fn v2_short_compressible_text_inlines_compressed() {
let (m, sent) = setup();
let text = "hello hello compressible world";
m.send_text(text, text_opts("chat", &["alice", "bob"]), &all_v2_room())
.await
.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
let h = header(&p[0]);
assert!(is_text_header(h));
assert_eq!(h.compression(), proto::data_stream::CompressionType::DeflateRaw);
let inline = h.inline_content.as_ref().unwrap();
assert_ne!(inline.as_slice(), text.as_bytes()); }
#[tokio::test]
async fn v2_short_incompressible_text_inlines_raw() {
let (m, sent) = setup();
m.send_text("short", text_opts("chat", &["alice", "bob"]), &all_v2_room())
.await
.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::None);
assert_eq!(h.inline_content.as_ref().unwrap().as_slice(), b"short");
}
#[tokio::test]
async fn v2_no_compression_cap_inlines_raw() {
let (m, sent) = setup();
let text = "hello hello compressible world";
m.send_text(text, text_opts("chat", &["noCompression"]), &all_v2_room())
.await
.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1); let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::None); assert_eq!(h.inline_content.as_ref().unwrap().as_slice(), text.as_bytes());
}
#[tokio::test]
async fn v2_large_highly_compressible_text_still_inlines() {
let (m, sent) = setup();
let text = "hello world".repeat(20_000);
m.send_text(&text, text_opts("chat", &["alice", "bob"]), &all_v2_room())
.await
.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::DeflateRaw);
assert!(h.inline_content.as_ref().unwrap().len() < text.len());
}
#[tokio::test]
async fn v2_somewhat_compressible_text_is_compressed_multipacket() {
let (m, sent) = setup();
let text = somewhat_compressible(50_000);
m.send_text(&text, text_opts("chat", &["alice", "bob"]), &all_v2_room())
.await
.unwrap();
let p = sent.lock().unwrap().clone();
let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::DeflateRaw);
assert!(h.inline_content.is_none());
let chunks: Vec<_> = p[1..p.len() - 1].iter().map(chunk).collect();
let uncompressed_chunks = text.len().div_ceil(constants::STREAM_CHUNK_SIZE_BYTES);
assert!(chunks.len() >= 2);
assert!(chunks.len() < uncompressed_chunks);
assert_eq!(chunks[0].content.len(), constants::STREAM_CHUNK_SIZE_BYTES); let total: usize = chunks.iter().map(|c| c.content.len()).sum();
assert!(total < text.len()); assert_trailer(p.last().unwrap());
}
#[tokio::test]
async fn v2_compress_false_short_inlines_raw() {
let (m, sent) = setup();
let text = "hello hello compressible world";
let opts = text_opts("chat", &["alice", "bob"]).with_compress(false);
m.send_text(text, opts, &all_v2_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::None);
assert_eq!(h.inline_content.as_ref().unwrap().as_slice(), text.as_bytes());
}
#[tokio::test]
async fn v2_compress_false_large_is_uncompressed_multipacket() {
let (m, sent) = setup();
let text = "B".repeat(50_000);
let opts = text_opts("chat", &["alice", "bob"]).with_compress(false);
m.send_text(&text, opts, &all_v2_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 6); assert_eq!(header(&p[0]).compression(), proto::data_stream::CompressionType::None);
assert_eq!(chunk(&p[1]).content.len(), 15_000);
}
#[tokio::test]
async fn v2_send_text_with_attachments_never_inlines() {
let (m, sent) = setup();
let opts = text_opts("chat", &["alice", "bob"]).with_attached_stream_id("att1");
m.send_text("hello hello compressible world", opts, &all_v2_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 3); let h = header(&p[0]);
assert!(h.inline_content.is_none());
assert_eq!(text_content_header(h).attached_stream_ids, vec!["att1".to_string()]);
assert_trailer(&p[2]);
}
#[tokio::test]
async fn v2_large_text_to_uncapable_recipient_is_uncompressed_multipacket() {
let (m, sent) = setup();
let text = "A".repeat(40_000);
m.send_text(&text, text_opts("chat", &["noCompression"]), &all_v2_room())
.await
.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 5);
let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::None);
assert!(h.inline_content.is_none());
assert_eq!(chunk(&p[1]).content.len(), 15_000);
assert_eq!(chunk(&p[2]).content.len(), 15_000);
assert_eq!(chunk(&p[3]).content.len(), 10_000);
assert_trailer(&p[4]);
}
#[tokio::test]
async fn v2_empty_text_sends_single_inline_packet() {
let (m, sent) = setup();
m.send_text("", text_opts("chat", &["alice", "bob"]), &all_v2_room())
.await
.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::None);
assert_eq!(h.inline_content.as_deref(), Some(&[][..]));
assert_eq!(h.total_length, Some(0));
}
}
mod send_bytes {
use super::*;
#[tokio::test]
async fn v2_send_bytes_short_incompressible_inlines_raw() {
let (m, sent) = setup();
m.send_bytes([0u8, 1, 2, 3], byte_opts("blob", &["alice", "bob"]), &all_v2_room())
.await
.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::None);
assert_eq!(h.inline_content.as_ref().unwrap().as_slice(), &[0u8, 1, 2, 3]);
}
#[tokio::test]
async fn v2_send_bytes_no_compression_cap_inlines_raw() {
let (m, sent) = setup();
let payload = "hello hello compressible world".as_bytes();
m.send_bytes(payload, byte_opts("blob", &["noCompression"]), &all_v2_room())
.await
.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1); let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::None); assert_eq!(h.inline_content.as_ref().unwrap().as_slice(), payload);
}
#[tokio::test]
async fn v2_send_bytes_large_highly_compressible_inlines() {
let (m, sent) = setup();
let payload = vec![0x01u8; 50_000];
m.send_bytes(&payload, byte_opts("blob", &["alice", "bob"]), &all_v2_room())
.await
.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1); let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::DeflateRaw);
assert!(h.inline_content.as_ref().unwrap().len() < payload.len());
assert_eq!(h.total_length, Some(50_000)); }
#[tokio::test]
async fn v2_send_bytes_somewhat_compressible_is_compressed_multipacket() {
let (m, sent) = setup();
let payload = somewhat_compressible(50_000).into_bytes();
m.send_bytes(&payload, byte_opts("blob", &["alice", "bob"]), &all_v2_room())
.await
.unwrap();
let p = sent.lock().unwrap().clone();
let h = header(&p[0]);
assert!(is_byte_header(h));
assert_eq!(h.compression(), proto::data_stream::CompressionType::DeflateRaw);
assert!(h.inline_content.is_none());
let chunks: Vec<_> = p[1..p.len() - 1].iter().map(chunk).collect();
assert!(chunks.len() >= 2);
assert!(chunks.len() < payload.len().div_ceil(constants::STREAM_CHUNK_SIZE_BYTES));
assert_eq!(chunks[0].content.len(), constants::STREAM_CHUNK_SIZE_BYTES);
assert_trailer(p.last().unwrap());
}
#[tokio::test]
async fn v2_send_bytes_compress_false_large_is_uncompressed_multipacket() {
let (m, sent) = setup();
let payload = vec![0x07u8; 40_000];
let opts = byte_opts("blob", &["alice", "bob"]).with_compress(false);
m.send_bytes(&payload, opts, &all_v2_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 5); let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::None);
assert!(h.inline_content.is_none());
assert_eq!(chunk(&p[1]).content.len(), 15_000);
assert_eq!(chunk(&p[2]).content.len(), 15_000);
assert_eq!(chunk(&p[3]).content.len(), 10_000);
assert!(chunk(&p[3]).content.iter().all(|b| *b == 0x07));
assert_trailer(&p[4]);
}
#[tokio::test]
async fn v2_send_bytes_short_compressible_inlines_compressed() {
let (m, sent) = setup();
let payload = "hello hello compressible world".as_bytes().to_vec();
let mut opts = byte_opts("blob", &["alice", "bob"]);
opts.attributes.insert("foo".to_string(), "bar".to_string());
let info = m.send_bytes(&payload, opts, &all_v2_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
let h = header(&p[0]);
assert!(is_byte_header(h));
assert_eq!(h.compression(), proto::data_stream::CompressionType::DeflateRaw);
assert_ne!(h.inline_content.as_ref().unwrap().as_slice(), payload.as_slice());
assert_eq!(info.name, "unknown");
assert_eq!(info.mime_type, "application/octet-stream");
assert_eq!(info.total_length, Some(payload.len() as u64));
assert_eq!(info.attributes().get("foo"), Some(&"bar".to_string()));
}
}
mod send_file {
use super::*;
async fn write_temp_file(bytes: &[u8]) -> std::path::PathBuf {
let path =
std::env::temp_dir().join(format!("lk_ds_test_{}.bin", create_random_uuid()));
tokio::fs::write(&path, bytes).await.unwrap();
path
}
#[tokio::test]
async fn send_file_never_inlines_and_compresses_when_eligible() {
let (m, sent) = setup();
let path = write_temp_file(&vec![0x01u8; 10_000]).await;
m.send_file(&path, byte_opts("file", &["alice", "bob"]), &all_v2_room())
.await
.unwrap();
let _ = tokio::fs::remove_file(&path).await;
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 3); let h = header(&p[0]);
assert!(is_byte_header(h));
assert_eq!(h.compression(), proto::data_stream::CompressionType::DeflateRaw);
assert!(h.inline_content.is_none());
assert!(chunk(&p[1]).content.len() < 10_000); assert_trailer(&p[2]);
}
#[tokio::test]
async fn send_file_uncompressed_splits_at_mtu() {
let (m, sent) = setup();
let path = write_temp_file(&vec![0x07u8; 20_000]).await;
m.send_file(&path, byte_opts("file", &[]).with_compress(false), &all_v2_room())
.await
.unwrap();
let _ = tokio::fs::remove_file(&path).await;
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 4); assert_eq!(header(&p[0]).compression(), proto::data_stream::CompressionType::None);
assert_eq!(chunk(&p[1]).content.len(), 15_000);
assert_eq!(chunk(&p[2]).content.len(), 5_000);
assert_eq!(chunk(&p[2]).chunk_index, 1);
assert_trailer(&p[3]);
}
#[tokio::test]
async fn send_file_incompressible_compressed_expands() {
let (m, sent) = setup();
let data = random_bytes(50_000);
let path = write_temp_file(&data).await;
m.send_file(&path, byte_opts("file", &["alice", "bob"]), &all_v2_room())
.await
.unwrap();
let _ = tokio::fs::remove_file(&path).await;
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 6); let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::DeflateRaw);
assert_eq!(chunk(&p[1]).content.len(), 15_000);
let total: usize =
p[1..p.len() - 1].iter().map(|packet| chunk(packet).content.len()).sum();
assert!(total > data.len());
assert_trailer(p.last().unwrap());
}
#[tokio::test]
async fn send_file_to_no_compression_recipient_is_uncompressed() {
let (m, sent) = setup();
let path = write_temp_file(&vec![0x07u8; 10_000]).await;
m.send_file(&path, byte_opts("file", &["noCompression"]), &all_v2_room())
.await
.unwrap();
let _ = tokio::fs::remove_file(&path).await;
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 3);
let h = header(&p[0]);
assert!(is_byte_header(h));
assert_eq!(h.compression(), proto::data_stream::CompressionType::None);
assert!(h.inline_content.is_none());
let c = chunk(&p[1]);
assert_eq!(c.content.len(), 10_000);
assert!(c.content.iter().all(|b| *b == 0x07));
assert_trailer(&p[2]);
}
#[tokio::test]
async fn send_file_empty_file() {
let (m, sent) = setup();
let path = write_temp_file(&[]).await;
m.send_file(&path, byte_opts("file", &["alice", "bob"]), &all_v2_room())
.await
.unwrap();
let _ = tokio::fs::remove_file(&path).await;
let p = sent.lock().unwrap().clone();
let h = header(&p[0]);
assert!(is_byte_header(h));
assert_eq!(h.total_length, Some(0));
assert_eq!(h.compression(), proto::data_stream::CompressionType::DeflateRaw);
assert_trailer(p.last().unwrap());
}
}
#[tokio::test]
async fn v2_broadcast_all_capable_inlines_compressed() {
let (m, sent) = setup();
m.send_text(
"hello hello compressible world",
text_opts("chat", &[]),
&all_v2_capable_room(),
)
.await
.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
assert_eq!(
header(&p[0]).compression(),
proto::data_stream::CompressionType::DeflateRaw
);
}
#[tokio::test]
async fn v2_broadcast_with_uncapable_member_inlines_raw() {
let (m, sent) = setup();
let text = "hello hello compressible world";
m.send_text(text, text_opts("chat", &[]), &all_v2_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::None);
assert_eq!(h.inline_content.as_ref().unwrap().as_slice(), text.as_bytes());
}
#[tokio::test]
async fn empty_room_broadcast_is_v2_eligible() {
let (m, sent) = setup();
m.send_text(
"hello hello compressible world",
text_opts("chat", &[]),
&FakeRegistry::new(),
)
.await
.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
assert_eq!(
header(&p[0]).compression(),
proto::data_stream::CompressionType::DeflateRaw
);
}
}
mod room_with_mixed_participants {
use super::*;
#[tokio::test]
async fn mixed_broadcast_falls_back_to_legacy() {
let (m, sent) = setup();
m.send_text("hello world", text_opts("chat", &[]), &mixed_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 3);
assert_eq!(header(&p[0]).compression(), proto::data_stream::CompressionType::None);
assert!(header(&p[0]).inline_content.is_none());
assert_eq!(chunk(&p[1]).content, b"hello world");
}
#[tokio::test]
async fn mixed_targeted_v2_subset_inlines_compressed() {
let (m, sent) = setup();
let text = "hello hello compressible world";
m.send_text(text, text_opts("chat", &["bob", "jim"]), &mixed_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::DeflateRaw);
assert_ne!(h.inline_content.as_ref().unwrap().as_slice(), text.as_bytes());
}
#[tokio::test]
async fn mixed_targeted_subset_missing_cap_inlines_uncompressed() {
let (m, sent) = setup();
let text = "hello hello compressible world";
m.send_text(text, text_opts("chat", &["bob", "jim", "noCompression"]), &mixed_room())
.await
.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
let h = header(&p[0]);
assert_eq!(h.compression(), proto::data_stream::CompressionType::None);
assert_eq!(h.inline_content.as_ref().unwrap().as_slice(), text.as_bytes());
}
}
#[tokio::test]
async fn stream_text_with_sender_identity_stamps_every_packet() {
let (m, sent) = setup();
let opts = text_opts("chat", &[]).with_sender_identity("impostor");
let writer = m.stream_text(opts).await.unwrap();
writer.write("hello").await.unwrap();
writer.close().await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 3);
assert!(p.iter().all(|pkt| pkt.participant_identity == "impostor"));
}
#[tokio::test]
async fn send_text_inline_with_sender_identity_stamps_packet() {
let (m, sent) = setup();
let opts = text_opts("chat", &["alice", "bob"]).with_sender_identity("impostor");
m.send_text("hello hello compressible world", opts, &all_v2_room()).await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 1);
assert_eq!(p[0].participant_identity, "impostor");
}
#[tokio::test]
async fn packets_carry_no_identity_when_sender_identity_unset() {
let (m, sent) = setup();
let writer = m.stream_text(text_opts("chat", &[])).await.unwrap();
writer.close().await.unwrap();
let p = sent.lock().unwrap().clone();
assert!(p.iter().all(|pkt| pkt.participant_identity.is_empty()));
}
#[tokio::test]
async fn stream_text_never_compresses_or_inlines() {
let (m, sent) = setup();
let writer = m.stream_text(text_opts("chat", &["noCompression"])).await.unwrap();
assert_eq!(sent.lock().unwrap().len(), 1);
let h0 = sent.lock().unwrap()[0].clone();
assert!(is_text_header(header(&h0)));
assert_eq!(header(&h0).compression(), proto::data_stream::CompressionType::None);
assert!(header(&h0).inline_content.is_none());
writer.write("hello world").await.unwrap();
assert_eq!(sent.lock().unwrap().len(), 2);
assert_eq!(chunk(&sent.lock().unwrap()[1]).content, b"hello world");
writer.close().await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 3);
assert_trailer(&p[2]);
}
#[tokio::test]
async fn stream_bytes_never_compresses_or_inlines() {
let (m, sent) = setup();
let writer = m.stream_bytes(byte_opts("blob", &["noCompression"])).await.unwrap();
assert_eq!(sent.lock().unwrap().len(), 1);
assert_eq!(
header(&sent.lock().unwrap()[0]).compression(),
proto::data_stream::CompressionType::None
);
writer.write(&[0u8, 1, 2, 3]).await.unwrap();
assert_eq!(chunk(&sent.lock().unwrap()[1]).content, vec![0, 1, 2, 3]);
writer.close().await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 3);
assert_trailer(&p[2]);
}
mod header_size_limit {
use super::*;
#[tokio::test]
async fn oversized_attributes_on_chunked_path_errors() {
let (m, _sent) = setup();
let mut opts = text_opts("chat", &[]); opts.attributes.insert("big".to_string(), "x".repeat(20_000));
let result = m.send_text("hello", opts, &pre_v2_room()).await;
assert!(matches!(result, Err(StreamError::HeaderTooLarge)));
}
fn header_with_attributes(attributes: HashMap<String, String>) -> Header {
Header {
stream_id: "s1".into(),
timestamp: 0,
topic: "chat".to_string(),
mime_type: constants::TEXT_MIME_TYPE.to_owned(),
total_length: None,
attributes,
content_header: Some(ContentHeader::TextHeader(TextHeader::default())),
inline_content: None,
compression: CompressionType::None,
}
}
#[test]
fn enforce_header_size_accepts_small_header() {
let header = header_with_attributes(HashMap::new());
assert!(enforce_header_size(&header, &[]).is_ok());
}
#[test]
fn enforce_header_size_rejects_large_header() {
let mut attributes = HashMap::new();
attributes.insert("big".to_string(), "x".repeat(20_000));
let header = header_with_attributes(attributes);
assert!(matches!(enforce_header_size(&header, &[]), Err(StreamError::HeaderTooLarge)));
}
}
#[test]
fn drop_raw_stream_on_non_tokio_thread_does_not_panic() {
let rt = tokio::runtime::Runtime::new().unwrap();
let raw_stream = rt.block_on(async {
let (packet_tx, mut packet_rx) =
bmrng::unbounded_channel::<proto::DataPacket, Result<(), SendError>>();
tokio::spawn(async move {
while let Ok((_packet, responder)) = packet_rx.recv().await {
let _ = responder.respond(Ok(()));
}
});
let header = Header {
stream_id: "gc-test-stream".into(),
timestamp: 0,
topic: "gc-test-topic".to_string(),
mime_type: constants::TEXT_MIME_TYPE.to_owned(),
total_length: None,
attributes: HashMap::new(),
content_header: None,
inline_content: None,
compression: CompressionType::None,
};
RawStream::open(RawStreamOpenOptions {
header,
destination_identities: vec![],
sender_identity: None,
packet_tx,
})
.await
.expect("RawStream should open")
});
let drop_thread = std::thread::spawn(move || drop(raw_stream));
drop_thread.join().expect("Dropping RawStream on a non-Tokio thread must not panic");
}
mod stream_text_bytes {
use super::*;
#[tokio::test]
async fn stream_bytes_multi_write_splits_at_mtu() {
let (m, sent) = setup();
let writer = m.stream_bytes(byte_opts("blob", &[])).await.unwrap();
writer.write(&vec![0x01u8; 20_000]).await.unwrap();
writer.write(&vec![0x01u8; 20_000]).await.unwrap();
writer.close().await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 6); for (i, expected_len) in [15_000usize, 5_000, 15_000, 5_000].iter().enumerate() {
let c = chunk(&p[i + 1]);
assert_eq!(c.chunk_index, i as u64);
assert_eq!(c.content.len(), *expected_len);
assert!(c.content.iter().all(|b| *b == 0x01));
}
assert_trailer(&p[5]);
}
#[tokio::test]
async fn close_with_options_sends_trailer_attributes() {
let (m, sent) = setup();
let writer = m.stream_text(text_opts("chat", &[])).await.unwrap();
writer.write("hello").await.unwrap();
let attributes = HashMap::from([("result".to_string(), "ok".to_string())]);
writer.close_with_options(None, Some(attributes.clone())).await.unwrap();
let p = sent.lock().unwrap().clone();
let trailer = match p.last().unwrap().value.as_ref().unwrap() {
proto::data_packet::Value::StreamTrailer(t) => t,
_ => panic!("expected stream trailer"),
};
assert_eq!(trailer.reason, "");
assert_eq!(trailer.attributes, attributes);
}
#[tokio::test]
async fn close_with_options_sends_reason_and_attributes() {
let (m, sent) = setup();
let writer = m.stream_text(text_opts("chat", &[])).await.unwrap();
let attributes = HashMap::from([("cause".to_string(), "cancelled".to_string())]);
writer.close_with_options(Some("aborted"), Some(attributes.clone())).await.unwrap();
let p = sent.lock().unwrap().clone();
let trailer = match p.last().unwrap().value.as_ref().unwrap() {
proto::data_packet::Value::StreamTrailer(t) => t,
_ => panic!("expected stream trailer"),
};
assert_eq!(trailer.reason, "aborted");
assert_eq!(trailer.attributes, attributes);
}
#[tokio::test]
async fn stream_text_oversized_attributes_errors() {
let (m, sent) = setup();
let mut opts = text_opts("chat", &[]);
opts.attributes.insert("big".to_string(), "x".repeat(20_000));
let result = m.stream_text(opts).await;
assert!(matches!(result, Err(StreamError::HeaderTooLarge)));
assert!(sent.lock().unwrap().is_empty());
}
#[tokio::test]
async fn stream_text_splits_on_utf8_boundaries_at_mtu() {
let (m, sent) = setup();
let text = format!("{}😀{}", "a".repeat(14_999), "b".repeat(10));
let writer = m.stream_text(text_opts("chat", &[])).await.unwrap();
writer.write(&text).await.unwrap();
writer.close().await.unwrap();
let p = sent.lock().unwrap().clone();
assert_eq!(p.len(), 4); let first = chunk(&p[1]);
let second = chunk(&p[2]);
assert_eq!(first.content.len(), 14_999);
let first_str =
std::str::from_utf8(&first.content).expect("chunk 0 must be valid UTF-8");
let second_str =
std::str::from_utf8(&second.content).expect("chunk 1 must be valid UTF-8");
assert_eq!(format!("{first_str}{second_str}"), text);
}
}
}