vector_calls/
link.rs

1//! The link between the media engine and whatever captures, encodes, decodes and
2//! paints video (a webview over a loopback socket, or an in-process port). Binary
3//! messages, the first byte says what follows.
4
5use serde::{Deserialize, Serialize};
6
7use super::wire::{Tracks, VideoCodec, VideoHeader, VideoKind};
8
9/// A video frame: the 16-byte wire header, then the encoded bytes. The same bytes
10/// go on the QUIC stream, so nothing is re-framed in between.
11pub const KIND_FRAME: u8 = 1;
12/// A JSON control message.
13pub const KIND_CONTROL: u8 = 2;
14/// Raw audio from a screen share: a 5-byte header (sample rate u32, channels u8) and
15/// interleaved little-endian f32 samples. Cancelled and encoded on the other side.
16pub const KIND_PCM: u8 = 3;
17pub const PCM_HEADER_LEN: usize = 5;
18
19/// Sent to the encoder side of the link.
20#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
21#[serde(tag = "t", rename_all = "snake_case")]
22pub enum ToLink {
23    /// Encode that track at this rate, size and frame rate from now on.
24    Rate { kind: VideoKind, kbps: u32, width: u32, height: u32, fps: u32 },
25    /// Too many frames in flight: drop that track's next capture.
26    Skip { kind: VideoKind },
27    /// Make that track's next frame a keyframe.
28    Keyframe { kind: VideoKind },
29    /// What the peer is sending now, so the stage can show or hide their pictures.
30    Peer { tracks: Tracks },
31    /// The peer cannot see us; stop capturing (or resume).
32    Pause { on: bool },
33    /// Send with this codec: the one the peer decodes.
34    Codec { codec: VideoCodec },
35}
36
37/// Received from the encoder side of the link.
38#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
39#[serde(tag = "t", rename_all = "snake_case")]
40pub enum FromLink {
41    /// What this device can encode and decode, from its boot probe.
42    Caps { encode: Vec<String>, decode: Vec<String> },
43    /// One encoder's own numbers, for the stats line.
44    Stats { kind: VideoKind, fps: u32, kbps: u32, width: u32, height: u32 },
45    /// My decoder for that track lost the chain: ask the peer for a keyframe.
46    Lost { kind: VideoKind },
47    /// My decoder refuses this codec outright: ask the peer to send another.
48    Unsupported { codec: VideoCodec },
49    #[serde(other)]
50    Unknown,
51}
52
53/// What one link message is.
54pub enum LinkMsg<'a> {
55    Frame { header: VideoHeader, frame: &'a [u8] },
56    Control(FromLink),
57    Pcm { rate: u32, channels: u8, samples: &'a [u8] },
58}
59
60/// Parses a message from the encoder side. `frame` is the header plus payload, the
61/// exact bytes to put on the wire.
62pub fn parse(msg: &[u8]) -> Option<LinkMsg<'_>> {
63    match msg.first()? {
64        &KIND_FRAME => {
65            let frame = &msg[1..];
66            let header = VideoHeader::parse(frame)?;
67            Some(LinkMsg::Frame { header, frame })
68        }
69        &KIND_CONTROL => serde_json::from_slice(&msg[1..]).ok().map(LinkMsg::Control),
70        &KIND_PCM => {
71            let body = &msg[1..];
72            if body.len() < PCM_HEADER_LEN {
73                return None;
74            }
75            let rate = u32::from_le_bytes([body[0], body[1], body[2], body[3]]);
76            let channels = body[4];
77            if !(1..=2).contains(&channels) || !(8000..=192_000).contains(&rate) {
78                return None;
79            }
80            Some(LinkMsg::Pcm { rate, channels, samples: &body[PCM_HEADER_LEN..] })
81        }
82        _ => None,
83    }
84}
85
86/// A control message ready to send to the encoder side.
87pub fn control(msg: &ToLink) -> Vec<u8> {
88    let mut out = vec![KIND_CONTROL];
89    if let Ok(body) = serde_json::to_vec(msg) {
90        out.extend_from_slice(&body);
91    }
92    out
93}
94
95#[cfg(test)]
96mod tests {
97    use super::*;
98    use crate::wire::{VIDEO_HEADER_LEN, VIDEO_KEY};
99
100    #[test]
101    fn frames_and_controls_parse_and_garbage_does_not() {
102        let mut msg = vec![KIND_FRAME];
103        let mut hdr = [0u8; VIDEO_HEADER_LEN];
104        VideoHeader { seq: 3, ts_us: 9, flags: VIDEO_KEY, codec: VideoCodec::H264 }.write(&mut hdr);
105        msg.extend_from_slice(&hdr);
106        msg.extend_from_slice(&[7, 7, 7]);
107        match parse(&msg) {
108            Some(LinkMsg::Frame { header, frame }) => {
109                assert_eq!(header.seq, 3);
110                assert_eq!(frame.len(), VIDEO_HEADER_LEN + 3);
111            }
112            _ => panic!("frame expected"),
113        }
114        let c = control(&ToLink::Rate { kind: VideoKind::Camera, kbps: 800, width: 640, height: 360, fps: 30 });
115        assert_eq!(c[0], KIND_CONTROL);
116        let mut lost = vec![KIND_CONTROL];
117        lost.extend_from_slice(br#"{"t":"lost","kind":"screen"}"#);
118        assert!(matches!(parse(&lost), Some(LinkMsg::Control(FromLink::Lost { kind: VideoKind::Screen }))));
119        let mut odd = vec![KIND_CONTROL];
120        odd.extend_from_slice(br#"{"t":"teleport"}"#);
121        assert!(matches!(parse(&odd), Some(LinkMsg::Control(FromLink::Unknown))));
122        assert!(parse(&[9, 9]).is_none());
123        assert!(parse(&[]).is_none());
124        let mut pcm = vec![KIND_PCM];
125        pcm.extend_from_slice(&48_000u32.to_le_bytes());
126        pcm.push(2);
127        pcm.extend_from_slice(&1.0f32.to_le_bytes());
128        match parse(&pcm) {
129            Some(LinkMsg::Pcm { rate, channels, samples }) => assert_eq!((rate, channels, samples.len()), (48_000, 2, 4)),
130            _ => panic!("pcm expected"),
131        }
132        pcm[5] = 7;
133        assert!(parse(&pcm).is_none(), "an impossible channel count is refused");
134    }
135}
vector_calls/link.rs

vector_calls/
link.rs
