Skip to main content

vector_calls/
link.rs

1//! The link between the media engine and whatever captures, encodes, decodes and
2//! paints video (a webview over a loopback socket, or an in-process port). Binary
3//! messages, the first byte says what follows.
4
5use serde::{Deserialize, Serialize};
6
7use super::wire::{Tracks, VideoCodec, VideoHeader, VideoKind};
8
9/// A video frame: the 16-byte wire header, then the encoded bytes. The same bytes
10/// go on the QUIC stream, so nothing is re-framed in between.
11pub const KIND_FRAME: u8 = 1;
12/// A JSON control message.
13pub const KIND_CONTROL: u8 = 2;
14/// Raw audio from a screen share: a 5-byte header (sample rate u32, channels u8) and
15/// interleaved little-endian f32 samples. Cancelled and encoded on the other side.
16pub const KIND_PCM: u8 = 3;
17pub const PCM_HEADER_LEN: usize = 5;
18/// A peer's video frame in a call of several: the sender's 32-byte key, then the 16-byte
19/// header and the encoded bytes. Only ever sent toward the decoding side.
20pub const KIND_PEER_FRAME: u8 = 4;
21
22/// Sent to the encoder side of the link.
23#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
24#[serde(tag = "t", rename_all = "snake_case")]
25pub enum ToLink {
26    /// Encode that track at this rate, size and frame rate from now on.
27    Rate { kind: VideoKind, kbps: u32, width: u32, height: u32, fps: u32 },
28    /// Too many frames in flight: drop that track's next capture.
29    Skip { kind: VideoKind },
30    /// Make that track's next frame a keyframe.
31    Keyframe { kind: VideoKind },
32    /// What the peer is sending now, so the stage can show or hide their pictures.
33    Peer { tracks: Tracks },
34    /// The peer cannot see us; stop capturing (or resume).
35    Pause { on: bool },
36    /// Send with this codec: the one the peer decodes.
37    Codec { codec: VideoCodec },
38}
39
40/// Received from the encoder side of the link.
41#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
42#[serde(tag = "t", rename_all = "snake_case")]
43pub enum FromLink {
44    /// What this device can encode and decode, from its boot probe.
45    Caps { encode: Vec<String>, decode: Vec<String> },
46    /// One encoder's own numbers, for the stats line.
47    Stats { kind: VideoKind, fps: u32, kbps: u32, width: u32, height: u32 },
48    /// My decoder for that track lost the chain: ask the peer for a keyframe.
49    Lost { kind: VideoKind },
50    /// My decoder refuses this codec outright: ask the peer to send another. In a call of
51    /// several, `peer` says whose (their key, hex).
52    Unsupported {
53        codec: VideoCodec,
54        #[serde(default)]
55        peer: Option<String>,
56    },
57    /// In a call of several, my decoder for this sender's (their key, hex) track lost the chain.
58    LostFrom { peer: String, kind: VideoKind },
59    #[serde(other)]
60    Unknown,
61}
62
63/// What one link message is.
64pub enum LinkMsg<'a> {
65    Frame { header: VideoHeader, frame: &'a [u8] },
66    Control(FromLink),
67    Pcm { rate: u32, channels: u8, samples: &'a [u8] },
68}
69
70/// Parses a message from the encoder side. `frame` is the header plus payload, the
71/// exact bytes to put on the wire.
72pub fn parse(msg: &[u8]) -> Option<LinkMsg<'_>> {
73    match msg.first()? {
74        &KIND_FRAME => {
75            let frame = &msg[1..];
76            let header = VideoHeader::parse(frame)?;
77            Some(LinkMsg::Frame { header, frame })
78        }
79        &KIND_CONTROL => serde_json::from_slice(&msg[1..]).ok().map(LinkMsg::Control),
80        &KIND_PCM => {
81            let body = &msg[1..];
82            if body.len() < PCM_HEADER_LEN {
83                return None;
84            }
85            let rate = u32::from_le_bytes([body[0], body[1], body[2], body[3]]);
86            let channels = body[4];
87            if !(1..=2).contains(&channels) || !(8000..=192_000).contains(&rate) {
88                return None;
89            }
90            Some(LinkMsg::Pcm { rate, channels, samples: &body[PCM_HEADER_LEN..] })
91        }
92        _ => None,
93    }
94}
95
96/// A control message ready to send to the encoder side.
97pub fn control(msg: &ToLink) -> Vec<u8> {
98    let mut out = vec![KIND_CONTROL];
99    if let Ok(body) = serde_json::to_vec(msg) {
100        out.extend_from_slice(&body);
101    }
102    out
103}
104
105#[cfg(test)]
106mod tests {
107    use super::*;
108    use crate::wire::{VIDEO_HEADER_LEN, VIDEO_KEY};
109
110    #[test]
111    fn frames_and_controls_parse_and_garbage_does_not() {
112        let mut msg = vec![KIND_FRAME];
113        let mut hdr = [0u8; VIDEO_HEADER_LEN];
114        VideoHeader { seq: 3, ts_us: 9, flags: VIDEO_KEY, codec: VideoCodec::H264 }.write(&mut hdr);
115        msg.extend_from_slice(&hdr);
116        msg.extend_from_slice(&[7, 7, 7]);
117        match parse(&msg) {
118            Some(LinkMsg::Frame { header, frame }) => {
119                assert_eq!(header.seq, 3);
120                assert_eq!(frame.len(), VIDEO_HEADER_LEN + 3);
121            }
122            _ => panic!("frame expected"),
123        }
124        let c = control(&ToLink::Rate { kind: VideoKind::Camera, kbps: 800, width: 640, height: 360, fps: 30 });
125        assert_eq!(c[0], KIND_CONTROL);
126        let mut lost = vec![KIND_CONTROL];
127        lost.extend_from_slice(br#"{"t":"lost","kind":"screen"}"#);
128        assert!(matches!(parse(&lost), Some(LinkMsg::Control(FromLink::Lost { kind: VideoKind::Screen }))));
129        let mut odd = vec![KIND_CONTROL];
130        odd.extend_from_slice(br#"{"t":"teleport"}"#);
131        assert!(matches!(parse(&odd), Some(LinkMsg::Control(FromLink::Unknown))));
132        assert!(parse(&[9, 9]).is_none());
133        assert!(parse(&[]).is_none());
134        let mut pcm = vec![KIND_PCM];
135        pcm.extend_from_slice(&48_000u32.to_le_bytes());
136        pcm.push(2);
137        pcm.extend_from_slice(&1.0f32.to_le_bytes());
138        match parse(&pcm) {
139            Some(LinkMsg::Pcm { rate, channels, samples }) => assert_eq!((rate, channels, samples.len()), (48_000, 2, 4)),
140            _ => panic!("pcm expected"),
141        }
142        pcm[5] = 7;
143        assert!(parse(&pcm).is_none(), "an impossible channel count is refused");
144    }
145}