use std::collections::HashMap;
use std::time::{Duration, Instant};
use rich_art::picture::{ByteBudget, Pacer, Picture, Pixels, over_ssh};
use rich_embed::ScreencastFrame;
use crate::engine::{NodeId, Rect};
pub const MAX_FPS: u32 = 30;
pub const HEADROOM: f64 = 0.1;
pub const MIN_INTERVAL: Duration = Duration::from_millis(1000 / MAX_FPS as u64);
#[derive(Clone, Debug, PartialEq)]
pub struct VideoFrame {
pub node: NodeId,
pub pixels: rich_embed::Pixels,
pub seq: u64,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Region {
pub node: NodeId,
pub rect: Rect,
pub playing: bool,
pub on_screen: bool,
pub interval: Option<Duration>,
}
impl Region {
pub fn moving(&self) -> bool {
self.playing && self.on_screen && !self.rect.is_empty()
}
}
#[derive(Debug, Default)]
pub struct Frames {
pub on: bool,
last: Vec<Last>,
seq: u64,
}
#[derive(Clone, Copy, Debug)]
struct Last {
node: NodeId,
slot: Instant,
taken: Instant,
every: Duration,
}
fn every(region: &Region) -> Duration {
region.interval.unwrap_or(MIN_INTERVAL).max(MIN_INTERVAL)
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Plan {
pub screencast: Option<bool>,
pub stopped: Vec<NodeId>,
pub due: bool,
}
impl Frames {
pub fn plan(&mut self, regions: &[Region], now: Instant) -> Plan {
let moving: Vec<&Region> = regions.iter().filter(|r| r.moving()).collect();
let mut plan = Plan::default();
let want = !moving.is_empty();
if want != self.on {
self.on = want;
plan.screencast = Some(want);
}
self.last.retain(|last| {
let keep = moving.iter().any(|r| r.node == last.node);
if !keep {
plan.stopped.push(last.node);
}
keep
});
plan.due = moving.iter().any(|r| self.due(r, now));
plan
}
fn due(&self, region: &Region, now: Instant) -> bool {
self.wait(region, now).is_zero()
}
fn wait(&self, region: &Region, now: Instant) -> Duration {
let every = every(region);
let Some(last) = self.last.iter().find(|l| l.node == region.node) else {
return Duration::ZERO;
};
let slot = (last.slot + every - every / 8).max(last.taken + every / 2);
slot.saturating_duration_since(now)
}
pub fn next_due(&self, regions: &[Region], now: Instant) -> Option<Duration> {
regions
.iter()
.filter(|r| r.moving())
.map(|r| self.wait(r, now))
.min()
}
pub fn cut(
&mut self,
regions: &[Region],
frame: Option<&ScreencastFrame>,
now: Instant,
) -> (Vec<VideoFrame>, Vec<Region>) {
let mut cut = Vec::new();
let mut missing = Vec::new();
for region in regions.iter().filter(|r| r.moving()) {
if !self.due(region, now) {
continue;
}
let r = region.rect;
match frame.and_then(|f| f.crop(r.x, r.y, r.width, r.height)) {
Some(pixels) if whole(&pixels, frame, r) => {
cut.push(self.stamp_every(region.node, every(region), pixels, now));
}
_ => missing.push(*region),
}
}
(cut, missing)
}
pub fn stamp(&mut self, node: NodeId, pixels: rich_embed::Pixels, now: Instant) -> VideoFrame {
let every = self
.last
.iter()
.find(|l| l.node == node)
.map_or(MIN_INTERVAL, |l| l.every);
self.stamp_every(node, every, pixels, now)
}
fn stamp_every(
&mut self,
node: NodeId,
every: Duration,
pixels: rich_embed::Pixels,
now: Instant,
) -> VideoFrame {
self.seq += 1;
let seq = self.seq;
let at = |slot: Option<Instant>| match slot.map(|s| s + every) {
Some(next) if now + every / 4 >= next && now < next + every => next,
_ => now,
};
let entry = |slot| Last {
node,
slot,
taken: now,
every,
};
match self.last.iter_mut().find(|l| l.node == node) {
Some(last) => *last = entry(at(Some(last.slot))),
None => self.last.push(entry(at(None))),
}
VideoFrame { node, pixels, seq }
}
}
pub struct Pacing {
pub budget: ByteBudget,
pub start: Pixels,
pub others: usize,
regions: HashMap<NodeId, Paced>,
}
struct Paced {
pacer: Pacer,
picture: Picture,
pending: Option<(Instant, u64)>,
moving: bool,
seq: u64,
}
pub type ByteCounter = fn(u32) -> u64;
impl Pacing {
pub fn new(budget: ByteBudget, start: Pixels) -> Pacing {
Pacing {
budget,
start,
others: 0,
regions: HashMap::new(),
}
}
pub fn start_for(asked: Option<Pixels>, detected: Pixels) -> Pixels {
match asked {
Some(pixels) => pixels,
None if over_ssh() => Pixels::Cells,
None => detected,
}
}
fn share(&self) -> f64 {
1.0 / (self.moving() + self.others).max(1) as f64
}
pub fn moving(&self) -> usize {
self.regions.values().filter(|r| r.moving).count()
}
fn paced_share(&self) -> f64 {
self.share() * (1.0 - HEADROOM)
}
pub fn offer(
&mut self,
frame: &VideoFrame,
cols: usize,
rows: usize,
now: Instant,
counted: ByteCounter,
) -> Option<Picture> {
let (w, h) = (frame.pixels.width, frame.pixels.height);
let rgba = frame.pixels.to_rgba();
if !self.regions.contains_key(&frame.node) {
let picture = Picture::from_rgba(w, h, rgba.clone())?;
let pacer = Pacer::new(self.start, cols, rows);
self.regions.insert(
frame.node,
Paced {
pacer,
picture,
pending: None,
moving: true,
seq: 0,
},
);
}
let share = {
let region = self.regions.get_mut(&frame.node)?;
region.moving = true;
region.pacer.resize(cols, rows);
if frame.seq <= region.seq {
return None;
}
self.paced_share()
};
let budget = self.budget;
let region = self.regions.get_mut(&frame.node)?;
if let Some((at, before)) = region.pending.take() {
let spent = counted(region.picture.id()).saturating_sub(before);
region.pacer.sent(at, (spent > 0).then_some(spent));
}
while region.pacer.settle(&budget, share) {}
if region.seq > 0 && !region.pacer.due(now, &budget, share) {
return None;
}
region.picture = region.picture.next_rgba(w, h, rgba)?;
region.seq = frame.seq;
region.pending = Some((now, counted(region.picture.id())));
Some(region.picture.clone())
}
pub fn still(&mut self, node: NodeId) {
if let Some(region) = self.regions.get_mut(&node) {
region.moving = false;
region.pending = None;
}
}
pub fn clear(&mut self) {
self.regions.clear();
}
pub fn forget(&mut self, node: NodeId) {
self.regions.remove(&node);
}
pub fn has(&self, node: NodeId) -> bool {
self.regions.contains_key(&node)
}
pub fn pixels(&self, node: NodeId) -> Pixels {
self.regions
.get(&node)
.map_or(self.start, |r| r.pacer.pixels())
}
pub fn recent_bytes(&self, node: NodeId) -> u64 {
self.regions
.get(&node)
.map_or(0, |r| r.pacer.recent_bytes())
}
pub fn cap(&self) -> f64 {
self.budget.bytes_per_second as f64 * self.share()
}
pub fn intervals(&self) -> Vec<(NodeId, Duration)> {
let share = self.paced_share();
let mut out: Vec<(NodeId, Duration)> = self
.regions
.iter()
.filter(|(_, r)| r.moving)
.map(|(node, r)| {
let fps = r.pacer.fps(&self.budget, share).max(0.1);
(*node, Duration::from_secs_f64(1.0 / fps).max(MIN_INTERVAL))
})
.collect();
out.sort_by_key(|(node, _)| *node);
out
}
}
#[derive(Clone, Debug)]
pub struct Job {
pub node: NodeId,
pub picture: Picture,
pub pixels: Pixels,
pub cols: usize,
pub rows: usize,
}
pub struct Encoder {
queue: std::sync::Arc<(std::sync::Mutex<Queue>, std::sync::Condvar)>,
}
#[derive(Default)]
struct Queue {
waiting: Vec<Job>,
closed: bool,
}
impl Encoder {
pub fn spawn(done: impl Fn(Job) + Send + 'static) -> Encoder {
let queue = std::sync::Arc::new((
std::sync::Mutex::new(Queue::default()),
std::sync::Condvar::new(),
));
let shared = queue.clone();
std::thread::spawn(move || {
loop {
let job = {
let (lock, ready) = &*shared;
let mut queue = lock.lock().unwrap_or_else(|e| e.into_inner());
while queue.waiting.is_empty() && !queue.closed {
queue = ready.wait(queue).unwrap_or_else(|e| e.into_inner());
}
if queue.closed {
return;
}
queue.waiting.remove(0)
};
job.picture
.prepare(crate::pictures::view(job.pixels), job.cols, job.rows);
done(job);
}
});
Encoder { queue }
}
pub fn encode(&self, job: Job) {
let (lock, ready) = &*self.queue;
let mut queue = lock.lock().unwrap_or_else(|e| e.into_inner());
queue.waiting.retain(|waiting| waiting.node != job.node);
queue.waiting.push(job);
ready.notify_one();
}
}
impl Drop for Encoder {
fn drop(&mut self) {
let (lock, ready) = &*self.queue;
lock.lock().unwrap_or_else(|e| e.into_inner()).closed = true;
ready.notify_one();
}
}
impl std::fmt::Debug for Encoder {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Encoder").finish_non_exhaustive()
}
}
fn whole(pixels: &rich_embed::Pixels, frame: Option<&ScreencastFrame>, rect: Rect) -> bool {
let Some(frame) = frame else {
return false;
};
let device = if frame.meta.device_width > 0.0 {
frame.meta.device_width
} else {
frame.pixels.width as f64
};
let scale = frame.pixels.width as f64 / device * frame.meta.page_scale.max(f64::MIN_POSITIVE);
(pixels.width as f64 + 1.5) >= rect.width * scale
&& (pixels.height as f64 + 1.5) >= rect.height * scale
}
#[cfg(test)]
mod tests {
use super::*;
use rich_embed::{FrameMeta, Pixels as Rgb};
fn frame(scroll_y: f64) -> ScreencastFrame {
ScreencastFrame {
pixels: Rgb::new(100, 100, vec![7; 100 * 100 * 3]).unwrap(),
meta: FrameMeta {
scroll_y,
page_scale: 1.0,
device_width: 100.0,
device_height: 100.0,
..FrameMeta::default()
},
}
}
fn region(node: NodeId, y: f64, playing: bool, on_screen: bool) -> Region {
Region {
node,
rect: Rect::new(10.0, y, 40.0, 20.0),
playing,
on_screen,
interval: None,
}
}
#[test]
fn the_screencast_runs_only_while_a_region_plays_on_screen() {
let mut frames = Frames::default();
let now = Instant::now();
let paused = [region(1, 10.0, false, true)];
assert_eq!(frames.plan(&paused, now).screencast, None);
let off_screen = [region(1, 10.0, true, false)];
assert_eq!(frames.plan(&off_screen, now).screencast, None);
let playing = [region(1, 10.0, true, true)];
let plan = frames.plan(&playing, now);
assert_eq!(plan.screencast, Some(true));
assert!(plan.due);
assert_eq!(frames.plan(&playing, now).screencast, None);
frames.cut(&playing, Some(&frame(0.0)), now);
let plan = frames.plan(&paused, now);
assert_eq!(plan.screencast, Some(false));
assert_eq!(plan.stopped, vec![1]);
}
#[test]
fn regions_are_cut_at_most_thirty_times_a_second() {
let mut frames = Frames::default();
let start = Instant::now();
let playing = [region(1, 10.0, true, true)];
let mut count = 0;
for tick in 0..400 {
let now = start + Duration::from_millis(tick * 5);
let (cut, missing) = frames.cut(&playing, Some(&frame(0.0)), now);
assert!(missing.is_empty());
count += cut.len();
}
assert!((59..=61).contains(&count), "{count}");
let mut frames = Frames::default();
let slow = [Region {
interval: Some(Duration::from_millis(500)),
..playing[0]
}];
let count: usize = (0..190)
.map(|tick| {
let now = start + Duration::from_millis(tick * 10);
frames.cut(&slow, Some(&frame(0.0)), now).0.len()
})
.sum();
assert_eq!(count, 4);
}
#[test]
fn the_next_frame_is_due_on_its_slot() {
let mut frames = Frames::default();
let start = Instant::now();
let playing = [region(1, 10.0, true, true)];
let paused = [region(1, 10.0, false, true)];
assert_eq!(frames.next_due(&paused, start), None);
assert_eq!(frames.next_due(&playing, start), Some(Duration::ZERO));
frames.cut(&playing, Some(&frame(0.0)), start);
let wait = frames.next_due(&playing, start).unwrap();
assert!(
wait <= MIN_INTERVAL && wait >= MIN_INTERVAL * 7 / 8,
"{wait:?}"
);
let late = start + MIN_INTERVAL + Duration::from_millis(3);
assert_eq!(frames.cut(&playing, Some(&frame(0.0)), late).0.len(), 1);
let wait = frames.next_due(&playing, late).unwrap();
assert!(wait < MIN_INTERVAL - Duration::from_millis(2), "{wait:?}");
}
#[test]
fn a_region_the_frame_does_not_hold_is_captured_on_its_own() {
let mut frames = Frames::default();
let now = Instant::now();
let regions = [region(1, 10.0, true, true), region(2, 500.0, true, true)];
let (cut, missing) = frames.cut(®ions, Some(&frame(0.0)), now);
assert_eq!(cut.len(), 1);
assert_eq!((cut[0].node, cut[0].seq), (1, 1));
assert_eq!((cut[0].pixels.width, cut[0].pixels.height), (40, 20));
assert_eq!(missing.len(), 1);
assert_eq!(missing[0].node, 2);
let mut frames = Frames::default();
let (cut, missing) = frames.cut(®ions[..1], Some(&frame(20.0)), now);
assert!(cut.is_empty());
assert_eq!(missing.len(), 1);
}
#[test]
fn the_encoder_makes_the_escape_ahead_and_keeps_the_newest_waiting() {
let (send, done) = std::sync::mpsc::channel();
let encoder = Encoder::spawn(move |job| drop(send.send(job)));
let first = Picture::from_rgba(8, 8, vec![9; 8 * 8 * 4]).unwrap();
let job = |picture: &Picture| Job {
node: 1,
picture: picture.clone(),
pixels: Pixels::Sixel,
cols: 4,
rows: 2,
};
encoder.encode(job(&first));
let made = done.recv_timeout(Duration::from_secs(10)).unwrap();
assert_eq!(made.picture, first);
let tag = rich_art::picture::PictureTag {
id: first.id(),
version: first.version(),
cols: 4,
rows: 2,
row: 0,
pixels: Pixels::Sixel,
fit: Some(rich_art::ImageFit::Cover),
anchor: rich_art::ImageAnchor::Center,
};
assert!(first.encode_tag(&tag, 0, 2).is_some());
let frames: Vec<Picture> = (0..20u8)
.scan(first.clone(), |last, k| {
*last = last.next_rgba(8, 8, vec![k; 8 * 8 * 4])?;
Some(last.clone())
})
.collect();
for frame in &frames {
encoder.encode(job(frame));
}
let mut last = None;
while let Ok(job) = done.recv_timeout(Duration::from_millis(500)) {
last = Some(job.picture);
}
assert_eq!(last.as_ref(), frames.last());
}
#[test]
fn a_region_that_moves_again_goes_on_numbering() {
let mut frames = Frames::default();
let now = Instant::now();
let playing = [region(1, 10.0, true, true)];
let paused = [region(1, 10.0, false, true)];
let mut last = 0;
for _ in 0..3 {
frames.plan(&playing, now);
let (cut, _) = frames.cut(&playing, Some(&frame(0.0)), now);
last = cut[0].seq;
frames.plan(&paused, now);
}
frames.plan(&playing, now);
let (cut, _) = frames.cut(&playing, Some(&frame(0.0)), now);
assert!(cut[0].seq > last, "{} after {last}", cut[0].seq);
}
}