use std::collections::HashMap;
use std::fmt::Write as _;
use std::ops::Range;
use std::sync::Arc;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use image::DynamicImage;
use super::screen::TerminalCellSize;
const MAX_TRANSMIT_BYTES: usize = 32 * 1024 * 1024;
const MAX_APC_BYTES: usize = MAX_TRANSMIT_BYTES;
const DEFAULT_IMAGE_BUDGET_BYTES: usize = 96 * 1024 * 1024;
const MAX_PLACEMENTS: usize = 256;
const MAX_IMAGE_DIMENSION: u32 = 16384;
const FIRST_AUTO_ID: u32 = 1 << 24;
#[derive(Clone)]
pub struct TerminalImage {
pixels: Arc<DynamicImage>,
source_hash: u64,
}
impl TerminalImage {
pub fn width(&self) -> u32 {
self.pixels.width()
}
pub fn height(&self) -> u32 {
self.pixels.height()
}
pub fn source_hash(&self) -> u64 {
self.source_hash
}
pub(crate) fn pixels(&self) -> &Arc<DynamicImage> {
&self.pixels
}
}
impl PartialEq for TerminalImage {
fn eq(&self, other: &Self) -> bool {
self.source_hash == other.source_hash
}
}
impl std::fmt::Debug for TerminalImage {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TerminalImage")
.field("width", &self.width())
.field("height", &self.height())
.field("source_hash", &self.source_hash)
.finish()
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct TerminalImagePlacement {
pub image_id: u32,
pub image: TerminalImage,
pub row: i32,
pub col: i32,
pub rows: u16,
pub cols: u16,
pub z: i32,
pub source_crop: Option<TerminalImageCrop>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TerminalImageCrop {
pub x: u32,
pub y: u32,
pub width: u32,
pub height: u32,
}
#[derive(Debug)]
pub(super) enum GraphicsSegment {
Text(Range<usize>),
HeldEscape,
Command(Box<GraphicsCommand>),
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
enum ScanState {
#[default]
Ground,
Escape,
Apc,
ApcEscape,
}
#[derive(Debug, Default)]
pub(super) struct GraphicsScanner {
state: ScanState,
apc: Vec<u8>,
overflowed: bool,
}
impl GraphicsScanner {
pub(super) fn is_plain(&self, bytes: &[u8]) -> bool {
self.state == ScanState::Ground
&& bytes.last() != Some(&0x1b)
&& !bytes.windows(2).any(|pair| pair == b"\x1b_")
}
pub(super) fn scan(&mut self, bytes: &[u8]) -> Vec<GraphicsSegment> {
let mut out = Vec::new();
let mut text_start = 0usize;
let mut idx = 0usize;
while idx < bytes.len() {
let byte = bytes[idx];
match self.state {
ScanState::Ground => {
if byte == 0x1b {
if text_start < idx {
out.push(GraphicsSegment::Text(text_start..idx));
}
text_start = idx;
self.state = ScanState::Escape;
}
idx += 1;
}
ScanState::Escape => {
if byte == b'_' {
self.state = ScanState::Apc;
self.apc.clear();
self.overflowed = false;
idx += 1;
text_start = idx;
} else {
self.state = ScanState::Ground;
if text_start == idx {
out.push(GraphicsSegment::HeldEscape);
}
}
}
ScanState::Apc => {
match byte {
0x1b => self.state = ScanState::ApcEscape,
0x07 => {
self.finish_apc(&mut out);
self.state = ScanState::Ground;
}
0x18 | 0x1a => {
self.apc.clear();
self.overflowed = false;
self.state = ScanState::Ground;
}
_ => self.push_apc(byte),
}
idx += 1;
text_start = idx;
}
ScanState::ApcEscape => {
if byte == b'\\' {
self.finish_apc(&mut out);
self.state = ScanState::Ground;
idx += 1;
text_start = idx;
} else {
self.apc.clear();
self.overflowed = false;
self.state = ScanState::Ground;
text_start = idx;
}
}
}
}
if self.state == ScanState::Ground && text_start < bytes.len() {
out.push(GraphicsSegment::Text(text_start..bytes.len()));
}
out
}
fn push_apc(&mut self, byte: u8) {
if self.overflowed {
return;
}
if self.apc.len() >= MAX_APC_BYTES {
self.apc.clear();
self.overflowed = true;
return;
}
self.apc.push(byte);
}
fn finish_apc(&mut self, out: &mut Vec<GraphicsSegment>) {
let body = std::mem::take(&mut self.apc);
let overflowed = std::mem::take(&mut self.overflowed);
if overflowed {
return;
}
let Some(rest) = body.strip_prefix(b"G") else {
return;
};
if let Some(command) = GraphicsCommand::parse(rest) {
out.push(GraphicsSegment::Command(Box::new(command)));
}
}
pub(super) fn reset(&mut self) {
self.state = ScanState::Ground;
self.apc.clear();
self.overflowed = false;
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
enum GraphicsAction {
#[default]
Transmit,
TransmitAndDisplay,
Display,
Delete,
Query,
Animate,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
enum GraphicsMedium {
#[default]
Direct,
OutOfBand,
}
#[derive(Clone, Debug)]
pub(super) struct GraphicsCommand {
action: GraphicsAction,
medium: GraphicsMedium,
format: u32,
width: u32,
height: u32,
id: u32,
number: u32,
placement: u32,
more: bool,
compressed: bool,
src_x: u32,
src_y: u32,
src_w: u32,
src_h: u32,
cols: u32,
rows: u32,
z: i32,
no_cursor_move: bool,
virtual_placement: bool,
delete: u8,
quiet: u32,
payload: Vec<u8>,
}
impl Default for GraphicsCommand {
fn default() -> Self {
Self {
action: GraphicsAction::default(),
medium: GraphicsMedium::default(),
format: 32,
width: 0,
height: 0,
id: 0,
number: 0,
placement: 0,
more: false,
compressed: false,
src_x: 0,
src_y: 0,
src_w: 0,
src_h: 0,
cols: 0,
rows: 0,
z: 0,
no_cursor_move: false,
virtual_placement: false,
delete: b'a',
quiet: 0,
payload: Vec::new(),
}
}
}
impl GraphicsCommand {
fn parse(body: &[u8]) -> Option<Self> {
let (control, payload) = match body.iter().position(|byte| *byte == b';') {
Some(at) => (&body[..at], &body[at + 1..]),
None => (body, &body[body.len()..]),
};
let mut command = Self::default();
for pair in control.split(|byte| *byte == b',') {
let mut halves = pair.splitn(2, |byte| *byte == b'=');
let ([key], Some(value)) = (halves.next()?, halves.next()) else {
continue;
};
command.apply_key(*key, value);
}
command.payload = BASE64.decode(payload).ok()?;
Some(command)
}
fn apply_key(&mut self, key: u8, value: &[u8]) {
let text = std::str::from_utf8(value).unwrap_or("");
let first = value.first().copied().unwrap_or(0);
match key {
b'a' => {
self.action = match first {
b'T' => GraphicsAction::TransmitAndDisplay,
b'p' => GraphicsAction::Display,
b'd' => GraphicsAction::Delete,
b'q' => GraphicsAction::Query,
b'a' | b'f' | b'c' => GraphicsAction::Animate,
_ => GraphicsAction::Transmit,
}
}
b't' => {
self.medium = match first {
b'f' | b't' | b's' => GraphicsMedium::OutOfBand,
_ => GraphicsMedium::Direct,
}
}
b'f' => self.format = text.parse().unwrap_or(32),
b's' => self.width = text.parse().unwrap_or(0),
b'v' => self.height = text.parse().unwrap_or(0),
b'i' => self.id = text.parse().unwrap_or(0),
b'I' => self.number = text.parse().unwrap_or(0),
b'p' => self.placement = text.parse().unwrap_or(0),
b'm' => self.more = text.parse().unwrap_or(0) == 1,
b'o' => self.compressed = first == b'z',
b'x' => self.src_x = text.parse().unwrap_or(0),
b'y' => self.src_y = text.parse().unwrap_or(0),
b'w' => self.src_w = text.parse().unwrap_or(0),
b'h' => self.src_h = text.parse().unwrap_or(0),
b'c' => self.cols = text.parse().unwrap_or(0),
b'r' => self.rows = text.parse().unwrap_or(0),
b'z' => self.z = text.parse().unwrap_or(0),
b'C' => self.no_cursor_move = text.parse().unwrap_or(0) == 1,
b'U' => self.virtual_placement = text.parse().unwrap_or(0) == 1,
b'd' => self.delete = first,
b'q' => self.quiet = text.parse().unwrap_or(0),
_ => {}
}
}
fn reports(&self, ok: bool) -> bool {
match self.quiet {
0 => true,
1 => !ok,
_ => false,
}
}
}
pub(super) const PLACEHOLDER: char = '\u{10EEEE}';
static ROWCOLUMN_DIACRITICS: [char; 297] = [
'\u{305}',
'\u{30d}',
'\u{30e}',
'\u{310}',
'\u{312}',
'\u{33d}',
'\u{33e}',
'\u{33f}',
'\u{346}',
'\u{34a}',
'\u{34b}',
'\u{34c}',
'\u{350}',
'\u{351}',
'\u{352}',
'\u{357}',
'\u{35b}',
'\u{363}',
'\u{364}',
'\u{365}',
'\u{366}',
'\u{367}',
'\u{368}',
'\u{369}',
'\u{36a}',
'\u{36b}',
'\u{36c}',
'\u{36d}',
'\u{36e}',
'\u{36f}',
'\u{483}',
'\u{484}',
'\u{485}',
'\u{486}',
'\u{487}',
'\u{592}',
'\u{593}',
'\u{594}',
'\u{595}',
'\u{597}',
'\u{598}',
'\u{599}',
'\u{59c}',
'\u{59d}',
'\u{59e}',
'\u{59f}',
'\u{5a0}',
'\u{5a1}',
'\u{5a8}',
'\u{5a9}',
'\u{5ab}',
'\u{5ac}',
'\u{5af}',
'\u{5c4}',
'\u{610}',
'\u{611}',
'\u{612}',
'\u{613}',
'\u{614}',
'\u{615}',
'\u{616}',
'\u{617}',
'\u{657}',
'\u{658}',
'\u{659}',
'\u{65a}',
'\u{65b}',
'\u{65d}',
'\u{65e}',
'\u{6d6}',
'\u{6d7}',
'\u{6d8}',
'\u{6d9}',
'\u{6da}',
'\u{6db}',
'\u{6dc}',
'\u{6df}',
'\u{6e0}',
'\u{6e1}',
'\u{6e2}',
'\u{6e4}',
'\u{6e7}',
'\u{6e8}',
'\u{6eb}',
'\u{6ec}',
'\u{730}',
'\u{732}',
'\u{733}',
'\u{735}',
'\u{736}',
'\u{73a}',
'\u{73d}',
'\u{73f}',
'\u{740}',
'\u{741}',
'\u{743}',
'\u{745}',
'\u{747}',
'\u{749}',
'\u{74a}',
'\u{7eb}',
'\u{7ec}',
'\u{7ed}',
'\u{7ee}',
'\u{7ef}',
'\u{7f0}',
'\u{7f1}',
'\u{7f3}',
'\u{816}',
'\u{817}',
'\u{818}',
'\u{819}',
'\u{81b}',
'\u{81c}',
'\u{81d}',
'\u{81e}',
'\u{81f}',
'\u{820}',
'\u{821}',
'\u{822}',
'\u{823}',
'\u{825}',
'\u{826}',
'\u{827}',
'\u{829}',
'\u{82a}',
'\u{82b}',
'\u{82c}',
'\u{82d}',
'\u{951}',
'\u{953}',
'\u{954}',
'\u{f82}',
'\u{f83}',
'\u{f86}',
'\u{f87}',
'\u{135d}',
'\u{135e}',
'\u{135f}',
'\u{17dd}',
'\u{193a}',
'\u{1a17}',
'\u{1a75}',
'\u{1a76}',
'\u{1a77}',
'\u{1a78}',
'\u{1a79}',
'\u{1a7a}',
'\u{1a7b}',
'\u{1a7c}',
'\u{1b6b}',
'\u{1b6d}',
'\u{1b6e}',
'\u{1b6f}',
'\u{1b70}',
'\u{1b71}',
'\u{1b72}',
'\u{1b73}',
'\u{1cd0}',
'\u{1cd1}',
'\u{1cd2}',
'\u{1cda}',
'\u{1cdb}',
'\u{1ce0}',
'\u{1dc0}',
'\u{1dc1}',
'\u{1dc3}',
'\u{1dc4}',
'\u{1dc5}',
'\u{1dc6}',
'\u{1dc7}',
'\u{1dc8}',
'\u{1dc9}',
'\u{1dcb}',
'\u{1dcc}',
'\u{1dd1}',
'\u{1dd2}',
'\u{1dd3}',
'\u{1dd4}',
'\u{1dd5}',
'\u{1dd6}',
'\u{1dd7}',
'\u{1dd8}',
'\u{1dd9}',
'\u{1dda}',
'\u{1ddb}',
'\u{1ddc}',
'\u{1ddd}',
'\u{1dde}',
'\u{1ddf}',
'\u{1de0}',
'\u{1de1}',
'\u{1de2}',
'\u{1de3}',
'\u{1de4}',
'\u{1de5}',
'\u{1de6}',
'\u{1dfe}',
'\u{20d0}',
'\u{20d1}',
'\u{20d4}',
'\u{20d5}',
'\u{20d6}',
'\u{20d7}',
'\u{20db}',
'\u{20dc}',
'\u{20e1}',
'\u{20e7}',
'\u{20e9}',
'\u{20f0}',
'\u{2cef}',
'\u{2cf0}',
'\u{2cf1}',
'\u{2de0}',
'\u{2de1}',
'\u{2de2}',
'\u{2de3}',
'\u{2de4}',
'\u{2de5}',
'\u{2de6}',
'\u{2de7}',
'\u{2de8}',
'\u{2de9}',
'\u{2dea}',
'\u{2deb}',
'\u{2dec}',
'\u{2ded}',
'\u{2dee}',
'\u{2def}',
'\u{2df0}',
'\u{2df1}',
'\u{2df2}',
'\u{2df3}',
'\u{2df4}',
'\u{2df5}',
'\u{2df6}',
'\u{2df7}',
'\u{2df8}',
'\u{2df9}',
'\u{2dfa}',
'\u{2dfb}',
'\u{2dfc}',
'\u{2dfd}',
'\u{2dfe}',
'\u{2dff}',
'\u{a66f}',
'\u{a67c}',
'\u{a67d}',
'\u{a6f0}',
'\u{a6f1}',
'\u{a8e0}',
'\u{a8e1}',
'\u{a8e2}',
'\u{a8e3}',
'\u{a8e4}',
'\u{a8e5}',
'\u{a8e6}',
'\u{a8e7}',
'\u{a8e8}',
'\u{a8e9}',
'\u{a8ea}',
'\u{a8eb}',
'\u{a8ec}',
'\u{a8ed}',
'\u{a8ee}',
'\u{a8ef}',
'\u{a8f0}',
'\u{a8f1}',
'\u{aab0}',
'\u{aab2}',
'\u{aab3}',
'\u{aab7}',
'\u{aab8}',
'\u{aabe}',
'\u{aabf}',
'\u{aac1}',
'\u{fe20}',
'\u{fe21}',
'\u{fe22}',
'\u{fe23}',
'\u{fe24}',
'\u{fe25}',
'\u{fe26}',
'\u{10a0f}',
'\u{10a38}',
'\u{1d185}',
'\u{1d186}',
'\u{1d187}',
'\u{1d188}',
'\u{1d189}',
'\u{1d1aa}',
'\u{1d1ab}',
'\u{1d1ac}',
'\u{1d1ad}',
'\u{1d242}',
'\u{1d243}',
'\u{1d244}',
];
#[cfg(test)]
pub(super) fn diacritic(index: u16) -> char {
ROWCOLUMN_DIACRITICS[usize::from(index).min(ROWCOLUMN_DIACRITICS.len() - 1)]
}
fn diacritic_value(mark: char) -> Option<u16> {
ROWCOLUMN_DIACRITICS
.binary_search(&mark)
.ok()
.map(|index| index as u16)
}
#[derive(Clone, Copy, Debug)]
pub(super) struct PlaceholderCell {
pub(super) row: u16,
pub(super) col: u16,
pub(super) id_low: u32,
pub(super) image_row: Option<u16>,
pub(super) image_col: Option<u16>,
pub(super) id_high: Option<u16>,
}
impl PlaceholderCell {
pub(super) fn new(row: u16, col: u16, id_low: u32, marks: &[char]) -> Self {
let mut values = marks.iter().filter_map(|mark| diacritic_value(*mark));
Self {
row,
col,
id_low,
image_row: values.next(),
image_col: values.next(),
id_high: values.next(),
}
}
}
#[derive(Clone, Copy, Debug)]
struct PlaceholderRun {
image_id: u32,
id_high: u16,
row: u16,
col: u16,
width: u16,
image_row: u16,
image_col: u16,
}
fn placeholder_runs(cells: &[PlaceholderCell]) -> Vec<PlaceholderRun> {
let mut runs: Vec<PlaceholderRun> = Vec::new();
let mut open: Option<PlaceholderRun> = None;
for cell in cells {
let adjacent =
open.is_some_and(|run| run.row == cell.row && run.col + run.width == cell.col);
let inherited_high = match (adjacent, open) {
(true, Some(run)) => run.id_high,
_ => 0,
};
let id_high = cell.id_high.unwrap_or(inherited_high);
let image_id = (u32::from(id_high) << 24) | (cell.id_low & 0x00ff_ffff);
let continues = adjacent
&& open.is_some_and(|run| {
run.image_id == image_id
&& cell.image_row.is_none_or(|value| value == run.image_row)
&& cell
.image_col
.is_none_or(|value| value == run.image_col + run.width)
});
if continues {
if let Some(run) = open.as_mut() {
run.width += 1;
}
continue;
}
if let Some(run) = open.take() {
runs.push(run);
}
open = Some(PlaceholderRun {
image_id,
id_high,
row: cell.row,
col: cell.col,
width: 1,
image_row: cell.image_row.unwrap_or(0),
image_col: cell.image_col.unwrap_or(0),
});
}
runs.extend(open);
runs
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct PlaceholderRect {
image_id: u32,
row: u16,
col: u16,
width: u16,
height: u16,
image_row: u16,
image_col: u16,
}
fn merge_placeholder_runs(runs: &[PlaceholderRun]) -> Vec<PlaceholderRect> {
let mut rects: Vec<PlaceholderRect> = Vec::new();
for run in runs {
let stackable = rects.iter_mut().find(|rect| {
rect.image_id == run.image_id
&& rect.col == run.col
&& rect.width == run.width
&& rect.image_col == run.image_col
&& rect.row + rect.height == run.row
&& rect.image_row + rect.height == run.image_row
});
if let Some(rect) = stackable {
rect.height += 1;
continue;
}
rects.push(PlaceholderRect {
image_id: run.image_id,
row: run.row,
col: run.col,
width: run.width,
height: 1,
image_row: run.image_row,
image_col: run.image_col,
});
}
rects
}
#[derive(Clone, Copy, Debug)]
pub(super) struct GraphicsContext {
pub(super) cursor_line: usize,
pub(super) cursor_col: u16,
pub(super) viewport_top_line: usize,
pub(super) alt_screen: bool,
pub(super) cell: TerminalCellSize,
pub(super) cols: u16,
}
#[derive(Debug, Default)]
pub(super) struct GraphicsOutcome {
pub(super) response: Option<Vec<u8>>,
pub(super) advance: Option<(u16, u16)>,
}
struct StoredImage {
image: TerminalImage,
bytes: usize,
used: u64,
}
#[derive(Clone, Debug)]
struct Placement {
image_id: u32,
placement_id: u32,
line: usize,
col: u16,
rows: u16,
cols: u16,
z: i32,
crop: Option<TerminalImageCrop>,
alt_screen: bool,
}
impl Placement {
fn covers_cell(&self, line: usize, col: u16) -> bool {
self.covers_line(line) && self.covers_column(col)
}
fn covers_line(&self, line: usize) -> bool {
line >= self.line && line < self.line.saturating_add(usize::from(self.rows))
}
fn covers_column(&self, col: u16) -> bool {
col >= self.col && col < self.col.saturating_add(self.cols)
}
}
struct PendingTransmit {
id: u32,
header: GraphicsCommand,
data: Vec<u8>,
}
pub(super) struct TerminalGraphics {
images: HashMap<u32, StoredImage>,
numbers: HashMap<u32, u32>,
placements: Vec<Placement>,
pending: Option<PendingTransmit>,
next_auto_id: u32,
budget: usize,
used_bytes: usize,
clock: u64,
}
impl Default for TerminalGraphics {
fn default() -> Self {
Self {
images: HashMap::new(),
numbers: HashMap::new(),
placements: Vec::new(),
pending: None,
next_auto_id: FIRST_AUTO_ID,
budget: DEFAULT_IMAGE_BUDGET_BYTES,
used_bytes: 0,
clock: 0,
}
}
}
impl TerminalGraphics {
pub(super) fn has_images(&self) -> bool {
!self.images.is_empty()
}
pub(super) fn set_budget(&mut self, bytes: usize) {
self.budget = bytes;
self.enforce_budget();
}
pub(super) fn reset(&mut self) {
self.images.clear();
self.numbers.clear();
self.placements.clear();
self.pending = None;
self.used_bytes = 0;
}
pub(super) fn clear_placements(&mut self) {
self.placements.clear();
}
pub(super) fn clear_alt_screen(&mut self) -> bool {
let before = self.placements.len();
self.placements.retain(|placement| !placement.alt_screen);
before != self.placements.len()
}
pub(super) fn drop_evicted(&mut self, evicted: usize) -> bool {
if evicted == 0 || self.placements.is_empty() {
return false;
}
self.placements
.retain(|placement| placement.line + usize::from(placement.rows) > evicted);
for placement in &mut self.placements {
placement.line = placement.line.saturating_sub(evicted);
}
true
}
pub(super) fn visible(
&self,
history_lines: usize,
display_offset: usize,
rows: u16,
alt_screen: bool,
) -> Vec<TerminalImagePlacement> {
let mut visible: Vec<_> = self
.placements
.iter()
.filter(|placement| placement.alt_screen == alt_screen)
.filter_map(|placement| {
let row = placement.line as i64 - history_lines as i64 + display_offset as i64;
if row + i64::from(placement.rows) <= 0 || row >= i64::from(rows) {
return None;
}
Some(TerminalImagePlacement {
image_id: placement.image_id,
image: self.images.get(&placement.image_id)?.image.clone(),
row: row.clamp(i32::MIN as i64, i32::MAX as i64) as i32,
col: i32::from(placement.col),
rows: placement.rows,
cols: placement.cols,
z: placement.z,
source_crop: placement.crop,
})
})
.collect();
visible.sort_by_key(|placement| placement.z);
visible
}
pub(super) fn placeholder_placements(
&self,
cells: &[PlaceholderCell],
cell: TerminalCellSize,
) -> Vec<TerminalImagePlacement> {
merge_placeholder_runs(&placeholder_runs(cells))
.into_iter()
.filter_map(|rect| {
let stored = self.images.get(&rect.image_id)?;
let (width, height) = (stored.image.width(), stored.image.height());
let x = u32::from(rect.image_col) * u32::from(cell.width);
let y = u32::from(rect.image_row) * u32::from(cell.height);
if x >= width || y >= height {
return None;
}
let crop = TerminalImageCrop {
x,
y,
width: (u32::from(rect.width) * u32::from(cell.width)).min(width - x),
height: (u32::from(rect.height) * u32::from(cell.height)).min(height - y),
};
Some(TerminalImagePlacement {
image_id: rect.image_id,
image: stored.image.clone(),
row: i32::from(rect.row),
col: i32::from(rect.col),
rows: rect.height,
cols: rect.width,
z: 0,
source_crop: Some(crop),
})
})
.collect()
}
pub(super) fn apply(
&mut self,
command: GraphicsCommand,
ctx: GraphicsContext,
) -> GraphicsOutcome {
self.clock = self.clock.wrapping_add(1);
match command.action {
GraphicsAction::Query => self.query(&command),
GraphicsAction::Delete => {
self.delete(&command, ctx);
GraphicsOutcome::default()
}
GraphicsAction::Display => self.display_stored(&command, ctx),
GraphicsAction::Transmit | GraphicsAction::TransmitAndDisplay => {
self.transmit(command, ctx)
}
GraphicsAction::Animate => GraphicsOutcome {
response: report(&command, command.id, Err("ENOTSUPP:animation")),
advance: None,
},
}
}
fn query(&mut self, command: &GraphicsCommand) -> GraphicsOutcome {
let result = match command.medium {
GraphicsMedium::OutOfBand => Err("ENOTSUPP:file transmission"),
GraphicsMedium::Direct => decode_payload(command, &command.payload).map(|_| ()),
};
GraphicsOutcome {
response: report(command, command.id, result),
advance: None,
}
}
fn transmit(&mut self, command: GraphicsCommand, ctx: GraphicsContext) -> GraphicsOutcome {
if command.medium == GraphicsMedium::OutOfBand {
self.pending = None;
return GraphicsOutcome {
response: report(&command, command.id, Err("ENOTSUPP:file transmission")),
advance: None,
};
}
if command.more || self.pending.is_some() {
return self.transmit_chunked(command, ctx);
}
let id = self.resolve_id(command.id, command.number);
let payload = command.payload.clone();
self.finish_transmit(id, &command, payload, ctx)
}
fn transmit_chunked(
&mut self,
command: GraphicsCommand,
ctx: GraphicsContext,
) -> GraphicsOutcome {
let mut pending = self.pending.take().unwrap_or_else(|| PendingTransmit {
id: 0,
header: command.clone(),
data: Vec::new(),
});
if pending.id == 0 {
pending.id = self.resolve_id(pending.header.id, pending.header.number);
}
if pending.data.len().saturating_add(command.payload.len()) > MAX_TRANSMIT_BYTES {
return GraphicsOutcome {
response: report(&command, pending.id, Err("EFBIG:payload too large")),
advance: None,
};
}
pending.data.extend_from_slice(&command.payload);
if command.more {
self.pending = Some(pending);
return GraphicsOutcome::default();
}
self.finish_transmit(pending.id, &pending.header, pending.data, ctx)
}
fn finish_transmit(
&mut self,
id: u32,
command: &GraphicsCommand,
payload: Vec<u8>,
ctx: GraphicsContext,
) -> GraphicsOutcome {
let decoded = match decode_payload(command, &payload) {
Ok(image) => image,
Err(error) => {
return GraphicsOutcome {
response: report(command, id, Err(error)),
advance: None,
};
}
};
let bytes = decoded_bytes(&decoded);
let image = TerminalImage {
pixels: Arc::new(decoded),
source_hash: hash_payload(command.format, &payload),
};
self.insert_image(id, image, bytes);
if command.number != 0 {
self.numbers.insert(command.number, id);
}
GraphicsOutcome {
response: report(command, id, Ok(())),
advance: (command.action == GraphicsAction::TransmitAndDisplay)
.then(|| self.place(id, command, ctx))
.flatten(),
}
}
fn display_stored(
&mut self,
command: &GraphicsCommand,
ctx: GraphicsContext,
) -> GraphicsOutcome {
let id = match self.lookup(command.id, command.number) {
Some(id) => id,
None => {
return GraphicsOutcome {
response: report(command, command.id, Err("ENOENT:no such image")),
advance: None,
};
}
};
let advance = self.place(id, command, ctx);
GraphicsOutcome {
response: report(command, id, Ok(())),
advance,
}
}
fn place(
&mut self,
id: u32,
command: &GraphicsCommand,
ctx: GraphicsContext,
) -> Option<(u16, u16)> {
let clock = self.clock;
let (image_w, image_h) = {
let stored = self.images.get_mut(&id)?;
stored.used = clock;
(stored.image.width(), stored.image.height())
};
if command.virtual_placement {
return None;
}
if image_w == 0 || image_h == 0 {
return None;
}
let crop = source_crop(command, image_w, image_h);
let (src_w, src_h) = crop
.map(|crop| (crop.width, crop.height))
.unwrap_or((image_w, image_h));
let cols = match command.cols {
0 => src_w.div_ceil(u32::from(ctx.cell.width)),
cols => cols,
};
let rows = match command.rows {
0 => src_h.div_ceil(u32::from(ctx.cell.height)),
rows => rows,
};
let cols = cols.clamp(1, u32::from(ctx.cols.max(1))) as u16;
let rows = rows.clamp(1, u32::from(u16::MAX)) as u16;
self.placements.retain(|placement| {
placement.image_id != id || placement.placement_id != command.placement
});
self.placements.push(Placement {
image_id: id,
placement_id: command.placement,
line: ctx.cursor_line,
col: ctx.cursor_col,
rows,
cols,
z: command.z,
crop,
alt_screen: ctx.alt_screen,
});
while self.placements.len() > MAX_PLACEMENTS {
self.placements.remove(0);
}
(!command.no_cursor_move).then_some((rows, cols))
}
fn delete(&mut self, command: &GraphicsCommand, ctx: GraphicsContext) {
let free_data = command.delete.is_ascii_uppercase();
let selector = command.delete.to_ascii_lowercase();
let target_col = command.src_x.saturating_sub(1).min(u32::from(u16::MAX)) as u16;
let target_line = ctx
.viewport_top_line
.saturating_add(command.src_y.saturating_sub(1) as usize);
let hit: Box<dyn Fn(&Placement) -> bool> = match selector {
b'a' => Box::new(|_| true),
b'i' => {
let (id, placement) = (command.id, command.placement);
Box::new(move |item| {
item.image_id == id && (placement == 0 || item.placement_id == placement)
})
}
b'n' => {
let id = self.numbers.get(&command.number).copied().unwrap_or(0);
Box::new(move |item| id != 0 && item.image_id == id)
}
b'c' => {
let (line, col) = (ctx.cursor_line, ctx.cursor_col);
Box::new(move |item| item.covers_cell(line, col))
}
b'z' => {
let z = command.z;
Box::new(move |item| item.z == z)
}
b'p' => Box::new(move |item| item.covers_cell(target_line, target_col)),
b'x' => Box::new(move |item| item.covers_column(target_col)),
b'y' => Box::new(move |item| item.covers_line(target_line)),
_ => return,
};
let mut freed: Vec<u32> = Vec::new();
self.placements.retain(|item| {
if !hit(item) {
return true;
}
if free_data {
freed.push(item.image_id);
}
false
});
if free_data {
match selector {
b'a' => {
let ids: Vec<u32> = self.images.keys().copied().collect();
for id in ids {
self.remove_image(id);
}
}
b'i' if command.placement == 0 => self.remove_image(command.id),
b'n' => {
if let Some(id) = self.numbers.get(&command.number).copied() {
self.remove_image(id);
}
}
_ => {
for id in freed {
self.remove_image(id);
}
}
}
}
}
fn insert_image(&mut self, id: u32, image: TerminalImage, bytes: usize) {
self.remove_image(id);
let clock = self.clock;
self.images.insert(
id,
StoredImage {
image,
bytes,
used: clock,
},
);
self.used_bytes = self.used_bytes.saturating_add(bytes);
self.enforce_budget();
}
fn remove_image(&mut self, id: u32) {
if let Some(stored) = self.images.remove(&id) {
self.used_bytes = self.used_bytes.saturating_sub(stored.bytes);
}
self.numbers.retain(|_, mapped| *mapped != id);
self.placements.retain(|placement| placement.image_id != id);
}
fn enforce_budget(&mut self) {
while self.used_bytes > self.budget && self.images.len() > 1 {
let victim = self
.images
.iter()
.min_by_key(|(_, stored)| (stored.used, stored.bytes))
.map(|(id, _)| *id);
let Some(victim) = victim else { break };
self.remove_image(victim);
}
}
fn lookup(&self, id: u32, number: u32) -> Option<u32> {
if id != 0 {
return self.images.contains_key(&id).then_some(id);
}
let mapped = *self.numbers.get(&number)?;
self.images.contains_key(&mapped).then_some(mapped)
}
fn resolve_id(&mut self, id: u32, number: u32) -> u32 {
if id != 0 {
return id;
}
if number != 0
&& let Some(existing) = self.numbers.get(&number).copied()
{
return existing;
}
let assigned = self.next_auto_id;
self.next_auto_id = self.next_auto_id.checked_add(1).unwrap_or(FIRST_AUTO_ID);
assigned
}
}
fn source_crop(command: &GraphicsCommand, width: u32, height: u32) -> Option<TerminalImageCrop> {
if command.src_x == 0 && command.src_y == 0 && command.src_w == 0 && command.src_h == 0 {
return None;
}
let x = command.src_x.min(width.saturating_sub(1));
let y = command.src_y.min(height.saturating_sub(1));
let w = match command.src_w {
0 => width - x,
requested => requested.min(width - x),
};
let h = match command.src_h {
0 => height - y,
requested => requested.min(height - y),
};
(w > 0 && h > 0).then_some(TerminalImageCrop {
x,
y,
width: w,
height: h,
})
}
fn decode_payload(command: &GraphicsCommand, payload: &[u8]) -> Result<DynamicImage, &'static str> {
let mut data = if command.compressed {
decompress(payload).ok_or("EINVAL:bad zlib payload")?
} else {
payload.to_vec()
};
match command.format {
100 => decode_png(&data),
format @ (24 | 32) => {
let channels = if format == 24 { 3usize } else { 4usize };
let (width, height) = (command.width, command.height);
if width == 0 || height == 0 {
return Err("EINVAL:missing s/v for raw pixels");
}
if width > MAX_IMAGE_DIMENSION || height > MAX_IMAGE_DIMENSION {
return Err("EFBIG:image too large");
}
let expected = (width as usize)
.checked_mul(height as usize)
.and_then(|pixels| pixels.checked_mul(channels))
.ok_or("EFBIG:image too large")?;
if data.len() < expected {
return Err("EINVAL:truncated pixel payload");
}
data.truncate(expected);
if channels == 3 {
image::RgbImage::from_raw(width, height, data).map(DynamicImage::ImageRgb8)
} else {
image::RgbaImage::from_raw(width, height, data).map(DynamicImage::ImageRgba8)
}
.ok_or("EINVAL:bad pixel payload")
}
_ => Err("ENOTSUPP:unsupported format"),
}
}
fn decode_png(data: &[u8]) -> Result<DynamicImage, &'static str> {
let mut reader =
image::ImageReader::with_format(std::io::Cursor::new(data), image::ImageFormat::Png);
let mut limits = image::Limits::default();
limits.max_image_width = Some(MAX_IMAGE_DIMENSION);
limits.max_image_height = Some(MAX_IMAGE_DIMENSION);
limits.max_alloc = Some(MAX_TRANSMIT_BYTES as u64);
reader.limits(limits);
reader.decode().map_err(|_| "EINVAL:bad PNG payload")
}
fn decompress(payload: &[u8]) -> Option<Vec<u8>> {
use std::io::Read as _;
let mut out = Vec::new();
flate2::read::ZlibDecoder::new(payload)
.take(MAX_TRANSMIT_BYTES as u64)
.read_to_end(&mut out)
.ok()?;
Some(out)
}
fn decoded_bytes(image: &DynamicImage) -> usize {
(image.width() as usize)
.saturating_mul(image.height() as usize)
.saturating_mul(4)
}
fn hash_payload(format: u32, payload: &[u8]) -> u64 {
use std::hash::{Hash as _, Hasher as _};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
format.hash(&mut hasher);
payload.hash(&mut hasher);
hasher.finish()
}
fn report(command: &GraphicsCommand, id: u32, result: Result<(), &str>) -> Option<Vec<u8>> {
if !command.reports(result.is_ok()) {
return None;
}
let mut response = format!("\x1b_Gi={id}");
if command.number != 0 {
let _ = write!(response, ",I={}", command.number);
}
if command.placement != 0 {
let _ = write!(response, ",p={}", command.placement);
}
let body = result.err().unwrap_or("OK");
let _ = write!(response, ";{body}\x1b\\");
Some(response.into_bytes())
}
#[cfg(test)]
mod tests {
use super::*;
fn rgb_command(keys: &str, width: u32, height: u32) -> Vec<u8> {
let pixels = vec![0xa0u8; (width * height * 3) as usize];
let payload = BASE64.encode(pixels);
format!("\x1b_Gf=24,s={width},v={height},t=d,{keys};{payload}\x1b\\").into_bytes()
}
fn context() -> GraphicsContext {
GraphicsContext {
cursor_line: 0,
cursor_col: 0,
viewport_top_line: 0,
alt_screen: false,
cell: TerminalCellSize::new(10, 20),
cols: 80,
}
}
fn scan_all(scanner: &mut GraphicsScanner, bytes: &[u8]) -> (Vec<u8>, Vec<GraphicsCommand>) {
let mut text = Vec::new();
let mut commands = Vec::new();
for segment in scanner.scan(bytes) {
match segment {
GraphicsSegment::Text(range) => text.extend_from_slice(&bytes[range]),
GraphicsSegment::HeldEscape => text.push(0x1b),
GraphicsSegment::Command(command) => commands.push(*command),
}
}
(text, commands)
}
#[test]
fn scanner_lifts_commands_out_of_surrounding_text() {
let mut scanner = GraphicsScanner::default();
let mut stream = b"before".to_vec();
stream.extend_from_slice(&rgb_command("a=T", 2, 2));
stream.extend_from_slice(b"after");
let (text, commands) = scan_all(&mut scanner, &stream);
assert_eq!(text, b"beforeafter");
assert_eq!(commands.len(), 1);
assert_eq!(commands[0].action, GraphicsAction::TransmitAndDisplay);
assert_eq!(commands[0].payload.len(), 2 * 2 * 3);
}
#[test]
fn scanner_survives_a_command_split_across_chunks() {
let command = rgb_command("a=T", 2, 2);
for split in 1..command.len() {
let mut scanner = GraphicsScanner::default();
let (head_text, head) = scan_all(&mut scanner, &command[..split]);
let (tail_text, tail) = scan_all(&mut scanner, &command[split..]);
assert!(
head_text.is_empty() && tail_text.is_empty(),
"split at {split} leaked graphics bytes into the grid stream"
);
assert_eq!(
head.len() + tail.len(),
1,
"split at {split} lost or duplicated the command"
);
}
}
#[test]
fn escape_that_is_not_a_command_reaches_the_grid() {
let mut scanner = GraphicsScanner::default();
assert!(!scanner.is_plain(b"red\x1b"));
let (first, _) = scan_all(&mut scanner, b"red\x1b");
let (second, commands) = scan_all(&mut scanner, b"[0m");
let mut text = first;
text.extend_from_slice(&second);
assert_eq!(text, b"red\x1b[0m");
assert!(commands.is_empty());
}
#[test]
fn non_graphics_apc_is_swallowed_like_the_vt_parser_would() {
let mut scanner = GraphicsScanner::default();
let (text, commands) = scan_all(&mut scanner, b"a\x1b_Xsomething\x1b\\b");
assert_eq!(text, b"ab");
assert!(commands.is_empty());
}
#[test]
fn transmit_and_display_places_the_image_and_moves_the_cursor() {
let mut graphics = TerminalGraphics::default();
let mut scanner = GraphicsScanner::default();
let (_, commands) = scan_all(&mut scanner, &rgb_command("a=T,i=7", 30, 40));
let outcome = graphics.apply(commands[0].clone(), context());
assert_eq!(outcome.advance, Some((2, 3)));
let visible = graphics.visible(0, 0, 24, false);
assert_eq!(visible.len(), 1);
assert_eq!((visible[0].row, visible[0].col), (0, 0));
assert_eq!((visible[0].rows, visible[0].cols), (2, 3));
}
#[test]
fn explicit_cell_size_overrides_the_pixel_size() {
let mut graphics = TerminalGraphics::default();
let mut scanner = GraphicsScanner::default();
let (_, commands) = scan_all(&mut scanner, &rgb_command("a=T,c=8,r=4", 30, 40));
let outcome = graphics.apply(commands[0].clone(), context());
assert_eq!(outcome.advance, Some((4, 8)));
}
#[test]
fn suppressed_cursor_movement_still_places() {
let mut graphics = TerminalGraphics::default();
let mut scanner = GraphicsScanner::default();
let (_, commands) = scan_all(&mut scanner, &rgb_command("a=T,C=1", 30, 40));
let outcome = graphics.apply(commands[0].clone(), context());
assert_eq!(outcome.advance, None);
assert_eq!(graphics.visible(0, 0, 24, false).len(), 1);
}
#[test]
fn a_probe_is_answered_without_storing_anything() {
let mut graphics = TerminalGraphics::default();
let mut scanner = GraphicsScanner::default();
let (_, commands) = scan_all(&mut scanner, &rgb_command("a=q,i=31", 1, 1));
let outcome = graphics.apply(commands[0].clone(), context());
assert_eq!(
outcome.response.as_deref(),
Some(b"\x1b_Gi=31;OK\x1b\\".as_ref())
);
assert!(graphics.visible(0, 0, 24, false).is_empty());
}
#[test]
fn out_of_band_transmission_is_refused_in_the_protocol_s_own_terms() {
let mut graphics = TerminalGraphics::default();
let mut scanner = GraphicsScanner::default();
let (_, commands) = scan_all(&mut scanner, b"\x1b_Ga=T,t=f,i=3;L3RtcC9pbWcucG5n\x1b\\");
let outcome = graphics.apply(commands[0].clone(), context());
let response = String::from_utf8(outcome.response.expect("a refusal is reported")).unwrap();
assert!(
response.contains("ENOTSUPP"),
"unexpected report: {response}"
);
}
#[test]
fn quiet_two_suppresses_even_failures() {
let mut graphics = TerminalGraphics::default();
let mut scanner = GraphicsScanner::default();
let (_, commands) = scan_all(&mut scanner, b"\x1b_Ga=T,t=f,q=2;Lw==\x1b\\");
assert!(
graphics
.apply(commands[0].clone(), context())
.response
.is_none()
);
}
#[test]
fn chunked_transmission_reassembles_before_decoding() {
let mut graphics = TerminalGraphics::default();
let pixels = vec![0x40u8; 30 * 40 * 3];
let encoded = BASE64.encode(&pixels);
let (head, tail) = encoded.split_at(encoded.len() / 2);
let mut scanner = GraphicsScanner::default();
let mut stream = format!("\x1b_Ga=T,f=24,s=30,v=40,t=d,i=9,m=1;{head}\x1b\\").into_bytes();
stream.extend_from_slice(format!("\x1b_Gm=0;{tail}\x1b\\").as_bytes());
let (_, commands) = scan_all(&mut scanner, &stream);
assert_eq!(commands.len(), 2);
assert!(
graphics
.apply(commands[0].clone(), context())
.advance
.is_none()
);
let outcome = graphics.apply(commands[1].clone(), context());
assert_eq!(outcome.advance, Some((2, 3)));
assert_eq!(graphics.visible(0, 0, 24, false).len(), 1);
}
#[test]
fn deleting_by_id_drops_the_placement() {
let mut graphics = TerminalGraphics::default();
let mut scanner = GraphicsScanner::default();
let (_, commands) = scan_all(&mut scanner, &rgb_command("a=T,i=4", 30, 40));
graphics.apply(commands[0].clone(), context());
let (_, deletes) = scan_all(&mut scanner, b"\x1b_Ga=d,d=i,i=4;\x1b\\");
graphics.apply(deletes[0].clone(), context());
assert!(graphics.visible(0, 0, 24, false).is_empty());
}
#[test]
fn evicted_scrollback_pulls_placements_up_and_then_off() {
let mut graphics = TerminalGraphics::default();
let mut scanner = GraphicsScanner::default();
let (_, commands) = scan_all(&mut scanner, &rgb_command("a=T", 30, 40));
let mut ctx = context();
ctx.cursor_line = 5;
graphics.apply(commands[0].clone(), ctx);
graphics.drop_evicted(3);
assert_eq!(graphics.visible(0, 0, 24, false)[0].row, 2);
graphics.drop_evicted(3);
assert_eq!(graphics.visible(0, 0, 24, false)[0].row, 0);
graphics.drop_evicted(4);
assert!(graphics.visible(0, 0, 24, false).is_empty());
}
#[test]
fn alt_screen_placements_are_kept_apart_from_the_primary_ones() {
let mut graphics = TerminalGraphics::default();
let mut scanner = GraphicsScanner::default();
let (_, commands) = scan_all(&mut scanner, &rgb_command("a=T,i=1", 30, 40));
graphics.apply(commands[0].clone(), context());
let (_, alt) = scan_all(&mut scanner, &rgb_command("a=T,i=2", 30, 40));
let mut alt_ctx = context();
alt_ctx.alt_screen = true;
graphics.apply(alt[0].clone(), alt_ctx);
assert_eq!(graphics.visible(0, 0, 24, true).len(), 1);
assert_eq!(graphics.visible(0, 0, 24, false).len(), 1);
graphics.clear_alt_screen();
assert!(graphics.visible(0, 0, 24, true).is_empty());
assert_eq!(graphics.visible(0, 0, 24, false).len(), 1);
}
#[test]
fn the_budget_evicts_least_recently_used_images() {
let mut graphics = TerminalGraphics::default();
graphics.set_budget(30 * 40 * 4);
let mut scanner = GraphicsScanner::default();
let (_, first) = scan_all(&mut scanner, &rgb_command("a=T,i=1", 30, 40));
graphics.apply(first[0].clone(), context());
let (_, second) = scan_all(&mut scanner, &rgb_command("a=T,i=2", 31, 40));
graphics.apply(second[0].clone(), context());
let visible = graphics.visible(0, 0, 24, false);
assert_eq!(visible.len(), 1, "the older image must have been evicted");
assert_eq!(visible[0].image.width(), 31);
}
#[test]
fn a_source_rectangle_is_carried_to_the_renderer() {
let mut graphics = TerminalGraphics::default();
let mut scanner = GraphicsScanner::default();
let (_, commands) = scan_all(&mut scanner, &rgb_command("a=T,x=5,y=6,w=10,h=12", 30, 40));
graphics.apply(commands[0].clone(), context());
let visible = graphics.visible(0, 0, 24, false);
assert_eq!(
visible[0].source_crop,
Some(TerminalImageCrop {
x: 5,
y: 6,
width: 10,
height: 12,
})
);
assert_eq!((visible[0].rows, visible[0].cols), (1, 1));
}
#[test]
fn a_large_unchunked_transmission_is_not_dropped() {
let mut graphics = TerminalGraphics::default();
let mut scanner = GraphicsScanner::default();
let (text, commands) = scan_all(&mut scanner, &rgb_command("a=T,i=1", 280, 160));
assert!(
text.is_empty(),
"the escape must not leak into the grid stream"
);
assert_eq!(
commands.len(),
1,
"a large single-escape transmit must survive scanning"
);
assert_eq!(
graphics.apply(commands[0].clone(), context()).advance,
Some((8, 28))
);
}
#[test]
fn a_truncated_raw_payload_is_reported_rather_than_drawn() {
let mut graphics = TerminalGraphics::default();
let mut scanner = GraphicsScanner::default();
let payload = BASE64.encode([1u8, 2, 3]);
let (_, commands) = scan_all(
&mut scanner,
format!("\x1b_Ga=T,f=24,s=30,v=40,t=d;{payload}\x1b\\").as_bytes(),
);
let outcome = graphics.apply(commands[0].clone(), context());
let response = String::from_utf8(outcome.response.expect("a refusal is reported")).unwrap();
assert!(response.contains("EINVAL"), "unexpected report: {response}");
assert!(graphics.visible(0, 0, 24, false).is_empty());
}
}