use core::pin::Pin;
use core::task::{Context, Poll, ready};
use std::collections::VecDeque;
use std::io;
use tokio::io::{AsyncRead, ReadBuf};
const SCRATCH: usize = 4096;
const STRING_LIMIT: usize = 4096;
const LINE_BREAKING_FINALS: &[u8] = b"ABCDEFGHJKXadef`";
const SCREEN_ROWS: usize = 256;
const SCREEN_ROW_BYTES: usize = SCRATCH;
#[derive(Clone, Copy, PartialEq, Eq)]
enum State {
Ground,
Escape,
EscapeIntermediate,
Csi,
Text {
bell_ends: bool,
seen: usize,
},
TextEscape,
}
pub(super) struct AnsiLines<R> {
inner: R,
state: State,
cr_pending: bool,
line_has_text: bool,
closed: Option<bool>,
row: Vec<u8>,
spilling: bool,
screen: VecDeque<Option<Vec<u8>>>,
replacing: usize,
blind: bool,
csi_param: u32,
csi_closed: bool,
carry: Vec<u8>,
ready: Vec<u8>,
sent: usize,
eof: bool,
}
impl<R> AnsiLines<R> {
pub(super) fn new(inner: R) -> Self {
Self {
inner,
state: State::Ground,
cr_pending: false,
line_has_text: false,
closed: None,
row: Vec::new(),
spilling: false,
screen: VecDeque::new(),
replacing: 0,
blind: false,
csi_param: 0,
csi_closed: false,
carry: Vec::new(),
ready: Vec::new(),
sent: 0,
eof: false,
}
}
fn break_line(&mut self, text: &mut Vec<u8>) {
if self.line_has_text {
let written = self.end_row(text, true);
self.closed = Some(written);
} else {
self.row.clear();
}
}
fn end_row(&mut self, text: &mut Vec<u8>, line_end: bool) -> bool {
let row = core::mem::take(&mut self.row);
let spilled = core::mem::replace(&mut self.spilling, false);
self.line_has_text = false;
let target = match self.replacing {
0 => None,
_ if self.blind => None,
n => Some(self.screen.len() - n),
};
let repeat = match target {
Some(i) if !spilled => self.screen[i].as_deref() == Some(row.as_slice()),
_ => false,
};
if !repeat {
text.extend_from_slice(&row);
if line_end {
text.push(b'\n');
}
}
let kept = (!spilled).then_some(row);
match (target, repeat) {
(Some(i), false) => self.screen[i] = kept,
(None, false) => {
if self.screen.len() == SCREEN_ROWS {
self.screen.pop_front();
}
self.screen.push_back(kept);
}
(_, true) => {}
}
self.replacing = self.replacing.saturating_sub(1);
if self.replacing == 0 {
self.blind = false;
}
!repeat
}
fn settle_cr(&mut self, text: &mut Vec<u8>) {
if self.cr_pending {
self.cr_pending = false;
self.break_line(text);
}
}
fn cursor_up(&mut self, rows: usize) {
let total = self.replacing.saturating_add(rows);
self.blind |= total > self.screen.len();
self.replacing = total.min(SCREEN_ROWS);
}
fn step(&mut self, byte: u8, text: &mut Vec<u8>) {
match self.state {
State::Ground => self.ground(byte, text),
State::Escape => match byte {
b'[' => {
self.state = State::Csi;
self.csi_param = 0;
self.csi_closed = false;
}
b']' => {
self.state = State::Text {
bell_ends: true,
seen: 0,
}
}
b'P' | b'X' | b'^' | b'_' => {
self.state = State::Text {
bell_ends: false,
seen: 0,
}
}
0x20..=0x2f => self.state = State::EscapeIntermediate,
0x1b => {}
0x30..=0x7e => self.state = State::Ground,
_ => {
self.state = State::Ground;
self.ground(byte, text);
}
},
State::EscapeIntermediate => match byte {
0x20..=0x2f => {}
0x30..=0x7e => self.state = State::Ground,
_ => {
self.state = State::Ground;
self.ground(byte, text);
}
},
State::Csi => match byte {
0x20..=0x3f => match byte {
b'0'..=b'9' if !self.csi_closed => {
self.csi_param = self
.csi_param
.saturating_mul(10)
.saturating_add(u32::from(byte - b'0'));
}
b';' => self.csi_closed = true,
_ => {}
},
0x40..=0x7e => {
self.state = State::Ground;
let up = byte == b'A';
let rows = usize::try_from(self.csi_param.max(1)).unwrap_or(usize::MAX);
if LINE_BREAKING_FINALS.contains(&byte) {
self.settle_cr(text);
self.break_line(text);
}
if up {
self.cursor_up(rows);
}
}
0x1b => self.state = State::Escape,
_ => {
self.state = State::Ground;
self.ground(byte, text);
}
},
State::Text { bell_ends, seen } => match byte {
0x07 if bell_ends => self.state = State::Ground,
0x1b => self.state = State::TextEscape,
_ if seen >= STRING_LIMIT => {
self.state = State::Ground;
self.ground(byte, text);
}
_ => {
self.state = State::Text {
bell_ends,
seen: seen + 1,
}
}
},
State::TextEscape => match byte {
b'\\' => self.state = State::Ground,
_ => {
self.state = State::Escape;
self.step(byte, text);
}
},
}
}
fn ground(&mut self, byte: u8, text: &mut Vec<u8>) {
match byte {
b'\r' => self.cr_pending = true,
b'\n' => {
self.cr_pending = false;
match self.closed.take() {
Some(written) if self.row.is_empty() => {
if written {
text.push(b'\n');
}
}
_ => {
self.end_row(text, true);
}
}
}
0x1b => {
self.settle_cr(text);
self.state = State::Escape;
}
0x00..=0x08 | 0x0b..=0x0c | 0x0e..=0x1a | 0x1c..=0x1f | 0x7f => {}
_ => {
self.settle_cr(text);
self.push_byte(byte, text);
if byte != b'\t' && byte != b' ' {
self.line_has_text = true;
}
}
}
}
fn push_byte(&mut self, byte: u8, text: &mut Vec<u8>) {
self.closed = None;
self.row.push(byte);
if self.row.len() < SCREEN_ROW_BYTES {
return;
}
text.extend_from_slice(&self.row);
self.row.clear();
self.spilling = true;
self.line_has_text = true;
}
fn decode(&mut self, text: &[u8], last: bool) {
let mut bytes = core::mem::take(&mut self.carry);
bytes.extend_from_slice(text);
let mut rest = bytes.as_slice();
loop {
match core::str::from_utf8(rest) {
Ok(valid) => {
self.ready.extend_from_slice(valid.as_bytes());
return;
}
Err(error) => {
let (valid, after) = rest.split_at(error.valid_up_to());
self.ready.extend_from_slice(valid);
match error.error_len() {
Some(skip) => {
self.ready.extend_from_slice("\u{fffd}".as_bytes());
rest = &after[skip..];
}
None if last => {
self.ready.extend_from_slice("\u{fffd}".as_bytes());
return;
}
None => {
self.carry = after.to_vec();
return;
}
}
}
}
}
}
}
impl<R: AsyncRead + Unpin> AsyncRead for AnsiLines<R> {
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
let this = self.get_mut();
while this.sent == this.ready.len() && !this.eof {
this.ready.clear();
this.sent = 0;
let mut scratch = [0u8; SCRATCH];
let mut read = ReadBuf::new(&mut scratch);
ready!(Pin::new(&mut this.inner).poll_read(cx, &mut read))?;
let mut text = Vec::with_capacity(read.filled().len());
if read.filled().is_empty() {
this.eof = true;
this.settle_cr(&mut text);
if !this.row.is_empty() {
this.end_row(&mut text, false);
}
}
for &byte in read.filled() {
this.step(byte, &mut text);
}
let last = this.eof;
this.decode(&text, last);
}
let pending = &this.ready[this.sent..];
let count = pending.len().min(buf.remaining());
buf.put_slice(&pending[..count]);
this.sent += count;
Poll::Ready(Ok(()))
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::VecDeque;
use tokio::io::AsyncReadExt as _;
struct Chunks(VecDeque<Vec<u8>>);
impl AsyncRead for Chunks {
fn poll_read(
mut self: Pin<&mut Self>,
_cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
if let Some(chunk) = self.0.pop_front() {
buf.put_slice(&chunk);
}
Poll::Ready(Ok(()))
}
}
async fn filtered(chunks: &[&[u8]]) -> String {
let reader = Chunks(chunks.iter().map(|chunk| chunk.to_vec()).collect());
let mut text = String::new();
AnsiLines::new(reader)
.read_to_string(&mut text)
.await
.expect("the filter's output must be valid UTF-8");
text
}
fn rows(n: usize) -> String {
(0..n).map(|row| format!("r{row}\n")).collect()
}
#[tokio::test]
async fn a_sequence_split_across_two_reads_is_removed_whole() {
assert_eq!(
filtered(&[b"\x1b[3", b"1mred\x1b[0", b"m\n"]).await,
"red\n"
);
assert_eq!(filtered(&[b"a\x1b]0;ti", b"tle\x07b\n"]).await, "ab\n");
assert_eq!(
filtered(&[b"a\x1b]8;;http://x\x1b", b"\\link\n"]).await,
"alink\n"
);
}
#[tokio::test]
async fn a_cursor_move_and_a_bare_carriage_return_end_a_line() {
assert_eq!(filtered(&[b"10%\r20%\r30%\n"]).await, "10%\n20%\n30%\n");
assert_eq!(filtered(&[b"one\x1b[2;1Htwo\n"]).await, "one\ntwo\n");
assert_eq!(filtered(&[b"gone\x1b[Kthere\n"]).await, "gone\nthere\n");
assert_eq!(filtered(&[b"a\x1b[1ab\x1b[1ec\n"]).await, "a\nb\nc\n");
}
#[tokio::test]
async fn a_multibyte_character_split_across_two_reads_decodes_whole() {
assert_eq!(
filtered(&[b"na\xc3", b"\xafve \xe2\x82", b"\xac\n"]).await,
"naïve €\n"
);
}
#[tokio::test]
async fn a_carriage_return_before_a_newline_is_just_the_newline() {
assert_eq!(filtered(&[b"a\r", b"\nb\r\n\r\n"]).await, "a\nb\n\n");
}
#[tokio::test]
async fn a_doubled_carriage_return_before_a_newline_is_just_the_newline() {
assert_eq!(filtered(&[b"a\r\r\nb\n"]).await, "a\nb\n");
assert_eq!(filtered(&[b"a\r", b"\r\nb\n"]).await, "a\nb\n");
assert_eq!(filtered(&[b"a\r\r\r\nb\n"]).await, "a\nb\n");
}
#[tokio::test]
async fn a_doubled_carriage_return_before_text_still_ends_the_line() {
assert_eq!(filtered(&[b"a\r\rb\n"]).await, "a\nb\n");
assert_eq!(filtered(&[b"a\r", b"\rb\n"]).await, "a\nb\n");
}
#[tokio::test]
async fn a_redraw_leaves_no_blank_line_between_its_parts() {
assert_eq!(filtered(&[b"a\n\r\x1b[Kb\n"]).await, "a\nb\n");
}
#[tokio::test]
async fn a_repainted_frame_identical_to_the_last_writes_nothing() {
assert_eq!(
filtered(&[b"one\ntwo\n\x1b[2Aone\ntwo\n"]).await,
"one\ntwo\n"
);
}
#[tokio::test]
async fn a_repainted_frame_writes_only_its_changed_rows() {
assert_eq!(
filtered(&[b"one\ntwo\n\x1b[2Aone\nTWO\n"]).await,
"one\ntwo\nTWO\n"
);
}
#[tokio::test]
async fn a_frame_longer_than_the_one_it_replaces_writes_its_extra_rows() {
assert_eq!(
filtered(&[b"one\ntwo\n\x1b[2Aone\ntwo\nthree\n"]).await,
"one\ntwo\nthree\n"
);
}
#[tokio::test]
async fn a_repainted_frame_split_across_reads_is_still_recognised() {
assert_eq!(
filtered(&[b"one\ntwo\n\x1b[2", b"Aone\ntw", b"o\n"]).await,
"one\ntwo\n"
);
}
#[tokio::test]
async fn a_cursor_up_past_the_kept_rows_writes_every_row() {
assert_eq!(
filtered(&[b"one\ntwo\n\x1b[9Aone\ntwo\n"]).await,
"one\ntwo\none\ntwo\n"
);
}
#[tokio::test]
async fn stacked_cursor_ups_open_one_window_past_the_kept_rows() {
assert_eq!(
filtered(&[b"one\ntwo\nthree\n\x1b[2A\x1b[2Aone\ntwo\nthree\n"]).await,
"one\ntwo\nthree\none\ntwo\nthree\n"
);
}
#[tokio::test]
async fn a_blind_window_stays_blind_until_it_ends() {
assert_eq!(
filtered(&[b"one\ntwo\n\x1b[3AX\n\x1b[1Aone\n"]).await,
"one\ntwo\nX\none\n"
);
assert_eq!(
filtered(&[b"one\ntwo\n\x1b[3Ax\ny\nz\n\x1b[3Ax\ny\nz\n"]).await,
"one\ntwo\nx\ny\nz\n"
);
}
#[tokio::test]
async fn a_repaint_inside_the_kept_rows_writes_nothing_at_the_cap() {
let screen = rows(SCREEN_ROWS);
let input = format!("{screen}\x1b[{SCREEN_ROWS}A{screen}");
assert_eq!(filtered(&[input.as_bytes()]).await, screen);
}
#[tokio::test]
async fn a_repaint_past_the_cap_writes_every_row() {
let screen = rows(SCREEN_ROWS + 1);
let input = format!("{screen}\x1b[{}A{screen}", SCREEN_ROWS + 1);
let chunks: Vec<&[u8]> = input.as_bytes().chunks(512).collect();
assert_eq!(filtered(&chunks).await, format!("{screen}{screen}"));
}
#[tokio::test]
async fn a_row_past_the_kept_cap_goes_out_as_it_is_read() {
let long: String = core::iter::repeat_n('x', SCREEN_ROW_BYTES * 3).collect();
let input = format!("{long}\n{long}\n\x1b[1A{long}\n");
let chunks: Vec<&[u8]> = input.as_bytes().chunks(512).collect();
assert_eq!(filtered(&chunks).await, format!("{long}\n{long}\n{long}\n"));
}
#[tokio::test]
async fn a_repaint_that_ends_the_stream_drops_its_unchanged_last_row() {
assert_eq!(
filtered(&[b"one\ntwo\n\x1b[2Aone\ntwo"]).await,
"one\ntwo\n"
);
assert_eq!(
filtered(&[b"one\ntwo\n\x1b[2Aone\nTWO"]).await,
"one\ntwo\nTWO"
);
}
#[tokio::test]
async fn whitespace_a_cursor_move_leaves_behind_is_not_part_of_the_next_row() {
assert_eq!(filtered(&[b"one\r\n \x1b[A\rone\r\n"]).await, "one\n");
assert_eq!(filtered(&[b"a\n \nb\n"]).await, "a\n \nb\n");
}
#[tokio::test]
async fn a_row_ended_by_an_erase_and_a_newline_is_one_screen_row() {
assert_eq!(
filtered(&[b"a\x1b[K\nb\x1b[K\n\x1b[2Aa\x1b[K\nb\x1b[K\n"]).await,
"a\n\nb\n\n"
);
assert_eq!(
filtered(&[b"a\x1b[K\r\nb\x1b[K\r\n\x1b[2Aa\x1b[K\r\nb\x1b[K\r\n"]).await,
"a\n\nb\n\n"
);
}
#[tokio::test]
async fn a_changed_row_after_an_erase_and_a_newline_is_written() {
assert_eq!(
filtered(&[b"a\x1b[K\nb\x1b[K\n\x1b[2Ab\x1b[K\nc\x1b[K\n"]).await,
"a\n\nb\n\nb\n\nc\n\n"
);
}
#[tokio::test]
async fn invalid_bytes_and_a_cut_short_end_become_replacement_characters() {
assert_eq!(filtered(&[b"a\xffb\xc3"]).await, "a\u{fffd}b\u{fffd}");
}
#[tokio::test]
async fn colour_and_other_escapes_leave_only_the_text() {
assert_eq!(
filtered(&[b"\x1b[1;32mok\x1b[0m \x1b(B\x1b=done\x1bPq\x1b\\\n"]).await,
"ok done\n"
);
}
#[tokio::test]
async fn an_unterminated_string_sequence_gives_the_log_back() {
let mut input = b"\x1b]0;".to_vec();
input.extend(core::iter::repeat_n(b'x', STRING_LIMIT + 8));
input.extend_from_slice(b"\nlater\n");
let chunks: Vec<&[u8]> = input.chunks(512).collect();
assert!(filtered(&chunks).await.ends_with("later\n"));
}
}