use newt_core::tty::str_width;
#[derive(Debug, Clone, PartialEq, Eq)]
enum Token {
Text(Vec<u8>),
Newline,
CarriageReturn,
Sgr(Vec<u8>),
EraseInLine(u8),
MoveToColumn(usize),
PrivateMode(Vec<u8>),
Other,
}
fn scan(bytes: &[u8]) -> (Vec<Token>, Vec<u8>) {
let mut out = Vec::new();
let mut text = Vec::new();
let mut i = 0;
let flush_text = |text: &mut Vec<u8>, out: &mut Vec<Token>| {
if !text.is_empty() {
out.push(Token::Text(std::mem::take(text)));
}
};
while i < bytes.len() {
match bytes[i] {
b'\n' => {
flush_text(&mut text, &mut out);
out.push(Token::Newline);
i += 1;
}
b'\r' => {
flush_text(&mut text, &mut out);
out.push(Token::CarriageReturn);
i += 1;
}
0x1b => {
flush_text(&mut text, &mut out);
match scan_escape(&bytes[i..]) {
Some((tok, len)) => {
out.push(tok);
i += len;
}
None => return (out, bytes[i..].to_vec()),
}
}
b'\t' => {
text.push(b'\t');
i += 1;
}
0x00..=0x1f | 0x7f => {
i += 1;
}
_ => {
text.push(bytes[i]);
i += 1;
}
}
}
let carry = split_incomplete_utf8(&mut text);
flush_text(&mut text, &mut out);
(out, carry)
}
fn split_incomplete_utf8(text: &mut Vec<u8>) -> Vec<u8> {
match std::str::from_utf8(text) {
Ok(_) => Vec::new(),
Err(e) if e.error_len().is_none() => text.split_off(e.valid_up_to()),
Err(_) => Vec::new(),
}
}
fn scan_escape(bytes: &[u8]) -> Option<(Token, usize)> {
let &second = bytes.get(1)?;
match second {
b'[' => {
let mut j = 2;
while j < bytes.len() && (0x30..=0x3f).contains(&bytes[j]) {
j += 1;
}
while j < bytes.len() && (0x20..=0x2f).contains(&bytes[j]) {
j += 1;
}
let &fin = bytes.get(j)?;
if !(0x40..=0x7e).contains(&fin) {
return Some((Token::Other, 1));
}
let seq = &bytes[..=j];
let params = &bytes[2..j];
let tok = match fin {
b'm' => Token::Sgr(seq.to_vec()),
b'K' => Token::EraseInLine(first_param(params).unwrap_or(0) as u8),
b'G' => Token::MoveToColumn(first_param(params).unwrap_or(1).max(1)),
b'h' | b'l' if params.first() == Some(&b'?') => {
if private_mode_forwarded(¶ms[1..]) {
Token::PrivateMode(seq.to_vec())
} else {
Token::Other
}
}
_ => Token::Other,
};
Some((tok, j + 1))
}
b']' => {
let mut j = 2;
while j < bytes.len() {
if bytes[j] == 0x07 {
return Some((Token::Other, j + 1));
}
if bytes[j] == 0x1b && bytes.get(j + 1) == Some(&b'\\') {
return Some((Token::Other, j + 2));
}
j += 1;
}
None
}
_ => Some((Token::Other, 2)),
}
}
fn first_param(params: &[u8]) -> Option<usize> {
let digits: Vec<u8> = params
.iter()
.copied()
.take_while(u8::is_ascii_digit)
.collect();
if digits.is_empty() {
return None;
}
std::str::from_utf8(&digits).ok()?.parse().ok()
}
const FORWARDED_PRIVATE_MODES: &[u16] = &[1000, 1002, 1003, 1005, 1006, 1015, 1016];
fn private_mode_forwarded(params: &[u8]) -> bool {
let mut any = false;
for part in params.split(|&b| b == b';') {
any = true;
let Ok(text) = std::str::from_utf8(part) else {
return false;
};
let Ok(mode) = text.parse::<u16>() else {
return false;
};
if !FORWARDED_PRIVATE_MODES.contains(&mode) {
return false;
}
}
any
}
pub(crate) type Row = Vec<u8>;
#[derive(Debug, Default, PartialEq, Eq)]
pub(crate) struct Drained {
pub(crate) lines: Vec<Row>,
pub(crate) passthrough: Vec<u8>,
pub(crate) partial_changed: bool,
}
#[derive(Debug, Default)]
pub(crate) struct TranscriptStream {
carry: Vec<u8>,
partial: Row,
styling: SgrState,
col: usize,
}
impl TranscriptStream {
pub(crate) fn new() -> Self {
Self::default()
}
pub(crate) fn partial(&self) -> &[u8] {
&self.partial
}
pub(crate) fn feed(&mut self, chunk: &[u8]) -> Drained {
let mut input = std::mem::take(&mut self.carry);
input.extend_from_slice(chunk);
let (tokens, carry) = scan(&input);
self.carry = carry;
let mut out = Drained::default();
for tok in tokens {
match tok {
Token::Text(t) => {
self.overwrite_at_col();
self.partial.extend_from_slice(&t);
self.col += str_width(&String::from_utf8_lossy(&t));
out.partial_changed = true;
}
Token::Sgr(seq) => {
self.overwrite_at_col();
self.styling.apply(&seq);
self.partial.extend_from_slice(&seq);
out.partial_changed = true;
}
Token::Newline => {
out.lines.push(std::mem::take(&mut self.partial));
self.col = 0;
out.partial_changed = true;
}
Token::CarriageReturn => {
self.col = 0;
}
Token::MoveToColumn(n) => {
self.col = n - 1;
}
Token::EraseInLine(mode) => {
match mode {
0 => self.truncate_to_col(),
_ => {
self.partial.clear();
self.col = 0;
}
}
out.partial_changed = true;
}
Token::PrivateMode(seq) => out.passthrough.extend_from_slice(&seq),
Token::Other => {}
}
}
out
}
fn overwrite_at_col(&mut self) {
if self.col < visible_width(&self.partial) {
self.truncate_to_col();
}
if self.partial.is_empty() {
self.partial = self.styling.prefix();
}
}
fn truncate_to_col(&mut self) {
let col = self.col;
self.partial = clip_to_width(&self.partial, col);
}
}
pub(crate) fn visible_width(row: &[u8]) -> usize {
let (tokens, _) = scan(row);
tokens
.iter()
.map(|t| match t {
Token::Text(t) => str_width(&String::from_utf8_lossy(t)),
_ => 0,
})
.sum()
}
pub(crate) fn clip_to_width(row: &[u8], cols: usize) -> Row {
let (tokens, _) = scan(row);
let mut out = Vec::new();
let mut used = 0usize;
for tok in tokens {
match tok {
Token::Text(t) => {
if used >= cols {
continue;
}
let s = String::from_utf8_lossy(&t);
let mut taken = String::new();
for ch in s.chars() {
let w = newt_core::tty::ch_width(ch);
if used + w > cols {
break;
}
used += w;
taken.push(ch);
}
out.extend_from_slice(taken.as_bytes());
}
Token::Sgr(seq) => out.extend_from_slice(&seq),
_ => {}
}
}
out
}
#[derive(Debug, Default)]
struct SgrState(std::collections::BTreeMap<u16, Vec<u16>>);
impl SgrState {
fn apply(&mut self, seq: &[u8]) {
let Ok(body) = std::str::from_utf8(&seq[2..seq.len() - 1]) else {
return;
};
let Some(params): Option<Vec<u16>> = body
.split(';')
.map(|part| {
if part.is_empty() {
Some(0)
} else {
part.parse().ok()
}
})
.collect()
else {
return;
};
let mut i = 0;
while i < params.len() {
let code = params[i];
let mut width = 1;
let key = match code {
0 => {
self.0.clear();
None
}
1..=9 | 20 | 21 | 51..=53 | 60..=64 | 73 | 74 => Some(match code {
21 => 4,
6 => 5,
52 => 51,
61..=64 => 60,
74 => 73,
_ => code,
}),
11..=19 => Some(10),
30..=37 | 90..=97 => Some(38),
40..=47 | 100..=107 => Some(48),
38 | 48 | 58 => {
width = match params.get(i + 1) {
Some(2) => 5,
Some(5) => 3,
_ => return,
};
if i + width > params.len() {
return;
}
Some(code)
}
10 | 22..=29 | 39 | 49 | 54 | 55 | 59 | 65 | 75 => {
let reset: &[u16] = match code {
10 => &[10],
22 => &[1, 2],
23 => &[3, 20],
24 => &[4],
25 => &[5],
27 => &[7],
28 => &[8],
29 => &[9],
39 => &[38],
49 => &[48],
54 => &[51],
55 => &[53],
59 => &[58],
65 => &[60],
75 => &[73],
_ => &[],
};
for key in reset {
self.0.remove(key);
}
None
}
_ => None,
};
if let Some(key) = key {
self.0.insert(key, params[i..i + width].to_vec());
}
i += width;
}
}
fn prefix(&self) -> Vec<u8> {
if self.0.is_empty() {
return Vec::new();
}
let params = self
.0
.values()
.flatten()
.map(u16::to_string)
.collect::<Vec<_>>()
.join(";");
format!("\x1b[{params}m").into_bytes()
}
}
pub(crate) fn wrap_row(row: &[u8], cols: usize) -> Vec<Row> {
let cols = cols.max(1);
let (tokens, _) = scan(row);
let mut rows: Vec<Row> = Vec::new();
let mut cur: Row = Vec::new();
let mut used = 0usize;
let mut styling = SgrState::default();
for tok in tokens {
match tok {
Token::Text(t) => {
for ch in String::from_utf8_lossy(&t).chars() {
let w = newt_core::tty::ch_width(ch);
if used + w > cols && used > 0 {
rows.push(std::mem::take(&mut cur));
cur.extend_from_slice(&styling.prefix());
used = 0;
}
let mut buf = [0u8; 4];
cur.extend_from_slice(ch.encode_utf8(&mut buf).as_bytes());
used += w;
}
}
Token::Sgr(seq) => {
styling.apply(&seq);
cur.extend_from_slice(&seq);
}
_ => {}
}
}
rows.push(cur);
rows
}
#[cfg(test)]
mod tests {
use super::*;
fn feed_all(s: &mut TranscriptStream, chunks: &[&[u8]]) -> Vec<String> {
let mut lines = Vec::new();
for c in chunks {
lines.extend(
s.feed(c)
.lines
.into_iter()
.map(|l| String::from_utf8_lossy(&l).into_owned()),
);
}
lines
}
#[test]
fn plain_lines_split_on_newline_and_keep_the_partial() {
let mut s = TranscriptStream::new();
assert_eq!(feed_all(&mut s, &[b"one\ntwo\nthr"]), ["one", "two"]);
assert_eq!(s.partial(), b"thr");
assert_eq!(feed_all(&mut s, &[b"ee\n"]), ["three"]);
assert_eq!(s.partial(), b"");
}
#[test]
fn spinner_frames_replace_the_partial_and_never_become_lines() {
let mut s = TranscriptStream::new();
let d = s.feed(b"\r\x1b[K\xe2\xa0\x8b thinking\xe2\x80\xa6 0.1s");
assert!(d.lines.is_empty());
assert!(d.partial_changed);
let d = s.feed(b"\r\x1b[K\xe2\xa0\x99 thinking\xe2\x80\xa6 0.2s");
assert!(d.lines.is_empty());
assert_eq!(
String::from_utf8_lossy(s.partial()),
"⠙ thinking… 0.2s",
"the second frame fully replaced the first"
);
}
#[test]
fn erase_then_permanent_line_yields_a_clean_line() {
let mut s = TranscriptStream::new();
s.feed(b"\r\x1b[K\xe2\xa0\x8b thinking\xe2\x80\xa6 0.1s");
let d = s.feed(b"\r\x1b[K retrying in 1s\n");
assert_eq!(d.lines, vec![b" retrying in 1s".to_vec()]);
assert_eq!(s.partial(), b"");
}
#[test]
fn sgr_is_kept_inline_and_ignored_for_width() {
let mut s = TranscriptStream::new();
let d = s.feed(b"\x1b[38;5;208m\xe2\x9a\x99 read_file\x1b[0m: x\n");
assert_eq!(d.lines.len(), 1);
assert!(d.lines[0].starts_with(b"\x1b[38;5;208m"));
assert_eq!(
visible_width(&d.lines[0]),
"⚙ read_file: x".chars().count() + 1 - 1
);
assert_eq!(visible_width(b"\x1b[1mab\x1b[0m"), 2);
}
#[test]
fn stream_style_survives_paragraphs_chunk_boundaries_and_physical_wraps() {
let mut stream = TranscriptStream::new();
let style = "\x1b[2;3;38;2;70;80;90m";
let lines = feed_all(
&mut stream,
&[b"\x1b[2;3;38;2;70;80", b";90mone\n\ntw", b"o\r\nthree"],
);
assert_eq!(
lines,
[format!("{style}one"), String::new(), format!("{style}two")]
);
assert_eq!(stream.partial(), format!("{style}three").as_bytes());
assert_eq!(
wrap_row(stream.partial(), 3),
[
format!("{style}thr").into_bytes(),
format!("{style}ee").into_bytes()
]
);
}
#[test]
fn stream_style_respects_selective_and_full_resets_between_lines() {
let mut stream = TranscriptStream::new();
let lines = feed_all(
&mut stream,
&[b"\x1b[2;3;38;5;8ma\n\x1b[22;39mb\nc\n\x1b[0m"],
);
assert_eq!(lines[2], "\x1b[3mc");
let answer = feed_all(&mut stream, &[b"\x1b[38;5;7manswer\x1b[0m\nplain\n"]);
assert!(answer[0].ends_with("\x1b[0m\x1b[38;5;7manswer\x1b[0m"));
assert_eq!(answer[1], "plain");
assert!(stream.partial().is_empty());
}
#[test]
fn stream_style_survives_carriage_return_and_erased_status_rows() {
let mut stream = TranscriptStream::new();
let lines = feed_all(
&mut stream,
&[b"\x1b[38;5;8mold\r\x1b[2Knew\nnext\r\x1b[Kreplacement\n"],
);
assert_eq!(lines, ["\x1b[38;5;8mnew", "\x1b[38;5;8mreplacement"]);
}
#[test]
fn allowlisted_private_modes_pass_through_and_leave_no_trace_in_rows() {
let mut s = TranscriptStream::new();
let d = s.feed(b"\x1b[?1000h\x1b[?1006hhello\n\x1b[?1000l");
assert_eq!(d.passthrough, b"\x1b[?1000h\x1b[?1006h\x1b[?1000l");
assert_eq!(d.lines, vec![b"hello".to_vec()]);
}
#[test]
fn presenter_owned_private_modes_are_dropped_not_forwarded() {
let mut s = TranscriptStream::new();
let d = s.feed(b"\x1b[?1049h\x1b[?25l\x1b[?2004l\x1b[?7lhi\n\x1b[?1049l\x1b[?25h\x1b[?7h");
assert!(
d.passthrough.is_empty(),
"alt-screen/cursor/paste/wrap must never reach the terminal: {:?}",
d.passthrough
);
assert_eq!(d.lines, vec![b"hi".to_vec()], "text between them survives");
}
#[test]
fn mouse_tracking_private_modes_are_forwarded() {
let mut s = TranscriptStream::new();
let d = s.feed(b"\x1b[?1000h\x1b[?1002;1003;1006h\x1b[?1000;1006l");
assert_eq!(
d.passthrough,
b"\x1b[?1000h\x1b[?1002;1003;1006h\x1b[?1000;1006l",
);
}
#[test]
fn a_mixed_private_mode_set_is_dropped_whole() {
let mut s = TranscriptStream::new();
let d = s.feed(b"\x1b[?1000;25hx\n");
assert!(
d.passthrough.is_empty(),
"1000 is allowlisted but 25 is not — deny the set: {:?}",
d.passthrough
);
assert_eq!(d.lines, vec![b"x".to_vec()]);
}
#[test]
fn cursor_motion_is_dropped_never_forwarded() {
let mut s = TranscriptStream::new();
let d = s.feed(b"\x1b[3A\x1b[Jgone\n\x1b[2Bx");
assert!(d.passthrough.is_empty());
assert_eq!(d.lines, vec![b"gone".to_vec()]);
assert_eq!(s.partial(), b"x");
}
#[test]
fn move_to_column_and_erase_whole_line_redraw_the_partial() {
let mut s = TranscriptStream::new();
s.feed(b"allow? y");
s.feed(b"\x1b[1G\x1b[2Kallow? ye");
assert_eq!(s.partial(), b"allow? ye");
}
#[test]
fn an_escape_split_across_chunks_is_reassembled() {
let mut s = TranscriptStream::new();
let d1 = s.feed(b"a\x1b[3");
assert!(d1.lines.is_empty());
assert_eq!(
s.partial(),
b"a",
"the half-sequence is held back, not printed"
);
let d2 = s.feed(b"8;5;1mb\n");
assert_eq!(d2.lines, vec![b"a\x1b[38;5;1mb".to_vec()]);
}
#[test]
fn a_spinner_glyph_split_across_chunks_is_reassembled_not_mangled() {
let mut s = TranscriptStream::new();
let d1 = s.feed(b"\r\x1b[K\xe2\xa0");
assert!(d1.lines.is_empty());
assert_eq!(s.partial(), b"", "the half glyph is held back, not shown");
s.feed(b"\x8b thinking");
let partial = String::from_utf8_lossy(s.partial());
assert_eq!(partial, "⠋ thinking", "reassembled cleanly");
assert!(
!partial.contains('\u{fffd}'),
"no replacement char: {partial:?}"
);
}
#[test]
fn a_cjk_char_split_across_chunks_keeps_its_width() {
let mut s = TranscriptStream::new();
s.feed(b"ab\xe4\xb8"); assert_eq!(
s.partial(),
b"ab",
"the half CJK char is not in the row yet"
);
let d = s.feed(b"\x96 world\n");
assert_eq!(d.lines, vec!["ab世 world".as_bytes().to_vec()]);
assert_eq!(
visible_width(&d.lines[0]),
2 + 2 + 6,
"ab + 世(2) + ' world'"
);
}
#[test]
fn a_combining_mark_split_from_its_base_stays_zero_width() {
let mut s = TranscriptStream::new();
s.feed(b"e\xcc"); let d = s.feed(b"\x81\n");
assert_eq!(d.lines, vec!["e\u{301}".as_bytes().to_vec()]);
assert_eq!(
visible_width(&d.lines[0]),
1,
"base + combining is one cell"
);
}
#[test]
fn overwrite_after_carriage_return_without_erase_truncates_then_appends() {
let mut s = TranscriptStream::new();
s.feed(b"12345");
s.feed(b"\rab");
assert_eq!(s.partial(), b"ab");
}
#[test]
fn clip_keeps_leading_sgr_and_cuts_on_a_cell_boundary() {
let row = b"\x1b[1mhello \xe4\xb8\x96\xe7\x95\x8c!\x1b[0m".to_vec();
let clipped = clip_to_width(&row, 7);
assert_eq!(clipped, b"\x1b[1mhello \x1b[0m".to_vec());
assert_eq!(visible_width(&clipped), 6);
}
#[test]
fn wrap_row_never_exceeds_cols_and_keeps_styling() {
let row = b"\x1b[2mabcdefghij\x1b[0m".to_vec();
let rows = wrap_row(&row, 4);
assert_eq!(rows.len(), 3);
for r in &rows {
assert!(visible_width(r) <= 4, "{r:?}");
}
assert_eq!(rows[0], b"\x1b[2mabcd".to_vec());
assert_eq!(rows[1], b"\x1b[2mefgh".to_vec());
assert_eq!(rows[2], b"\x1b[2mij\x1b[0m".to_vec());
assert_eq!(wrap_row(b"", 10), vec![Vec::<u8>::new()]);
}
#[test]
fn wrapped_rows_replay_color_changes_and_resets() {
let rows = wrap_row(b"\x1b[38;2;100;110;120mabcd\x1b[1mefgh\x1b[0mijkl", 4);
assert_eq!(rows[1], b"\x1b[1;38;2;100;110;120mefgh\x1b[0m");
assert_eq!(rows[2], b"ijkl");
}
#[test]
fn frequent_color_changes_do_not_accumulate_replay_history() {
let input = "\x1b[31mx\x1b[32my".repeat(50_000);
let rows = wrap_row(input.as_bytes(), 80);
let bytes: usize = rows.iter().map(Vec::len).sum();
assert!(
bytes < input.len() + rows.len() * 32,
"replay grew to {bytes} bytes"
);
assert!(rows.iter().all(|row| visible_width(row) <= 80));
}
#[test]
fn wide_glyph_wraps_before_the_margin_not_across_it() {
let rows = wrap_row("ab世".as_bytes(), 3);
assert_eq!(rows, vec![b"ab".to_vec(), "世".as_bytes().to_vec()]);
}
}