use crate::sanitize::{sanitize_with_policy, str_display_width, truncate_str, SanitizePolicy};
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
const MAX_COPY_TEXT_BYTES: usize = 64 * 1024;
const MAX_CODE_LINE_WIDTH: usize = 4096;
const MAX_CODE_BLOCKS: usize = 64;
const MAX_CODE_BLOCK_LINES: usize = 4096;
const MAX_CODE_BLOCK_BYTES: usize = 64 * 1024;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct OutputOptions {
pub width: usize,
pub wrap: bool,
pub code_line_numbers: bool,
pub max_rows: usize,
pub sanitize: SanitizePolicy,
}
impl OutputOptions {
pub fn for_width(width: usize) -> Self {
Self {
width: width.max(1),
wrap: true,
code_line_numbers: true,
max_rows: usize::MAX,
sanitize: SanitizePolicy::code_output(SanitizePolicy::UNLIMITED_WIDTH, usize::MAX),
}
}
pub fn with_wrap(mut self, wrap: bool) -> Self {
self.wrap = wrap;
self
}
pub fn with_code_line_numbers(mut self, show: bool) -> Self {
self.code_line_numbers = show;
self
}
pub fn with_max_rows(mut self, max_rows: usize) -> Self {
self.max_rows = max_rows.max(1);
self
}
}
impl Default for OutputOptions {
fn default() -> Self {
Self::for_width(88)
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct OutputDocument {
pub rows: Vec<OutputRow>,
pub copy_text: String,
pub removed_control_chars: usize,
pub removed_escape_sequences: usize,
pub removed_zero_width_chars: usize,
pub truncated: bool,
}
impl OutputDocument {
pub fn visible_rows(&self, scroll: usize, height: usize) -> &[OutputRow] {
let height = height.max(1);
let max_scroll = self.rows.len().saturating_sub(height);
let scroll = scroll.min(max_scroll);
let end = self.rows.len().saturating_sub(scroll);
let start = end.saturating_sub(height);
&self.rows[start..end]
}
pub fn code_block_refs(&self) -> Vec<CodeBlockRef<'_>> {
let mut blocks = Vec::new();
let mut language = None;
let mut start = 0usize;
let mut in_code = false;
for (index, row) in self.rows.iter().enumerate() {
match row {
OutputRow::CodeFenceStart { language: lang } => {
language = safe_info_string(lang);
start = index.saturating_add(1);
in_code = true;
}
OutputRow::CodeFenceEnd if in_code => {
push_code_block_ref(&mut blocks, language, &self.rows[start..index], true);
language = None;
in_code = false;
if blocks.len() >= MAX_CODE_BLOCKS {
break;
}
}
_ => {}
}
}
if in_code && blocks.len() < MAX_CODE_BLOCKS {
push_code_block_ref(&mut blocks, language, &self.rows[start..], false);
}
blocks
}
pub fn code_blocks(&self) -> Vec<CodeBlockOutput> {
let mut blocks = Vec::new();
let mut current = CodeBlockBuilder::default();
let mut in_code = false;
for row in &self.rows {
match row {
OutputRow::CodeFenceStart { language: lang } => {
current = CodeBlockBuilder::new(lang);
in_code = true;
}
OutputRow::Code { text, .. } if in_code => current.push_line(text),
OutputRow::CodeFenceEnd if in_code => {
push_code_block(&mut blocks, ¤t);
current = CodeBlockBuilder::default();
in_code = false;
if blocks.len() >= MAX_CODE_BLOCKS {
break;
}
}
_ => {}
}
}
if in_code && current.line_count > 0 && blocks.len() < MAX_CODE_BLOCKS {
push_code_block(&mut blocks, ¤t);
}
blocks
}
pub fn code_block_analyses(&self) -> Vec<CodeBlockAnalysis> {
self.code_block_refs()
.into_iter()
.map(CodeBlockRef::analysis)
.collect()
}
}
fn push_code_block_ref<'a>(
blocks: &mut Vec<CodeBlockRef<'a>>,
language: Option<&'a str>,
rows: &'a [OutputRow],
allow_empty: bool,
) {
if blocks.len() >= MAX_CODE_BLOCKS {
return;
}
let (line_count, byte_count, truncated) = code_ref_metrics(rows);
if line_count == 0 && !allow_empty {
return;
}
blocks.push(CodeBlockRef {
language,
rows,
line_count,
byte_count,
truncated,
});
}
fn code_ref_metrics(rows: &[OutputRow]) -> (usize, usize, bool) {
let mut line_count = 0usize;
let mut byte_count = 0usize;
let mut truncated = false;
for row in rows {
let OutputRow::Code { text, .. } = row else {
continue;
};
if line_count >= MAX_CODE_BLOCK_LINES {
truncated = true;
break;
}
let separator = usize::from(line_count > 0);
let additional = separator.saturating_add(text.len());
if byte_count.saturating_add(additional) > MAX_CODE_BLOCK_BYTES {
truncated = true;
if line_count == 0 {
line_count = 1;
byte_count = MAX_CODE_BLOCK_BYTES.min(text.len());
}
break;
}
byte_count += additional;
line_count += 1;
}
(line_count, byte_count, truncated)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct CodeBlockRef<'a> {
pub language: Option<&'a str>,
pub rows: &'a [OutputRow],
pub line_count: usize,
pub byte_count: usize,
pub truncated: bool,
}
impl<'a> CodeBlockRef<'a> {
pub fn is_empty(self) -> bool {
self.line_count == 0
}
pub fn lines(self) -> CodeBlockLines<'a> {
CodeBlockLines {
rows: self.rows.iter(),
remaining: self.line_count.min(MAX_CODE_BLOCK_LINES),
}
}
pub fn line_analyses(self) -> Vec<CodeLineAnalysis> {
self.lines().map(analyze_code_line).collect()
}
pub fn analysis(self) -> CodeBlockAnalysis {
analyze_code_block_lines(
self.lines(),
self.line_count,
self.truncated || self.line_count > MAX_CODE_BLOCK_LINES,
)
}
}
pub struct CodeBlockLines<'a> {
rows: std::slice::Iter<'a, OutputRow>,
remaining: usize,
}
impl<'a> Iterator for CodeBlockLines<'a> {
type Item = &'a str;
fn next(&mut self) -> Option<Self::Item> {
if self.remaining == 0 {
return None;
}
for row in self.rows.by_ref() {
if let OutputRow::Code { text, .. } = row {
self.remaining -= 1;
return Some(text);
}
}
None
}
}
#[derive(Default)]
struct CodeBlockBuilder {
language: String,
code: String,
line_count: usize,
}
impl CodeBlockBuilder {
fn new(language: &str) -> Self {
Self {
language: sanitize_info_string(language),
code: String::new(),
line_count: 0,
}
}
fn push_line(&mut self, line: &str) {
if self.line_count >= MAX_CODE_BLOCK_LINES || self.code.len() >= MAX_CODE_BLOCK_BYTES {
return;
}
if self.line_count > 0 {
append_capped(&mut self.code, "\n", MAX_CODE_BLOCK_BYTES);
}
append_capped(&mut self.code, line, MAX_CODE_BLOCK_BYTES);
self.line_count += 1;
}
}
fn push_code_block(blocks: &mut Vec<CodeBlockOutput>, block: &CodeBlockBuilder) {
if blocks.len() >= MAX_CODE_BLOCKS {
return;
}
blocks.push(CodeBlockOutput {
language: if block.language.is_empty() {
None
} else {
Some(block.language.clone())
},
code: block.code.clone(),
line_count: block.line_count,
});
}
#[derive(Clone, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub enum OutputRow {
Blank,
Paragraph {
text: String,
continuation: bool,
},
Heading {
level: usize,
text: String,
},
Bullet {
text: String,
continuation: bool,
},
Quote {
text: String,
continuation: bool,
},
CodeFenceStart {
language: String,
},
Code {
language: String,
line_number: usize,
gutter: Option<String>,
text: String,
tokens: Vec<CodeToken>,
},
CodeFenceEnd,
}
#[derive(Clone, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct CodeBlockOutput {
pub language: Option<String>,
pub code: String,
pub line_count: usize,
}
impl CodeBlockOutput {
pub fn line_analyses(&self) -> Vec<CodeLineAnalysis> {
if self.line_count == 0 {
return Vec::new();
}
self.code
.split('\n')
.take(self.line_count.min(MAX_CODE_BLOCK_LINES))
.map(analyze_code_line)
.collect()
}
pub fn analysis(&self) -> CodeBlockAnalysis {
analyze_code_block_lines(
self.code.split('\n'),
self.line_count,
self.line_count > MAX_CODE_BLOCK_LINES,
)
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct CodeBlockAnalysis {
pub line_count: usize,
pub blank_line_count: usize,
pub comment_line_count: usize,
pub semantic_line_count: usize,
pub max_indent_columns: usize,
pub max_display_width: usize,
pub token_count: usize,
pub keyword_count: usize,
pub string_count: usize,
pub number_count: usize,
pub comment_count: usize,
pub punctuation_count: usize,
pub opens_block_count: usize,
pub closes_block_count: usize,
pub truncated: bool,
}
impl CodeBlockAnalysis {
pub fn non_blank_line_count(self) -> usize {
self.line_count.saturating_sub(self.blank_line_count)
}
pub fn has_comments(self) -> bool {
self.comment_count > 0
}
pub fn has_semantic_tokens(self) -> bool {
self.semantic_line_count > 0
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct CodeLineAnalysis {
pub indent_columns: usize,
pub display_width: usize,
pub token_count: usize,
pub keyword_count: usize,
pub string_count: usize,
pub number_count: usize,
pub comment_count: usize,
pub punctuation_count: usize,
pub blank: bool,
pub opens_block: bool,
pub closes_block: bool,
pub trailing_punctuation: Option<char>,
}
impl CodeLineAnalysis {
pub fn has_comment(self) -> bool {
self.comment_count > 0
}
pub fn has_semantic_token(self) -> bool {
self.keyword_count > 0 || self.string_count > 0 || self.number_count > 0
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct CodeToken {
pub kind: CodeTokenKind,
pub text: String,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub enum CodeTokenKind {
Text,
Keyword,
String,
Number,
Comment,
Punctuation,
}
pub fn analyze_code_line(line: &str) -> CodeLineAnalysis {
let line = bounded_code_line(line);
let trimmed_start = line.trim_start();
let trimmed_end = line.trim_end();
let tokens = tokenize_code_line(&line);
let mut analysis = CodeLineAnalysis {
indent_columns: indent_columns(&line),
display_width: str_display_width(&line),
token_count: tokens.len(),
blank: trimmed_start.is_empty(),
opens_block: opens_block(trimmed_end),
closes_block: closes_block(trimmed_start),
trailing_punctuation: trailing_punctuation(trimmed_end),
..CodeLineAnalysis::default()
};
for token in tokens {
match token.kind {
CodeTokenKind::Keyword => analysis.keyword_count += 1,
CodeTokenKind::String => analysis.string_count += 1,
CodeTokenKind::Number => analysis.number_count += 1,
CodeTokenKind::Comment => analysis.comment_count += 1,
CodeTokenKind::Punctuation => analysis.punctuation_count += 1,
CodeTokenKind::Text => {}
}
}
analysis
}
fn analyze_code_block_lines<'a, I>(
lines: I,
expected_lines: usize,
truncated: bool,
) -> CodeBlockAnalysis
where
I: IntoIterator<Item = &'a str>,
{
let expected_lines = expected_lines.min(MAX_CODE_BLOCK_LINES);
let mut analysis = CodeBlockAnalysis {
truncated,
..CodeBlockAnalysis::default()
};
for line in lines.into_iter().take(expected_lines) {
push_code_line_analysis(&mut analysis, analyze_code_line(line));
}
if analysis.line_count < expected_lines {
analysis.truncated = true;
}
analysis
}
fn push_code_line_analysis(block: &mut CodeBlockAnalysis, line: CodeLineAnalysis) {
block.line_count += 1;
block.blank_line_count += usize::from(line.blank);
block.comment_line_count += usize::from(line.has_comment());
block.semantic_line_count += usize::from(line.has_semantic_token());
block.max_indent_columns = block.max_indent_columns.max(line.indent_columns);
block.max_display_width = block.max_display_width.max(line.display_width);
block.token_count += line.token_count;
block.keyword_count += line.keyword_count;
block.string_count += line.string_count;
block.number_count += line.number_count;
block.comment_count += line.comment_count;
block.punctuation_count += line.punctuation_count;
block.opens_block_count += usize::from(line.opens_block);
block.closes_block_count += usize::from(line.closes_block);
}
pub fn materialize_output(input: &str, options: OutputOptions) -> OutputDocument {
let max_rows = options.max_rows.max(1);
let sanitized = sanitize_with_policy(input, output_sanitize_policy(options, max_rows));
let mut rows = Vec::new();
let mut copy_text = String::new();
let mut copy_truncated = false;
let mut in_code = false;
let mut code_line = 1usize;
let mut language = String::new();
let mut truncated = sanitized.truncated;
for raw in sanitized.text.lines() {
if rows.len() >= max_rows {
truncated = true;
break;
}
let trimmed_end = raw.trim_end();
if let Some(lang) = trimmed_end.strip_prefix("```") {
if in_code {
rows.push(OutputRow::CodeFenceEnd);
push_copy_line_segments(&mut copy_text, &["```"], &mut copy_truncated);
in_code = false;
language.clear();
} else {
language = sanitize_info_string(lang.trim());
rows.push(OutputRow::CodeFenceStart {
language: language.clone(),
});
if language.is_empty() {
push_copy_line_segments(&mut copy_text, &["```"], &mut copy_truncated);
} else {
push_copy_line_segments(
&mut copy_text,
&["```", &language],
&mut copy_truncated,
);
}
in_code = true;
code_line = 1;
}
if truncate_rows_to_limit(&mut rows, max_rows) {
truncated = true;
break;
}
continue;
}
if in_code {
let code_width = code_text_width(&options, code_line);
let text = if options.wrap {
truncate_str(trimmed_end, code_width)
} else {
truncate_str(trimmed_end, MAX_CODE_LINE_WIDTH)
};
if str_display_width(trimmed_end) > str_display_width(&text) {
truncated = true;
}
rows.push(OutputRow::Code {
language: language.clone(),
line_number: code_line,
gutter: if options.code_line_numbers {
Some(format!("{:>3} |", code_line))
} else {
None
},
tokens: tokenize_code_line(&text),
text: text.clone(),
});
push_copy_line_segments(&mut copy_text, &[trimmed_end], &mut copy_truncated);
code_line += 1;
if truncate_rows_to_limit(&mut rows, max_rows) {
truncated = true;
break;
}
continue;
}
let trimmed = trimmed_end.trim_start();
if trimmed.is_empty() {
rows.push(OutputRow::Blank);
push_copy_line_segments(&mut copy_text, &[""], &mut copy_truncated);
} else if let Some(text) = heading_text(trimmed, 1) {
truncated |= push_heading(
&mut rows,
&mut copy_text,
&mut copy_truncated,
1,
text,
options.width,
);
} else if let Some(text) = heading_text(trimmed, 2) {
truncated |= push_heading(
&mut rows,
&mut copy_text,
&mut copy_truncated,
2,
text,
options.width,
);
} else if let Some(text) = heading_text(trimmed, 3) {
truncated |= push_heading(
&mut rows,
&mut copy_text,
&mut copy_truncated,
3,
text,
options.width,
);
} else if let Some(text) = trimmed
.strip_prefix("- ")
.or_else(|| trimmed.strip_prefix("* "))
.or_else(|| trimmed.strip_prefix("+ "))
{
let row_budget = max_rows.saturating_sub(rows.len());
truncated |= push_wrapped(
&mut rows,
text,
options.width.saturating_sub(2),
options.wrap,
OutputKind::Bullet,
row_budget,
);
push_copy_line_segments(&mut copy_text, &["- ", text], &mut copy_truncated);
} else if let Some(text) = trimmed.strip_prefix("> ") {
let row_budget = max_rows.saturating_sub(rows.len());
truncated |= push_wrapped(
&mut rows,
text,
options.width.saturating_sub(2),
options.wrap,
OutputKind::Quote,
row_budget,
);
push_copy_line_segments(&mut copy_text, &["> ", text], &mut copy_truncated);
} else {
let row_budget = max_rows.saturating_sub(rows.len());
truncated |= push_wrapped(
&mut rows,
trimmed,
options.width,
options.wrap,
OutputKind::Paragraph,
row_budget,
);
push_copy_line_segments(&mut copy_text, &[trimmed], &mut copy_truncated);
}
if truncate_rows_to_limit(&mut rows, max_rows) {
truncated = true;
break;
}
}
if in_code {
if rows.len() < max_rows {
rows.push(OutputRow::CodeFenceEnd);
}
push_copy_line_segments(&mut copy_text, &["```"], &mut copy_truncated);
truncated = true;
}
if copy_truncated {
truncated = true;
}
if rows.is_empty() {
rows.push(OutputRow::Blank);
}
OutputDocument {
rows,
copy_text,
removed_control_chars: sanitized.removed_control_chars,
removed_escape_sequences: sanitized.removed_escape_sequences,
removed_zero_width_chars: sanitized.removed_zero_width_chars,
truncated,
}
}
fn output_sanitize_policy(options: OutputOptions, max_rows: usize) -> SanitizePolicy {
let mut policy = options.sanitize;
if max_rows != usize::MAX {
policy.max_lines = policy.max_lines.min(max_rows.saturating_mul(4).max(1));
if policy.max_line_width == SanitizePolicy::UNLIMITED_WIDTH {
policy.max_line_width = options
.width
.saturating_mul(8)
.clamp(256, MAX_CODE_LINE_WIDTH);
}
}
policy
}
pub fn tokenize_code_line(line: &str) -> Vec<CodeToken> {
let line = bounded_code_line(line);
let trimmed = line.trim_start();
if trimmed.starts_with("//") || trimmed.starts_with('#') || trimmed.starts_with("--") {
return vec![CodeToken {
kind: CodeTokenKind::Comment,
text: line,
}];
}
let mut tokens = Vec::new();
let mut chars = line.chars().peekable();
while let Some(ch) = chars.next() {
if ch == '"' || ch == '\'' || ch == '`' {
let mut text = String::from(ch);
let mut escaped = false;
for next in chars.by_ref() {
text.push(next);
if escaped {
escaped = false;
} else if next == '\\' {
escaped = true;
} else if next == ch {
break;
}
}
tokens.push(CodeToken {
kind: CodeTokenKind::String,
text,
});
} else if ch.is_ascii_digit() {
let mut text = String::from(ch);
while let Some(next) = chars.peek().copied() {
if next.is_ascii_digit() || matches!(next, '_' | '.') {
text.push(next);
chars.next();
} else {
break;
}
}
tokens.push(CodeToken {
kind: CodeTokenKind::Number,
text,
});
} else if ch == '/' && chars.peek() == Some(&'/') {
let mut text = String::from(ch);
text.push(chars.next().unwrap_or('/'));
text.extend(chars.by_ref());
tokens.push(CodeToken {
kind: CodeTokenKind::Comment,
text,
});
break;
} else if ch == '-' && chars.peek() == Some(&'-') {
let mut text = String::from(ch);
text.push(chars.next().unwrap_or('-'));
text.extend(chars.by_ref());
tokens.push(CodeToken {
kind: CodeTokenKind::Comment,
text,
});
break;
} else if ch == '#' {
let mut text = String::from(ch);
text.extend(chars.by_ref());
tokens.push(CodeToken {
kind: CodeTokenKind::Comment,
text,
});
break;
} else if is_ident_start(ch) {
let mut text = String::from(ch);
while let Some(next) = chars.peek().copied() {
if is_ident_continue(next) {
text.push(next);
chars.next();
} else {
break;
}
}
tokens.push(CodeToken {
kind: if is_keyword(&text) {
CodeTokenKind::Keyword
} else {
CodeTokenKind::Text
},
text,
});
} else {
tokens.push(CodeToken {
kind: if ch.is_ascii_punctuation() {
CodeTokenKind::Punctuation
} else {
CodeTokenKind::Text
},
text: ch.to_string(),
});
}
}
tokens
}
fn bounded_code_line(line: &str) -> String {
if line.len() > MAX_CODE_LINE_WIDTH {
truncate_str(line, MAX_CODE_LINE_WIDTH)
} else {
line.to_string()
}
}
#[derive(Clone, Copy)]
enum OutputKind {
Paragraph,
Bullet,
Quote,
}
fn push_heading(
rows: &mut Vec<OutputRow>,
copy_text: &mut String,
copy_truncated: &mut bool,
level: usize,
text: &str,
width: usize,
) -> bool {
let width = width.saturating_sub(level + 1);
let visible = truncate_str(text, width);
let truncated = str_display_width(text) > str_display_width(&visible);
rows.push(OutputRow::Heading {
level,
text: visible,
});
push_copy_line_segments(copy_text, &[&"#".repeat(level), " ", text], copy_truncated);
truncated
}
fn push_wrapped(
rows: &mut Vec<OutputRow>,
text: &str,
width: usize,
wrap: bool,
kind: OutputKind,
row_budget: usize,
) -> bool {
if row_budget == 0 {
return true;
}
if !wrap || width == 0 || str_display_width(text) <= width {
let visible = truncate_str(text, width);
let truncated = str_display_width(text) > str_display_width(&visible);
rows.push(make_output_row(kind, visible, false));
return truncated;
}
let mut line = String::new();
let mut line_width = 0usize;
let mut continuation = false;
let mut truncated = false;
let mut emitted = 0usize;
for word in text.split_whitespace() {
let word_width = str_display_width(word);
let spacing = if line.is_empty() { 0 } else { 1 };
if !line.is_empty() && line_width + spacing + word_width > width {
if push_output_row_limited(rows, kind, line, continuation, &mut emitted, row_budget) {
return true;
}
line = String::new();
line_width = 0;
continuation = true;
}
if !line.is_empty() {
line.push(' ');
line_width += 1;
}
if word_width > width {
line.push_str(&truncate_str(word, width.saturating_sub(line_width)));
line_width = width;
truncated = true;
} else {
line.push_str(word);
line_width += word_width;
}
}
if !line.is_empty()
&& push_output_row_limited(rows, kind, line, continuation, &mut emitted, row_budget)
{
truncated = true;
}
truncated
}
fn push_output_row_limited(
rows: &mut Vec<OutputRow>,
kind: OutputKind,
text: String,
continuation: bool,
emitted: &mut usize,
row_budget: usize,
) -> bool {
if *emitted >= row_budget {
true
} else {
rows.push(make_output_row(kind, text, continuation));
*emitted += 1;
false
}
}
fn truncate_rows_to_limit(rows: &mut Vec<OutputRow>, max_rows: usize) -> bool {
if rows.len() > max_rows {
rows.truncate(max_rows);
true
} else {
false
}
}
fn make_output_row(kind: OutputKind, text: String, continuation: bool) -> OutputRow {
match kind {
OutputKind::Paragraph => OutputRow::Paragraph { text, continuation },
OutputKind::Bullet => OutputRow::Bullet { text, continuation },
OutputKind::Quote => OutputRow::Quote { text, continuation },
}
}
fn push_copy_line_segments(copy_text: &mut String, segments: &[&str], truncated: &mut bool) {
if copy_text.len() >= MAX_COPY_TEXT_BYTES {
*truncated = true;
return;
}
if !copy_text.is_empty() {
append_capped(copy_text, "\n", MAX_COPY_TEXT_BYTES);
}
for segment in segments {
let before = copy_text.len();
append_capped(copy_text, segment, MAX_COPY_TEXT_BYTES);
if copy_text.len() - before < segment.len() {
*truncated = true;
return;
}
}
}
fn append_capped(output: &mut String, input: &str, max_bytes: usize) {
let remaining = max_bytes.saturating_sub(output.len());
if remaining == 0 {
return;
}
if input.len() <= remaining {
output.push_str(input);
return;
}
for ch in input.chars() {
if output.len() + ch.len_utf8() > max_bytes {
break;
}
output.push(ch);
}
}
fn heading_text(input: &str, level: usize) -> Option<&str> {
let prefix = match level {
1 => "# ",
2 => "## ",
3 => "### ",
_ => return None,
};
input.strip_prefix(prefix)
}
fn code_text_width(options: &OutputOptions, line_number: usize) -> usize {
if !options.code_line_numbers {
return options.width;
}
let gutter = format!("{:>3} |", line_number);
options.width.saturating_sub(gutter.len() + 1).max(1)
}
fn sanitize_info_string(input: &str) -> String {
input
.chars()
.filter(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '+' | '.' | '#'))
.take(32)
.collect()
}
fn safe_info_string(input: &str) -> Option<&str> {
(!input.is_empty()
&& input.chars().count() <= 32
&& input
.chars()
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '+' | '.' | '#')))
.then_some(input)
}
fn is_ident_start(ch: char) -> bool {
ch == '_' || ch.is_ascii_alphabetic()
}
fn is_ident_continue(ch: char) -> bool {
ch == '_' || ch.is_ascii_alphanumeric()
}
fn indent_columns(line: &str) -> usize {
let mut columns = 0usize;
for ch in line.chars() {
match ch {
' ' => columns += 1,
'\t' => columns += 4,
ch if ch.is_whitespace() => columns += 1,
_ => break,
}
if columns >= MAX_CODE_LINE_WIDTH {
return MAX_CODE_LINE_WIDTH;
}
}
columns
}
fn opens_block(trimmed_end: &str) -> bool {
matches!(trimmed_end.chars().last(), Some('{' | '[' | '(' | ':'))
}
fn closes_block(trimmed_start: &str) -> bool {
trimmed_start.starts_with('}')
|| trimmed_start.starts_with(']')
|| trimmed_start.starts_with(')')
|| trimmed_start == "end"
|| trimmed_start
.strip_prefix("end")
.is_some_and(|rest| rest.starts_with(char::is_whitespace) || rest.starts_with(';'))
}
fn trailing_punctuation(trimmed_end: &str) -> Option<char> {
trimmed_end
.chars()
.next_back()
.filter(|ch| ch.is_ascii_punctuation())
}
fn is_keyword(token: &str) -> bool {
matches!(
token,
"as" | "async"
| "await"
| "break"
| "case"
| "class"
| "const"
| "continue"
| "crate"
| "default"
| "else"
| "enum"
| "export"
| "false"
| "fn"
| "for"
| "from"
| "function"
| "if"
| "impl"
| "import"
| "in"
| "let"
| "loop"
| "match"
| "mod"
| "mut"
| "pub"
| "return"
| "self"
| "static"
| "struct"
| "super"
| "trait"
| "true"
| "type"
| "use"
| "where"
| "while"
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn materialized_output_strips_ansi_and_parses_code() {
let doc = materialize_output(
"# Hi\n\x1b[31mred\x1b[0m\n```rust\nfn main() { println!(\"ok\"); }\n```",
OutputOptions::for_width(80),
);
assert_eq!(doc.removed_escape_sequences, 2);
assert!(matches!(doc.rows[0], OutputRow::Heading { .. }));
assert_eq!(doc.code_blocks().len(), 1);
assert_eq!(doc.code_block_refs()[0].language, Some("rust"));
assert!(!doc.copy_text.contains('\x1b'));
}
#[test]
fn code_tokens_mark_keywords_strings_and_numbers() {
let tokens = tokenize_code_line("let answer = 42; // comment");
assert!(tokens
.iter()
.any(|token| token.kind == CodeTokenKind::Keyword));
assert!(tokens
.iter()
.any(|token| token.kind == CodeTokenKind::Number));
assert!(tokens
.iter()
.any(|token| token.kind == CodeTokenKind::Punctuation));
assert!(tokens
.iter()
.any(|token| token.kind == CodeTokenKind::Comment));
}
#[test]
fn code_line_analysis_counts_semantic_tokens_and_structure() {
let analysis = analyze_code_line(" let answer = call(42); // ok");
assert_eq!(analysis.indent_columns, 4);
assert!(analysis.display_width > 20);
assert!(analysis.token_count >= 8);
assert_eq!(analysis.keyword_count, 1);
assert_eq!(analysis.number_count, 1);
assert_eq!(analysis.comment_count, 1);
assert!(analysis.has_comment());
assert!(analysis.has_semantic_token());
assert_eq!(analysis.trailing_punctuation, None);
assert!(!analysis.opens_block);
assert!(!analysis.closes_block);
}
#[test]
fn code_line_analysis_detects_block_edges_and_blank_lines() {
let opening = analyze_code_line("if ready:");
let closing = analyze_code_line("}");
let blank = analyze_code_line(" ");
assert!(opening.opens_block);
assert!(!opening.closes_block);
assert!(closing.closes_block);
assert!(blank.blank);
assert_eq!(blank.indent_columns, 4);
}
#[test]
fn code_block_refs_expose_line_analyses_without_joining() {
let doc = materialize_output(
"```rust\nfn main() {\n println!(\"ok\");\n}\n```",
OutputOptions::for_width(120),
);
let block = doc.code_block_refs()[0];
let analyses = block.line_analyses();
assert_eq!(analyses.len(), 3);
assert!(analyses[0].opens_block);
assert_eq!(analyses[1].string_count, 1);
assert!(analyses[2].closes_block);
}
#[test]
fn code_block_outputs_expose_line_analyses() {
let doc = materialize_output(
"```python\ndef run():\n return 42\n```",
OutputOptions::for_width(120),
);
let block = &doc.code_blocks()[0];
let analyses = block.line_analyses();
assert_eq!(analyses.len(), 2);
assert!(analyses[0].opens_block);
assert_eq!(analyses[1].keyword_count, 1);
assert_eq!(analyses[1].number_count, 1);
}
#[test]
fn code_block_outputs_preserve_blank_line_analyses() {
let doc = materialize_output("```\n\n```", OutputOptions::for_width(120));
let block = &doc.code_blocks()[0];
let analyses = block.line_analyses();
assert_eq!(block.line_count, 1);
assert_eq!(analyses.len(), 1);
assert!(analyses[0].blank);
}
#[test]
fn code_block_refs_expose_block_analysis_without_joining() {
let doc = materialize_output(
"```rust\nfn main() {\n // explain\n let answer = 42; // ok\n\n}\n```",
OutputOptions::for_width(120),
);
let analysis = doc.code_block_refs()[0].analysis();
assert_eq!(analysis.line_count, 5);
assert_eq!(analysis.blank_line_count, 1);
assert_eq!(analysis.non_blank_line_count(), 4);
assert_eq!(analysis.comment_line_count, 2);
assert_eq!(analysis.semantic_line_count, 2);
assert_eq!(analysis.keyword_count, 2);
assert_eq!(analysis.number_count, 1);
assert_eq!(analysis.comment_count, 2);
assert_eq!(analysis.opens_block_count, 1);
assert_eq!(analysis.closes_block_count, 1);
assert_eq!(analysis.max_indent_columns, 4);
assert!(analysis.max_display_width > 20);
assert!(analysis.has_comments());
assert!(analysis.has_semantic_tokens());
assert!(!analysis.truncated);
}
#[test]
fn code_block_outputs_mark_mismatched_public_line_counts_truncated() {
let block = CodeBlockOutput {
language: None,
code: "let answer = 42;".into(),
line_count: 3,
};
let analysis = block.analysis();
assert_eq!(analysis.line_count, 1);
assert_eq!(analysis.keyword_count, 1);
assert_eq!(analysis.number_count, 1);
assert!(analysis.truncated);
}
#[test]
fn output_documents_collect_code_block_analyses() {
let doc = materialize_output(
"```rust\nlet answer = 42;\n```\n```sql\n-- explain\n```",
OutputOptions::for_width(120),
);
let analyses = doc.code_block_analyses();
assert_eq!(analyses.len(), 2);
assert_eq!(analyses[0].number_count, 1);
assert_eq!(analyses[0].comment_count, 0);
assert_eq!(analyses[1].comment_line_count, 1);
assert!(analyses[1].has_comments());
}
#[test]
fn public_code_tokenization_is_bounded_for_huge_lines() {
let line = "a".repeat(MAX_CODE_LINE_WIDTH * 4);
let tokens = tokenize_code_line(&line);
let joined_len = tokens.iter().map(|token| token.text.len()).sum::<usize>();
assert!(joined_len <= MAX_CODE_LINE_WIDTH);
assert!(analyze_code_line(&line).display_width <= MAX_CODE_LINE_WIDTH);
}
#[test]
fn visible_rows_clamp_scroll() {
let doc = materialize_output("one\ntwo\nthree\nfour", OutputOptions::for_width(20));
let rows = doc.visible_rows(999, 2);
assert_eq!(rows.len(), 2);
}
#[test]
fn max_rows_limits_wrapped_paragraph_output() {
let doc = materialize_output(
"alpha beta gamma delta epsilon zeta",
OutputOptions::for_width(8).with_max_rows(2),
);
assert_eq!(doc.rows.len(), 2);
assert!(doc.truncated);
}
#[test]
fn max_rows_stops_wrapped_row_production_early() {
let input = vec!["word"; 10_000].join(" ");
let doc = materialize_output(&input, OutputOptions::for_width(8).with_max_rows(2));
assert_eq!(doc.rows.len(), 2);
assert!(doc.truncated);
}
#[test]
fn max_rows_bounds_copy_text_for_huge_lines() {
let input = format!("# title\n{}", "a".repeat(MAX_COPY_TEXT_BYTES * 2));
let doc = materialize_output(&input, OutputOptions::for_width(16).with_max_rows(2));
assert!(doc.copy_text.len() <= MAX_COPY_TEXT_BYTES);
assert!(doc.truncated);
}
#[test]
fn no_wrap_code_rows_are_bounded_before_tokenization() {
let input = format!("```rust\n{}\n```", "a".repeat(MAX_CODE_LINE_WIDTH * 2));
let doc = materialize_output(
&input,
OutputOptions::for_width(80)
.with_wrap(false)
.with_max_rows(3),
);
let OutputRow::Code { text, tokens, .. } = &doc.rows[1] else {
panic!("expected bounded code row");
};
assert!(text.len() <= MAX_CODE_LINE_WIDTH);
assert!(tokens.len() <= MAX_CODE_LINE_WIDTH);
assert!(doc.truncated);
}
#[test]
fn max_rows_limits_unclosed_code_fence() {
let doc = materialize_output(
"```rust\nfn one() {}\nfn two() {}",
OutputOptions::for_width(80).with_max_rows(2),
);
let blocks = doc.code_blocks();
assert_eq!(doc.rows.len(), 2);
assert!(doc.truncated);
assert!(doc.copy_text.ends_with("```"));
assert_eq!(blocks.len(), 1);
assert_eq!(blocks[0].language.as_deref(), Some("rust"));
assert_eq!(blocks[0].line_count, 1);
let refs = doc.code_block_refs();
assert_eq!(refs.len(), 1);
assert_eq!(refs[0].language, Some("rust"));
assert_eq!(refs[0].lines().collect::<Vec<_>>(), vec!["fn one() {}"]);
}
#[test]
fn code_blocks_ignore_orphan_code_rows_and_fence_ends() {
let doc = OutputDocument {
rows: vec![
OutputRow::Code {
language: "rust".into(),
line_number: 1,
gutter: None,
text: "let orphan = true;".into(),
tokens: Vec::new(),
},
OutputRow::CodeFenceEnd,
],
copy_text: String::new(),
removed_control_chars: 0,
removed_escape_sequences: 0,
removed_zero_width_chars: 0,
truncated: false,
};
assert!(doc.code_blocks().is_empty());
assert!(doc.code_block_refs().is_empty());
}
#[test]
fn code_block_refs_include_closed_empty_fences() {
let doc = materialize_output("```\n```", OutputOptions::for_width(80));
let refs = doc.code_block_refs();
assert_eq!(refs.len(), 1);
assert!(refs[0].is_empty());
assert_eq!(refs[0].lines().count(), 0);
}
#[test]
fn code_blocks_bound_public_malformed_documents() {
let mut rows = vec![OutputRow::CodeFenceStart {
language: "rust".to_string(),
}];
for index in 0..(MAX_CODE_BLOCK_LINES + 100) {
rows.push(OutputRow::Code {
language: "rust".to_string(),
line_number: index + 1,
gutter: None,
text: "x".repeat(128),
tokens: Vec::new(),
});
}
let doc = OutputDocument {
rows,
copy_text: String::new(),
removed_control_chars: 0,
removed_escape_sequences: 0,
removed_zero_width_chars: 0,
truncated: false,
};
let blocks = doc.code_blocks();
assert_eq!(blocks.len(), 1);
assert!(blocks[0].line_count <= MAX_CODE_BLOCK_LINES);
assert!(blocks[0].code.len() <= MAX_CODE_BLOCK_BYTES);
}
#[test]
fn code_block_refs_bound_public_malformed_documents_without_joining_code() {
let mut rows = vec![OutputRow::CodeFenceStart {
language: "rust\x1b[31m".to_string(),
}];
for index in 0..(MAX_CODE_BLOCK_LINES + 100) {
rows.push(OutputRow::Code {
language: "rust".to_string(),
line_number: index + 1,
gutter: None,
text: "x".repeat(128),
tokens: Vec::new(),
});
}
let doc = OutputDocument {
rows,
copy_text: String::new(),
removed_control_chars: 0,
removed_escape_sequences: 0,
removed_zero_width_chars: 0,
truncated: false,
};
let blocks = doc.code_block_refs();
assert_eq!(blocks.len(), 1);
assert_eq!(blocks[0].language, None);
assert!(blocks[0].line_count <= MAX_CODE_BLOCK_LINES);
assert!(blocks[0].byte_count <= MAX_CODE_BLOCK_BYTES);
assert!(blocks[0].truncated);
assert_eq!(blocks[0].lines().count(), blocks[0].line_count);
}
#[test]
fn public_code_block_refs_bound_line_analysis() {
let mut rows = Vec::new();
for index in 0..(MAX_CODE_BLOCK_LINES + 100) {
rows.push(OutputRow::Code {
language: "rust".to_string(),
line_number: index + 1,
gutter: None,
text: "let x = 1;".to_string(),
tokens: Vec::new(),
});
}
let block = CodeBlockRef {
language: Some("rust"),
rows: &rows,
line_count: MAX_CODE_BLOCK_LINES + 100,
byte_count: usize::MAX,
truncated: false,
};
let analysis = block.analysis();
assert_eq!(block.line_analyses().len(), MAX_CODE_BLOCK_LINES);
assert_eq!(analysis.line_count, MAX_CODE_BLOCK_LINES);
assert!(analysis.truncated);
}
#[test]
fn truncated_flag_tracks_heading_width_truncation() {
let doc = materialize_output("# exceptionally-long-heading", OutputOptions::for_width(10));
assert!(doc.truncated);
assert!(matches!(doc.rows[0], OutputRow::Heading { .. }));
}
#[test]
fn truncated_flag_tracks_wrapped_bullet_word_truncation() {
let doc = materialize_output("- supercalifragilistic", OutputOptions::for_width(8));
assert!(doc.truncated);
assert!(matches!(doc.rows[0], OutputRow::Bullet { .. }));
}
#[test]
fn truncated_flag_tracks_no_wrap_paragraph_truncation() {
let doc = materialize_output(
"alpha beta gamma",
OutputOptions::for_width(8).with_wrap(false),
);
assert!(doc.truncated);
assert!(matches!(doc.rows[0], OutputRow::Paragraph { .. }));
}
}