use crate::vocab::Vocab;
use crate::Channel;
pub(crate) trait Sink {
fn text(&mut self, s: &str);
fn boundary(&mut self) {}
}
pub(crate) struct StringSink<'a>(pub &'a mut String);
impl Sink for StringSink<'_> {
fn text(&mut self, s: &str) {
self.0.push_str(s);
}
}
pub(crate) struct PartsSink {
v: Vocab,
limit: usize,
parts: Vec<String>,
cur: String,
cur_len: usize,
pending: String,
}
impl PartsSink {
pub fn new(v: Vocab) -> Self {
Self {
limit: v.limit,
v,
parts: Vec::new(),
cur: String::new(),
cur_len: 0,
pending: String::new(),
}
}
pub fn into_parts(mut self) -> Vec<String> {
self.commit();
if !self.cur.is_empty() {
self.parts.push(self.cur);
}
self.parts
}
fn commit(&mut self) {
if self.pending.is_empty() {
return;
}
let pending = std::mem::take(&mut self.pending);
let piece = if self.cur.is_empty() { pending.trim_start_matches('\n') } else { &pending };
let n = piece.chars().count();
if n == 0 {
self.pending = pending;
self.pending.clear();
return;
}
if self.cur_len + n <= self.limit {
self.cur.push_str(piece);
self.cur_len += n;
} else {
if !self.cur.is_empty() {
self.parts.push(std::mem::take(&mut self.cur));
self.cur_len = 0;
}
let piece = piece.trim_start_matches('\n');
let n = piece.chars().count();
if n <= self.limit {
self.cur.push_str(piece);
self.cur_len = n;
} else {
let mut chunks = split_hard(piece, self.limit, &self.v);
if let Some(last) = chunks.pop() {
self.cur_len = last.chars().count();
self.cur = last;
}
self.parts.extend(chunks);
}
}
self.pending = pending;
self.pending.clear();
}
}
impl Sink for PartsSink {
fn text(&mut self, s: &str) {
self.pending.push_str(s);
}
fn boundary(&mut self) {
self.commit();
}
}
fn split_hard(text: &str, limit: usize, v: &Vocab) -> Vec<String> {
let mut parts = Vec::new();
let mut cur = String::new();
let mut len = 0usize;
let mut markup = Markup::default();
if v.channel != Channel::TelegramHtml && !v.is_plain() {
markup.spans = std::rc::Rc::from(scan_spans(text));
}
for line in text.split_inclusive('\n') {
for word in line.split_inclusive(' ') {
let mut rest = word;
while !rest.is_empty() {
let mut probe = markup.clone();
probe.feed(rest, v);
let budget = limit.saturating_sub(probe.reserve(v));
let n = rest.chars().count();
if len + n <= budget {
markup = probe;
len += push_after_reopen(&mut cur, rest, &mut markup);
break;
}
if len > 0 {
let before = len;
cut(&mut parts, &mut cur, &mut len, &mut markup, v, limit);
if len < before {
continue;
}
}
let room = if budget == 0 { limit.saturating_sub(markup.reserve(v)) } else { budget };
let mut take = room.saturating_sub(len).max(1);
let (end, probe) = loop {
let end = rest.char_indices().nth(take).map_or(rest.len(), |(i, _)| i);
let end = if v.html_out() { entity_cut(rest, end) } else { end };
let mut probe = markup.clone();
probe.feed(&rest[..end], v);
let reserve = probe.reserve(v);
let need = len + rest[..end].chars().count() + reserve;
if need <= limit || take == 1 || budget == 0 || reserve >= limit {
break (end, probe);
}
take -= (need - limit).min(take - 1);
};
markup = probe;
len += push_after_reopen(&mut cur, &rest[..end], &mut markup);
rest = &rest[end..];
if !rest.is_empty() {
cut(&mut parts, &mut cur, &mut len, &mut markup, v, limit);
}
}
}
}
if !cur.is_empty() {
parts.push(cur);
}
parts
}
fn entity_cut(s: &str, end: usize) -> usize {
let Some(at) = s[..end].rfind('&') else { return end };
let tail = &s[at + 1..];
let Some(semi) = tail.find(';') else { return end };
let is_entity = semi > 0
&& semi <= 8
&& tail[..semi].bytes().all(|b| b.is_ascii_alphanumeric() || b == b'#');
if !is_entity || at + 1 + semi < end {
return end;
}
if at > 0 { at } else { at + 1 + semi + 1 }
}
fn push_after_reopen(cur: &mut String, s: &str, markup: &mut Markup) -> usize {
let s = if markup.fresh { s.trim_start() } else { s };
if !s.is_empty() {
markup.fresh = false;
}
cur.push_str(s);
s.chars().count()
}
fn cut(
parts: &mut Vec<String>,
cur: &mut String,
len: &mut usize,
markup: &mut Markup,
v: &Vocab,
limit: usize,
) {
if cur.is_empty() {
return;
}
let mut part = std::mem::take(cur);
let carry = match part.len().checked_sub(markup.partial.len()) {
Some(at) if !markup.partial.is_empty() && at > 0 && part.ends_with(&markup.partial) => {
part.split_off(at)
}
_ => String::new(),
};
if v.channel != Channel::TelegramHtml {
let kept = part.trim_end().len();
part.truncate(kept);
}
markup.close_all(&mut part, v);
parts.push(part);
markup.reopen(cur, v);
markup.fresh = v.channel != Channel::TelegramHtml && !cur.is_empty();
let reopened = cur.chars().count() + carry.chars().count();
if reopened >= limit {
cur.clear();
markup.forget();
*len = 0;
return;
}
cur.push_str(&carry);
*len = reopened;
}
#[derive(Clone)]
struct Markup {
tags: Vec<(String, String)>,
fence: Option<String>,
partial: String,
at_line_start: bool,
spans: std::rc::Rc<[Span]>,
pos: usize,
dropped: bool,
fresh: bool,
}
#[derive(Clone, Copy, Debug, PartialEq)]
struct Span {
kind: SpanKind,
start: usize,
end: usize,
}
#[derive(Clone, Copy, Debug, PartialEq)]
enum SpanKind {
Code(usize),
Emph(&'static str),
Tag(&'static str),
}
fn emph_tags() -> impl Iterator<Item = &'static str> {
["strong", "em", "del"].into_iter().chain(crate::vocab::INLINE_TAGS.iter().map(|(name, _, _)| *name))
}
impl SpanKind {
fn write_open(self, out: &mut String) {
match self {
SpanKind::Code(run) => {
for _ in 0..run {
out.push('`');
}
}
SpanKind::Emph(m) => out.push_str(m),
SpanKind::Tag(name) => {
out.push('<');
out.push_str(name);
out.push('>');
}
}
}
fn write_close(self, out: &mut String) {
match self {
SpanKind::Tag(name) => {
out.push_str("</");
out.push_str(name);
out.push('>');
}
_ => self.write_open(out),
}
}
fn reopen_len(self) -> usize {
match self {
SpanKind::Code(run) => run * 2,
SpanKind::Emph(m) => m.len() * 2,
SpanKind::Tag(name) => name.len() * 2 + 5,
}
}
}
fn emph_tag_at(ch: &[char], at: usize) -> Option<(&'static str, bool, usize)> {
let rest = &ch[at..];
let closing = rest.get(1) == Some(&'/');
let from = if closing { 2 } else { 1 };
emph_tags().find_map(|name| {
let end = from + name.len();
let hit = rest.len() > end
&& rest[from..end].iter().copied().eq(name.chars())
&& rest[end] == '>';
hit.then_some((name, closing, end + 1))
})
}
fn scan_spans(text: &str) -> Vec<Span> {
let ch: Vec<char> = text.chars().collect();
let mut fenced = vec![false; ch.len()];
{
let mut fence = false;
let mut i = 0;
while i < ch.len() {
let mut j = i;
while j < ch.len() && (ch[j] == ' ' || ch[j] == '\t') {
j += 1;
}
let line_end = ch[i..].iter().position(|&x| x == '\n').map_or(ch.len(), |p| i + p);
let is_fence = ch[j..].starts_with(&['`', '`', '`']);
if is_fence || fence {
for f in &mut fenced[i..line_end] {
*f = true;
}
}
if is_fence {
fence = !fence;
}
i = line_end + 1;
}
}
let mut spans = Vec::new();
let mut open: Vec<(SpanKind, usize)> = Vec::new();
let mut i = 0;
while i < ch.len() {
let c = ch[i];
if fenced[i] {
i += 1;
continue;
}
if c == '\n' {
let mut j = i + 1;
while j < ch.len() && (ch[j] == ' ' || ch[j] == '\t') {
j += 1;
}
let next = &ch[j..];
let item = next.is_empty()
|| next[0] == '\n'
|| matches!(next, ['-' | '*' | '+' | '>' | '#', ' ', ..])
|| (next[0].is_ascii_digit()
&& next.iter().skip(1).find(|c| !c.is_ascii_digit()).is_some_and(|c| *c == '.'));
if item {
open.clear();
}
i += 1;
continue;
}
if c == '\\' {
i += 2;
continue;
}
if c == '`' {
let run = ch[i..].iter().take_while(|&&x| x == '`').count();
if run >= 3 {
i += run;
continue;
}
let mut j = i + run;
let close = loop {
match ch[j..].iter().position(|&x| x == '`' || x == '\n') {
None => break None,
Some(p) if ch[j + p] == '\n' => break None,
Some(p) if fenced[j + p] => break None,
Some(p) => {
let at = j + p;
let n = ch[at..].iter().take_while(|&&x| x == '`').count();
if n == run {
break Some(at);
}
j = at + n;
}
}
};
match close {
Some(at) => {
spans.push(Span { kind: SpanKind::Code(run), start: i, end: at + run });
i = at + run;
}
None => i += run,
}
continue;
}
if c == '<' {
match emph_tag_at(&ch, i) {
Some((name, false, len)) => {
open.push((SpanKind::Tag(name), i));
i += len;
}
Some((name, true, len)) => {
if open.last().map(|o| o.0) == Some(SpanKind::Tag(name)) {
let (kind, start) = open.pop().expect("방금 확인했다");
spans.push(Span { kind, start, end: i + len });
}
i += len;
}
None => i += 1,
}
continue;
}
if !matches!(c, '*' | '~') {
i += 1;
continue;
}
let run = ch[i..].iter().take_while(|&&x| x == c).count();
let prev = if i > 0 { Some(ch[i - 1]) } else { None };
let next = ch.get(i + run).copied();
let opens = next.is_some_and(|n| !n.is_whitespace());
let closes = prev.is_some_and(|p| !p.is_whitespace());
let markers: &[&'static str] = match (c, run) {
('~', 2) => &["~~"],
('*', 1) => &["*"],
('*', 2) => &["**"],
('*', 3) => {
if open.last().map(|o| o.0) == Some(SpanKind::Emph("*")) {
&["*", "**"]
} else {
&["**", "*"]
}
}
_ => &[],
};
let mut at = i;
for m in markers {
let kind = SpanKind::Emph(m);
if closes && open.last().map(|o| o.0) == Some(kind) {
let (_, start) = open.pop().expect("방금 확인했다");
spans.push(Span { kind, start, end: at + m.len() });
} else if opens {
open.push((kind, at));
}
at += m.len();
}
i += run;
}
spans.sort_by_key(|s| s.start);
spans
}
impl Default for Markup {
fn default() -> Self {
Self {
tags: Vec::new(),
fence: None,
partial: String::new(),
at_line_start: true,
spans: std::rc::Rc::from(Vec::new()),
pos: 0,
dropped: false,
fresh: false,
}
}
}
impl Markup {
fn feed(&mut self, s: &str, v: &Vocab) {
if v.channel == Channel::TelegramHtml {
self.feed_html(s);
} else {
self.feed_fence(s);
self.pos += s.chars().count();
}
}
fn open_spans(&self) -> impl Iterator<Item = &Span> {
let pos = self.pos;
let dropped = self.dropped;
self.spans.iter().filter(move |sp| !dropped && sp.start < pos && pos < sp.end)
}
fn feed_html(&mut self, s: &str) {
let joined;
let text = if self.partial.is_empty() {
s
} else {
joined = format!("{}{}", self.partial, s);
self.partial.clear();
&joined
};
let ch: Vec<char> = text.chars().collect();
let mut i = 0;
while i < ch.len() {
if ch[i] != '<' {
i += 1;
continue;
}
let start = i;
let closing = ch.get(i + 1) == Some(&'/');
let from = if closing { i + 2 } else { i + 1 };
let mut j = from;
while j < ch.len() && (ch[j].is_ascii_alphanumeric() || ch[j] == '-') {
j += 1;
}
let name: String = ch[from..j].iter().collect();
while j < ch.len() && ch[j] != '>' {
j += 1;
}
if j >= ch.len() {
self.partial = ch[start..].iter().collect();
return;
}
if name.is_empty() {
i += 1;
continue;
}
if closing {
if let Some(at) = self.tags.iter().rposition(|(n, _)| *n == name) {
self.tags.truncate(at);
}
} else {
self.tags.push((name, ch[start..=j].iter().collect()));
}
i = j + 1;
}
}
fn feed_fence(&mut self, s: &str) {
for (i, line) in s.split('\n').enumerate() {
if i == 0 && !self.at_line_start {
continue;
}
let t = line.trim_start();
if t.starts_with("```") {
self.fence = match self.fence {
Some(_) => None,
None => Some(t.trim_start_matches('`').to_string()),
};
}
}
if !s.is_empty() {
self.at_line_start = s.ends_with('\n') || (self.at_line_start && s.trim().is_empty());
}
}
fn reserve(&self, v: &Vocab) -> usize {
let mut n: usize = self.tags.iter().map(|(name, full)| name.chars().count() + 3 + full.chars().count()).sum();
if let Some(info) = &self.fence {
n += 4 + 4 + info.chars().count();
}
n += self.open_spans().map(|sp| sp.kind.reopen_len()).sum::<usize>();
let _ = v;
n
}
fn close_all(&self, out: &mut String, v: &Vocab) {
let open: Vec<&Span> = self.open_spans().collect();
for sp in open.iter().rev() {
sp.kind.write_close(out);
}
for (name, _) in self.tags.iter().rev() {
out.push_str("</");
out.push_str(name);
out.push('>');
}
if self.fence.is_some() {
v.verbatim_close("", out);
}
}
fn forget(&mut self) {
self.tags.clear();
self.fence = None;
self.partial.clear();
self.dropped = true;
}
fn reopen(&self, out: &mut String, _v: &Vocab) {
if let Some(info) = &self.fence {
out.push_str("```");
out.push_str(info);
out.push('\n');
}
for (_, full) in &self.tags {
out.push_str(full);
}
for sp in self.open_spans() {
sp.kind.write_open(out);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn html(limit: usize, text: &str) -> Vec<String> {
split_hard(text, limit, &Vocab::new(Channel::TelegramHtml))
}
fn balanced(part: &str) -> bool {
let ch: Vec<char> = part.chars().collect();
let mut stack: Vec<String> = Vec::new();
let mut i = 0;
while i < ch.len() {
if ch[i] != '<' {
i += 1;
continue;
}
let closing = ch.get(i + 1) == Some(&'/');
let from = if closing { i + 2 } else { i + 1 };
let mut j = from;
while j < ch.len() && ch[j].is_ascii_alphanumeric() {
j += 1;
}
let name: String = ch[from..j].iter().collect();
while j < ch.len() && ch[j] != '>' {
j += 1;
}
if j >= ch.len() {
return false; }
if closing {
if stack.pop().as_deref() != Some(name.as_str()) {
return false;
}
} else {
stack.push(name);
}
i = j + 1;
}
stack.is_empty()
}
#[test]
fn splitting_makes_progress_inside_open_markup() {
let text = format!("<blockquote>글 {} 끝</blockquote>", "y".repeat(600));
let parts = html(200, &text);
assert!(parts.len() < 10, "조각이 {}개 — 진도가 안 나갔다", parts.len());
for (i, p) in parts.iter().enumerate() {
assert!(balanced(p), "조각 {i} 의 태그가 안 맞는다: {p:?}");
}
assert!(parts.concat().contains("끝"), "뒤쪽 내용이 사라졌다");
}
#[test]
fn splitting_gives_up_on_markup_that_cannot_reopen() {
let url = "https://e.com/".to_string() + &"x".repeat(400);
let text = format!("<a href=\"{url}\">아주 긴 링크</a> 뒤에 오는 글");
let parts = html(200, &text);
assert!(parts.len() < 10, "조각이 {}개 — 진도가 안 나갔다", parts.len());
assert!(parts.concat().contains("뒤에 오는 글"), "뒤쪽 내용이 사라졌다");
}
}