#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum State {
LookingForOpening,
ThinkingStartedEatingWhitespace,
Thinking,
ThinkingDoneEatingWhitespace,
ThinkingDone,
}
#[derive(Debug, Clone)]
pub struct Parser {
state: State,
pub opening_tag: String,
pub closing_tag: String,
acc: String,
}
impl Parser {
pub fn new(opening_tag: impl Into<String>, closing_tag: impl Into<String>) -> Self {
Self {
state: State::LookingForOpening,
opening_tag: opening_tag.into(),
closing_tag: closing_tag.into(),
acc: String::new(),
}
}
pub fn add_content(&mut self, content: &str) -> (String, String) {
self.acc.push_str(content);
let mut thinking_out = String::new();
let mut remaining_out = String::new();
loop {
let (thinking, remaining, keep_looping) = self.eat();
thinking_out.push_str(&thinking);
remaining_out.push_str(&remaining);
if !keep_looping {
break;
}
}
(thinking_out, remaining_out)
}
fn eat(&mut self) -> (String, String, bool) {
match self.state {
State::LookingForOpening => {
let trimmed = self.acc.trim_start();
if let Some(after_tag) = strip_all_leading_occurrences(trimmed, &self.opening_tag) {
let after = after_tag.trim_start().to_string();
self.acc.clear();
self.acc.push_str(&after);
self.state = if after.is_empty() {
State::ThinkingStartedEatingWhitespace
} else {
State::Thinking
};
(String::new(), String::new(), true)
} else if !self.opening_tag.is_empty() && self.opening_tag.starts_with(trimmed) {
(String::new(), String::new(), false)
} else if trimmed.is_empty() {
(String::new(), String::new(), false)
} else {
self.state = State::ThinkingDone;
let untrimmed = std::mem::take(&mut self.acc);
(String::new(), untrimmed, false)
}
}
State::ThinkingStartedEatingWhitespace => {
let trimmed = self.acc.trim_start().to_string();
self.acc.clear();
if trimmed.is_empty() {
(String::new(), String::new(), false)
} else {
self.state = State::Thinking;
self.acc.push_str(&trimmed);
(String::new(), String::new(), true)
}
}
State::Thinking => {
let acc = self.acc.clone();
if let Some(pos) = acc.find(&self.closing_tag) {
let thinking = acc[..pos].to_string();
let remaining = acc[pos + self.closing_tag.len()..].to_string();
let remaining = remaining.trim_start().to_string();
self.acc.clear();
self.state = if remaining.is_empty() {
State::ThinkingDoneEatingWhitespace
} else {
State::ThinkingDone
};
(thinking, remaining, false)
} else {
let overlap_len = overlap(&acc, &self.closing_tag);
if overlap_len > 0 {
let split = acc.len() - overlap_len;
let thinking = acc[..split].to_string();
let remaining = acc[split..].to_string();
self.acc.clear();
self.acc.push_str(&remaining);
(thinking, String::new(), false)
} else {
self.acc.clear();
(acc, String::new(), false)
}
}
}
State::ThinkingDoneEatingWhitespace => {
let trimmed = self.acc.trim_start().to_string();
self.acc.clear();
if !trimmed.is_empty() {
self.state = State::ThinkingDone;
}
(String::new(), trimmed, false)
}
State::ThinkingDone => {
let acc = std::mem::take(&mut self.acc);
(String::new(), acc, false)
}
}
}
}
fn strip_all_leading_occurrences<'a>(trimmed: &'a str, tag: &str) -> Option<&'a str> {
if tag.is_empty() {
return None;
}
trimmed.strip_prefix(tag)
}
fn overlap(s: &str, delim: &str) -> usize {
let max = delim.len().min(s.len());
for i in (1..=max).rev() {
if s.ends_with(&delim[..i]) {
return i;
}
}
0
}
#[cfg(test)]
mod tests {
use super::*;
fn parser() -> Parser {
Parser::new("<think>", "</think>")
}
#[test]
fn extract_thinking_whole_string_at_once() {
let cases = [
("<think> internal </think> world", "internal ", "world"),
(
"<think>a</think><think>b</think>c",
"a",
"<think>b</think>c",
),
("no think", "", "no think"),
];
for (input, want_think, want_content) in cases {
let mut p = parser();
let (thinking, content) = p.add_content(input);
assert_eq!(thinking, want_think, "thinking for {input:?}");
assert_eq!(content, want_content, "content for {input:?}");
}
}
#[test]
fn thinking_streaming_token_by_token() {
let mut p = parser();
let steps: &[(&str, &str, &str, State)] = &[
("<think>", "", "", State::ThinkingStartedEatingWhitespace),
("\n", "", "", State::ThinkingStartedEatingWhitespace),
("</think>", "", "", State::ThinkingDoneEatingWhitespace),
("\n\n", "", "", State::ThinkingDoneEatingWhitespace),
("Hi", "", "Hi", State::ThinkingDone),
(" there", "", " there", State::ThinkingDone),
];
for (input, want_thinking, want_content, want_state) in steps {
let (thinking, content) = p.add_content(input);
assert_eq!(&thinking, want_thinking, "thinking for {input:?}");
assert_eq!(&content, want_content, "content for {input:?}");
assert_eq!(p.state, *want_state, "state after {input:?}");
}
}
#[test]
fn content_without_a_thinking_tag_passes_through_untouched() {
let mut p = parser();
let (thinking, content) = p.add_content(" abc");
assert_eq!(thinking, "");
assert_eq!(content, " abc");
assert_eq!(p.state, State::ThinkingDone);
let (thinking, content) = p.add_content("def");
assert_eq!(thinking, "");
assert_eq!(content, "def");
}
#[test]
fn content_before_a_thinking_tag_nerfs_it() {
let mut p = parser();
let (thinking, content) = p.add_content(" abc <think>def</think> ghi");
assert_eq!(thinking, "");
assert_eq!(content, " abc <think>def</think> ghi");
}
#[test]
fn partial_opening_tag_builds_up_across_calls() {
let mut p = parser();
assert_eq!(p.add_content(" <th"), ("".into(), "".into()));
assert_eq!(p.state, State::LookingForOpening);
assert_eq!(p.add_content("in"), ("".into(), "".into()));
assert_eq!(p.state, State::LookingForOpening);
assert_eq!(p.add_content("k>a"), ("a".into(), "".into()));
assert_eq!(p.state, State::Thinking);
}
#[test]
fn partial_closing_tag_fakeout_recovers() {
let mut p = parser();
assert_eq!(p.add_content("<think>abc</th"), ("abc".into(), "".into()));
assert_eq!(p.state, State::Thinking);
assert_eq!(p.add_content("ing>def"), ("</thing>def".into(), "".into()));
assert_eq!(p.state, State::Thinking);
assert_eq!(p.add_content("ghi</thi"), ("ghi".into(), "".into()));
assert_eq!(p.add_content("nk>jkl"), ("".into(), "jkl".into()));
assert_eq!(p.state, State::ThinkingDone);
}
#[test]
fn whitespace_after_closing_tag_is_eaten() {
let mut p = parser();
let (thinking, content) = p.add_content(" <think>abc</think>\n\ndef");
assert_eq!(thinking, "abc");
assert_eq!(content, "def");
}
#[test]
fn leading_thinking_whitespace_is_eaten_but_inner_preserved() {
let mut p = parser();
assert_eq!(p.add_content(" <think> \t "), ("".into(), "".into()));
assert_eq!(p.state, State::ThinkingStartedEatingWhitespace);
assert_eq!(
p.add_content(" these are some "),
("these are some ".into(), "".into())
);
assert_eq!(
p.add_content("thoughts </think> "),
("thoughts ".into(), "".into())
);
assert_eq!(p.state, State::ThinkingDoneEatingWhitespace);
assert_eq!(
p.add_content(" more content"),
("".into(), "more content".into())
);
}
#[test]
fn overlap_finds_longest_suffix_prefix_match() {
assert_eq!(overlap("abc</thi", "</think>"), 5);
assert_eq!(overlap("abcdef", "</think>"), 0);
assert_eq!(overlap("", "</think>"), 0);
}
}