use anyhow::anyhow;
use axum::body::{Body, Bytes};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use axum::Json;
use futures::StreamExt;
use reqwest::Client;
use serde::Serialize;
use tokio::time::{Duration, Instant};
use super::sched::ActivityGuard;
use super::types::*;
use super::{post_chat, AppError, Target};
pub(super) fn bytes_to_lines(
stream: impl futures::Stream<Item = reqwest::Result<Bytes>> + Send + 'static,
) -> impl futures::Stream<Item = String> + Send + 'static {
futures::stream::unfold(
(stream.boxed(), Vec::<u8>::new()),
|(mut stream, mut buf)| async move {
loop {
if let Some(pos) = buf.iter().position(|&b| b == b'\n') {
let line = String::from_utf8_lossy(&buf[..pos])
.trim_end_matches('\r')
.to_string();
buf.drain(..=pos);
return Some((line, (stream, buf)));
}
match futures::StreamExt::next(&mut stream).await {
Some(Ok(chunk)) => buf.extend_from_slice(&chunk),
Some(Err(_)) | None => {
if buf.is_empty() {
return None;
}
let line = String::from_utf8_lossy(&buf).into_owned();
buf.clear();
return Some((line, (stream, buf)));
}
}
}
},
)
}
pub(super) fn oai_chunk_to_content(
payload: &str,
) -> Option<(String, Option<String>, Option<String>)> {
if payload == "[DONE]" {
return Some((String::new(), None, Some(String::new())));
}
let chunk = serde_json::from_str::<OAIChunk>(payload).ok()?;
let choice = chunk.choices.first()?;
let content = choice.delta.content.as_deref().unwrap_or("").to_string();
let thinking = choice
.delta
.reasoning_content
.clone()
.or_else(|| choice.delta.reasoning.clone())
.or_else(|| choice.delta.thinking.clone())
.filter(|s| !s.is_empty());
let finish = choice
.finish_reason
.clone()
.filter(|r| !r.is_empty() && r != "null");
Some((content, thinking, finish))
}
pub(super) enum RawContentExtractor {
Undetermined(String),
Passthrough,
Harmony(Box<crate::harmony::HarmonyMessageHandler>),
PlainThink(Box<crate::thinking::Parser>),
}
pub(super) const CANDIDATE_TAGS: [&str; 3] = ["<|start|>", "<|channel|>", "<think>"];
impl RawContentExtractor {
pub(super) fn new() -> Self {
RawContentExtractor::Undetermined(String::new())
}
pub(super) fn process(
&mut self,
content: String,
backend_thinking: Option<String>,
) -> (String, Option<String>) {
if backend_thinking.is_some() {
let buffered = self.flush();
*self = RawContentExtractor::Passthrough;
return (buffered + &content, backend_thinking);
}
match self {
RawContentExtractor::Passthrough => (content, None),
RawContentExtractor::Harmony(h) => {
let (c, t, tool) = h.add_content(&content);
(c, non_empty_thinking(t, tool))
}
RawContentExtractor::PlainThink(p) => {
let (t, c) = p.add_content(&content);
(c, (!t.is_empty()).then_some(t))
}
RawContentExtractor::Undetermined(buf) => {
buf.push_str(&content);
let trimmed = buf.trim_start();
if trimmed.is_empty()
|| CANDIDATE_TAGS
.iter()
.any(|tag| tag.starts_with(trimmed) && trimmed.len() < tag.len())
{
return (String::new(), None);
}
let buffered = std::mem::take(buf);
let trimmed_starts_with = |tag: &str| buffered.trim_start().starts_with(tag);
if trimmed_starts_with("<|start|>") || trimmed_starts_with("<|channel|>") {
let mut h = crate::harmony::HarmonyMessageHandler::new();
if trimmed_starts_with("<|channel|>") {
h.parser.add_implicit_start();
}
let (c, t, tool) = h.add_content(&buffered);
let thinking = non_empty_thinking(t, tool);
*self = RawContentExtractor::Harmony(Box::new(h));
(c, thinking)
} else {
let mut p = crate::thinking::Parser::new("<think>", "</think>");
let (t, c) = p.add_content(&buffered);
*self = RawContentExtractor::PlainThink(Box::new(p));
(c, (!t.is_empty()).then_some(t))
}
}
}
}
pub(super) fn flush(&mut self) -> String {
match self {
RawContentExtractor::Undetermined(buf) => std::mem::take(buf),
_ => String::new(),
}
}
}
pub(super) fn non_empty_thinking(thinking: String, tool: String) -> Option<String> {
let combined = thinking + &tool;
(!combined.is_empty()).then_some(combined)
}
#[derive(Default, Debug)]
pub(super) struct OllamaDelta {
pub(super) content: String,
pub(super) thinking: Option<String>,
pub(super) tool_calls: Option<Vec<OllamaToolCall>>,
pub(super) done: bool,
pub(super) done_reason: Option<String>,
pub(super) metrics: OllamaMetrics,
}
pub(super) struct OllamaLineDecoder {
pub(super) tool_calls_acc:
std::cell::RefCell<std::collections::BTreeMap<usize, ToolCallAccumulator>>,
pub(super) content_extractor: std::cell::RefCell<RawContentExtractor>,
pub(super) finish_reason: std::cell::RefCell<Option<String>>,
pub(super) usage: std::cell::Cell<Option<OAIUsage>>,
pub(super) timings: std::cell::Cell<Option<LlamaTimings>>,
pub(super) done_sent: std::cell::Cell<bool>,
pub(super) started: Instant,
pub(super) load_duration: Duration,
}
impl OllamaLineDecoder {
pub(super) fn new(started: Instant, load_duration: Duration) -> Self {
Self {
tool_calls_acc: std::cell::RefCell::new(std::collections::BTreeMap::new()),
content_extractor: std::cell::RefCell::new(RawContentExtractor::new()),
finish_reason: std::cell::RefCell::new(None),
usage: std::cell::Cell::new(None),
timings: std::cell::Cell::new(None),
done_sent: std::cell::Cell::new(false),
started,
load_duration,
}
}
pub(super) fn decode(&self, line: &str) -> Option<OllamaDelta> {
let payload = line.strip_prefix("data: ")?;
accumulate_tool_call_deltas(payload, &self.tool_calls_acc);
if let Ok(chunk) = serde_json::from_str::<OAIChunk>(payload) {
if let Some(usage) = chunk.usage {
self.usage.set(Some(usage));
}
if let Some(timings) = chunk.timings {
self.timings.set(Some(timings));
}
}
let (content, thinking, finish) = oai_chunk_to_content(payload)?;
let (content, thinking) = self
.content_extractor
.borrow_mut()
.process(content, thinking);
match finish {
Some(reason) if reason.is_empty() => {
if self.finish_reason.borrow().is_none() {
return Some(OllamaDelta {
content,
thinking,
..Default::default()
});
}
Some(self.done_delta(content, thinking))
}
Some(reason) => {
*self.finish_reason.borrow_mut() = Some(reason);
Some(OllamaDelta {
content,
thinking,
..Default::default()
})
}
None => Some(OllamaDelta {
content,
thinking,
..Default::default()
}),
}
}
pub(super) fn finish(&self) -> Option<OllamaDelta> {
if self.finish_reason.borrow().is_none() {
return None;
}
(!self.done_sent.get()).then(|| self.done_delta(String::new(), None))
}
pub(super) fn done_delta(&self, mut content: String, thinking: Option<String>) -> OllamaDelta {
if self.done_sent.replace(true) {
return OllamaDelta {
content,
thinking,
..Default::default()
};
}
content.push_str(&self.content_extractor.borrow_mut().flush());
let drained = std::mem::take(&mut *self.tool_calls_acc.borrow_mut());
let reason = self.finish_reason.borrow().clone().unwrap_or_default();
OllamaDelta {
content,
thinking,
tool_calls: finalize_tool_calls(&drained),
done: true,
done_reason: Some(ollama_done_reason(&reason)),
metrics: self.metrics(),
}
}
pub(super) fn metrics(&self) -> OllamaMetrics {
let ns = |ms: f64| (ms * 1_000_000.0) as u64;
let usage = self.usage.get();
let timings = self.timings.get();
let total = self.started.elapsed();
let cached = usage.map(|u| {
let details = &u.prompt_tokens_details;
let read = details.cached_tokens.unwrap_or(0).min(u.prompt_tokens);
let written = details
.cache_write_tokens
.unwrap_or(0)
.min(u.prompt_tokens - read);
(
u.prompt_tokens,
details.cached_tokens.map(|_| read),
written,
)
});
OllamaMetrics {
total_duration: Some(total.as_nanos() as u64),
load_duration: Some(self.load_duration.as_nanos() as u64),
prompt_eval_count: timings
.map(|t| t.prompt_n.saturating_add(t.cache_n.unwrap_or(0)))
.or(cached.map(|c| c.0)),
prompt_eval_cached_count: timings.and_then(|t| t.cache_n).or(cached.and_then(|c| c.1)),
prompt_eval_cache_write_count: cached.map(|c| c.2).filter(|&n| n > 0),
prompt_eval_duration: timings.map(|t| ns(t.prompt_ms)),
eval_count: timings
.map(|t| t.predicted_n)
.or(usage.map(|u| u.completion_tokens)),
eval_duration: timings.map(|t| ns(t.predicted_ms)),
}
}
}
pub(super) fn ollama_done_reason(finish_reason: &str) -> String {
match finish_reason {
"length" => "length".to_string(),
_ => "stop".to_string(),
}
}
#[derive(Default)]
pub(super) struct OllamaFold {
pub(super) content: String,
pub(super) thinking: Option<String>,
pub(super) tool_calls: Option<Vec<OllamaToolCall>>,
pub(super) done: bool,
pub(super) done_reason: Option<String>,
pub(super) metrics: OllamaMetrics,
}
impl OllamaFold {
pub(super) fn push(&mut self, delta: OllamaDelta) {
self.content.push_str(&delta.content);
if let Some(t) = delta.thinking {
self.thinking.get_or_insert_with(String::new).push_str(&t);
}
if delta.done {
self.tool_calls = delta.tool_calls;
self.done = true;
self.done_reason = delta.done_reason;
self.metrics = delta.metrics;
}
}
pub(super) fn into_delta(self) -> OllamaDelta {
OllamaDelta {
content: self.content,
thinking: self.thinking,
tool_calls: self.tool_calls,
done: true,
done_reason: self.done_reason,
metrics: self.metrics,
}
}
}
#[cfg(test)]
pub(super) fn fold_ollama_lines<I: IntoIterator<Item = String>>(lines: I) -> OllamaFold {
let decoder = OllamaLineDecoder::new(Instant::now(), Duration::ZERO);
let mut fold = OllamaFold::default();
for line in lines {
if let Some(delta) = decoder.decode(&line) {
fold.push(delta);
}
}
if let Some(delta) = decoder.finish() {
fold.push(delta);
}
fold
}
#[allow(clippy::too_many_arguments)]
pub(super) async fn stream_ollama<T: Serialize + Send + 'static>(
streaming: bool,
client: Client,
target: Target,
mut oai_req: OAIChatRequest,
activity: ActivityGuard,
started: Instant,
load_duration: Duration,
build_chunk: impl Fn(OllamaDelta) -> T + Send + 'static,
) -> Result<Response, AppError> {
oai_req.stream_options = Some(serde_json::json!({ "include_usage": true }));
let resp = post_chat(&client, &target, &mut oai_req).await?;
if !streaming {
let _activity = activity;
let decoder = OllamaLineDecoder::new(started, load_duration);
let mut fold = OllamaFold::default();
let mut lines = Box::pin(bytes_to_lines(resp));
while let Some(line) = lines.next().await {
if let Some(delta) = decoder.decode(&line) {
fold.push(delta);
}
}
if let Some(delta) = decoder.finish() {
fold.push(delta);
}
if !fold.done {
return Err(AppError(
anyhow!("inference backend closed the connection before finishing"),
StatusCode::BAD_GATEWAY,
));
}
return Ok(Json(build_chunk(fold.into_delta())).into_response());
}
let decoder = OllamaLineDecoder::new(started, load_duration);
let stream = bytes_to_lines(resp)
.map(Some)
.chain(futures::stream::once(futures::future::ready(None)))
.map(move |line| {
let _activity = &activity;
let delta = match line {
Some(line) => decoder.decode(&line),
None => decoder.finish(),
};
let out = delta
.map(|delta| serde_json::to_string(&build_chunk(delta)).unwrap_or_default() + "\n")
.unwrap_or_default();
Ok::<_, std::convert::Infallible>(Bytes::from(out))
});
Ok(Response::builder()
.header("content-type", "application/x-ndjson")
.body(Body::from_stream(stream))
.unwrap())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn draining_the_accumulator_on_finalize_prevents_delivering_a_tool_call_twice() {
let acc = std::cell::RefCell::new(std::collections::BTreeMap::new());
accumulate_tool_call_deltas(
r#"{"choices":[{"delta":{"tool_calls":[
{"index":0,"function":{"name":"get_weather","arguments":"{}"}}
]},"finish_reason":"tool_calls"}]}"#,
&acc,
);
let first = finalize_tool_calls(&std::mem::take(&mut *acc.borrow_mut()));
assert!(
first.is_some(),
"the first done signal must still deliver the tool call"
);
let second = finalize_tool_calls(&std::mem::take(&mut *acc.borrow_mut()));
assert_eq!(
second, None,
"a second done signal (the trailing [DONE] line) must not re-deliver it"
);
}
#[test]
fn fold_ollama_lines_concatenates_deltas() {
let lines = [
"",
r#"data: {"choices":[{"delta":{"reasoning_content":"let me "},"finish_reason":null}]}"#,
r#"data: {"choices":[{"delta":{"reasoning_content":"think"},"finish_reason":null}]}"#,
r#"data: {"choices":[{"delta":{"content":"Hello"},"finish_reason":null}]}"#,
": keep-alive",
r#"data: {"choices":[{"delta":{"content":", world"},"finish_reason":"stop"}]}"#,
"data: [DONE]",
]
.map(String::from);
let fold = fold_ollama_lines(lines);
assert_eq!(fold.content, "Hello, world");
assert_eq!(fold.thinking.as_deref(), Some("let me think"));
assert_eq!(fold.tool_calls, None);
assert!(fold.done);
}
#[test]
fn fold_ollama_lines_keeps_tool_calls_across_a_second_done_line() {
let lines = [
r#"data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_abc","function":{"name":"get_weather","arguments":"{\"city\":"}}]},"finish_reason":null}]}"#,
r#"data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"Ankara\"}"}}]},"finish_reason":null}]}"#,
r#"data: {"choices":[{"delta":{"content":""},"finish_reason":"tool_calls"}]}"#,
r#"data: {"choices":[],"usage":{"prompt_tokens":12,"completion_tokens":7}}"#,
"data: [DONE]",
]
.map(String::from);
let fold = fold_ollama_lines(lines);
let calls = fold.tool_calls.expect("survive the trailing [DONE]");
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].id.as_deref(), Some("call_abc"));
assert_eq!(calls[0].function.index, 0);
assert_eq!(calls[0].function.name, "get_weather");
assert_eq!(calls[0].function.arguments["city"], "Ankara");
assert_eq!(fold.done_reason.as_deref(), Some("stop"));
assert_eq!(fold.metrics.prompt_eval_count, Some(12));
assert_eq!(fold.metrics.eval_count, Some(7));
assert!(fold.metrics.total_duration.is_some());
}
#[test]
fn fold_ollama_lines_keeps_a_tool_call_sent_without_an_index() {
let lines = [
r#"data: {"choices":[{"delta":{"role":"assistant","tool_calls":[{"id":"0","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}]},"finish_reason":"tool_calls","index":0}]}"#,
r#"data: {"choices":[],"usage":{"prompt_tokens":20,"completion_tokens":5}}"#,
"data: [DONE]",
]
.map(String::from);
let fold = fold_ollama_lines(lines);
assert!(fold.done, "the finish chunk was dropped");
let calls = fold.tool_calls.expect("the tool call was dropped");
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].id.as_deref(), Some("0"));
assert_eq!(calls[0].function.index, 0);
assert_eq!(calls[0].function.name, "get_weather");
assert_eq!(calls[0].function.arguments["city"], "Paris");
}
#[test]
fn a_providers_cached_prefix_is_part_of_the_prompt_count() {
let metrics = |usage: &str| {
let lines = [
r#"data: {"choices":[{"delta":{"content":"a"},"finish_reason":"stop"}]}"#
.to_string(),
format!(r#"data: {{"choices":[],"usage":{usage}}}"#),
"data: [DONE]".to_string(),
];
fold_ollama_lines(lines).metrics
};
let m = metrics(
r#"{"prompt_tokens":100,"completion_tokens":2,"prompt_tokens_details":{"cached_tokens":80,"cache_write_tokens":15}}"#,
);
assert_eq!(m.prompt_eval_count, Some(100));
assert_eq!(m.prompt_eval_cached_count, Some(80));
assert_eq!(m.prompt_eval_cache_write_count, Some(15));
let m = metrics(r#"{"prompt_tokens":10,"prompt_tokens_details":{"cached_tokens":80}}"#);
assert_eq!(
(m.prompt_eval_count, m.prompt_eval_cached_count),
(Some(10), Some(10))
);
assert_eq!(m.prompt_eval_cache_write_count, None);
}
#[test]
fn a_cache_hit_leaves_the_prompt_count_whole() {
let counts = |timings: &str| {
let lines = [
r#"data: {"choices":[{"delta":{"content":"a"},"finish_reason":"stop"}]}"#
.to_string(),
format!(r#"data: {{"choices":[],"timings":{timings}}}"#),
"data: [DONE]".to_string(),
];
let m = fold_ollama_lines(lines).metrics;
(m.prompt_eval_count, m.prompt_eval_cached_count)
};
let timings = |prompt_n: u64, cache_n: &str| {
format!(
r#"{{"prompt_n":{prompt_n},"prompt_ms":1.0,{cache_n}"predicted_n":1,"predicted_ms":1.0}}"#
)
};
assert_eq!(
counts(&timings(5422, r#""cache_n":0,"#)),
(Some(5422), Some(0))
);
assert_eq!(
counts(&timings(4, r#""cache_n":5418,"#)),
(Some(5422), Some(5418))
);
assert_eq!(counts(&timings(7, "")), (Some(7), None));
}
#[test]
fn the_decoder_emits_one_done_chunk() {
let decoder = OllamaLineDecoder::new(Instant::now(), Duration::from_millis(5));
assert!(
!decoder
.decode(r#"data: {"choices":[{"delta":{"content":"a"},"finish_reason":null}]}"#)
.unwrap()
.done
);
let finish = decoder
.decode(
r#"data: {"choices":[{"delta":{"content":"b"},"finish_reason":"length"}],"timings":{"prompt_n":3,"prompt_ms":10.0,"cache_n":1,"predicted_n":2,"predicted_ms":20.0}}"#,
)
.unwrap();
assert_eq!(finish.content, "b");
assert!(!finish.done, "the finish chunk is not the done chunk");
let done = decoder.decode("data: [DONE]").unwrap();
assert!(done.done);
assert_eq!(done.done_reason.as_deref(), Some("length"));
assert_eq!(done.metrics.prompt_eval_count, Some(4));
assert_eq!(done.metrics.prompt_eval_cached_count, Some(1));
assert_eq!(done.metrics.eval_count, Some(2));
assert_eq!(done.metrics.prompt_eval_duration, Some(10_000_000));
assert_eq!(done.metrics.eval_duration, Some(20_000_000));
assert_eq!(done.metrics.load_duration, Some(5_000_000));
assert!(!decoder.decode("data: [DONE]").unwrap().done);
assert!(decoder.finish().is_none());
let without_done = OllamaLineDecoder::new(Instant::now(), Duration::ZERO);
without_done
.decode(r#"data: {"choices":[{"delta":{"content":"a"},"finish_reason":"stop"}]}"#);
let closed = without_done
.finish()
.expect("a seen finish closes the stream");
assert!(closed.done);
assert_eq!(closed.done_reason.as_deref(), Some("stop"));
assert!(without_done.finish().is_none());
let cut = OllamaLineDecoder::new(Instant::now(), Duration::ZERO);
assert!(!cut.decode("data: [DONE]").unwrap().done);
assert!(cut.finish().is_none());
}
#[test]
fn fold_ollama_lines_handles_an_empty_response() {
let lines = [r#"data: {"choices":[{"delta":{"content":""},"finish_reason":"stop"}]}"#]
.map(String::from);
let fold = fold_ollama_lines(lines);
assert!(fold.content.is_empty() && fold.thinking.is_none() && fold.tool_calls.is_none());
assert!(fold.done);
}
#[test]
fn fold_ollama_lines_reports_a_missing_terminal_chunk() {
let lines =
[r#"data: {"choices":[{"delta":{"content":"half a sen"},"finish_reason":null}]}"#]
.map(String::from);
let fold = fold_ollama_lines(lines);
assert_eq!(fold.content, "half a sen");
assert!(!fold.done, "no terminal chunk means the reply is truncated");
}
#[test]
fn oai_chunk_to_content_ported_ollama_stream_decoding_cases() {
assert_eq!(
oai_chunk_to_content(
r#"{"choices":[{"delta":{"content":"hi"},"finish_reason":null}]}"#
),
Some(("hi".into(), None, None))
);
assert_eq!(
oai_chunk_to_content(
r#"{"choices":[{"delta":{"content":""},"finish_reason":"stop"}]}"#
),
Some((String::new(), None, Some("stop".into())))
);
assert_eq!(
oai_chunk_to_content("[DONE]"),
Some((String::new(), None, Some(String::new())))
);
for field in ["reasoning_content", "reasoning", "thinking"] {
assert_eq!(
oai_chunk_to_content(&format!(
r#"{{"choices":[{{"delta":{{"{field}":"hmm"}},"finish_reason":null}}]}}"#
)),
Some((String::new(), Some("hmm".into()), None))
);
}
assert_eq!(
oai_chunk_to_content(
r#"{"choices":[{"delta":{"content":"x","reasoning_content":""},"finish_reason":null}]}"#
),
Some(("x".into(), None, None))
);
assert_eq!(oai_chunk_to_content("not json"), None);
assert_eq!(oai_chunk_to_content(r#"{"choices":[]}"#), None);
}
#[test]
fn raw_content_extractor_passes_plain_content_through_untouched() {
let mut ext = RawContentExtractor::new();
assert_eq!(
ext.process("hello there".into(), None),
("hello there".into(), None)
);
assert_eq!(
ext.process(" friend".into(), None),
(" friend".into(), None)
);
}
#[test]
fn raw_content_extractor_locks_into_passthrough_once_backend_thinking_seen() {
let mut ext = RawContentExtractor::new();
assert_eq!(
ext.process(String::new(), Some("reasoning".into())),
(String::new(), Some("reasoning".into()))
);
assert_eq!(
ext.process("<think>literal text</think>".into(), None),
("<think>literal text</think>".into(), None)
);
}
#[test]
fn raw_content_extractor_recovers_a_buffered_prefix_when_backend_thinking_appears_later() {
let mut ext = RawContentExtractor::new();
assert_eq!(ext.process("<".into(), None), (String::new(), None));
assert_eq!(
ext.process("hello".into(), Some("reasoning".into())),
("<hello".into(), Some("reasoning".into()))
);
assert_eq!(ext.flush(), "");
}
#[test]
fn raw_content_extractor_falls_back_to_plain_think_tags() {
let mut ext = RawContentExtractor::new();
let (c1, t1) = ext.process("<think>".into(), None);
assert_eq!((c1, t1), (String::new(), None));
let (c2, t2) = ext.process("hmm".into(), None);
assert_eq!((c2, t2), (String::new(), Some("hmm".into())));
let (c3, t3) = ext.process("</think>answer".into(), None);
assert_eq!((c3, t3), ("answer".into(), None));
}
#[test]
fn raw_content_extractor_falls_back_to_harmony_channels() {
let mut ext = RawContentExtractor::new();
let (content, thinking) = ext.process(
"<|start|>assistant<|channel|>analysis<|message|>thinking...<|end|>\
<|start|>assistant<|channel|>final<|message|>the answer<|end|>"
.into(),
None,
);
assert_eq!(content, "the answer");
assert_eq!(thinking, Some("thinking...".into()));
}
#[test]
fn raw_content_extractor_leaves_content_without_any_tag_untouched() {
let mut ext = RawContentExtractor::new();
let (content, thinking) = ext.process("just a normal reply".into(), None);
assert_eq!(content, "just a normal reply");
assert_eq!(thinking, None);
}
#[test]
fn raw_content_extractor_buffers_across_calls_to_classify_a_token_split_harmony_stream() {
let mut ext = RawContentExtractor::new();
let whole = "<|start|>assistant<|channel|>analysis<|message|>thinking...<|end|>\
<|start|>assistant<|channel|>final<|message|>the answer<|end|>";
let mut content = String::new();
let mut thinking = String::new();
for ch in whole.chars() {
let mut buf = [0u8; 4];
let (c, t) = ext.process(ch.encode_utf8(&mut buf).to_string(), None);
content.push_str(&c);
if let Some(t) = t {
thinking.push_str(&t);
}
}
assert_eq!(content, "the answer");
assert_eq!(thinking, "thinking...");
}
#[test]
fn raw_content_extractor_primes_harmony_when_a_stream_starts_mid_message() {
let mut ext = RawContentExtractor::new();
let (content, thinking) = ext.process(
"<|channel|>analysis<|message|>thinking...<|end|>\
<|start|>assistant<|channel|>final<|message|>the answer<|end|>"
.into(),
None,
);
assert_eq!(content, "the answer");
assert_eq!(thinking, Some("thinking...".into()));
}
#[test]
fn raw_content_extractor_flush_recovers_a_buffered_prefix_at_stream_end() {
let mut ext = RawContentExtractor::new();
let (content, thinking) = ext.process("<".into(), None);
assert_eq!(content, "");
assert_eq!(thinking, None);
assert_eq!(ext.flush(), "<");
assert_eq!(ext.flush(), "");
}
#[test]
fn raw_content_extractor_flush_is_a_no_op_once_a_mode_is_decided() {
let mut ext = RawContentExtractor::new();
ext.process("just a normal reply".into(), None);
assert_eq!(ext.flush(), "");
let mut ext = RawContentExtractor::new();
ext.process(String::new(), Some("reasoning".into()));
assert_eq!(ext.flush(), "");
}
#[test]
fn bytes_to_lines_preserves_utf8_split_across_chunks() {
let line = "data: g\u{fc}nayd\u{131}n \u{1f600}";
let raw = format!("{line}\n");
let chunks: Vec<reqwest::Result<Bytes>> = raw
.as_bytes()
.iter()
.map(|b| Ok(Bytes::copy_from_slice(&[*b])))
.collect();
let stream = bytes_to_lines(futures::stream::iter(chunks));
let lines: Vec<String> = futures::executor::block_on(StreamExt::collect::<Vec<_>>(stream));
assert_eq!(lines, vec![line.to_string()]);
}
#[test]
fn bytes_to_lines_ported_ollama_client_stream_chunking() {
let chunks: Vec<reqwest::Result<Bytes>> = vec![
Ok(Bytes::from("data: {\"a\":")),
Ok(Bytes::from("1}\r\ndata: {\"b\":2}\n")),
Ok(Bytes::from("data: tail")),
];
let stream = bytes_to_lines(futures::stream::iter(chunks));
let lines: Vec<String> = futures::executor::block_on(StreamExt::collect::<Vec<_>>(stream));
assert_eq!(
lines,
vec![
"data: {\"a\":1}".to_string(),
"data: {\"b\":2}".to_string(),
"data: tail".to_string(),
]
);
}
}