1use std::collections::HashMap;
5
6use futures::StreamExt;
7use reqwest::header::{HeaderMap, HeaderValue, ACCEPT};
8use serde_json::{json, Value};
9use tokio_util::sync::CancellationToken;
10use tracing::warn;
11
12use super::stream::{
13 channel_stream, parse_streaming_json, AssistantMessageEvent, AssistantStream, SseDecoder,
14};
15use super::types::{
16 AssistantMessage, ContentBlock, Message, Model, StopReason, ThinkingLevel, ToolSchema, Usage,
17};
18
19#[derive(Debug, Clone, Default)]
21pub struct SimpleStreamOptions {
22 pub api_key: Option<String>,
23 pub auth_token: Option<String>,
25 pub max_tokens: Option<usize>,
26 pub temperature: Option<f64>,
27 pub reasoning: Option<ThinkingLevel>,
28 pub thinking_budget_tokens: Option<usize>,
29 pub cache_control: bool,
32 pub timeout_ms: Option<u64>,
33}
34
35impl SimpleStreamOptions {
36 pub fn cache_control(mut self, on: bool) -> Self {
37 self.cache_control = on;
38 self
39 }
40}
41
42pub struct LlmContext<'a> {
44 pub system_prompt: Option<&'a str>,
45 pub messages: &'a [Message],
46 pub tools: &'a [ToolSchema],
47}
48
49const ANTHROPIC_VERSION: &str = "2023-06-01";
50
51#[derive(Default)]
54struct BlockScratch {
55 our_index: usize,
56 kind: String,
57 partial_json: String,
58}
59
60pub fn stream_simple(
67 client: &reqwest::Client,
68 model: &Model,
69 context: &LlmContext<'_>,
70 options: &SimpleStreamOptions,
71 cancel: CancellationToken,
72) -> AssistantStream {
73 let body = build_request_body(model, context, options);
74 let headers = build_headers(model, options);
75 let url = format!("{}/v1/messages", model.base_url.trim_end_matches('/'));
76 let model = model.clone();
77 let timeout = options.timeout_ms;
78 let cancel_inner = cancel.clone();
79 let (tx, stream) = channel_stream(64);
80
81 let client = client.clone();
82 tokio::spawn(async move {
83 let cancel = cancel_inner;
84 let req = client
85 .post(&url)
86 .headers(headers)
87 .json(&body);
88 let req = if let Some(ms) = timeout {
89 req.timeout(std::time::Duration::from_millis(ms))
90 } else {
91 req
92 };
93
94 let response = tokio::select! {
95 _ = cancel.cancelled() => {
96 emit_terminal(&tx, StopReason::Aborted, "Request aborted").await;
97 return;
98 }
99 res = req.send() => match res {
100 Ok(r) => r,
101 Err(e) => {
102 emit_terminal(&tx, StopReason::Error, &format!("Request failed: {e}")).await;
103 return;
104 }
105 },
106 };
107
108 let status = response.status();
109 if !status.is_success() {
110 let text = response.text().await.unwrap_or_default();
111 emit_terminal(
112 &tx,
113 StopReason::Error,
114 &format!("Anthropic API error ({}): {}", status, truncate(&text, 2000)),
115 )
116 .await;
117 return;
118 }
119
120 let _ = tx.send(AssistantMessageEvent::Start).await;
121 run_body(response, &model, cancel, tx).await;
122 });
123
124 stream
125}
126
127async fn run_body(
128 response: reqwest::Response,
129 model: &Model,
130 cancel: CancellationToken,
131 tx: tokio::sync::mpsc::Sender<AssistantMessageEvent>,
132) {
133 let mut output = AssistantMessage::empty();
134 output.model = model.id.clone();
135 output.provider = model.provider.clone();
136
137 let mut decoder = SseDecoder::new();
138 let mut blocks: HashMap<usize, BlockScratch> = HashMap::new();
139 let mut saw_start = false;
140 let mut saw_stop = false;
141
142 let mut body = response.bytes_stream();
143 let mut stream_error: Option<String> = None;
144
145 loop {
146 let chunk = tokio::select! {
147 _ = cancel.cancelled() => {
148 emit_terminal(&tx, StopReason::Aborted, "Request aborted").await;
149 return;
150 }
151 c = body.next() => c,
152 };
153 match chunk {
154 Some(Ok(bytes)) => {
155 for ev in decoder.feed(&bytes) {
156 handle_sse(
157 ev, &mut output, &mut blocks, &mut saw_start, &mut saw_stop, &tx,
158 )
159 .await;
160 }
161 }
162 Some(Err(e)) => {
163 stream_error = Some(format!("stream read error: {e}"));
164 break;
165 }
166 None => break,
167 }
168 }
169
170 for ev in decoder.finish() {
171 handle_sse(
172 ev, &mut output, &mut blocks, &mut saw_start, &mut saw_stop, &tx,
173 )
174 .await;
175 }
176
177 if let Some(msg) = stream_error {
178 if saw_start && !saw_stop {
179 warn!(error = %msg, "anthropic stream ended prematurely");
180 }
181 }
182
183 if saw_start && !saw_stop {
184 finalize_message(&mut output, model, StopReason::Error);
185 output.error_message = Some("Anthropic stream ended before message_stop".into());
186 let _ = tx
187 .send(AssistantMessageEvent::Error {
188 reason: StopReason::Error,
189 message: output,
190 })
191 .await;
192 return;
193 }
194
195 let reason = if cancel.is_cancelled() {
196 StopReason::Aborted
197 } else if output.stop_reason == StopReason::default() {
198 StopReason::Stop
199 } else {
200 output.stop_reason
201 };
202 output.stop_reason = reason;
203 finalize_message(&mut output, model, reason);
204 let _ = tx
205 .send(AssistantMessageEvent::Done {
206 reason,
207 message: output,
208 })
209 .await;
210}
211
212async fn handle_sse(
213 sse: super::stream::ServerSentEvent,
214 output: &mut AssistantMessage,
215 blocks: &mut HashMap<usize, BlockScratch>,
216 saw_start: &mut bool,
217 saw_stop: &mut bool,
218 tx: &tokio::sync::mpsc::Sender<AssistantMessageEvent>,
219) {
220 if sse.event.as_deref() == Some("error") {
221 output.error_message = Some(sse.data.clone());
223 return;
224 }
225 let parsed: Value = match serde_json::from_str(&sse.data) {
226 Ok(v) => v,
227 Err(_) => return,
228 };
229 let ty = parsed.get("type").and_then(|v| v.as_str()).unwrap_or("");
230 match ty {
231 "message_start" => {
232 *saw_start = true;
233 if let Some(msg) = parsed.get("message") {
234 if let Some(id) = msg.get("id").and_then(|v| v.as_str()) {
235 output.response_id = Some(id.to_string());
236 }
237 if let Some(u) = msg.get("usage") {
238 apply_usage(&mut output.usage, u);
239 }
240 }
241 }
242 "content_block_start" => {
243 let index = parsed.get("index").and_then(|v| v.as_u64()).unwrap_or(0) as usize;
244 let block = parsed.get("content_block").cloned().unwrap_or(Value::Null);
245 let kind = block
246 .get("type")
247 .and_then(|v| v.as_str())
248 .unwrap_or("text")
249 .to_string();
250 let our_index = output.content.len();
251 match kind.as_str() {
252 "text" => {
253 output.content.push(ContentBlock::Text { text: String::new() });
254 let _ = tx
255 .send(AssistantMessageEvent::TextStart { content_index: our_index })
256 .await;
257 }
258 "thinking" => {
259 output.content.push(ContentBlock::Thinking {
260 thinking: String::new(),
261 thinking_signature: String::new(),
262 redacted: false,
263 });
264 let _ = tx
265 .send(AssistantMessageEvent::ThinkingStart { content_index: our_index })
266 .await;
267 }
268 "redacted_thinking" => {
269 let data = block
270 .get("data")
271 .and_then(|v| v.as_str())
272 .unwrap_or("")
273 .to_string();
274 output.content.push(ContentBlock::Thinking {
275 thinking: "[Reasoning redacted]".into(),
276 thinking_signature: data,
277 redacted: true,
278 });
279 let _ = tx
280 .send(AssistantMessageEvent::ThinkingStart { content_index: our_index })
281 .await;
282 }
283 "tool_use" => {
284 let id = block
285 .get("id")
286 .and_then(|v| v.as_str())
287 .unwrap_or("")
288 .to_string();
289 let name = block
290 .get("name")
291 .and_then(|v| v.as_str())
292 .unwrap_or("")
293 .to_string();
294 output.content.push(ContentBlock::ToolCall {
295 id,
296 name,
297 arguments: Value::Object(Default::default()),
298 });
299 let _ = tx
300 .send(AssistantMessageEvent::ToolCallStart { content_index: our_index })
301 .await;
302 }
303 _ => return,
304 }
305 blocks.insert(
306 index,
307 BlockScratch {
308 our_index,
309 kind,
310 partial_json: String::new(),
311 },
312 );
313 }
314 "content_block_delta" => {
315 let index = parsed.get("index").and_then(|v| v.as_u64()).unwrap_or(0) as usize;
316 let delta = parsed.get("delta").cloned().unwrap_or(Value::Null);
317 let dtype = delta.get("type").and_then(|v| v.as_str()).unwrap_or("");
318 let Some(scratch) = blocks.get_mut(&index) else {
319 return;
320 };
321 let our_index = scratch.our_index;
322 let Some(block) = output.content.get_mut(our_index) else {
323 return;
324 };
325 match (dtype, block) {
326 ("text_delta", ContentBlock::Text { text }) => {
327 if let Some(t) = delta.get("text").and_then(|v| v.as_str()) {
328 text.push_str(t);
329 let _ = tx
330 .send(AssistantMessageEvent::TextDelta {
331 content_index: our_index,
332 delta: t.to_string(),
333 })
334 .await;
335 }
336 }
337 ("thinking_delta", ContentBlock::Thinking { thinking, .. }) => {
338 if let Some(t) = delta.get("thinking").and_then(|v| v.as_str()) {
339 thinking.push_str(t);
340 let _ = tx
341 .send(AssistantMessageEvent::ThinkingDelta {
342 content_index: our_index,
343 delta: t.to_string(),
344 })
345 .await;
346 }
347 }
348 ("signature_delta", ContentBlock::Thinking { thinking_signature, .. }) => {
349 if let Some(s) = delta.get("signature").and_then(|v| v.as_str()) {
350 thinking_signature.push_str(s);
351 }
352 }
353 ("input_json_delta", ContentBlock::ToolCall { arguments, .. }) => {
354 if let Some(p) = delta.get("partial_json").and_then(|v| v.as_str()) {
355 scratch.partial_json.push_str(p);
356 let _ = tx
357 .send(AssistantMessageEvent::ToolCallDelta {
358 content_index: our_index,
359 delta: p.to_string(),
360 })
361 .await;
362 *arguments = parse_streaming_json(&scratch.partial_json);
363 }
364 }
365 _ => {}
366 }
367 }
368 "content_block_stop" => {
369 let index = parsed.get("index").and_then(|v| v.as_u64()).unwrap_or(0) as usize;
370 if let Some(scratch) = blocks.remove(&index) {
371 let our_index = scratch.our_index;
372 if let Some(ContentBlock::ToolCall { arguments, .. }) = output.content.get_mut(our_index)
373 {
374 *arguments = parse_streaming_json(&scratch.partial_json);
375 }
376 let event = match scratch.kind.as_str() {
377 "text" => AssistantMessageEvent::TextEnd { content_index: our_index },
378 "thinking" | "redacted_thinking" => {
379 AssistantMessageEvent::ThinkingEnd { content_index: our_index }
380 }
381 "tool_use" => AssistantMessageEvent::ToolCallEnd { content_index: our_index },
382 _ => return,
383 };
384 let _ = tx.send(event).await;
385 }
386 }
387 "message_delta" => {
388 if let Some(reason) = parsed
389 .get("delta")
390 .and_then(|d| d.get("stop_reason"))
391 .and_then(|v| v.as_str())
392 {
393 output.stop_reason = map_stop_reason(reason);
394 }
395 if let Some(u) = parsed.get("usage") {
396 apply_usage(&mut output.usage, u);
397 }
398 }
399 "message_stop" => {
400 *saw_stop = true;
401 }
402 _ => {}
403 }
404}
405
406fn finalize_message(output: &mut AssistantMessage, model: &Model, _reason: StopReason) {
407 output.usage.recompute_total();
408 let cost = model.cost_for(&output.usage);
409 output.usage.cost = cost;
410}
411
412fn apply_usage(usage: &mut Usage, value: &Value) {
413 if let Some(v) = value.get("input_tokens").and_then(|v| v.as_u64()) {
414 usage.input = v;
415 }
416 if let Some(v) = value.get("output_tokens").and_then(|v| v.as_u64()) {
417 usage.output = v;
418 }
419 if let Some(v) = value.get("cache_read_input_tokens").and_then(|v| v.as_u64()) {
420 usage.cache_read = v;
421 }
422 if let Some(v) = value.get("cache_creation_input_tokens").and_then(|v| v.as_u64()) {
423 usage.cache_write = v;
424 }
425 usage.recompute_total();
426}
427
428fn map_stop_reason(reason: &str) -> StopReason {
429 match reason {
430 "end_turn" | "pause_turn" | "stop_sequence" => StopReason::Stop,
431 "max_tokens" => StopReason::Length,
432 "tool_use" => StopReason::ToolUse,
433 "refusal" | "sensitive" => StopReason::Error,
434 _ => StopReason::Stop,
435 }
436}
437
438async fn emit_terminal(
439 tx: &tokio::sync::mpsc::Sender<AssistantMessageEvent>,
440 reason: StopReason,
441 message: &str,
442) {
443 let mut msg = AssistantMessage::empty();
444 msg.stop_reason = reason;
445 msg.error_message = Some(message.to_string());
446 let _ = tx
447 .send(AssistantMessageEvent::Error {
448 reason,
449 message: msg,
450 })
451 .await;
452}
453
454fn truncate(s: &str, max: usize) -> String {
455 if s.chars().count() <= max {
458 return s.to_string();
459 }
460 let truncated: String = s.chars().take(max).collect();
461 format!("{truncated}…")
462}
463
464fn build_headers(model: &Model, options: &SimpleStreamOptions) -> HeaderMap {
469 let mut headers = HeaderMap::new();
470 headers.insert(ACCEPT, HeaderValue::from_static("application/json"));
471 headers.insert(
472 "anthropic-version",
473 HeaderValue::from_static(ANTHROPIC_VERSION),
474 );
475 headers.insert(
476 "anthropic-dangerous-direct-browser-access",
477 HeaderValue::from_static("true"),
478 );
479 if let Some(token) = &options.auth_token {
480 if let Ok(v) = HeaderValue::from_str(&format!("Bearer {token}")) {
481 headers.insert("authorization", v);
482 }
483 } else if let Some(key) = &options.api_key {
484 if let Ok(v) = HeaderValue::from_str(key) {
485 headers.insert("x-api-key", v);
486 }
487 }
488 let _ = model;
490 headers
491}
492
493fn cache_ephemeral() -> Value {
494 json!({ "type": "ephemeral" })
495}
496
497fn level_to_effort(level: ThinkingLevel) -> Option<&'static str> {
500 match level {
501 ThinkingLevel::Off => None,
502 ThinkingLevel::Minimal | ThinkingLevel::Low => Some("low"),
503 ThinkingLevel::Medium => Some("medium"),
504 ThinkingLevel::High => Some("high"),
505 ThinkingLevel::Xhigh => Some("xhigh"),
506 }
507}
508
509fn level_to_budget(level: ThinkingLevel) -> usize {
510 match level {
511 ThinkingLevel::Off => 0,
512 ThinkingLevel::Minimal => 1024,
513 ThinkingLevel::Low => 4_096,
514 ThinkingLevel::Medium => 8_192,
515 ThinkingLevel::High => 16_000,
516 ThinkingLevel::Xhigh => 24_000,
517 }
518}
519
520fn build_request_body(
521 model: &Model,
522 context: &LlmContext<'_>,
523 options: &SimpleStreamOptions,
524) -> Value {
525 let max_tokens = options.max_tokens.unwrap_or(model.max_tokens);
526 let cache = if options.cache_control {
527 Some(cache_ephemeral())
528 } else {
529 None
530 };
531
532 let system: Vec<Value> = match context.system_prompt {
534 Some(prompt) if !prompt.trim().is_empty() => {
535 let mut block = json!({ "type": "text", "text": prompt });
536 if let Some(cc) = &cache {
537 block["cache_control"] = cc.clone();
538 }
539 vec![block]
540 }
541 _ => Vec::new(),
542 };
543
544 let messages = convert_messages(context.messages, cache.as_ref());
545
546 let mut body = json!({
547 "model": model.id,
548 "messages": messages,
549 "max_tokens": max_tokens,
550 "stream": true,
551 });
552 if !system.is_empty() {
553 body["system"] = Value::Array(system);
554 }
555
556 if model.reasoning {
557 match options.reasoning {
558 None => {
559 body["thinking"] = json!({ "type": "disabled" });
560 }
561 Some(ThinkingLevel::Off) => {
562 body["thinking"] = json!({ "type": "disabled" });
563 }
564 Some(level) if model.force_adaptive_thinking => {
565 let mut thinking = json!({ "type": "adaptive", "display": "summarized" });
566 if let Some(effort) = level_to_effort(level) {
567 thinking["effort"] = json!(effort);
568 body["output_config"] = json!({ "effort": effort });
569 }
570 body["thinking"] = thinking;
571 }
572 Some(level) => {
573 let budget = options
574 .thinking_budget_tokens
575 .unwrap_or_else(|| level_to_budget(level));
576 let budget = budget.min(max_tokens.saturating_sub(1)).max(1024);
577 body["thinking"] = json!({
578 "type": "enabled",
579 "budget_tokens": budget,
580 "display": "summarized",
581 });
582 }
583 }
584 } else if let Some(temp) = options.temperature {
585 if model.supports_temperature {
586 body["temperature"] = json!(temp);
587 }
588 } else if model.supports_temperature {
589 }
591
592 if !context.tools.is_empty() {
593 body["tools"] = Value::Array(convert_tools(context.tools, cache.as_ref()));
594 }
595
596 body
597}
598
599fn convert_messages(messages: &[Message], cache: Option<&Value>) -> Vec<Value> {
603 let mut out: Vec<Value> = Vec::new();
604 let mut i = 0;
605 while i < messages.len() {
606 match &messages[i] {
607 Message::User(u) => {
608 let has_images = u.content.iter().any(|b| matches!(b, super::types::UserBlock::Image { .. }));
609 if !has_images {
610 let text = u.text_content();
611 if text.trim().is_empty() {
612 i += 1;
613 continue;
614 }
615 out.push(json!({ "role": "user", "content": text }));
616 } else {
617 let blocks: Vec<Value> = u
618 .content
619 .iter()
620 .map(|b| match b {
621 super::types::UserBlock::Text { text } => {
622 json!({ "type": "text", "text": text })
623 }
624 super::types::UserBlock::Image { data, mime_type } => json!({
625 "type": "image",
626 "source": {
627 "type": "base64",
628 "media_type": mime_type,
629 "data": data,
630 }
631 }),
632 })
633 .filter(|b| !(b.get("type").and_then(|v| v.as_str()) == Some("text")
634 && b.get("text").and_then(|v| v.as_str()).unwrap_or("").trim().is_empty()))
635 .collect();
636 if blocks.is_empty() {
637 i += 1;
638 continue;
639 }
640 out.push(json!({ "role": "user", "content": blocks }));
641 }
642 i += 1;
643 }
644 Message::Assistant(a) => {
645 let mut blocks: Vec<Value> = Vec::new();
646 for c in &a.content {
647 match c {
648 ContentBlock::Text { text } => {
649 if !text.trim().is_empty() {
650 blocks.push(json!({ "type": "text", "text": text }));
651 }
652 }
653 ContentBlock::Thinking {
654 thinking,
655 thinking_signature,
656 redacted,
657 } => {
658 if *redacted {
659 blocks.push(json!({ "type": "redacted_thinking", "data": thinking_signature }));
660 } else if !thinking.trim().is_empty() {
661 if thinking_signature.is_empty() {
662 blocks.push(json!({ "type": "text", "text": thinking }));
663 } else {
664 blocks.push(json!({
665 "type": "thinking",
666 "thinking": thinking,
667 "signature": thinking_signature,
668 }));
669 }
670 }
671 }
672 ContentBlock::ToolCall { id, name, arguments } => {
673 blocks.push(json!({
674 "type": "tool_use",
675 "id": id,
676 "name": name,
677 "input": arguments,
678 }));
679 }
680 }
681 }
682 if !blocks.is_empty() {
683 out.push(json!({ "role": "assistant", "content": blocks }));
684 }
685 i += 1;
686 }
687 Message::ToolResult(_) => {
688 let mut results: Vec<Value> = Vec::new();
690 while i < messages.len() {
691 if let Message::ToolResult(tr) = &messages[i] {
692 let content = convert_result_content(&tr.content);
693 results.push(json!({
694 "type": "tool_result",
695 "tool_use_id": tr.tool_call_id,
696 "content": content,
697 "is_error": tr.is_error,
698 }));
699 i += 1;
700 } else {
701 break;
702 }
703 }
704 if !results.is_empty() {
705 out.push(json!({ "role": "user", "content": results }));
706 }
707 }
708 }
709 }
710
711 if let Some(cc) = cache {
713 if let Some(last) = out.last_mut() {
714 if last.get("role").and_then(|v| v.as_str()) == Some("user") {
715 if let Some(content) = last.get_mut("content") {
716 if let Some(arr) = content.as_array_mut() {
717 if let Some(last_block) = arr.last_mut() {
718 last_block["cache_control"] = cc.clone();
719 }
720 } else if let Some(s) = content.as_str() {
721 let block = json!({ "type": "text", "text": s, "cache_control": cc });
722 *content = Value::Array(vec![block]);
723 }
724 }
725 }
726 }
727 }
728
729 out
730}
731
732fn convert_result_content(content: &[super::types::ToolResultContent]) -> Value {
733 let has_image = content
734 .iter()
735 .any(|c| matches!(c, super::types::ToolResultContent::Image { .. }));
736 if !has_image {
737 let text = content
738 .iter()
739 .filter_map(|c| match c {
740 super::types::ToolResultContent::Text { text } => Some(text.as_str()),
741 _ => None,
742 })
743 .collect::<Vec<_>>()
744 .join("\n");
745 Value::String(text)
746 } else {
747 Value::Array(
748 content
749 .iter()
750 .map(|c| match c {
751 super::types::ToolResultContent::Text { text } => {
752 json!({ "type": "text", "text": text })
753 }
754 super::types::ToolResultContent::Image { data, mime_type } => json!({
755 "type": "image",
756 "source": { "type": "base64", "media_type": mime_type, "data": data },
757 }),
758 })
759 .collect(),
760 )
761 }
762}
763
764fn convert_tools(tools: &[ToolSchema], cache: Option<&Value>) -> Vec<Value> {
765 tools
766 .iter()
767 .enumerate()
768 .map(|(i, t)| {
769 let schema = &t.input_schema;
770 let properties = schema
771 .get("properties")
772 .cloned()
773 .unwrap_or_else(|| Value::Object(Default::default()));
774 let required = schema
775 .get("required")
776 .cloned()
777 .unwrap_or_else(|| Value::Array(Vec::new()));
778 let mut entry = json!({
779 "name": t.name,
780 "description": t.description,
781 "input_schema": {
782 "type": "object",
783 "properties": properties,
784 "required": required,
785 },
786 });
787 if i == tools.len() - 1 {
788 if let Some(cc) = cache {
789 entry["cache_control"] = cc.clone();
790 }
791 }
792 entry
793 })
794 .collect()
795}
796
797#[cfg(test)]
798mod tests {
799 use super::truncate;
800
801 #[test]
802 fn truncate_is_char_safe_on_multibyte() {
803 let s = "中".repeat(3000); let t = truncate(&s, 2000);
806 assert!(t.chars().count() <= 2001); assert!(t.ends_with('…'));
808 }
809
810 #[test]
811 fn truncate_short_input_unchanged() {
812 assert_eq!(truncate("abc", 10), "abc");
813 assert_eq!(truncate("ab", 2), "ab");
814 }
815
816 #[test]
817 fn truncate_cuts_at_char_boundary() {
818 let t = truncate("a中b", 2);
819 assert_eq!(t, "a中…");
820 }
821}