llm_output_parser/
text.rs

1//! Clean text extraction from LLM responses.
2//!
3//! Provides [`parse_text`] for extracting clean prose from LLM output,
4//! stripping think blocks and common boilerplate prefixes.
5
6use crate::error::{
7    ensure_input_within_limits, record_extracted_span, ParseError, ParseOptions, ParseTrace,
8};
9use crate::extract::preprocess_opts;
10
11/// Common boilerplate prefixes that LLMs add to responses.
12const SIMPLE_PREFIXES: &[&str] = &[
13    "Sure! ",
14    "Sure, ",
15    "Sure.\n",
16    "Of course! ",
17    "Of course, ",
18    "Of course.\n",
19    "Certainly! ",
20    "Certainly, ",
21    "Certainly.\n",
22    "Absolutely! ",
23    "Absolutely, ",
24];
25
26/// Prefixes that consume up to the next newline or colon.
27const LINE_PREFIXES: &[&str] = &["Here's ", "Here is "];
28
29/// Clean an LLM response for use as plain text.
30///
31/// Processing:
32/// 1. Strip `<think>` blocks
33/// 2. Trim whitespace
34/// 3. Strip common LLM boilerplate prefixes:
35///    "Sure!", "Here's...", "Of course!", "Certainly!", etc.
36///
37/// Returns the cleaned text or `EmptyResponse` if nothing remains.
38///
39/// # Examples
40///
41/// ```
42/// use llm_output_parser::parse_text;
43///
44/// let result = parse_text("Sure! Paris is the capital.").unwrap();
45/// assert_eq!(result, "Paris is the capital.");
46/// ```
47pub fn parse_text(response: &str) -> Result<String, ParseError> {
48    let opts = ParseOptions::default();
49    let (text, _trace) = parse_text_with_trace(response, &opts)?;
50    Ok(text)
51}
52
53/// Clean an LLM response as plain text with diagnostic trace.
54pub fn parse_text_with_trace(
55    response: &str,
56    opts: &ParseOptions,
57) -> Result<(String, ParseTrace), ParseError> {
58    ensure_input_within_limits(response, opts)?;
59
60    let cleaned = preprocess_opts(response, opts.strip_think_tags);
61    let mut trace = ParseTrace::default();
62
63    if cleaned.is_empty() {
64        return Err(ParseError::EmptyResponse);
65    }
66
67    let mut text = cleaned.as_str();
68
69    trace.strategies_tried.push("simple_prefix_strip");
70    for prefix in SIMPLE_PREFIXES {
71        if let Some(rest) = text.strip_prefix(prefix) {
72            text = rest;
73            break;
74        }
75    }
76
77    trace.strategies_tried.push("line_prefix_strip");
78    if text == cleaned.as_str() {
79        // Only if no simple prefix was stripped
80        for prefix in LINE_PREFIXES {
81            if let Some(rest) = text.strip_prefix(prefix) {
82                // Find the next newline or colon and skip past it
83                if let Some(pos) = rest.find('\n') {
84                    text = rest[pos + 1..].trim_start();
85                    break;
86                } else if let Some(pos) = rest.find(':') {
87                    text = rest[pos + 1..].trim_start();
88                    break;
89                }
90            }
91        }
92    }
93
94    let result = text.trim().to_string();
95
96    if result.is_empty() {
97        return Err(ParseError::EmptyResponse);
98    }
99
100    record_extracted_span(&mut trace, &cleaned, &result);
101    Ok((result, trace))
102}
103
104#[cfg(test)]
105mod tests {
106    use super::*;
107
108    #[test]
109    fn clean_text() {
110        let result = parse_text("Paris is the capital.").unwrap();
111        assert_eq!(result, "Paris is the capital.");
112    }
113
114    #[test]
115    fn with_think() {
116        let result = parse_text("<think>reasoning</think>Paris.").unwrap();
117        assert_eq!(result, "Paris.");
118    }
119
120    #[test]
121    fn sure_prefix() {
122        let result = parse_text("Sure! Paris is the capital.").unwrap();
123        assert_eq!(result, "Paris is the capital.");
124    }
125
126    #[test]
127    fn heres_prefix() {
128        let result = parse_text("Here's the answer:\nParis.").unwrap();
129        assert_eq!(result, "Paris.");
130    }
131
132    #[test]
133    fn empty_after_strip() {
134        let result = parse_text("<think>just thinking</think>");
135        assert!(result.is_err());
136    }
137
138    #[test]
139    fn already_clean() {
140        let result = parse_text("No prefix here.").unwrap();
141        assert_eq!(result, "No prefix here.");
142    }
143
144    #[test]
145    fn with_trace_records_prefix_strategy() {
146        let opts = ParseOptions::default();
147        let (text, trace) = parse_text_with_trace("Sure! Paris is the capital.", &opts).unwrap();
148        assert_eq!(text, "Paris is the capital.");
149        assert!(trace.strategies_tried.contains(&"simple_prefix_strip"));
150        assert!(trace.extracted_span.is_some());
151    }
152
153    #[test]
154    fn with_trace_rejects_oversized_input() {
155        let opts = ParseOptions {
156            max_input_bytes: 8,
157            ..ParseOptions::default()
158        };
159        let err = parse_text_with_trace("This is long enough", &opts).unwrap_err();
160        assert_eq!(err.kind(), "too_large");
161    }
162}
llm_output_parser/text.rs

llm_output_parser/
text.rs
