goose_context_management/
structured.rs

1use crate::templates::{self, COMPACTION_SUMMARY_TEMPLATE};
2use goose_provider_types::json::safely_parse_json;
3use serde::{Deserialize, Serialize};
4
5/// Structured output of the compaction LLM call.
6///
7/// Every list is ordered most-important-first so consumers (the render
8/// template, experiments that truncate sections) can cut from the tail.
9/// Fields deserialize leniently - omitted fields default to empty, and an
10/// object or number where a string was asked for is stringified rather than
11/// failing - because models routinely enrich the schema (e.g. `{"error": ..,
12/// "fix": ..}` entries) and one such field must not discard a good summary.
13#[derive(Debug, Clone, Default, Serialize, Deserialize)]
14pub struct StructuredSummary {
15    #[serde(default, deserialize_with = "lenient_string_list")]
16    pub user_intent: Vec<String>,
17    #[serde(default, deserialize_with = "lenient_string_list")]
18    pub technical_concepts: Vec<String>,
19    #[serde(default, deserialize_with = "lenient_file_list")]
20    pub files: Vec<FileActivity>,
21    #[serde(default, deserialize_with = "lenient_string_list")]
22    pub errors_and_fixes: Vec<String>,
23    #[serde(default, deserialize_with = "lenient_string_list")]
24    pub problem_solving: Vec<String>,
25    #[serde(default, deserialize_with = "lenient_string_list")]
26    pub user_messages: Vec<String>,
27    #[serde(default, deserialize_with = "lenient_string_list")]
28    pub pending_tasks: Vec<String>,
29    #[serde(default, deserialize_with = "lenient_string_opt")]
30    pub current_work: Option<String>,
31    #[serde(default, deserialize_with = "lenient_string_opt")]
32    pub next_step: Option<String>,
33    /// Unknown top-level fields, kept so a user-customized compaction prompt
34    /// that adds fields can still reach them from a customized render
35    /// template. Not counted when deciding whether a summary is empty.
36    #[serde(flatten)]
37    pub extra: serde_json::Map<String, serde_json::Value>,
38}
39
40#[derive(Debug, Clone, Serialize, Deserialize)]
41pub struct FileActivity {
42    #[serde(default, deserialize_with = "lenient_string")]
43    pub path: String,
44    #[serde(default, deserialize_with = "lenient_string")]
45    pub summary: String,
46    #[serde(default, deserialize_with = "lenient_string_opt")]
47    pub key_code: Option<String>,
48}
49
50fn stringify_lenient(value: &serde_json::Value) -> String {
51    match value {
52        serde_json::Value::String(s) => s.clone(),
53        serde_json::Value::Null => String::new(),
54        serde_json::Value::Object(map) => map
55            .iter()
56            .map(|(k, v)| format!("{k}: {}", stringify_lenient(v)))
57            .collect::<Vec<_>>()
58            .join("; "),
59        serde_json::Value::Array(items) => items
60            .iter()
61            .map(stringify_lenient)
62            .collect::<Vec<_>>()
63            .join("; "),
64        other => other.to_string(),
65    }
66}
67
68fn lenient_string_list<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
69where
70    D: serde::Deserializer<'de>,
71{
72    let value = serde_json::Value::deserialize(deserializer)?;
73    Ok(match value {
74        serde_json::Value::Array(items) => items.iter().map(stringify_lenient).collect(),
75        serde_json::Value::Null => Vec::new(),
76        other => vec![stringify_lenient(&other)],
77    })
78}
79
80fn lenient_string<'de, D>(deserializer: D) -> Result<String, D::Error>
81where
82    D: serde::Deserializer<'de>,
83{
84    let value = serde_json::Value::deserialize(deserializer)?;
85    Ok(stringify_lenient(&value))
86}
87
88fn lenient_string_opt<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
89where
90    D: serde::Deserializer<'de>,
91{
92    let value = serde_json::Value::deserialize(deserializer)?;
93    Ok(match value {
94        serde_json::Value::Null => None,
95        other => Some(stringify_lenient(&other)),
96    })
97}
98
99/// `files` entries should be objects, but a model that over-applies the
100/// "plain strings" rule may emit them as strings; render those as path-only
101/// activities rather than discarding the whole summary.
102fn lenient_file_list<'de, D>(deserializer: D) -> Result<Vec<FileActivity>, D::Error>
103where
104    D: serde::Deserializer<'de>,
105{
106    let value = serde_json::Value::deserialize(deserializer)?;
107    let items = match value {
108        serde_json::Value::Array(items) => items,
109        serde_json::Value::Null => return Ok(Vec::new()),
110        other => vec![other],
111    };
112    Ok(items
113        .into_iter()
114        .filter_map(|item| match item {
115            serde_json::Value::Object(_) => serde_json::from_value(item).ok(),
116            other => {
117                let path = stringify_lenient(&other);
118                (!path.trim().is_empty()).then_some(FileActivity {
119                    path,
120                    summary: String::new(),
121                    key_code: None,
122                })
123            }
124        })
125        .collect())
126}
127
128impl StructuredSummary {
129    /// Returns `None` when no usable JSON document is found so the caller can
130    /// keep the raw response text - the lossless fallback.
131    pub fn parse(response_text: &str) -> Option<Self> {
132        json_candidates(response_text).into_iter().find_map(|c| {
133            let value = safely_parse_json(c).ok()?;
134            let mut summary: Self = serde_json::from_value(value).ok()?;
135            summary.normalize();
136            (!summary.is_empty()).then_some(summary)
137        })
138    }
139
140    pub fn render(&self) -> Result<String, minijinja::Error> {
141        self.render_with(
142            &templates::builtin_template(COMPACTION_SUMMARY_TEMPLATE)
143                .expect("builtin compaction summary template"),
144        )
145    }
146
147    pub fn render_with(&self, template: &str) -> Result<String, minijinja::Error> {
148        templates::render(template, self)
149    }
150
151    /// Drops blank entries so a response of blank strings counts as empty
152    /// (raw-text fallback) rather than rendering a summary of nothing.
153    fn normalize(&mut self) {
154        fn blank(s: &str) -> bool {
155            s.trim().is_empty()
156        }
157        for list in [
158            &mut self.user_intent,
159            &mut self.technical_concepts,
160            &mut self.errors_and_fixes,
161            &mut self.problem_solving,
162            &mut self.user_messages,
163            &mut self.pending_tasks,
164        ] {
165            list.retain(|s| !blank(s));
166        }
167        for file in &mut self.files {
168            if file.key_code.as_deref().is_some_and(blank) {
169                file.key_code = None;
170            }
171        }
172        self.files
173            .retain(|f| !blank(&f.path) || !blank(&f.summary) || f.key_code.is_some());
174        if self.current_work.as_deref().is_some_and(blank) {
175            self.current_work = None;
176        }
177        if self.next_step.as_deref().is_some_and(blank) {
178            self.next_step = None;
179        }
180    }
181
182    fn is_empty(&self) -> bool {
183        self.user_intent.is_empty()
184            && self.technical_concepts.is_empty()
185            && self.files.is_empty()
186            && self.errors_and_fixes.is_empty()
187            && self.problem_solving.is_empty()
188            && self.user_messages.is_empty()
189            && self.pending_tasks.is_empty()
190            && self.current_work.is_none()
191            && self.next_step.is_none()
192    }
193}
194
195/// Candidate JSON documents in the model's response, tried in order until one
196/// parses: after each `</analysis>` terminator (last first) the
197/// post-terminator ```json fences (last first) then a leading object, and
198/// finally a leading object of the whole text.
199///
200/// Terminators before the last are retried because the summary JSON may
201/// itself quote `</analysis>` (e.g. a session editing compaction prompts),
202/// hiding the real terminator from a plain rfind. Such a candidate is
203/// accepted only if it contains every later terminator occurrence - proof
204/// they were quoted inside it - so a fenced example inside the scratchpad,
205/// which precedes the real terminator, can never leak through.
206///
207/// Every candidate must sit directly at its marker and brace-balance to a
208/// close. Anything looser erodes the lossless fallback: JSON merely quoted in
209/// a prose response, or a fenced example inside the discarded scratchpad,
210/// would silently replace the raw text. Extraction is brace-balanced rather
211/// than fence-delimited because string values may legally contain ```, and an
212/// unterminated object (output cut off mid-JSON) is not repaired - repair
213/// would drop the late, continuation-critical sections that the raw-text
214/// fallback preserves.
215#[allow(clippy::string_slice)] // All markers are ASCII; indices are byte offsets of ASCII matches.
216fn json_candidates(text: &str) -> Vec<&str> {
217    const TERMINATOR: &str = "</analysis>";
218
219    let mut cuts: Vec<usize> = text
220        .match_indices(TERMINATOR)
221        .map(|(idx, _)| idx + TERMINATOR.len())
222        .collect();
223    if cuts.is_empty() {
224        cuts.push(0);
225    }
226
227    let mut candidates: Vec<&str> = Vec::new();
228    for &cut in cuts.iter().rev() {
229        let tail = &text[cut..];
230        let later_terminators = tail.matches(TERMINATOR).count();
231        candidates.extend(
232            fenced_json_blocks(tail)
233                .chain(leading_object(tail))
234                .filter(|candidate| candidate.matches(TERMINATOR).count() == later_terminators),
235        );
236    }
237    candidates.extend(leading_object(text));
238    candidates.dedup();
239    candidates
240}
241
242/// Every fence is tried, last first, because a string value may itself quote
243/// a fenced JSON snippet, and such an embedded fence must not shadow the real
244/// one.
245#[allow(clippy::string_slice)] // The marker is ASCII; indices are byte offsets of ASCII matches.
246fn fenced_json_blocks(text: &str) -> impl Iterator<Item = &str> {
247    text.match_indices("```json")
248        .collect::<Vec<_>>()
249        .into_iter()
250        .rev()
251        .filter_map(|(idx, marker)| leading_object(&text[idx + marker.len()..]))
252}
253
254#[allow(clippy::string_slice)] // Indices come from char_indices(); slicing is safe.
255fn leading_object(text: &str) -> Option<&str> {
256    let text = text.trim_start();
257    if !text.starts_with('{') {
258        return None;
259    }
260
261    let mut depth = 0usize;
262    let mut in_string = false;
263    let mut escaped = false;
264
265    for (idx, ch) in text.char_indices() {
266        if in_string {
267            if escaped {
268                escaped = false;
269            } else if ch == '\\' {
270                escaped = true;
271            } else if ch == '"' {
272                in_string = false;
273            }
274            continue;
275        }
276        match ch {
277            '"' => in_string = true,
278            '{' => depth += 1,
279            '}' => {
280                depth -= 1;
281                if depth == 0 {
282                    return Some(&text[..=idx]);
283                }
284            }
285            _ => {}
286        }
287    }
288
289    None
290}
291
292#[cfg(test)]
293mod tests {
294    use super::*;
295
296    const FULL_RESPONSE: &str = r#"<analysis>
297The user asked to fix a bug in parser.rs. I traced it to an off-by-one
298in {brace handling} and patched it.
299</analysis>
300
301```json
302{
303  "user_intent": ["Fix the parser bug", "Add a regression test"],
304  "technical_concepts": ["off-by-one", "tokenizer"],
305  "files": [
306    {"path": "src/parser.rs", "summary": "Fixed off-by-one in scan loop", "key_code": "fn scan(&mut self) { .. }"}
307  ],
308  "errors_and_fixes": ["Panic on empty input, fixed with early return"],
309  "problem_solving": ["Root-caused via failing unit test"],
310  "user_messages": ["fix the parser bug", "add a test"],
311  "pending_tasks": ["Add a regression test"],
312  "current_work": "Writing the regression test in tests/parser.rs",
313  "next_step": "Finish the regression test"
314}
315```"#;
316
317    #[test]
318    fn parses_fenced_json_after_analysis() {
319        let summary = StructuredSummary::parse(FULL_RESPONSE).expect("should parse");
320        assert_eq!(
321            summary.user_intent,
322            vec!["Fix the parser bug", "Add a regression test"]
323        );
324        assert_eq!(summary.files.len(), 1);
325        assert_eq!(summary.files[0].path, "src/parser.rs");
326        assert_eq!(
327            summary.current_work.as_deref(),
328            Some("Writing the regression test in tests/parser.rs")
329        );
330    }
331
332    #[test]
333    fn unusable_responses_fall_back_to_raw_text() {
334        for text in [
335            // freeform prose, no JSON document
336            "Here is a summary of the conversation. The user asked about compaction.",
337            // no visible content
338            "{}",
339            r#"{"notes": "unknown fields alone are not a summary"}"#,
340            r#"{"current_work": ""}"#,
341            r#"{"files": [{}], "user_intent": [" "]}"#,
342            // output cut off mid-JSON: never repaired
343            "```json\n{\"user_intent\": [\"Fix the bug\"], \"pending_tasks\": [\"Write tests\", \"Update docs",
344            // JSON quoted inside prose is not anchored at a marker
345            r#"The session focused on the parser migration. The tracker entry {"current_work": "migrate parser"} is unchanged, and tests still need porting."#,
346            "<analysis>reviewing</analysis>\nA prose recap: the config was set to {\"user_intent\": [\"quoted example\"]} per the docs, then the run passed.",
347            // a fenced example inside the scratchpad is not the summary
348            "<analysis>\nThe target shape is:\n```json\n{\"user_intent\": [\"example only\"]}\n```\nNow let me review the conversation.\n</analysis>\nSorry, I ran out of room and could not produce the summary document.",
349            // a quoted terminator inside the scratchpad must not expose its fenced example
350            "<analysis>\nThe prompt ends with </analysis> and shows the shape:\n```json\n{\"user_intent\": [\"example only\"]}\n```\nNow let me review the conversation.\n</analysis>\nSorry, I ran out of room and could not produce the summary document.",
351        ] {
352            assert!(
353                StructuredSummary::parse(text).is_none(),
354                "should fall back to raw text for: {text}"
355            );
356        }
357    }
358
359    #[test]
360    fn embedded_fences_in_string_values_do_not_break_extraction() {
361        let text = "```json\n{\"user_intent\": [\"Document the build\"], \"files\": [{\"path\": \"README.md\", \"summary\": \"Added build docs\", \"key_code\": \"```bash\\ncargo build\\n```\"}], \"pending_tasks\": [\"Publish the docs\"]}\n```";
362        let summary = StructuredSummary::parse(text).expect("should parse");
363        assert_eq!(
364            summary.files[0].key_code.as_deref(),
365            Some("```bash\ncargo build\n```")
366        );
367        assert_eq!(summary.pending_tasks, vec!["Publish the docs"]);
368
369        let quoted_json_fence = "```json\n{\"user_intent\": [\"Document the config\"], \"files\": [{\"path\": \"docs/config.md\", \"summary\": \"Added config examples\", \"key_code\": \"```json\\n{\\\"retries\\\": 3}\\n```\"}]}\n```";
370        let summary =
371            StructuredSummary::parse(quoted_json_fence).expect("should parse via the outer fence");
372        assert_eq!(
373            summary.files[0].key_code.as_deref(),
374            Some("```json\n{\"retries\": 3}\n```")
375        );
376    }
377
378    #[test]
379    fn quoted_terminator_inside_summary_json_does_not_hide_it() {
380        let text = "<analysis>\nThe session edited the compaction prompt itself.\n</analysis>\n```json\n{\"user_intent\": [\"Rework the scratchpad prompt\"], \"files\": [{\"path\": \"prompts/compact.md\", \"summary\": \"Tightened the <analysis>...</analysis> instructions\"}]}\n```";
381        let summary = StructuredSummary::parse(text).expect("should parse");
382        assert_eq!(summary.user_intent, vec!["Rework the scratchpad prompt"]);
383        assert_eq!(
384            summary.files[0].summary,
385            "Tightened the <analysis>...</analysis> instructions"
386        );
387    }
388
389    #[test]
390    fn retries_next_candidate_when_fenced_extraction_fails() {
391        let text = "<analysis>the model was told to emit ```json with {braces</analysis>\n{\"user_intent\": [\"Real goal\"]}";
392        let summary = StructuredSummary::parse(text).expect("should parse");
393        assert_eq!(summary.user_intent, vec!["Real goal"]);
394    }
395
396    #[test]
397    fn lenient_shapes_are_stringified_not_rejected() {
398        let text = r#"{
399            "user_intent": "fix the flaky test",
400            "errors_and_fixes": [
401                {"error": "cursor drifted after replay batch 34", "fix": "bounded mpsc channel"},
402                "plain string entry",
403                null
404            ],
405            "pending_tasks": [42],
406            "current_work": {"task": "regression test", "status": "in progress"}
407        }"#;
408        let summary = StructuredSummary::parse(text).expect("should parse leniently");
409        assert_eq!(summary.user_intent, vec!["fix the flaky test"]);
410        assert_eq!(
411            summary.errors_and_fixes,
412            vec![
413                "error: cursor drifted after replay batch 34; fix: bounded mpsc channel",
414                "plain string entry",
415            ]
416        );
417        assert_eq!(summary.pending_tasks, vec!["42"]);
418        assert_eq!(
419            summary.current_work.as_deref(),
420            Some("task: regression test; status: in progress")
421        );
422    }
423
424    #[test]
425    fn file_entries_parse_leniently() {
426        let text = r#"{"files": [
427            "src/parser.rs",
428            {"path": "tests/parser.rs", "summary": "Added regression test"},
429            {"path": "src/scan.rs", "summary": 42, "key_code": ["fn a() {}", "fn b() {}"]},
430            ""
431        ]}"#;
432        let summary = StructuredSummary::parse(text).expect("should parse");
433        assert_eq!(summary.files.len(), 3);
434        assert_eq!(summary.files[0].path, "src/parser.rs");
435        assert_eq!(summary.files[0].summary, "");
436        assert_eq!(summary.files[1].summary, "Added regression test");
437        assert_eq!(summary.files[2].summary, "42");
438        assert_eq!(
439            summary.files[2].key_code.as_deref(),
440            Some("fn a() {}; fn b() {}")
441        );
442    }
443
444    #[test]
445    fn drops_blank_entries_but_keeps_content() {
446        let text = r#"{"user_intent": ["", "Fix the bug"], "files": [{"path": "a.rs", "summary": "Patched", "key_code": "  "}], "next_step": " "}"#;
447        let summary = StructuredSummary::parse(text).expect("should parse");
448        assert_eq!(summary.user_intent, vec!["Fix the bug"]);
449        assert_eq!(summary.files[0].key_code, None);
450        assert_eq!(summary.next_step, None);
451    }
452
453    #[test]
454    fn renders_markdown_sections() {
455        let summary = StructuredSummary::parse(FULL_RESPONSE).unwrap();
456        let rendered = summary.render().expect("should render");
457        assert!(rendered.contains("## User Intent"));
458        assert!(rendered.contains("- Fix the parser bug"));
459        assert!(rendered.contains("### src/parser.rs"));
460        assert!(rendered.contains("fn scan(&mut self) { .. }"));
461        assert!(rendered.contains("## Next Step"));
462    }
463
464    #[test]
465    fn render_fences_exceed_backtick_runs_in_key_code() {
466        let summary = StructuredSummary {
467            files: vec![FileActivity {
468                path: "docs/build.md".to_string(),
469                summary: "Documented the build".to_string(),
470                key_code: Some(
471                    "```bash\ncargo build\n```\n````\nnested fence docs\n````".to_string(),
472                ),
473            }],
474            errors_and_fixes: vec!["None".to_string()],
475            ..Default::default()
476        };
477        let rendered = summary.render().expect("should render");
478        // key_code's longest backtick run is four, so the fence must be five
479        assert_eq!(rendered.matches("\n`````\n").count(), 2);
480        let errors_heading = rendered.find("## Errors + Fixes").unwrap();
481        let closing_fence = rendered.rfind("\n`````\n").unwrap();
482        assert!(errors_heading > closing_fence);
483    }
484}
goose_context_management/structured.rs

goose_context_management/
structured.rs
