remem/memory/format/
parse.rs

1use super::{extract_field, ParsedObservation, OBSERVATION_TYPES};
2
3const INVALID_TYPE_PREVIEW_BYTES: usize = 120;
4
5#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6pub(crate) enum InvalidObservationTypeDrop {
7    Missing,
8    Unknown,
9}
10
11#[derive(Debug, Clone, PartialEq)]
12pub(crate) struct ParseObservationsOutcome {
13    pub(crate) observations: Vec<ParsedObservation>,
14    pub(crate) invalid_type_drops: Vec<InvalidObservationTypeDrop>,
15}
16
17impl ParseObservationsOutcome {
18    pub(crate) fn had_invalid_type(&self) -> bool {
19        !self.invalid_type_drops.is_empty()
20    }
21}
22
23/// Find `needle` in `haystack` using ASCII case-insensitive comparison.
24/// Returns the byte offset of the first match, or `None`.
25pub(crate) fn find_ascii_ci(haystack: &str, needle: &str) -> Option<usize> {
26    let needle = needle.as_bytes();
27    haystack
28        .as_bytes()
29        .windows(needle.len())
30        .position(|w| w.eq_ignore_ascii_case(needle))
31}
32
33fn extract_array(content: &str, array_name: &str, element_name: &str) -> Vec<String> {
34    let open = format!("<{}>", array_name);
35    let close = format!("</{}>", array_name);
36    let Some(start) = content.find(&open) else {
37        return vec![];
38    };
39    let start = start + open.len();
40    let Some(end_rel) = content[start..].find(&close) else {
41        return vec![];
42    };
43    let end = start + end_rel;
44    let inner = &content[start..end];
45
46    let elem_open = format!("<{}>", element_name);
47    let elem_close = format!("</{}>", element_name);
48    let mut results = Vec::new();
49    let mut pos = 0;
50    while let Some(found) = inner[pos..].find(&elem_open) {
51        let value_start = pos + found + elem_open.len();
52        let Some(end_rel) = inner[value_start..].find(&elem_close) else {
53            break;
54        };
55        let value_end = value_start + end_rel;
56        let value = inner[value_start..value_end].trim().to_string();
57        if !value.is_empty() {
58            results.push(value);
59        }
60        pos = value_end + elem_close.len();
61    }
62    results
63}
64
65/// Parse the optional `<confidence>` field. A missing field is normal model
66/// output and stays silent; a present-but-unparseable value (e.g. "very high")
67/// is logged at warn level before falling back, so silent quality degradation
68/// stays observable (the caller substitutes the default confidence).
69fn parse_confidence(content: &str) -> Option<f64> {
70    let raw = extract_field(content, "confidence")?;
71    match raw.parse::<f64>() {
72        Ok(value) if value.is_finite() => Some(value.clamp(0.0, 1.0)),
73        _ => {
74            crate::log::warn(
75                "observation-parse",
76                &format!("invalid <confidence> value {raw:?}; falling back to default"),
77            );
78            None
79        }
80    }
81}
82
83pub fn parse_observations(text: &str) -> Vec<ParsedObservation> {
84    parse_observations_with_outcome(text).observations
85}
86
87pub(crate) fn parse_observations_with_outcome(text: &str) -> ParseObservationsOutcome {
88    let mut observations = Vec::new();
89    let mut invalid_type_drops = Vec::new();
90    let mut pos = 0;
91
92    while let Some(tag_start_rel) = find_ascii_ci(&text[pos..], "<observation") {
93        let tag_start = pos + tag_start_rel;
94        let Some(open_end_rel) = text[tag_start..].find('>') else {
95            break;
96        };
97        let content_start = tag_start + open_end_rel + 1;
98        let Some(close_rel) = find_ascii_ci(&text[content_start..], "</observation>") else {
99            break;
100        };
101        let content_end = content_start + close_rel;
102        let content = &text[content_start..content_end];
103
104        let Some(raw_type) = extract_field(content, "type") else {
105            crate::log::error(
106                "observation-parse",
107                "dropping observation: drop_reason=missing_type raw_type=\"\"",
108            );
109            invalid_type_drops.push(InvalidObservationTypeDrop::Missing);
110            pos = content_end + "</observation>".len();
111            continue;
112        };
113        let obs_type = raw_type.trim().to_ascii_lowercase();
114        if !OBSERVATION_TYPES.contains(&obs_type.as_str()) {
115            let raw_type_preview = crate::adapter::redaction::redact_and_truncate(
116                &raw_type,
117                INVALID_TYPE_PREVIEW_BYTES,
118            );
119            crate::log::error(
120                "observation-parse",
121                &format!(
122                    "dropping observation: drop_reason=unknown_type raw_type_preview={raw_type_preview:?} raw_type_bytes={}",
123                    raw_type.len()
124                ),
125            );
126            invalid_type_drops.push(InvalidObservationTypeDrop::Unknown);
127            pos = content_end + "</observation>".len();
128            continue;
129        }
130
131        let mut concepts = extract_array(content, "concepts", "concept");
132        concepts.retain(|concept| !concept.eq_ignore_ascii_case(&obs_type));
133
134        observations.push(ParsedObservation {
135            obs_type,
136            title: extract_field(content, "title"),
137            subtitle: extract_field(content, "subtitle"),
138            facts: extract_array(content, "facts", "fact"),
139            narrative: extract_field(content, "narrative"),
140            concepts,
141            files_read: extract_array(content, "files_read", "file"),
142            files_modified: extract_array(content, "files_modified", "file"),
143            confidence: parse_confidence(content),
144        });
145
146        pos = content_end + "</observation>".len();
147    }
148
149    ParseObservationsOutcome {
150        observations,
151        invalid_type_drops,
152    }
153}
remem/memory/format/parse.rs

remem/memory/format/
parse.rs
