aimem_core/convo/
normalize.rs

1//! normalize.rs — convert chat export formats to plain transcript.
2
3use serde_json::Value;
4use std::path::Path;
5
6/// Normalize file content to plain transcript string.
7pub fn normalize_content(content: &str, path: &Path) -> String {
8    let content = content.trim();
9    if content.is_empty() {
10        return content.to_string();
11    }
12
13    // Already has `>` markers — pass through
14    let quote_count = content
15        .lines()
16        .filter(|l| l.trim_start().starts_with('>'))
17        .count();
18    if quote_count >= 3 {
19        return content.to_string();
20    }
21
22    let ext = path
23        .extension()
24        .map(|e| e.to_string_lossy().to_lowercase())
25        .unwrap_or_default();
26
27    if ext == "json"
28        || ext == "jsonl"
29        || content.trim_start().starts_with('{')
30        || content.trim_start().starts_with('[')
31    {
32        if let Some(normalized) = try_normalize_json(content) {
33            return normalized;
34        }
35    }
36
37    content.to_string()
38}
39
40fn try_normalize_json(content: &str) -> Option<String> {
41    if let Some(s) = try_claude_code_jsonl(content) {
42        return Some(s);
43    }
44    let data: Value = serde_json::from_str(content).ok()?;
45    try_claude_ai(&data)
46        .or_else(|| try_chatgpt(&data))
47        .or_else(|| try_slack(&data))
48}
49
50fn try_claude_code_jsonl(content: &str) -> Option<String> {
51    let mut messages = Vec::new();
52    let mut any_ok = false;
53    for line in content.lines() {
54        let line = line.trim();
55        if line.is_empty() {
56            continue;
57        }
58        let Ok(val): Result<Value, _> = serde_json::from_str(line) else {
59            continue;
60        };
61        any_ok = true;
62        let role = val["role"].as_str().unwrap_or("");
63        let text = extract_text(&val["content"]);
64        if !text.is_empty() {
65            if role == "user" {
66                messages.push(format!("> {}", text.replace('\n', "\n> ")));
67            } else {
68                messages.push(text);
69            }
70        }
71    }
72    if any_ok && !messages.is_empty() {
73        Some(messages.join("\n\n"))
74    } else {
75        None
76    }
77}
78
79fn try_claude_ai(data: &Value) -> Option<String> {
80    let arr = data
81        .get("conversation")
82        .or(data.get("messages"))?
83        .as_array()?;
84    if arr.is_empty() {
85        return None;
86    }
87    let mut messages = Vec::new();
88    for msg in arr {
89        let sender = msg["sender"]
90            .as_str()
91            .unwrap_or(msg["role"].as_str().unwrap_or("unknown"));
92        let text = extract_text(&msg["text"]);
93        let text = if text.is_empty() {
94            extract_text(&msg["content"])
95        } else {
96            text
97        };
98        if text.is_empty() {
99            continue;
100        }
101        if sender == "human" || sender == "user" {
102            messages.push(format!("> {}", text.replace('\n', "\n> ")));
103        } else {
104            messages.push(text);
105        }
106    }
107    if messages.is_empty() {
108        None
109    } else {
110        Some(messages.join("\n\n"))
111    }
112}
113
114fn try_chatgpt(data: &Value) -> Option<String> {
115    let conversations = if data.is_array() {
116        data.as_array()?.clone()
117    } else if data.get("mapping").is_some() {
118        vec![data.clone()]
119    } else {
120        return None;
121    };
122
123    let mut all_messages: Vec<String> = Vec::new();
124    for conv in &conversations {
125        let mapping = conv.get("mapping")?.as_object()?;
126        let mut nodes: Vec<(&str, &Value)> = mapping.iter().map(|(k, v)| (k.as_str(), v)).collect();
127        nodes.sort_by(|a, b| {
128            let ta = a.1["message"]["create_time"].as_f64().unwrap_or(0.0);
129            let tb = b.1["message"]["create_time"].as_f64().unwrap_or(0.0);
130            ta.partial_cmp(&tb).unwrap_or(std::cmp::Ordering::Equal)
131        });
132        for (_, node) in nodes {
133            let role = node["message"]["author"]["role"].as_str().unwrap_or("");
134            let text = extract_text(&node["message"]["content"]);
135            if text.is_empty() || role == "system" {
136                continue;
137            }
138            if role == "user" {
139                all_messages.push(format!("> {}", text.replace('\n', "\n> ")));
140            } else {
141                all_messages.push(text);
142            }
143        }
144    }
145    if all_messages.is_empty() {
146        None
147    } else {
148        Some(all_messages.join("\n\n"))
149    }
150}
151
152fn try_slack(data: &Value) -> Option<String> {
153    let arr = data.as_array()?;
154    if !arr.iter().any(|m| m["type"].as_str() == Some("message")) {
155        return None;
156    }
157    let messages: Vec<String> = arr
158        .iter()
159        .filter(|m| m["type"].as_str() == Some("message"))
160        .filter_map(|m| {
161            let text = m["text"].as_str()?;
162            let user = m["user"].as_str().unwrap_or("unknown");
163            Some(format!("[{user}] {text}"))
164        })
165        .collect();
166    if messages.is_empty() {
167        None
168    } else {
169        Some(messages.join("\n"))
170    }
171}
172
173fn extract_text(val: &Value) -> String {
174    match val {
175        Value::String(s) => s.trim().to_string(),
176        Value::Array(parts) => parts
177            .iter()
178            .filter_map(|p| {
179                if p["type"].as_str() == Some("text") {
180                    p["text"].as_str().map(|s| s.to_string())
181                } else {
182                    p.as_str().map(|s| s.to_string())
183                }
184            })
185            .collect::<Vec<_>>()
186            .join("\n"),
187        _ => String::new(),
188    }
189}
aimem_core/convo/normalize.rs

aimem_core/convo/
normalize.rs
