mempal_runtime/ingest/
detect.rs

1use serde_json::Value;
2
3#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4pub enum Format {
5    ClaudeJsonl,
6    ChatGptJson,
7    CodexJsonl,
8    SlackJson,
9    PlainText,
10}
11
12pub fn detect_format(content: &str) -> Format {
13    if is_claude_jsonl(content) {
14        return Format::ClaudeJsonl;
15    }
16
17    if is_codex_jsonl(content) {
18        return Format::CodexJsonl;
19    }
20
21    if is_slack_json(content) {
22        return Format::SlackJson;
23    }
24
25    if is_chatgpt_json(content) {
26        return Format::ChatGptJson;
27    }
28
29    Format::PlainText
30}
31
32fn is_codex_jsonl(content: &str) -> bool {
33    let mut has_session_meta = false;
34    let mut event_msg_count = 0;
35
36    for line in content.lines().map(str::trim).filter(|l| !l.is_empty()) {
37        let Ok(value) = serde_json::from_str::<Value>(line) else {
38            return false;
39        };
40        match value.get("type").and_then(Value::as_str) {
41            Some("session_meta") => has_session_meta = true,
42            Some("event_msg") => event_msg_count += 1,
43            Some("response_item") => {} // skip but don't reject
44            _ => return false,
45        }
46    }
47
48    has_session_meta && event_msg_count >= 2
49}
50
51fn is_slack_json(content: &str) -> bool {
52    let Ok(value) = serde_json::from_str::<Value>(content) else {
53        return false;
54    };
55    let Some(arr) = value.as_array() else {
56        return false;
57    };
58    // Slack messages have "type": "message" and "user"/"username" + "text"
59    arr.iter().take(5).any(|msg| {
60        msg.get("type").and_then(Value::as_str) == Some("message")
61            && (msg.get("user").is_some() || msg.get("username").is_some())
62            && msg.get("text").is_some()
63    })
64}
65
66fn is_claude_jsonl(content: &str) -> bool {
67    let mut saw_line = false;
68
69    for line in content
70        .lines()
71        .map(str::trim)
72        .filter(|line| !line.is_empty())
73    {
74        let Ok(value) = serde_json::from_str::<Value>(line) else {
75            return false;
76        };
77
78        if value.get("type").and_then(Value::as_str).is_none() {
79            return false;
80        }
81        if extract_message_text(&value).is_none() {
82            return false;
83        }
84
85        saw_line = true;
86    }
87
88    saw_line
89}
90
91fn is_chatgpt_json(content: &str) -> bool {
92    let Ok(value) = serde_json::from_str::<Value>(content) else {
93        return false;
94    };
95
96    matches!(value, Value::Array(_))
97        || value.get("messages").is_some()
98        || value.get("mapping").is_some()
99}
100
101pub(crate) fn extract_message_text(value: &Value) -> Option<String> {
102    value
103        .get("message")
104        .and_then(Value::as_str)
105        .map(ToOwned::to_owned)
106        .or_else(|| value.get("content").and_then(extract_content_text))
107}
108
109pub(crate) fn extract_content_text(value: &Value) -> Option<String> {
110    match value {
111        Value::String(text) => Some(text.clone()),
112        Value::Array(items) => Some(
113            items
114                .iter()
115                .filter_map(Value::as_str)
116                .collect::<Vec<_>>()
117                .join("\n"),
118        ),
119        Value::Object(map) => map
120            .get("parts")
121            .and_then(Value::as_array)
122            .map(|parts| {
123                parts
124                    .iter()
125                    .filter_map(Value::as_str)
126                    .collect::<Vec<_>>()
127                    .join("\n")
128            })
129            .filter(|text| !text.is_empty()),
130        _ => None,
131    }
132}
mempal_runtime/ingest/detect.rs

mempal_runtime/ingest/
detect.rs
