use crate::ingest::{Turn, TurnTokens};
use serde_json::Value;
use std::path::Path;
const EXTRA_TOKEN_FIELDS: &[&str] = &["reasoning_output_tokens"];
pub fn parse_rollout(path: &Path) -> Vec<Turn> {
let Ok(content) = std::fs::read_to_string(path) else {
eprintln!(" warn: could not read {}", path.display());
return vec![];
};
let mut turns: Vec<Turn> = vec![];
let mut cur: Option<Turn> = None;
let mut model = String::new();
let mut prov = Prov::default();
for line in content.lines().filter(|l| !l.trim().is_empty()) {
let Ok(v) = serde_json::from_str::<Value>(line) else {
continue; };
let ts = v.get("timestamp").and_then(|t| t.as_str()).unwrap_or("");
let payload = v.get("payload").unwrap_or(&Value::Null);
let outer = v.get("type").and_then(|t| t.as_str()).unwrap_or("");
let inner = payload.get("type").and_then(|t| t.as_str()).unwrap_or("");
match (outer, inner) {
("session_meta", _) => {
if let Some(m) = payload.get("model").and_then(|m| m.as_str()) {
model = m.to_string();
}
prov.absorb(payload);
}
("turn_context", _) => {
if let Some(m) = payload.get("model").and_then(|m| m.as_str()) {
model = m.to_string();
}
prov.absorb(payload);
}
("response_item", "message") => {
let role = payload.get("role").and_then(|r| r.as_str()).unwrap_or("");
let text = join_text(payload.get("content"));
if text.is_empty() {
continue;
}
match role {
"user" => {
if let Some(t) = cur.take() {
turns.push(t);
}
cur = Some(Turn {
user_text: text,
assistant_text: String::new(),
tool_calls: vec![],
tool_calls_raw: vec![],
tokens: TurnTokens::default(),
model: model.clone(),
timestamp: ts.to_string(),
cwd: prov.cwd.clone(),
git_branches: prov.branch.clone().into_iter().collect(),
git_commit: prov.commit.clone(),
});
}
"assistant" => {
let t = cur.get_or_insert_with(|| empty_turn(&model, ts, &prov));
if !t.assistant_text.is_empty() {
t.assistant_text.push_str("\n\n");
}
t.assistant_text.push_str(&text);
if t.model.is_empty() {
t.model = model.clone();
}
}
_ => {}
}
}
("response_item", "function_call") => {
let name = payload
.get("name")
.and_then(|n| n.as_str())
.unwrap_or("tool");
let t = cur.get_or_insert_with(|| empty_turn(&model, ts, &prov));
t.tool_calls.push(name.to_string());
t.tool_calls_raw.push(serde_json::json!({
"name": name,
"input": parse_arguments(payload.get("arguments")),
"id": payload.get("call_id").cloned().unwrap_or(Value::Null),
"source": "codex",
}));
}
("event_msg", "token_count") => {
if let Some(last) = payload.get("info").and_then(|i| i.get("last_token_usage")) {
let t = cur.get_or_insert_with(|| empty_turn(&model, ts, &prov));
t.tokens.add(&tokens_from(last));
}
}
_ => {}
}
}
if let Some(t) = cur.take() {
turns.push(t);
}
turns
}
fn empty_turn(model: &str, ts: &str, prov: &Prov) -> Turn {
Turn {
user_text: String::new(),
assistant_text: String::new(),
tool_calls: vec![],
tool_calls_raw: vec![],
tokens: TurnTokens::default(),
model: model.to_string(),
timestamp: ts.to_string(),
cwd: prov.cwd.clone(),
git_branches: prov.branch.clone().into_iter().collect(),
git_commit: prov.commit.clone(),
}
}
#[derive(Default, Clone)]
struct Prov {
cwd: Option<String>,
branch: Option<String>,
commit: Option<String>,
}
impl Prov {
fn absorb(&mut self, payload: &Value) {
if let Some(c) = non_empty(payload.get("cwd")) {
self.cwd = Some(c);
}
let git = payload.get("git");
if let Some(b) = git.and_then(|g| non_empty(g.get("branch"))) {
self.branch = Some(b);
}
if let Some(h) = git.and_then(|g| non_empty(g.get("commit_hash"))) {
self.commit = Some(h);
}
}
}
fn non_empty(v: Option<&Value>) -> Option<String> {
v.and_then(|x| x.as_str())
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string)
}
fn join_text(content: Option<&Value>) -> String {
let Some(items) = content.and_then(|c| c.as_array()) else {
return content
.and_then(|c| c.as_str())
.unwrap_or_default()
.to_string();
};
items
.iter()
.filter_map(|c| c.get("text").and_then(|t| t.as_str()))
.collect::<Vec<_>>()
.join("\n")
.trim()
.to_string()
}
fn parse_arguments(args: Option<&Value>) -> Value {
match args.and_then(|a| a.as_str()) {
Some(s) => serde_json::from_str(s).unwrap_or_else(|_| Value::String(s.to_string())),
None => args.cloned().unwrap_or(Value::Null),
}
}
fn tokens_from(u: &Value) -> TurnTokens {
let get = |k: &str| u.get(k).and_then(|v| v.as_u64()).unwrap_or(0);
let mut t = TurnTokens {
input: get("input_tokens"),
output: get("output_tokens"),
cache_read: get("cached_input_tokens"),
cache_creation: 0,
extra: Default::default(),
};
for f in EXTRA_TOKEN_FIELDS {
let v = get(f);
if v > 0 {
t.extra.insert((*f).to_string(), v);
}
}
t
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
fn write_rollout(lines: &[&str]) -> tempfile::NamedTempFile {
let mut f = tempfile::NamedTempFile::new().expect("tempfile");
for l in lines {
writeln!(f, "{}", l).expect("write");
}
f.flush().expect("flush");
f
}
#[test]
fn splits_turns_on_user_messages() {
let f = write_rollout(&[
r#"{"timestamp":"t0","type":"session_meta","payload":{"id":"s1","cwd":"/a/b","model":"gpt-5.2"}}"#,
r#"{"timestamp":"t1","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"first"}]}}"#,
r#"{"timestamp":"t2","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"reply one"}]}}"#,
r#"{"timestamp":"t3","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"second"}]}}"#,
]);
let turns = parse_rollout(f.path());
assert_eq!(turns.len(), 2);
assert_eq!(turns[0].user_text, "first");
assert_eq!(turns[0].assistant_text, "reply one");
assert_eq!(turns[0].model, "gpt-5.2");
assert_eq!(turns[1].user_text, "second");
}
#[test]
fn developer_messages_are_not_user_input() {
let f = write_rollout(&[
r#"{"timestamp":"t0","type":"response_item","payload":{"type":"message","role":"developer","content":[{"type":"input_text","text":"<permissions instructions>"}]}}"#,
r#"{"timestamp":"t1","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"real question"}]}}"#,
]);
let turns = parse_rollout(f.path());
assert_eq!(turns.len(), 1);
assert_eq!(turns[0].user_text, "real question");
}
#[test]
fn reasoning_tokens_are_preserved_not_folded_into_output() {
let f = write_rollout(&[
r#"{"timestamp":"t1","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"q"}]}}"#,
r#"{"timestamp":"t2","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":21461,"cached_input_tokens":14592,"output_tokens":258,"reasoning_output_tokens":171,"total_tokens":21719}}}}"#,
]);
let turns = parse_rollout(f.path());
assert_eq!(turns[0].tokens.input, 21461);
assert_eq!(
turns[0].tokens.output, 258,
"reasoning must not inflate output"
);
assert_eq!(turns[0].tokens.cache_read, 14592);
assert_eq!(
turns[0].tokens.extra.get("reasoning_output_tokens"),
Some(&171)
);
}
#[test]
fn token_counts_sum_across_a_session() {
let f = write_rollout(&[
r#"{"timestamp":"t1","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"q"}]}}"#,
r#"{"timestamp":"t2","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":100,"output_tokens":10,"reasoning_output_tokens":1}}}}"#,
r#"{"timestamp":"t3","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":50,"output_tokens":5,"reasoning_output_tokens":2}}}}"#,
]);
let turns = parse_rollout(f.path());
assert_eq!(turns[0].tokens.input, 150);
assert_eq!(turns[0].tokens.output, 15);
assert_eq!(
turns[0].tokens.extra.get("reasoning_output_tokens"),
Some(&3)
);
}
#[test]
fn function_call_arguments_are_parsed_not_left_as_a_string() {
let f = write_rollout(&[
r#"{"timestamp":"t1","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"q"}]}}"#,
r#"{"timestamp":"t2","type":"response_item","payload":{"type":"function_call","name":"exec_command","arguments":"{\"cmd\":\"git commit -m \\\"x\\\"\"}","call_id":"c1"}}"#,
]);
let turns = parse_rollout(f.path());
assert_eq!(turns[0].tool_calls, vec!["exec_command"]);
let raw = &turns[0].tool_calls_raw[0];
assert_eq!(raw["input"]["cmd"], "git commit -m \"x\"");
}
#[test]
fn malformed_lines_do_not_abort_the_parse() {
let f = write_rollout(&[
r#"{"timestamp":"t1","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"q"}]}}"#,
r#"{"timestamp":"t2","type":"resp"#, ]);
let turns = parse_rollout(f.path());
assert_eq!(turns.len(), 1);
assert_eq!(turns[0].user_text, "q");
}
}
#[cfg(test)]
mod real_data_probe {
use super::*;
#[test]
#[ignore]
fn codex_real_rollouts_parse() {
use crate::sources::{Codex, SessionSource};
let src = Codex;
if !src.is_available() {
eprintln!("no ~/.codex on this machine; skipping");
return;
}
let refs = src.discover_all();
let mut turns = 0usize;
let mut input = 0u64;
let mut output = 0u64;
let mut reasoning = 0u64;
let mut empty_files = 0usize;
for r in &refs {
let ts = src.parse(r);
if ts.is_empty() {
empty_files += 1;
}
for t in &ts {
turns += 1;
input += t.tokens.input;
output += t.tokens.output;
reasoning += t
.tokens
.extra
.get("reasoning_output_tokens")
.copied()
.unwrap_or(0);
}
}
eprintln!(
"sessions={} turns={} input={} output={} reasoning={} files_with_no_turns={}",
refs.len(),
turns,
input,
output,
reasoning,
empty_files
);
if !refs.is_empty() {
assert!(turns > 0, "discovered rollouts but parsed no turns");
}
}
}