magi-code 0.77.1

use super::{CodecReport, FileAggregate, MAX_JSONL_LINES, u64_from_usize};
use crate::{
    context::count_supported_provider_conversation_item_tokens,
    providers::{OPENAI_CODEX_PROVIDER, ProviderConversationItem},
    sessions::SessionEvent,
};
use serde::{Deserialize, Serialize};
use serde_json::Value;

const MAX_CANDIDATE_BYTES: usize = 1024 * 1024;
const MAX_TEMPLATE_LINE_BYTES: usize = 4 * 1024;
const MIN_TEMPLATE_LITERAL_BYTES: usize = 4;
const LINE_RLE_INSTRUCTION: &str =
    "Concatenate items in order; emit raw once; repeat text count times.";
const TEMPLATE_INSTRUCTION: &str =
    "Concatenate items in order; emit raw once; emit prefix, each value, and suffix.";

#[derive(Debug)]
struct ExtractedToolResult {
    call_id: String,
    content: String,
    legacy: bool,
}

#[derive(Debug)]
struct CandidateBuild {
    output: Option<String>,
    limit_hit: bool,
}

#[derive(Debug)]
struct CandidateEvaluation {
    validation_passed: bool,
    tokens: Option<u64>,
    limit_hit: bool,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LineEndingClass {
    Lf,
    CrLf,
    NoNewline,
}

impl LineEndingClass {
    const fn as_str(self) -> &'static str {
        match self {
            Self::Lf => "lf",
            Self::CrLf => "crlf",
            Self::NoNewline => "none",
        }
    }

    fn from_str(value: &str) -> Option<Self> {
        match value {
            "lf" => Some(Self::Lf),
            "crlf" => Some(Self::CrLf),
            "none" => Some(Self::NoNewline),
            _ => None,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct LineRleEnvelope {
    marker: String,
    instruction: String,
    items: Vec<LineRleItem>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "kind", deny_unknown_fields)]
enum LineRleItem {
    #[serde(rename = "raw")]
    Raw { text: String },
    #[serde(rename = "repeat")]
    Repeat { text: String, count: u32 },
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct TemplateEnvelope {
    marker: String,
    instruction: String,
    items: Vec<TemplateItem>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "kind", deny_unknown_fields)]
enum TemplateItem {
    #[serde(rename = "raw")]
    Raw { text: String },
    #[serde(rename = "template")]
    Template {
        prefix: String,
        suffix: String,
        ending: String,
        values: Vec<String>,
    },
}

#[derive(Debug, Clone)]
struct TemplatePattern {
    prefix: String,
    suffix: String,
    ending: LineEndingClass,
}

pub(super) fn process_jsonl_line(line: &[u8], model: &str, aggregate: &mut FileAggregate) {
    let line = line.strip_suffix(b"\n").unwrap_or(line);
    let line = line.strip_suffix(b"\r").unwrap_or(line);
    let event = match parse_session_event(line) {
        Some(event) => event,
        None => {
            aggregate.jsonl.malformed = aggregate.jsonl.malformed.saturating_add(1);
            return;
        }
    };
    aggregate.jsonl.valid_events = aggregate.jsonl.valid_events.saturating_add(1);
    if event.event_type != "tool_result" {
        return;
    }
    aggregate.tool_results.found = aggregate.tool_results.found.saturating_add(1);
    let Some(extracted) = extract_tool_result(&event) else {
        aggregate.tool_results.invalid = aggregate.tool_results.invalid.saturating_add(1);
        return;
    };
    if extracted.legacy {
        aggregate.tool_results.legacy = aggregate.tool_results.legacy.saturating_add(1);
    }
    if extracted.content.is_empty() {
        aggregate.tool_results.empty = aggregate.tool_results.empty.saturating_add(1);
    }
    measure_tool_result(extracted, model, aggregate);
}

/// Parses each JSONL line with the canonical persisted session-event type.
/// Serde accepts additive top-level fields, rejects duplicate fields, and
/// preserves the same required-field behavior as session replay.
fn parse_session_event(line: &[u8]) -> Option<SessionEvent> {
    serde_json::from_slice::<SessionEvent>(line).ok()
}

fn extract_tool_result(event: &SessionEvent) -> Option<ExtractedToolResult> {
    if event.event_type != "tool_result" {
        return None;
    }
    let payload = event.payload.as_object()?;
    let result = payload.get("result")?.as_object()?;
    let call_id = result
        .get("call_id")
        .or_else(|| payload.get("call_id"))
        .and_then(Value::as_str)?;
    if call_id.is_empty() {
        return None;
    }
    let tool_name = result
        .get("tool_name")
        .or_else(|| payload.get("tool_name"))
        .and_then(Value::as_str)?;
    if tool_name.is_empty() {
        return None;
    }
    let _success = result.get("success").and_then(Value::as_bool)?;
    let (content, legacy) = match (result.get("content"), result.get("output")) {
        (Some(content), None) => (content.as_str()?.to_string(), false),
        (None, Some(output)) => (output.as_str()?.to_string(), true),
        _ => return None,
    };
    Some(ExtractedToolResult {
        call_id: call_id.to_string(),
        content,
        legacy,
    })
}

fn measure_tool_result(extracted: ExtractedToolResult, model: &str, aggregate: &mut FileAggregate) {
    if extracted.content.len() > MAX_CANDIDATE_BYTES {
        aggregate.tool_results.skipped_limit =
            aggregate.tool_results.skipped_limit.saturating_add(1);
        aggregate.limit_hit = true;
        return;
    }
    let baseline_item = codex_responses_tool_result_item(&extracted.call_id, &extracted.content);
    let Some(baseline_tokens) = count_item_tokens(model, &baseline_item) else {
        // The model was validated before discovery. If a future tokenizer
        // lookup still fails, fail closed without misreporting a result-cap
        // skip.
        return;
    };
    aggregate.tool_results.measured = aggregate.tool_results.measured.saturating_add(1);
    aggregate.baseline.bytes = aggregate
        .baseline
        .bytes
        .saturating_add(u64_from_usize(extracted.content.len()));
    aggregate.baseline.tokens = aggregate.baseline.tokens.saturating_add(baseline_tokens);

    let json = evaluate_candidate(
        build_json_minification(&extracted.content),
        &baseline_item,
        model,
    );
    let rle = evaluate_candidate(build_line_rle(&extracted.content), &baseline_item, model);
    let template = evaluate_candidate(
        build_template_folding(&extracted.content),
        &baseline_item,
        model,
    );
    aggregate.limit_hit |= json.limit_hit || rle.limit_hit || template.limit_hit;
    record_codec_evaluation(
        &mut aggregate.codecs.json_minification,
        &json,
        baseline_tokens,
    );
    record_codec_evaluation(&mut aggregate.codecs.line_rle, &rle, baseline_tokens);
    record_codec_evaluation(
        &mut aggregate.codecs.template_folding,
        &template,
        baseline_tokens,
    );

    let candidates = [json.tokens, rle.tokens, template.tokens];
    let selected = select_unique_best(&candidates, baseline_tokens);
    let selected_tokens = selected
        .and_then(|index| candidates[index])
        .unwrap_or(baseline_tokens);
    aggregate.portfolio.selected_tokens = aggregate
        .portfolio
        .selected_tokens
        .saturating_add(selected_tokens);
    let saved = baseline_tokens.saturating_sub(selected_tokens);
    aggregate.portfolio.saved_tokens = aggregate.portfolio.saved_tokens.saturating_add(saved);
    if let Some(index) = selected {
        aggregate.portfolio.outputs_transformed =
            aggregate.portfolio.outputs_transformed.saturating_add(1);
        let codec = match index {
            0 => &mut aggregate.codecs.json_minification,
            1 => &mut aggregate.codecs.line_rle,
            _ => &mut aggregate.codecs.template_folding,
        };
        codec.selected = codec.selected.saturating_add(1);
        codec.saved_tokens_when_selected = codec.saved_tokens_when_selected.saturating_add(saved);
    }
}

/// Counts the Codex Responses-style local BPE projection: type, call_id,
/// output, and the standard item framing. It is not a provider billing count.
fn codex_responses_tool_result_item(call_id: &str, output: &str) -> ProviderConversationItem {
    ProviderConversationItem::ResponseItem(serde_json::json!({
        "type": "function_call_output",
        "call_id": call_id,
        "output": output,
    }))
}

fn count_item_tokens(model: &str, item: &ProviderConversationItem) -> Option<u64> {
    count_supported_provider_conversation_item_tokens(OPENAI_CODEX_PROVIDER, model, item)
        .map(u64_from_usize)
}
fn evaluate_candidate(
    candidate: CandidateBuild,
    baseline_item: &ProviderConversationItem,
    model: &str,
) -> CandidateEvaluation {
    let Some(output) = candidate.output else {
        return CandidateEvaluation {
            validation_passed: false,
            tokens: None,
            limit_hit: candidate.limit_hit,
        };
    };
    let candidate_item = item_with_output(baseline_item, &output);
    let tokens = count_item_tokens(model, &candidate_item);
    CandidateEvaluation {
        validation_passed: tokens.is_some(),
        tokens,
        limit_hit: candidate.limit_hit,
    }
}

fn item_with_output(item: &ProviderConversationItem, output: &str) -> ProviderConversationItem {
    let ProviderConversationItem::ResponseItem(value) = item else {
        return item.clone();
    };
    let mut value = value.clone();
    let Some(object) = value.as_object_mut() else {
        return item.clone();
    };
    object.insert("output".to_string(), Value::String(output.to_string()));
    ProviderConversationItem::ResponseItem(value)
}

fn record_codec_evaluation(
    codec: &mut CodecReport,
    evaluation: &CandidateEvaluation,
    baseline_tokens: u64,
) {
    codec.attempted = codec.attempted.saturating_add(1);
    if !evaluation.validation_passed {
        return;
    }
    codec.applicable_validation_passed = codec.applicable_validation_passed.saturating_add(1);
    codec.baseline_tokens_when_applicable = codec
        .baseline_tokens_when_applicable
        .saturating_add(baseline_tokens);
    let Some(tokens) = evaluation.tokens else {
        return;
    };
    codec.candidate_tokens_when_applicable = codec
        .candidate_tokens_when_applicable
        .saturating_add(tokens);
    if tokens < baseline_tokens {
        codec.strict_improvements = codec.strict_improvements.saturating_add(1);
        codec.saved_tokens_when_strict_improvement = codec
            .saved_tokens_when_strict_improvement
            .saturating_add(baseline_tokens.saturating_sub(tokens));
    }
}

fn select_unique_best(candidate_tokens: &[Option<u64>], baseline: u64) -> Option<usize> {
    let mut best: Option<(usize, u64)> = None;
    let mut tied = false;
    for (index, candidate) in candidate_tokens.iter().enumerate() {
        let Some(tokens) = candidate else {
            continue;
        };
        if *tokens >= baseline {
            continue;
        }
        match best {
            None => {
                best = Some((index, *tokens));
                tied = false;
            }
            Some((_, best_tokens)) if *tokens < best_tokens => {
                best = Some((index, *tokens));
                tied = false;
            }
            Some((_, best_tokens)) if *tokens == best_tokens => {
                tied = true;
            }
            Some(_) => {}
        }
    }
    if tied {
        None
    } else {
        best.map(|(index, _)| index)
    }
}

fn build_json_minification(input: &str) -> CandidateBuild {
    if input.len() > MAX_CANDIDATE_BYTES {
        return CandidateBuild {
            output: None,
            limit_hit: true,
        };
    }
    let Some(()) = validate_complete_json(input) else {
        return CandidateBuild {
            output: None,
            limit_hit: false,
        };
    };
    let mut output = Vec::with_capacity(input.len());
    let mut in_string = false;
    let mut escaped = false;
    for byte in input.bytes() {
        if in_string {
            output.push(byte);
            if escaped {
                escaped = false;
            } else if byte == b'\\' {
                escaped = true;
            } else if byte == b'"' {
                in_string = false;
            }
            continue;
        }
        if byte == b'"' {
            in_string = true;
            output.push(byte);
        } else if !matches!(byte, b' ' | b'\t' | b'\n' | b'\r') {
            output.push(byte);
        }
    }
    let Ok(output) = String::from_utf8(output) else {
        return CandidateBuild {
            output: None,
            limit_hit: false,
        };
    };
    if validate_complete_json(&output).is_none() {
        return CandidateBuild {
            output: None,
            limit_hit: false,
        };
    }
    CandidateBuild {
        output: Some(output),
        limit_hit: false,
    }
}

fn validate_complete_json(input: &str) -> Option<()> {
    use serde::Deserialize;
    use serde::de::IgnoredAny;
    let mut deserializer = serde_json::Deserializer::from_str(input);
    IgnoredAny::deserialize(&mut deserializer).ok()?;
    deserializer.end().ok()?;
    Some(())
}

fn build_line_rle(input: &str) -> CandidateBuild {
    if input.len() > MAX_CANDIDATE_BYTES {
        return CandidateBuild {
            output: None,
            limit_hit: true,
        };
    }
    let chunks = split_lf_chunks(input);
    if chunks.len() > MAX_JSONL_LINES {
        return CandidateBuild {
            output: None,
            limit_hit: true,
        };
    }
    let mut items = Vec::new();
    let mut raw = String::new();
    let mut index = 0;
    let mut has_run = false;
    while index < chunks.len() {
        let mut end = index.saturating_add(1);
        while end < chunks.len() && chunks[end] == chunks[index] {
            end = end.saturating_add(1);
        }
        let run_length = end.saturating_sub(index);
        if run_length >= 2 {
            flush_rle_raw(&mut items, &mut raw);
            let Some(count) = u32::try_from(run_length).ok() else {
                return CandidateBuild {
                    output: None,
                    limit_hit: true,
                };
            };
            items.push(LineRleItem::Repeat {
                text: chunks[index].to_string(),
                count,
            });
            has_run = true;
        } else {
            raw.push_str(chunks[index]);
        }
        index = end;
    }
    flush_rle_raw(&mut items, &mut raw);
    if !has_run {
        return CandidateBuild {
            output: None,
            limit_hit: false,
        };
    }
    let envelope = LineRleEnvelope {
        marker: "magi-line-rle-v0".to_string(),
        instruction: LINE_RLE_INSTRUCTION.to_string(),
        items,
    };
    let Ok(encoded) = serde_json::to_string(&envelope) else {
        return CandidateBuild {
            output: None,
            limit_hit: true,
        };
    };
    if encoded.len() > MAX_CANDIDATE_BYTES || decode_line_rle(&encoded).as_deref() != Some(input) {
        return CandidateBuild {
            output: None,
            limit_hit: encoded.len() > MAX_CANDIDATE_BYTES,
        };
    }
    CandidateBuild {
        output: Some(encoded),
        limit_hit: false,
    }
}

fn flush_rle_raw(items: &mut Vec<LineRleItem>, raw: &mut String) {
    if raw.is_empty() {
        return;
    }
    items.push(LineRleItem::Raw {
        text: std::mem::take(raw),
    });
}

fn split_lf_chunks(input: &str) -> Vec<&str> {
    let mut chunks = Vec::new();
    let mut start = 0;
    for (index, byte) in input.bytes().enumerate() {
        if byte == b'\n' {
            chunks.push(&input[start..index.saturating_add(1)]);
            start = index.saturating_add(1);
        }
    }
    if start < input.len() {
        chunks.push(&input[start..]);
    }
    chunks
}

fn decode_line_rle(encoded: &str) -> Option<String> {
    if encoded.len() > MAX_CANDIDATE_BYTES {
        return None;
    }
    let value = serde_json::from_str::<Value>(encoded).ok()?;
    validate_exact_object_keys(value.as_object()?, &["marker", "instruction", "items"])?;
    let items = value.get("items")?.as_array()?;
    if items.is_empty()
        || items
            .iter()
            .any(|item| validate_line_rle_item(item).is_none())
    {
        return None;
    }
    let envelope = serde_json::from_value::<LineRleEnvelope>(value).ok()?;
    if envelope.marker != "magi-line-rle-v0" || envelope.instruction != LINE_RLE_INSTRUCTION {
        return None;
    }
    let canonical = serde_json::to_string(&envelope).ok()?;
    if canonical != encoded {
        return None;
    }
    let mut output = String::new();
    let mut line_count = 0usize;
    let mut previous_was_raw = false;
    for item in envelope.items {
        match item {
            LineRleItem::Raw { text } => {
                if text.is_empty() || previous_was_raw {
                    return None;
                }
                line_count = line_count.checked_add(split_lf_chunks(&text).len())?;
                if line_count > MAX_JSONL_LINES {
                    return None;
                }
                push_bounded(&mut output, &text, MAX_CANDIDATE_BYTES)?;
                previous_was_raw = true;
            }
            LineRleItem::Repeat { text, count } => {
                if text.is_empty()
                    || !text.ends_with('\n')
                    || split_lf_chunks(&text).len() != 1
                    || count < 2
                {
                    return None;
                }
                let repeat_count = usize::try_from(count).ok()?;
                line_count = line_count.checked_add(repeat_count)?;
                if line_count > MAX_JSONL_LINES {
                    return None;
                }
                for _ in 0..repeat_count {
                    push_bounded(&mut output, &text, MAX_CANDIDATE_BYTES)?;
                }
                previous_was_raw = false;
            }
        }
    }
    Some(output)
}

fn build_template_folding(input: &str) -> CandidateBuild {
    if input.len() > MAX_CANDIDATE_BYTES {
        return CandidateBuild {
            output: None,
            limit_hit: true,
        };
    }
    let chunks = split_lf_chunks(input);
    if chunks.len() > MAX_JSONL_LINES
        || chunks
            .iter()
            .any(|chunk| chunk.len() > MAX_TEMPLATE_LINE_BYTES)
    {
        return CandidateBuild {
            output: None,
            limit_hit: true,
        };
    }
    let mut items = Vec::new();
    let mut raw = String::new();
    let mut index = 0;
    let mut has_template = false;
    while index < chunks.len() {
        if index.saturating_add(1) < chunks.len()
            && let Some((pattern, first, second)) =
                derive_template_pattern(chunks[index], chunks[index.saturating_add(1)])
        {
            let mut values = vec![first, second];
            let mut end = index.saturating_add(2);
            while end < chunks.len() {
                let Some(value) = template_value(&pattern, chunks[end]) else {
                    break;
                };
                values.push(value);
                end = end.saturating_add(1);
            }
            if values.len() >= 2 {
                flush_template_raw(&mut items, &mut raw);
                items.push(TemplateItem::Template {
                    prefix: pattern.prefix,
                    suffix: pattern.suffix,
                    ending: pattern.ending.as_str().to_string(),
                    values,
                });
                has_template = true;
                index = end;
                continue;
            }
        }
        raw.push_str(chunks[index]);
        index = index.saturating_add(1);
    }
    flush_template_raw(&mut items, &mut raw);
    if !has_template {
        return CandidateBuild {
            output: None,
            limit_hit: false,
        };
    }
    let envelope = TemplateEnvelope {
        marker: "magi-line-template-v0".to_string(),
        instruction: TEMPLATE_INSTRUCTION.to_string(),
        items,
    };
    let Ok(encoded) = serde_json::to_string(&envelope) else {
        return CandidateBuild {
            output: None,
            limit_hit: true,
        };
    };
    if encoded.len() > MAX_CANDIDATE_BYTES || decode_template(&encoded).as_deref() != Some(input) {
        return CandidateBuild {
            output: None,
            limit_hit: encoded.len() > MAX_CANDIDATE_BYTES,
        };
    }
    CandidateBuild {
        output: Some(encoded),
        limit_hit: false,
    }
}

fn flush_template_raw(items: &mut Vec<TemplateItem>, raw: &mut String) {
    if raw.is_empty() {
        return;
    }
    items.push(TemplateItem::Raw {
        text: std::mem::take(raw),
    });
}

fn derive_template_pattern(first: &str, second: &str) -> Option<(TemplatePattern, String, String)> {
    if first == second
        || first.len() > MAX_TEMPLATE_LINE_BYTES
        || second.len() > MAX_TEMPLATE_LINE_BYTES
        || ending_class(first) != ending_class(second)
    {
        return None;
    }
    let ending = ending_class(first);
    let min_len = first.len().min(second.len());
    let mut prefix = first
        .bytes()
        .zip(second.bytes())
        .take_while(|(left, right)| left == right)
        .count();
    while prefix > 0 && (!first.is_char_boundary(prefix) || !second.is_char_boundary(prefix)) {
        prefix = prefix.saturating_sub(1);
    }
    let mut suffix = 0usize;
    while suffix < min_len
        && first.as_bytes()[first.len().saturating_sub(1).saturating_sub(suffix)]
            == second.as_bytes()[second.len().saturating_sub(1).saturating_sub(suffix)]
    {
        suffix = suffix.saturating_add(1);
    }
    let max_suffix = min_len.saturating_sub(prefix);
    suffix = suffix.min(max_suffix);
    while suffix > 0
        && (!first.is_char_boundary(first.len().saturating_sub(suffix))
            || !second.is_char_boundary(second.len().saturating_sub(suffix)))
    {
        suffix = suffix.saturating_sub(1);
    }
    let first_end = first.len().saturating_sub(suffix);
    let second_end = second.len().saturating_sub(suffix);
    if prefix > first_end || prefix > second_end {
        return None;
    }
    let first_value = &first[prefix..first_end];
    let second_value = &second[prefix..second_end];
    if first_value == second_value
        || prefix.saturating_add(suffix) < MIN_TEMPLATE_LITERAL_BYTES
        || prefix.saturating_add(suffix).saturating_mul(2) < first.len().max(second.len())
    {
        return None;
    }
    Some((
        TemplatePattern {
            prefix: first[..prefix].to_string(),
            suffix: first[first_end..].to_string(),
            ending,
        },
        first_value.to_string(),
        second_value.to_string(),
    ))
}

fn template_value(pattern: &TemplatePattern, line: &str) -> Option<String> {
    if line.len() > MAX_TEMPLATE_LINE_BYTES
        || ending_class(line) != pattern.ending
        || !line.starts_with(&pattern.prefix)
        || !line.ends_with(&pattern.suffix)
    {
        return None;
    }
    let start = pattern.prefix.len();
    let end = line.len().checked_sub(pattern.suffix.len())?;
    if start > end || !line.is_char_boundary(start) || !line.is_char_boundary(end) {
        return None;
    }
    Some(line[start..end].to_string())
}

fn ending_class(line: &str) -> LineEndingClass {
    if line.ends_with("\r\n") {
        LineEndingClass::CrLf
    } else if line.ends_with('\n') {
        LineEndingClass::Lf
    } else {
        LineEndingClass::NoNewline
    }
}

fn is_single_line_chunk(line: &str, ending: LineEndingClass) -> bool {
    match ending {
        LineEndingClass::Lf | LineEndingClass::CrLf => {
            line.ends_with('\n') && split_lf_chunks(line).len() == 1
        }
        LineEndingClass::NoNewline => !line.contains('\n'),
    }
}

fn decode_template(encoded: &str) -> Option<String> {
    if encoded.len() > MAX_CANDIDATE_BYTES {
        return None;
    }
    let value = serde_json::from_str::<Value>(encoded).ok()?;
    validate_exact_object_keys(value.as_object()?, &["marker", "instruction", "items"])?;
    let items = value.get("items")?.as_array()?;
    if items.is_empty()
        || items
            .iter()
            .any(|item| validate_template_item(item).is_none())
    {
        return None;
    }
    let envelope = serde_json::from_value::<TemplateEnvelope>(value).ok()?;
    if envelope.marker != "magi-line-template-v0" || envelope.instruction != TEMPLATE_INSTRUCTION {
        return None;
    }
    let canonical = serde_json::to_string(&envelope).ok()?;
    if canonical != encoded {
        return None;
    }
    let mut output = String::new();
    let mut line_count = 0usize;
    let mut previous_was_raw = false;
    for item in envelope.items {
        match item {
            TemplateItem::Raw { text } => {
                if text.is_empty() || previous_was_raw {
                    return None;
                }
                line_count = line_count.checked_add(split_lf_chunks(&text).len())?;
                if line_count > MAX_JSONL_LINES {
                    return None;
                }
                push_bounded(&mut output, &text, MAX_CANDIDATE_BYTES)?;
                previous_was_raw = true;
            }
            TemplateItem::Template {
                prefix,
                suffix,
                ending,
                values,
            } => {
                let ending = LineEndingClass::from_str(&ending)?;
                if values.len() < 2
                    || prefix.len().saturating_add(suffix.len()) < MIN_TEMPLATE_LITERAL_BYTES
                {
                    return None;
                }
                if !values.windows(2).any(|pair| pair[0] != pair[1]) {
                    return None;
                }
                for value in values {
                    let line = format!("{prefix}{value}{suffix}");
                    if line.len() > MAX_TEMPLATE_LINE_BYTES
                        || ending_class(&line) != ending
                        || !is_single_line_chunk(&line, ending)
                        || !line.starts_with(&prefix)
                        || !line.ends_with(&suffix)
                    {
                        return None;
                    }
                    line_count = line_count.checked_add(1)?;
                    if line_count > MAX_JSONL_LINES {
                        return None;
                    }
                    push_bounded(&mut output, &line, MAX_CANDIDATE_BYTES)?;
                }
                previous_was_raw = false;
            }
        }
    }
    Some(output)
}

fn validate_line_rle_item(value: &Value) -> Option<()> {
    let object = value.as_object()?;
    match object.get("kind")?.as_str()? {
        "raw" => validate_exact_object_keys(object, &["kind", "text"]),
        "repeat" => validate_exact_object_keys(object, &["kind", "text", "count"]),
        _ => None,
    }
}

fn validate_template_item(value: &Value) -> Option<()> {
    let object = value.as_object()?;
    match object.get("kind")?.as_str()? {
        "raw" => validate_exact_object_keys(object, &["kind", "text"]),
        "template" => {
            validate_exact_object_keys(object, &["kind", "prefix", "suffix", "ending", "values"])
        }
        _ => None,
    }
}

fn validate_exact_object_keys(
    object: &serde_json::Map<String, Value>,
    keys: &[&str],
) -> Option<()> {
    if object.len() != keys.len() || keys.iter().any(|key| !object.contains_key(*key)) {
        return None;
    }
    Some(())
}

fn push_bounded(output: &mut String, value: &str, max_bytes: usize) -> Option<()> {
    let next_len = output.len().checked_add(value.len())?;
    if next_len > max_bytes {
        return None;
    }
    output.push_str(value);
    Some(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::providers::ProviderConversationItem;
    use proptest::prelude::*;
    use serde_json::{Value, json};
    use std::{fs, path::Path};
    use tempfile::TempDir;

    fn event_with_content(content: &str) -> String {
        serde_json::to_string(&json!({
            "event_type": "tool_result",
            "timestamp": "2025-01-01T00:00:00Z",
            "session_id": "private-session-id",
            "cwd": "/private/session-cwd",
            "payload": {
                "call_id": "private-call-id",
                "result": {
                    "tool_name": "private-tool-name",
                    "success": true,
                    "content": content
                }
            }
        }))
        .unwrap()
    }

    fn write_session(root: &Path, name: &str, lines: &[String]) {
        fs::create_dir_all(root).unwrap();
        fs::write(root.join(name), format!("{}\n", lines.join("\n"))).unwrap();
    }

    fn measure_one(content: &str) -> super::super::MeasurementReport {
        let temp = TempDir::new().unwrap();
        let sessions = temp.path().join("sessions");
        write_session(&sessions, "one.jsonl", &[event_with_content(content)]);
        super::super::measure_sessions(&sessions, "gpt-4o").unwrap()
    }

    fn generated_text() -> impl Strategy<Value = String> {
        prop::collection::vec(
            prop::sample::select(vec!["", "a", "é", "中", "🙂", ":", "\\", "\""]),
            0..16,
        )
        .prop_map(|parts| parts.join(""))
    }

    #[test]
    fn event_parser_uses_canonical_session_event_deserialization() {
        let valid = serde_json::from_str::<Value>(&event_with_content("ok")).unwrap();
        assert!(parse_session_event(serde_json::to_string(&valid).unwrap().as_bytes()).is_some());

        let mut missing_timestamp = valid.clone();
        missing_timestamp
            .as_object_mut()
            .unwrap()
            .remove("timestamp");
        assert!(
            parse_session_event(
                serde_json::to_string(&missing_timestamp)
                    .unwrap()
                    .as_bytes()
            )
            .is_none()
        );

        let mut unknown = valid;
        unknown
            .as_object_mut()
            .unwrap()
            .insert("private_extra".to_string(), json!(true));
        assert!(parse_session_event(serde_json::to_string(&unknown).unwrap().as_bytes()).is_some());

        let duplicate = br#"{"event_type":"tool_result","event_type":"tool_result","timestamp":"2025-01-01T00:00:00Z","session_id":"private-session-id","cwd":"/private/session-cwd","payload":{}}"#;
        assert!(parse_session_event(duplicate).is_none());
    }

    #[test]
    fn json_minification_preserves_strings_escapes_duplicates_and_number_spelling() {
        let input = r#" { "a" : "  \"keep\" ", "a" : 1e+02, "escaped": "\\n" } "#;
        let output = build_json_minification(input).output.unwrap();
        assert_eq!(output, r#"{"a":"  \"keep\" ","a":1e+02,"escaped":"\\n"}"#);
        assert!(validate_complete_json(&output).is_some());
    }

    #[test]
    fn json_minification_rejects_malformed_trailing_and_bom() {
        for input in ["{", "{} {}", "\u{feff}{}"] {
            assert!(build_json_minification(input).output.is_none());
        }
    }

    #[test]
    fn line_rle_round_trips_line_endings_and_literal_bytes() {
        for input in [
            "A\nA\nB\nA\n",
            "A\r\nA\r\nlast",
            "\n\n\0\"\\\n\n",
            "A\nB\nA\n",
            "é\r\né\r\n",
        ] {
            let candidate = build_line_rle(input);
            if let Some(encoded) = candidate.output.as_deref() {
                assert_eq!(decode_line_rle(encoded), Some(input.to_string()));
            }
        }
        assert!(build_line_rle("A\nB\nA\n").output.is_none());
    }

    #[test]
    fn line_rle_decoder_rejects_unknown_fields_and_noncanonical_json() {
        let encoded = r#"{"marker":"magi-line-rle-v0","instruction":"Concatenate items in order; emit raw once; repeat text count times.","items":[{"kind":"repeat","text":"x\n","count":2}]}"#;
        assert_eq!(decode_line_rle(encoded), Some("x\nx\n".to_string()));
        assert!(decode_line_rle(
            r#"{"marker":"magi-line-rle-v0","instruction":"Concatenate items in order; emit raw once; repeat text count times.","items":[{"kind":"repeat","text":"x\n","count":2,"extra":1}]}"#
        )
        .is_none());
        assert!(decode_line_rle(
            r#"{ "marker":"magi-line-rle-v0", "instruction":"Concatenate items in order; emit raw once; repeat text count times.", "items":[{"kind":"repeat","text":"x\n","count":2}]}"#
        )
        .is_none());
        assert!(decode_line_rle(
            r#"{"marker":"magi-line-rle-v0","instruction":"Concatenate items in order; emit raw once; repeat text count times.","items":[{"kind":"repeat","text":"x\ny\n","count":2}]}"#
        )
        .is_none());
    }

    #[test]
    fn template_folding_round_trips_adjacent_unicode_and_mixed_endings() {
        for input in [
            "item=one\nitem=two\nitem=three\n",
            "item=one\r\nitem=two\r\nitem=three\r\n",
            "é:one\né:two\nlast",
        ] {
            let candidate = build_template_folding(input);
            assert_eq!(
                candidate
                    .output
                    .as_ref()
                    .and_then(|value| decode_template(value)),
                Some(input.to_string())
            );
        }
    }

    #[test]
    fn template_folding_is_adjacent_and_does_not_group_a_b_a() {
        let input = "kind=A\nother line\nkind=B\n";
        assert!(build_template_folding(input).output.is_none());
        let input = "kind=A\nkind=B\nother line\nkind=C\n";
        let candidate = build_template_folding(input);
        assert_eq!(
            candidate
                .output
                .as_ref()
                .and_then(|value| decode_template(value)),
            Some(input.to_string())
        );
    }

    #[test]
    fn template_folding_rejects_ambiguous_short_literals_and_bounds() {
        assert!(build_template_folding("a\nb\n").output.is_none());
        let long = format!(
            "{}=a\n{}=b\n",
            "x".repeat(MAX_TEMPLATE_LINE_BYTES),
            "x".repeat(MAX_TEMPLATE_LINE_BYTES)
        );
        assert!(build_template_folding(&long).limit_hit);
    }

    #[test]
    fn template_decoder_rejects_unknown_fields_and_mixed_line_endings() {
        assert!(decode_template(
            r#"{"marker":"magi-line-template-v0","instruction":"Concatenate items in order; emit raw once; emit prefix, each value, and suffix.","items":[{"kind":"template","prefix":"item=","suffix":"\n","ending":"lf","values":["one","two"],"extra":true}]}"#
        )
        .is_none());
        assert!(
            build_template_folding("item=one\nitem=two\r\n")
                .output
                .is_none()
        );
        assert!(
            build_template_folding("é-value=one\né-value=two\n")
                .output
                .is_some()
        );

        let encoded = build_template_folding("item=one\nitem=two\n")
            .output
            .unwrap();
        let mut value = serde_json::from_str::<Value>(&encoded).unwrap();
        value["instruction"] = json!("wrong instruction");
        assert!(decode_template(&serde_json::to_string(&value).unwrap()).is_none());
        value.as_object_mut().unwrap().remove("instruction");
        assert!(decode_template(&serde_json::to_string(&value).unwrap()).is_none());
    }

    #[test]
    fn codex_response_output_count_ignores_tool_name_metadata() {
        let short_name = ProviderConversationItem::ResponseItem(json!({
            "type": "function_call_output",
            "call_id": "call-1",
            "tool_name": "a",
            "output": "same output"
        }));
        let long_name = ProviderConversationItem::ResponseItem(json!({
            "type": "function_call_output",
            "call_id": "call-1",
            "tool_name": "a-tool-name-that-must-not-affect-counting",
            "output": "same output"
        }));

        assert_eq!(
            count_item_tokens("gpt-4o", &short_name),
            count_item_tokens("gpt-4o", &long_name)
        );
        assert_eq!(
            codex_responses_tool_result_item("call-1", "same output"),
            ProviderConversationItem::ResponseItem(json!({
                "type": "function_call_output",
                "call_id": "call-1",
                "output": "same output"
            }))
        );
    }

    #[test]
    fn portfolio_selection_uses_only_independent_candidates() {
        let content = "fixed\nfixed\nitem=one\nitem=two\nitem=three\n";
        let item = ProviderConversationItem::ResponseItem(serde_json::json!({
            "type": "function_call_output",
            "call_id": "private-call-id",
            "output": content,
        }));
        let baseline = count_item_tokens("gpt-4o", &item).unwrap();
        let candidates = [
            evaluate_candidate(build_json_minification(content), &item, "gpt-4o").tokens,
            evaluate_candidate(build_line_rle(content), &item, "gpt-4o").tokens,
            evaluate_candidate(build_template_folding(content), &item, "gpt-4o").tokens,
        ];
        let expected = select_unique_best(&candidates, baseline)
            .and_then(|index| candidates[index])
            .unwrap_or(baseline);

        let report = measure_one(content);

        assert_eq!(report.tool_results.measured, 1);
        assert_eq!(report.portfolio.selected_tokens, expected);
        assert!(report.codecs.line_rle.applicable_validation_passed > 0);
        assert!(report.codecs.template_folding.applicable_validation_passed > 0);
    }

    proptest! {
        #[test]
        fn line_rle_round_trips_bounded_generated_lines(
            lines in prop::collection::vec(generated_text(), 0..64),
            ending in prop::sample::select(vec!["\n", "\r\n"]),
            final_newline in any::<bool>(),
        ) {
            let mut input = lines.join(ending);
            if final_newline && !lines.is_empty() {
                input.push_str(ending);
            }
            let candidate = build_line_rle(&input);
            prop_assert!(!candidate.limit_hit);
            if let Some(encoded) = candidate.output {
                prop_assert_eq!(decode_line_rle(&encoded), Some(input));
            }
        }

        #[test]
        fn template_round_trips_bounded_generated_lines(
            prefix in proptest::string::string_regex("[A-Za-z]{2,8}").unwrap(),
            suffix in proptest::string::string_regex("[=:,; ]{2,8}").unwrap(),
            values in prop::collection::vec(generated_text(), 2..32),
            ending in prop::sample::select(vec!["\n", "\r\n"]),
            final_newline in any::<bool>(),
        ) {
            let mut input = values
                .iter()
                .map(|value| format!("{prefix}{value}{suffix}{ending}"))
                .collect::<String>();
            if !final_newline {
                input.truncate(input.len().saturating_sub(ending.len()));
            }
            let candidate = build_template_folding(&input);
            prop_assert!(!candidate.limit_hit);
            if let Some(encoded) = candidate.output {
                prop_assert_eq!(decode_template(&encoded), Some(input));
            }
        }
    }
}