thirdshift 0.35.1

Turn a GitHub issue into a ready-for-review pull request, unattended
//! Price every retained Session once; compare complete logical Trials in pairs.
use std::collections::BTreeMap;
use std::io::{BufReader, Cursor};
use std::path::Path;

use anyhow::Result;
use serde::Serialize;

use super::trial_results::files::{Native, metadata};
use super::{TrialResult, trial_sessions};
use crate::otlp_jsonl::{AnyValue, Reader, Spans};
use crate::session::usage;
use crate::trial::accounting::{Attempt, Prices, Session, SessionStart};
use crate::trial::{Outcome, Output, VoidCause};

pub(super) fn capture(
    path: &Path,
    output: &Output,
    role: &str,
    prices: &Prices,
    stopped_at: u64,
) -> Result<Attempt> {
    let mut accounting = Attempt::default();
    for session in trial_sessions::attempt(&Native, path)?.sessions {
        let launched: Option<SessionStart> =
            metadata(&Native, &SessionStart::path(&session.original))?;
        let bytes = session.spans.as_ref().map(std::fs::read).transpose()?;
        let summary = bytes
            .as_ref()
            .map(|bytes| usage::from_spans(BufReader::new(Cursor::new(bytes))))
            .transpose()?
            .unwrap_or_default();
        let mut started = 0;
        let mut time_ms = 0;
        let mut recorded_role = None;
        let mut ended = false;
        if let Some(bytes) = &bytes {
            for request in Reader::<_, Spans>::new(Cursor::new(bytes)) {
                for span in request?
                    .resource_spans
                    .into_iter()
                    .flat_map(|resource| resource.scope_spans)
                    .flat_map(|scope| scope.spans)
                    .filter(|span| span.name.starts_with("invoke_agent "))
                {
                    started = span.start_time_unix_nano;
                    ended = span.status.is_some();
                    recorded_role = span.attributes.iter().find_map(|attribute| {
                        match (&*attribute.key, &attribute.value) {
                            ("thirdshift.trial.role", Some(AnyValue::String(role))) => {
                                Some(role.clone())
                            }
                            _ => None,
                        }
                    });
                    time_ms =
                        u128::from(span.end_time_unix_nano.saturating_sub(started)) / 1_000_000;
                }
            }
        }
        if !ended && let Some(launched) = &launched {
            time_ms = (u128::from(stopped_at.saturating_sub(launched.at_unix_nano)) / 1_000_000)
                .min(output.timings.total_ms);
        }
        let role = recorded_role
            .as_deref()
            .or_else(|| launched.as_ref().map(|start| start.role.as_str()))
            .unwrap_or_else(|| {
                output
                    .steps
                    .iter()
                    .find(|step| {
                        let nanos = |text: &str| {
                            chrono::DateTime::parse_from_rfc3339(text)
                                .ok()
                                .and_then(|date| date.timestamp_nanos_opt())
                                .and_then(|n| u64::try_from(n).ok())
                        };
                        nanos(&step.started_at)
                            .zip(nanos(&step.ended_at))
                            .is_some_and(|(begin, end)| begin <= started && started <= end)
                    })
                    .map_or(role, |step| step.name.as_str())
            });
        let priced = prices.cost(&summary);
        let gap = priced.as_ref().err().map(|error| {
            format!(
                "accounting gap in {} ({role}): {error:#}",
                session.original.display()
            )
        });
        accounting.sessions.push(Session {
            path: session.original.to_string_lossy().into_owned(),
            role: role.into(),
            time_ms,
            models: summary
                .models
                .into_iter()
                .map(|model| {
                    (
                        model.model.unwrap_or_else(|| "unknown".into()),
                        model.measurement,
                    )
                })
                .collect(),
            cost_usd: priced.ok(),
            gap,
        });
    }
    // A legacy Arm can report success without retaining any Session family.
    // Command-only Processes legitimately start none.
    if accounting.sessions.is_empty()
        && (output.log.is_some()
            || output.session_id.is_some()
            || (matches!(
                output.outcome,
                Outcome::Succeeded | Outcome::Delivered | Outcome::StoppedToAsk
            ) && role != "process"
                && output.timings.session_ms > 0))
        || (output.void_cause == Some(VoidCause::AccountingGap) && accounting.gap().is_none())
    {
        accounting.sessions.push(Session {
            path: path.join("session").to_string_lossy().into_owned(),
            role: role.into(),
            time_ms: output.timings.session_ms,
            models: BTreeMap::new(),
            cost_usd: None,
            gap: Some(
                output
                    .cause
                    .as_deref()
                    .filter(|_| output.void_cause == Some(VoidCause::AccountingGap))
                    .unwrap_or("accounting gap: missing usage and retained Session evidence")
                    .into(),
            ),
        });
    }
    Ok(accounting)
}

#[derive(Default, Serialize)]
pub(crate) struct Report {
    pub baseline: Arm,
    pub candidate: Arm,
    pub trials: Vec<Trial>,
    pub paired_median_ratios: Ratios,
}

#[derive(Default, Serialize)]
pub(crate) struct Ratios {
    pub cost: Option<f64>,
    pub time: Option<f64>,
    pub pairs: usize,
    pub time_pairs: usize,
}

#[derive(Default, Serialize)]
pub(crate) struct Arm {
    #[serde(flatten)]
    pub total: Total,
    pub roles: BTreeMap<String, Total>,
    pub timing_excluded_trials: usize,
}

#[derive(Serialize)]
pub(crate) struct Total {
    pub sessions: usize,
    pub cost_usd: Option<f64>,
    pub time_ms: u128,
}

impl Default for Total {
    fn default() -> Self {
        Self {
            sessions: 0,
            cost_usd: Some(0.0),
            time_ms: 0,
        }
    }
}

impl Total {
    fn include(&mut self, cost: Option<f64>, time_ms: u128, sessions: usize) {
        self.sessions += sessions;
        self.time_ms += time_ms;
        self.cost_usd = self.cost_usd.zip(cost).map(|(left, right)| left + right);
    }
}

#[derive(Serialize)]
pub(crate) struct Trial {
    pub id: String,
    pub arm: super::Arm,
    pub cost_usd: Option<f64>,
    pub time_ms: u128,
    pub suspended: bool,
    pub roles: BTreeMap<String, Total>,
    pub sessions: Vec<SessionRef>,
}

#[derive(Serialize)]
pub(crate) struct SessionRef {
    pub attempt: usize,
    #[serde(flatten)]
    pub session: Session,
}

pub(super) fn trial_total(result: &TrialResult) -> Total {
    let mut total = Total::default();
    for output in &result.attempts {
        let cost = output.accounting.as_ref().and_then(|accounting| {
            accounting
                .sessions
                .iter()
                .try_fold(0.0, |sum, session| session.cost_usd.map(|cost| sum + cost))
        });
        total.include(
            cost,
            output.timings.total_ms,
            output
                .accounting
                .as_ref()
                .map_or(0, |accounting| accounting.sessions.len()),
        );
    }
    total
}

impl Report {
    /// Shared by CLI and Console, including Experiments without frozen prices.
    pub(crate) fn timing_note(&self) -> String {
        [("Baseline", &self.baseline), ("Candidate", &self.candidate)]
            .into_iter()
            .filter(|(_, arm)| arm.timing_excluded_trials > 0)
            .map(|(name, arm)| {
                let count = arm.timing_excluded_trials;
                format!(
                    "{name}: {count} {} left out of timing (suspended)",
                    if count == 1 { "Trial" } else { "Trials" }
                )
            })
            .collect::<Vec<_>>()
            .join("; ")
    }

    pub(super) fn render(&self) -> String {
        let cost =
            |amount: Option<f64>| amount.map_or("unknown".into(), |amount| format!("${amount:.6}"));
        let ratio =
            |value: Option<f64>| value.map_or("unknown".into(), |value| format!("{value:.3}"));
        let mut text = format!(
            "\nFrozen-price accounting: Baseline {}, Candidate {}\nPaired median Candidate/Baseline: cost {}, time {} ({} pairs)",
            cost(self.baseline.total.cost_usd),
            cost(self.candidate.total.cost_usd),
            ratio(self.paired_median_ratios.cost),
            ratio(self.paired_median_ratios.time),
            self.paired_median_ratios.pairs
        );
        for (name, arm) in [("Baseline", &self.baseline), ("Candidate", &self.candidate)] {
            for (role, total) in &arm.roles {
                text.push_str(&format!(
                    "\n{name} {role}: {} sessions, {}, {:.3} minutes",
                    total.sessions,
                    cost(total.cost_usd),
                    total.time_ms as f64 / 60_000.0
                ));
            }
        }
        text
    }

    pub(super) fn of(results: &[TrialResult], unfinished: &[TrialResult]) -> Self {
        let mut report = Self::default();
        for result in results.iter().chain(unfinished) {
            let total = trial_total(result);
            let suspended = result.suspended();
            let mut trial = Trial {
                id: result.key(),
                arm: result.arm,
                cost_usd: total.cost_usd,
                time_ms: total.time_ms,
                suspended,
                roles: BTreeMap::new(),
                sessions: Vec::new(),
            };
            for (attempt, output) in result.attempts.iter().enumerate() {
                if let Some(accounting) = &output.accounting {
                    for session in &accounting.sessions {
                        trial
                            .roles
                            .entry(session.role.clone())
                            .or_default()
                            .include(session.cost_usd, session.time_ms, 1);
                        trial.sessions.push(SessionRef {
                            attempt,
                            session: session.clone(),
                        });
                    }
                }
            }
            let arm = match result.arm {
                super::Arm::Baseline => &mut report.baseline,
                super::Arm::Candidate => &mut report.candidate,
            };
            arm.timing_excluded_trials += usize::from(suspended);
            arm.total.include(
                total.cost_usd,
                if suspended { 0 } else { total.time_ms },
                total.sessions,
            );
            for (role, total) in &trial.roles {
                arm.roles.entry(role.clone()).or_default().include(
                    total.cost_usd,
                    if suspended { 0 } else { total.time_ms },
                    total.sessions,
                );
            }
            report.trials.push(trial);
        }
        let mut costs = Vec::new();
        let mut times = Vec::new();
        for pair in paired_ratios(results) {
            report.paired_median_ratios.pairs += 1;
            costs.extend(pair.cost);
            times.extend(pair.time);
        }
        report.paired_median_ratios.cost = median(&mut costs);
        report.paired_median_ratios.time_pairs = times.len();
        report.paired_median_ratios.time = median(&mut times);
        report
    }
}

pub(super) struct PairedRatios<'a> {
    pub suite: &'a str,
    pub task: &'a str,
    pub cost: Option<f64>,
    pub time: Option<f64>,
    pub timing_excluded: bool,
}

pub(super) fn paired_ratios(results: &[TrialResult]) -> impl Iterator<Item = PairedRatios<'_>> {
    results
        .iter()
        .filter(|result| result.arm == super::Arm::Baseline && result.outcome != Outcome::Void)
        .filter_map(|baseline| {
            results
                .iter()
                .find(|result| {
                    result.arm == super::Arm::Candidate
                        && result.suite == baseline.suite
                        && result.task == baseline.task
                        && result.trial == baseline.trial
                        && result.outcome != Outcome::Void
                })
                .map(|candidate| {
                    let base = trial_total(baseline);
                    let cand = trial_total(candidate);
                    let timing_excluded = baseline.suspended() || candidate.suspended();
                    PairedRatios {
                        suite: &baseline.suite,
                        task: &baseline.task,
                        cost: base
                            .cost_usd
                            .zip(cand.cost_usd)
                            .filter(|(base, _)| *base > 0.0)
                            .map(|(base, cand)| cand / base),
                        time: (base.time_ms > 0 && !timing_excluded)
                            .then(|| cand.time_ms as f64 / base.time_ms as f64),
                        timing_excluded,
                    }
                })
        })
}

pub(super) fn median(values: &mut [f64]) -> Option<f64> {
    values.sort_by(f64::total_cmp);
    let middle = values.len() / 2;
    if values.is_empty() {
        None
    } else if values.len().is_multiple_of(2) {
        Some(values[middle - 1] / 2.0 + values[middle] / 2.0)
    } else {
        Some(values[middle])
    }
}