expensive 0.5.5

Terminal dashboard for AI coding harness token usage and spend
Documentation
use std::{
    fs::{self, File, Metadata},
    io::{BufRead, BufReader, Read, Seek, SeekFrom},
    path::{Path, PathBuf},
    time::UNIX_EPOCH,
};

use anyhow::{Context, Result};

use crate::index::{ArtifactCheckpoint, ArtifactRecord};

const BOUNDARY_BYTES: i64 = 4 * 1_024;

#[derive(Clone, Debug)]
pub struct FileMetadata {
    pub device: Option<i64>,
    pub inode: Option<i64>,
    pub size: i64,
    pub modified_ns: Option<i64>,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ScanPlan {
    Unchanged,
    Append(i64),
    Full,
}

#[derive(Clone, Debug)]
pub struct LineScan {
    pub parsed_offset: i64,
    pub full_hash: Option<Vec<u8>>,
}

pub fn discover(roots: &[PathBuf]) -> Result<Vec<PathBuf>> {
    let mut paths = Vec::new();
    for root in roots {
        collect(root, &mut paths)?;
    }
    paths.sort();
    Ok(paths)
}

fn collect(path: &Path, paths: &mut Vec<PathBuf>) -> Result<()> {
    let metadata = match fs::symlink_metadata(path) {
        Ok(metadata) => metadata,
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
        Err(error) => return Err(error).with_context(|| format!("reading {}", path.display())),
    };
    if metadata.file_type().is_symlink() {
        return Ok(());
    }
    if metadata.is_file() {
        if path
            .extension()
            .is_some_and(|extension| extension == "jsonl")
        {
            paths.push(path.to_path_buf());
        }
        return Ok(());
    }
    if !metadata.is_dir() {
        return Ok(());
    }
    for entry in fs::read_dir(path).with_context(|| format!("listing {}", path.display()))? {
        collect(&entry?.path(), paths)?;
    }
    Ok(())
}

pub fn artifact_key(path: &Path) -> String {
    path.canonicalize()
        .unwrap_or_else(|_| path.to_path_buf())
        .display()
        .to_string()
}

pub fn metadata(path: &Path) -> Result<FileMetadata> {
    let metadata = fs::metadata(path).with_context(|| format!("reading {}", path.display()))?;
    let (device, inode) = file_identity(&metadata);
    Ok(FileMetadata {
        device,
        inode,
        size: i64::try_from(metadata.len()).unwrap_or(i64::MAX),
        modified_ns: modified_ns(&metadata),
    })
}

pub fn plan(
    path: &Path,
    metadata: &FileMetadata,
    checkpoint: Option<&ArtifactCheckpoint>,
    parser_version: i64,
    force_full: bool,
) -> Result<ScanPlan> {
    if force_full {
        return Ok(ScanPlan::Full);
    }
    let Some(checkpoint) = checkpoint else {
        return Ok(ScanPlan::Full);
    };
    if checkpoint.parser_version != parser_version {
        return Ok(ScanPlan::Full);
    }
    if checkpoint.size == Some(metadata.size) && checkpoint.modified_ns == metadata.modified_ns {
        return Ok(ScanPlan::Unchanged);
    }
    let same_file = checkpoint
        .device
        .zip(metadata.device)
        .is_none_or(|(old, new)| old == new)
        && checkpoint
            .inode
            .zip(metadata.inode)
            .is_none_or(|(old, new)| old == new);
    if !same_file || metadata.size < checkpoint.parsed_offset {
        return Ok(ScanPlan::Full);
    }
    if metadata.size <= checkpoint.size.unwrap_or(checkpoint.parsed_offset) {
        return Ok(ScanPlan::Full);
    }
    let boundary_matches = match &checkpoint.boundary_hash {
        Some(expected) => boundary_hash(path, checkpoint.parsed_offset)? == *expected,
        None => checkpoint.parsed_offset == 0,
    };
    if boundary_matches {
        Ok(ScanPlan::Append(checkpoint.parsed_offset))
    } else {
        Ok(ScanPlan::Full)
    }
}

pub fn scan_lines(
    path: &Path,
    start_offset: i64,
    hash_full_file: bool,
    mut visit: impl FnMut(&[u8]),
) -> Result<LineScan> {
    let mut file = File::open(path).with_context(|| format!("opening {}", path.display()))?;
    file.seek(SeekFrom::Start(start_offset.max(0) as u64))?;
    let mut reader = BufReader::new(file);
    let mut line = Vec::new();
    let mut parsed_offset = start_offset.max(0);
    let mut hasher = hash_full_file.then(blake3::Hasher::new);

    loop {
        line.clear();
        let bytes = reader.read_until(b'\n', &mut line)?;
        if bytes == 0 {
            break;
        }
        if let Some(hasher) = &mut hasher {
            hasher.update(&line);
        }
        if !line.ends_with(b"\n") {
            break;
        }
        visit(&line);
        parsed_offset = parsed_offset.saturating_add(i64::try_from(bytes).unwrap_or(i64::MAX));
    }

    Ok(LineScan {
        parsed_offset,
        full_hash: hasher.map(|hasher| hasher.finalize().as_bytes().to_vec()),
    })
}

pub fn first_line(path: &Path) -> Result<Vec<u8>> {
    let file = File::open(path).with_context(|| format!("opening {}", path.display()))?;
    let mut reader = BufReader::new(file);
    let mut line = Vec::new();
    reader.read_until(b'\n', &mut line)?;
    Ok(line)
}

pub fn boundary_hash(path: &Path, parsed_offset: i64) -> Result<Vec<u8>> {
    let end = parsed_offset.max(0);
    let start = end.saturating_sub(BOUNDARY_BYTES).max(0);
    let mut file = File::open(path).with_context(|| format!("opening {}", path.display()))?;
    file.seek(SeekFrom::Start(start as u64))?;
    let length = usize::try_from(end.saturating_sub(start)).unwrap_or(0);
    let mut bytes = vec![0; length];
    file.read_exact(&mut bytes)?;
    Ok(blake3::hash(&bytes).as_bytes().to_vec())
}

pub fn artifact(
    key: String,
    path: &Path,
    metadata: &FileMetadata,
    scan: &LineScan,
    cursor: Option<String>,
    parser_version: i64,
    scanned_at_ms: i64,
) -> Result<ArtifactRecord> {
    Ok(ArtifactRecord {
        key,
        path: Some(path.display().to_string()),
        device: metadata.device,
        inode: metadata.inode,
        size: Some(metadata.size),
        modified_ns: metadata.modified_ns,
        parsed_offset: scan.parsed_offset,
        boundary_hash: Some(boundary_hash(path, scan.parsed_offset)?),
        full_hash: scan.full_hash.clone(),
        cursor,
        parser_version,
        scanned_at_ms,
    })
}

fn modified_ns(metadata: &Metadata) -> Option<i64> {
    let duration = metadata.modified().ok()?.duration_since(UNIX_EPOCH).ok()?;
    i64::try_from(duration.as_nanos()).ok()
}

#[cfg(unix)]
fn file_identity(metadata: &Metadata) -> (Option<i64>, Option<i64>) {
    use std::os::unix::fs::MetadataExt;
    (
        i64::try_from(metadata.dev()).ok(),
        i64::try_from(metadata.ino()).ok(),
    )
}

#[cfg(not(unix))]
fn file_identity(_metadata: &Metadata) -> (Option<i64>, Option<i64>) {
    (None, None)
}

#[cfg(test)]
mod tests {
    use std::io::Write;

    use super::*;

    #[test]
    fn resumes_only_after_complete_lines() {
        let mut file = tempfile::NamedTempFile::new().unwrap();
        file.write_all(b"{\"one\":1}\n{\"partial\":").unwrap();
        file.flush().unwrap();
        let mut lines = Vec::new();

        let scan = scan_lines(file.path(), 0, true, |line| lines.push(line.to_vec())).unwrap();

        assert_eq!(lines, vec![b"{\"one\":1}\n".to_vec()]);
        assert_eq!(scan.parsed_offset, 10);
        assert!(scan.full_hash.is_some());
    }

    #[test]
    fn detects_safe_append_and_in_place_rewrite() {
        let mut file = tempfile::NamedTempFile::new().unwrap();
        file.write_all(b"first\n").unwrap();
        file.flush().unwrap();
        let initial_metadata = metadata(file.path()).unwrap();
        let checkpoint = ArtifactCheckpoint {
            device: initial_metadata.device,
            inode: initial_metadata.inode,
            size: Some(initial_metadata.size),
            modified_ns: initial_metadata.modified_ns,
            parsed_offset: initial_metadata.size,
            boundary_hash: Some(boundary_hash(file.path(), initial_metadata.size).unwrap()),
            full_hash: None,
            cursor: None,
            parser_version: 1,
        };

        file.write_all(b"second\n").unwrap();
        file.flush().unwrap();
        let appended = metadata(file.path()).unwrap();
        assert_eq!(
            plan(file.path(), &appended, Some(&checkpoint), 1, false).unwrap(),
            ScanPlan::Append(6)
        );

        std::fs::write(file.path(), b"other!\nmore\n").unwrap();
        let rewritten = metadata(file.path()).unwrap();
        assert_eq!(
            plan(file.path(), &rewritten, Some(&checkpoint), 1, false).unwrap(),
            ScanPlan::Full
        );
    }
}