car-inference 0.47.0

//! LIVE verification that local inference reports real token usage —
//! Parslee-ai/car#795. Ignored by default: it loads real weights off disk and
//! runs the actual decode loop, which is the only way to prove this. Nothing
//! is mocked, because the bug was precisely that a path which *had* the counts
//! discarded them; a fixture would assert the plumbing, not the counting.
//!
//!   cargo test -p car-inference --test live_local_usage -- --ignored --nocapture
//!
//! Requires a local model on disk (`car models pull mlx/qwen3-0.6b:6bit`, or
//! the candle GGUF equivalent on non-Apple platforms). Skips with a printed
//! reason when the weights aren't there, rather than failing.
//!
//! Both directions are checked, because they broke independently: the
//! non-streaming path builds `TokenUsage` from the decode loop's own counters,
//! while the streaming path has to *emit* a `StreamEvent::Usage` for the
//! accumulator to see one at all. Before the fix, streaming local generation
//! reported `usage: null` on every platform.

use car_inference::tasks::generate::{GenerateParams, GenerateRequest};
use car_inference::{InferenceConfig, InferenceEngine, StreamEvent};

/// The smallest builtin local model, so the test stays a few seconds.
#[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
const MODEL: &str = "mlx/qwen3-0.6b:6bit";
#[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
const MODEL: &str = "qwen/qwen3-0.6b:q8_0";

fn request(max_tokens: usize) -> GenerateRequest {
    GenerateRequest {
        prompt: "Reply with the single word: OK. /no_think".to_string(),
        model: Some(MODEL.to_string()),
        params: GenerateParams {
            max_tokens,
            temperature: 0.0,
            strict_model: true,
            ..Default::default()
        },
        ..Default::default()
    }
}

/// `true` when the weights are on disk. Same check the runtime itself uses to
/// decide whether a local model can serve a turn without downloading, so a
/// skip here means the same thing `car models list` would report.
fn model_available(engine: &InferenceEngine) -> bool {
    engine
        .unified_registry
        .ready_without_download(MODEL)
        .unwrap_or(false)
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "loads real local weights; run with --ignored"]
async fn local_inference_reports_real_token_usage() {
    let engine = InferenceEngine::new(InferenceConfig::default());
    if !model_available(&engine) {
        eprintln!("[SKIP] {MODEL} is not installed — `car models pull {MODEL}`");
        return;
    }

    // ---- non-streaming ----------------------------------------------------
    let result = engine
        .generate_tracked(request(24))
        .await
        .expect("local generation");
    let usage = result
        .usage
        .expect("local generation must report usage, not the null of #795");

    assert!(
        usage.prompt_tokens > 0,
        "prompt_tokens must be the real post-truncation count, got 0 — a zero \
         is indistinguishable from 'this used no tokens' to a consumer summing \
         totals, which is the whole complaint in #795"
    );
    assert!(
        usage.completion_tokens > 0,
        "the model produced text ({:?}) so completion_tokens cannot be 0",
        result.text
    );
    assert_eq!(
        usage.total_tokens,
        usage.prompt_tokens + usage.completion_tokens,
        "total must be the sum of the two buckets"
    );
    assert!(
        usage.context_window > 0,
        "non-streaming local calls know the model's context window and must \
         report it"
    );
    assert_eq!(
        (
            usage.cache_read_input_tokens,
            usage.cache_creation_input_tokens
        ),
        (0, 0),
        "in-process inference has no remote prompt cache"
    );

    // ---- streaming --------------------------------------------------------
    let mut stream = engine
        .generate_tracked_stream(request(24))
        .await
        .expect("local stream");
    let mut streamed_usage: Option<(u64, u64)> = None;
    let mut saw_done = false;
    while let Some(event) = stream.events.recv().await {
        match event {
            StreamEvent::Usage {
                input_tokens,
                output_tokens,
                ..
            } => streamed_usage = Some((input_tokens, output_tokens)),
            StreamEvent::Done { .. } => saw_done = true,
            StreamEvent::Error(message) => panic!("local stream errored: {message}"),
            _ => {}
        }
    }
    assert!(saw_done, "the stream must terminate with Done");

    let (stream_prompt, stream_completion) = streamed_usage.expect(
        "a local stream must emit StreamEvent::Usage — without it the \
         accumulator's saw_usage stays false and every streamed local call \
         ends with usage: null (#795)",
    );
    assert!(
        stream_completion > 0,
        "streamed completion_tokens cannot be 0"
    );

    // The two paths tokenize the same prompt with the same tokenizer and the
    // same chat template, so their prompt counts must agree exactly. This is
    // the assertion that catches a future divergence — e.g. one path counting
    // pre-truncation and the other post-truncation.
    assert_eq!(
        stream_prompt, usage.prompt_tokens,
        "streaming and non-streaming must report the same prompt_tokens for \
         an identical request"
    );
}