turbolog 0.2.1

Ultralight log anomaly detection — no API key, no Python, pipe-friendly CLI with local LLM explain
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

use std::path::Path;

use anyhow::Result;

use crate::ingest::Embedder;

#[cfg(not(feature = "embedded-model"))]
use std::path::PathBuf;

#[cfg(not(feature = "embedded-model"))]
use anyhow::anyhow;

#[cfg(feature = "embedded-model")]
pub static MODEL_BYTES: &[u8] = include_bytes!("../models/model.onnx");

#[cfg(feature = "embedded-model")]
pub static TOKENIZER_BYTES: &[u8] = include_bytes!("../models/tokenizer.json");

/// Creates an Embedder from embedded bytes (feature = "embedded-model") or from disk.
/// When not embedded, looks for model files in (priority order):
///   1. `model_dir` (explicit --model-dir or TURBOLOG_MODEL_DIR)
///   2. $XDG_DATA_HOME/turbolog/models  (~/.local/share/turbolog/models)
/// If neither exists, downloads from Hugging Face automatically.
pub fn make_embedder(_model_dir: &Path) -> Result<Embedder> {
    #[cfg(feature = "embedded-model")]
    {
        Embedder::from_bytes(MODEL_BYTES, TOKENIZER_BYTES)
    }
    #[cfg(not(feature = "embedded-model"))]
    {
        let dir = resolve_model_dir(_model_dir);
        ensure_models(&dir)?;
        Embedder::new(dir.join("model.onnx"), dir.join("tokenizer.json"))
    }
}

#[cfg(not(feature = "embedded-model"))]
fn resolve_model_dir(explicit: &Path) -> PathBuf {
    if explicit.join("model.onnx").exists() {
        return explicit.to_path_buf();
    }
    let base = std::env::var("XDG_DATA_HOME")
        .map(PathBuf::from)
        .unwrap_or_else(|_| {
            PathBuf::from(std::env::var("HOME").unwrap_or_else(|_| ".".to_string()))
                .join(".local/share")
        });
    base.join("turbolog/models")
}

#[cfg(not(feature = "embedded-model"))]
fn ensure_models(dir: &Path) -> Result<()> {
    let model = dir.join("model.onnx");
    let tokenizer = dir.join("tokenizer.json");
    if model.exists() && tokenizer.exists() {
        return Ok(());
    }
    std::fs::create_dir_all(dir)?;
    const BASE: &str =
        "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main";
    if !model.exists() {
        eprintln!("[turbolog] First run: downloading model (~86 MB) to {} ...", dir.display());
        download(&format!("{BASE}/onnx/model.onnx"), &model)?;
    }
    if !tokenizer.exists() {
        download(&format!("{BASE}/tokenizer.json"), &tokenizer)?;
    }
    eprintln!("[turbolog] Model ready.");
    Ok(())
}

#[cfg(not(feature = "embedded-model"))]
fn download(url: &str, dest: &Path) -> Result<()> {
    let resp = ureq::get(url)
        .call()
        .map_err(|e| anyhow!("Failed to download {url}: {e}"))?;
    let mut file = std::fs::File::create(dest)?;
    std::io::copy(&mut resp.into_reader(), &mut file)?;
    Ok(())
}