hypersteeldb 0.3.1

//! Pre-projected text situations: one JSON object per line, `{ "tokens": [...] }`; the ingest
//! pipeline (EN/JA/KO SPLADE/SPO projection) already emits this. Tokens pass through unchanged.

use crate::projector::{CorpusKind, Projector, Situation};
use serde::Deserialize;
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::PathBuf;

#[derive(Deserialize)]
struct Row {
    #[serde(default)]
    tokens: Vec<String>,
}

pub struct JsonlProjector {
    path: PathBuf,
    max_lines: Option<usize>,
}

impl JsonlProjector {
    pub fn open(path: impl Into<PathBuf>, max_lines: Option<usize>) -> JsonlProjector {
        JsonlProjector { path: path.into(), max_lines }
    }
}

impl Projector for JsonlProjector {
    fn columns(&self) -> Vec<String> {
        vec!["tokens".to_string()]
    }
    fn kind(&self) -> CorpusKind {
        CorpusKind::Text
    }
    fn source(&self) -> String {
        self.path.display().to_string()
    }
    fn project(self: Box<Self>, sink: &mut dyn FnMut(Situation)) -> std::io::Result<()> {
        let reader = BufReader::with_capacity(1 << 20, File::open(&self.path)?);
        let mut n = 0usize;
        for line in reader.lines() {
            let line = line?;
            if line.trim().is_empty() {
                continue;
            }
            if let Some(max) = self.max_lines {
                if n >= max {
                    break;
                }
            }
            let row: Row = serde_json::from_str(&line).unwrap_or(Row { tokens: Vec::new() });
            let display = vec![row.tokens.join("  ")];
            sink(Situation::new(row.tokens, display));
            n += 1;
        }
        Ok(())
    }
}