steeldb/projectors/
jsonl_proj.rs

1//! Pre-projected text situations: one JSON object per line, `{ "tokens": [...] }`; the ingest
2//! pipeline (EN/JA/KO SPLADE/SPO projection) already emits this. Tokens pass through unchanged.
3
4use crate::projector::{CorpusKind, Projector, Situation};
5use serde::Deserialize;
6use std::fs::File;
7use std::io::{BufRead, BufReader};
8use std::path::PathBuf;
9
10#[derive(Deserialize)]
11struct Row {
12    #[serde(default)]
13    tokens: Vec<String>,
14}
15
16pub struct JsonlProjector {
17    path: PathBuf,
18    max_lines: Option<usize>,
19}
20
21impl JsonlProjector {
22    pub fn open(path: impl Into<PathBuf>, max_lines: Option<usize>) -> JsonlProjector {
23        JsonlProjector { path: path.into(), max_lines }
24    }
25}
26
27impl Projector for JsonlProjector {
28    fn columns(&self) -> Vec<String> {
29        vec!["tokens".to_string()]
30    }
31    fn kind(&self) -> CorpusKind {
32        CorpusKind::Text
33    }
34    fn source(&self) -> String {
35        self.path.display().to_string()
36    }
37    fn project(self: Box<Self>, sink: &mut dyn FnMut(Situation)) -> std::io::Result<()> {
38        let reader = BufReader::with_capacity(1 << 20, File::open(&self.path)?);
39        let mut n = 0usize;
40        for line in reader.lines() {
41            let line = line?;
42            if line.trim().is_empty() {
43                continue;
44            }
45            if let Some(max) = self.max_lines {
46                if n >= max {
47                    break;
48                }
49            }
50            let row: Row = serde_json::from_str(&line).unwrap_or(Row { tokens: Vec::new() });
51            let display = vec![row.tokens.join("  ")];
52            sink(Situation::new(row.tokens, display));
53            n += 1;
54        }
55        Ok(())
56    }
57}
steeldb/projectors/jsonl_proj.rs

steeldb/projectors/
jsonl_proj.rs
