steeldb/
projector.rs

1//! Projector — the pluggable ingest seam, analogous to DuckDB's table functions / replacement scans.
2//!
3//! DuckDB queries CSV/Parquet/JSON uniformly because every reader lowers to a common DataChunk that
4//! the SQL engine consumes. Here every file format lowers to a common **Situation stream** (tokens +
5//! display cells) that the IKL/bitmap engine consumes — so adding a format never touches the query
6//! layer. Projection is push-based: a reader emits one Situation per source record into a sink.
7//!
8//! The core owns the tabular/columnar/text projectors (CSV, JSONL text situations; Parquet/plain-text
9//! next). Heavy document formats (PDF/DOCX/PPTX/HTML) reuse the existing producer pipeline, which emits
10//! the same Situation stream over a sidecar bridge — no re-implementation of document parsing here.
11
12use serde::Serialize;
13
14#[derive(Serialize, Clone, Copy, PartialEq)]
15#[serde(rename_all = "lowercase")]
16pub enum CorpusKind {
17    Csv,
18    Text,
19}
20
21/// One projected record: the tokens it asserts (for set-algebra), the cells to show for it, and any
22/// numeric fields it carries (for `(num <field> <op> <value>)` range predicates — e.g. a CSV numeric
23/// column or a canonicalised quantity span like `qty-length` = 70 m).
24pub struct Situation {
25    pub tokens: Vec<String>,
26    pub display: Vec<String>,
27    pub numbers: Vec<(String, f64)>,
28    /// **Infon polarity** per token (paper §1.2): `(token, i)` where `i ∈ {-1, -0.5, +0.5, +1}`. Only
29    /// tokens whose polarity differs from the default `+1` need an entry; the rest are asserted-positive.
30    /// Populated from the tagger's epistemic head and consumed by `InfonIndex::add_infon_polar`.
31    pub beliefs: Vec<(String, f32)>,
32}
33
34impl Situation {
35    /// Token + display record with no numeric fields (the common case).
36    pub fn new(tokens: Vec<String>, display: Vec<String>) -> Situation {
37        Situation { tokens, display, numbers: Vec::new(), beliefs: Vec::new() }
38    }
39}
40
41/// A push-based reader over one source. `project` drives the sink once per source record.
42pub trait Projector {
43    fn columns(&self) -> Vec<String>;
44    fn kind(&self) -> CorpusKind;
45    fn source(&self) -> String;
46    fn project(self: Box<Self>, sink: &mut dyn FnMut(Situation)) -> std::io::Result<()>;
47}
48
49/// slugify a value into a token-safe form: trim, lowercase, collapse whitespace/`/` to `-`.
50pub fn slug(s: &str) -> String {
51    let t = s.trim().to_lowercase();
52    let mut out = String::with_capacity(t.len());
53    let mut prev_dash = false;
54    for ch in t.chars() {
55        if ch.is_whitespace() || ch == '/' {
56            if !prev_dash && !out.is_empty() {
57                out.push('-');
58                prev_dash = true;
59            }
60        } else {
61            out.push(ch);
62            prev_dash = false;
63        }
64    }
65    while out.ends_with('-') {
66        out.pop();
67    }
68    out
69}
steeldb/projector.rs

steeldb/
projector.rs
