vlajmaji_store/
lib.rs

1//! Concrete local persistence for the accepted source-preserving lexical archive.
2//!
3//! The archive remains canonical. SQLite/FTS5 and optional disk Oxigraph/RocksDB
4//! are rebuildable indexes; this crate does not normalize spelling or merge sources.
5
6use std::{collections::BTreeSet, error::Error, fmt, path::Path};
7
8use serde::{Deserialize, Serialize};
9use sha2::{Digest, Sha256};
10use vlajmaji_lexicon::{FieldPresence, FormRole, Lexicon, RecordIdentity, SourceCrossReference};
11
12mod sqlite;
13pub use sqlite::SqliteStore;
14
15mod dictionary;
16mod dictionary_schema;
17pub use dictionary::{DictionaryStore, DictionaryStoreError, DictionaryStoreInvalidity};
18
19#[cfg(feature = "graph")]
20mod graph;
21#[cfg(feature = "graph")]
22pub use graph::GraphStore;
23
24/// A finite, backend-independent query accepted by both local stores.
25#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
26#[non_exhaustive]
27pub enum SearchQuery {
28    /// Byte-exact captured headword lookup.
29    ExactSpelling(String),
30    /// Literal substring of an ASCII-lowercased source definition.
31    DefinitionContains(String),
32    /// Exact form string, optionally restricted to its source role.
33    Form {
34        /// Exact form bytes represented as a Rust string.
35        value: String,
36        /// Optional source assertion role.
37        role: Option<FormRole>,
38    },
39    /// Captured cross-reference traversal from an exact headword.
40    CrossReferences {
41        /// Exact start spelling.
42        spelling: String,
43        /// Maximum traversal depth, one or two.
44        depth: u8,
45    },
46    /// Query per-record field presence, optionally requiring exact retained value.
47    Field {
48        /// Exact top-level source field name.
49        name: String,
50        /// Required source presence state.
51        state: PresenceState,
52        /// Optional exact retained raw value.
53        raw_value: Option<String>,
54    },
55    /// Exact proposal spelling, kept separate from captured records.
56    ProposalExact(String),
57}
58
59/// Field-state selector for [`SearchQuery::Field`].
60#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
61#[non_exhaustive]
62pub enum PresenceState {
63    /// The source record has no field with this name.
64    Missing,
65    /// The JSON field exists and its value is JSON null.
66    Null,
67    /// The field exists and has a retained source value.
68    Value,
69}
70
71/// One edge traversed by a cross-reference query.
72#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
73pub struct EdgeHit {
74    /// Stable source-record ID of the edge origin.
75    pub origin_id: String,
76    /// Stable source-record ID of the target.
77    pub target_id: Option<String>,
78    /// Exact target spelling in the source note.
79    pub target_spelling: String,
80    /// Decoded-note UTF-8 start byte offset.
81    pub byte_start: usize,
82    /// Decoded-note UTF-8 exclusive end byte offset.
83    pub byte_end: usize,
84    /// Bounded parser rule identifier.
85    pub rule_version: String,
86}
87
88/// Complete, comparable result from one native query execution.
89#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
90pub struct SearchResult {
91    /// Sorted distinct source record identities.
92    pub record_ids: BTreeSet<String>,
93    /// Distinct traversed source edges, populated only for link queries.
94    pub edges: BTreeSet<EdgeHit>,
95    /// Backend-reported candidate count before any exact postfilter.
96    pub candidates: usize,
97    /// Native execution path label.
98    pub path: String,
99    /// Visible reason for the native SQL scan fallback, if selected.
100    pub fallback_reason: Option<String>,
101}
102
103/// Typed failure from opening, building, or querying a local native store.
104#[derive(Debug)]
105#[non_exhaustive]
106pub enum StoreError {
107    /// Caller-supplied path already exists; creation never replaces it.
108    ExistingPath,
109    /// Store metadata is absent, incomplete, has another schema, or another archive hash.
110    InvalidStore(String),
111    /// Accepted lexical archive failed validation.
112    InvalidArchive(String),
113    /// Backend operation failed with its original diagnostic.
114    Backend(String),
115    /// Filesystem operation failed.
116    Io(std::io::Error),
117}
118
119impl fmt::Display for StoreError {
120    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
121        match self {
122            Self::ExistingPath => write!(f, "store path already exists"),
123            Self::InvalidStore(message) => write!(f, "invalid store: {message}"),
124            Self::InvalidArchive(message) => write!(f, "invalid archive: {message}"),
125            Self::Backend(message) => write!(f, "backend error: {message}"),
126            Self::Io(error) => write!(f, "filesystem error: {error}"),
127        }
128    }
129}
130
131impl Error for StoreError {
132    fn source(&self) -> Option<&(dyn Error + 'static)> {
133        match self {
134            Self::Io(error) => Some(error),
135            _ => None,
136        }
137    }
138}
139
140/// Computes the lowercase hexadecimal SHA-256 of archive bytes.
141#[must_use]
142pub fn archive_sha256(bytes: &[u8]) -> String {
143    format!("{:x}", Sha256::digest(bytes))
144}
145
146/// Opens and validates the canonical versioned archive before building an index.
147pub fn validate_archive(bytes: &[u8]) -> Result<Lexicon, StoreError> {
148    Lexicon::from_archive_bytes(bytes)
149        .map_err(|error| StoreError::InvalidArchive(error.to_string()))
150}
151
152/// Computes the stable lowercase hexadecimal identifier used by this store for
153/// one source record identity. The encoding is the byte-exact tuple
154/// `(snapshot namespace, snapshot SHA-256, locator)`, with each UTF-8 component
155/// prefixed by its big-endian `u64` byte length. It is not a content hash.
156#[must_use]
157pub fn record_id(identity: &RecordIdentity) -> String {
158    let parts = [
159        identity.snapshot().namespace().as_bytes(),
160        identity.snapshot().sha256().as_bytes(),
161        identity.locator().as_bytes(),
162    ];
163    let mut tuple = Vec::new();
164    for part in parts {
165        tuple.extend_from_slice(&(part.len() as u64).to_be_bytes());
166        tuple.extend_from_slice(part);
167    }
168    let mut out = String::with_capacity(tuple.len() * 2);
169    for byte in tuple {
170        use fmt::Write as _;
171        let _ = write!(out, "{byte:02x}");
172    }
173    out
174}
175
176pub(crate) fn edge_hits(reference: &SourceCrossReference) -> Vec<EdgeHit> {
177    let targets: Vec<Option<String>> = if reference.targets().is_empty() {
178        vec![None]
179    } else {
180        reference
181            .targets()
182            .iter()
183            .map(|target| Some(record_id(target)))
184            .collect()
185    };
186    targets
187        .into_iter()
188        .map(|target_id| EdgeHit {
189            origin_id: record_id(reference.origin()),
190            target_id,
191            target_spelling: reference.target_spelling().to_owned(),
192            byte_start: reference.byte_start(),
193            byte_end: reference.byte_end(),
194            rule_version: reference.rule_version().to_owned(),
195        })
196        .collect()
197}
198
199pub(crate) fn role_name(role: FormRole) -> &'static str {
200    match role {
201        FormRole::SourceReportedForm => "source-reported",
202        FormRole::AssignedRafsi => "assigned-rafsi",
203        _ => "other-source-role",
204    }
205}
206
207/// Executes the separate in-memory full scan used as a correctness oracle.
208///
209/// This function is intended for controlled comparison and does not use a native index.
210#[must_use]
211pub fn scan_reference(lexicon: &Lexicon, query: &SearchQuery) -> SearchResult {
212    let mut result = empty_result("independent-rust-scan", None);
213    match query {
214        SearchQuery::ExactSpelling(spelling) => {
215            for record in lexicon.lookup_exact(spelling) {
216                result.record_ids.insert(record_id(record.identity()));
217            }
218        }
219        SearchQuery::DefinitionContains(term) => {
220            let needle = term.to_ascii_lowercase();
221            if needle.is_empty() {
222                return result;
223            }
224            for record in lexicon.records() {
225                if record
226                    .definition()
227                    .is_some_and(|definition| definition.to_ascii_lowercase().contains(&needle))
228                {
229                    result.record_ids.insert(record_id(record.identity()));
230                }
231            }
232        }
233        SearchQuery::Form { value, role } => {
234            for form in lexicon.forms() {
235                if form.form() == value && role.is_none_or(|expected| form.role() == expected) {
236                    result.record_ids.insert(record_id(form.source_record()));
237                }
238            }
239        }
240        SearchQuery::CrossReferences { spelling, depth } => {
241            if !(1..=2).contains(depth) {
242                return result;
243            }
244            let mut frontier: BTreeSet<String> = lexicon
245                .lookup_exact(spelling)
246                .into_iter()
247                .map(|record| record_id(record.identity()))
248                .collect();
249            for _ in 0..*depth {
250                let mut next = BTreeSet::new();
251                for reference in lexicon.cross_references() {
252                    for hit in edge_hits(reference) {
253                        if frontier.contains(&hit.origin_id) {
254                            result.edges.insert(hit.clone());
255                            if let Some(target_id) = hit.target_id {
256                                next.insert(target_id.clone());
257                                result.record_ids.insert(target_id);
258                            }
259                        }
260                    }
261                }
262                frontier = next;
263            }
264        }
265        SearchQuery::Field {
266            name,
267            state,
268            raw_value,
269        } => {
270            for record in lexicon.records() {
271                let presence = record.field(name);
272                let matches = match (state, presence) {
273                    (PresenceState::Missing, FieldPresence::Missing) => raw_value.is_none(),
274                    (PresenceState::Null, FieldPresence::Null) => raw_value
275                        .as_deref()
276                        .is_none_or(|expected| expected == "null"),
277                    (PresenceState::Value, FieldPresence::Value(raw)) => {
278                        raw_value.as_deref().is_none_or(|expected| expected == raw)
279                    }
280                    _ => false,
281                };
282                if matches {
283                    result.record_ids.insert(record_id(record.identity()));
284                }
285            }
286        }
287        SearchQuery::ProposalExact(spelling) => {
288            for proposal in lexicon.lookup_proposals(spelling) {
289                result.record_ids.insert(proposal.id().to_owned());
290            }
291        }
292    }
293    result.candidates = result.record_ids.len();
294    result
295}
296
297pub(crate) fn empty_result(path: &str, fallback_reason: Option<String>) -> SearchResult {
298    SearchResult {
299        record_ids: BTreeSet::new(),
300        edges: BTreeSet::new(),
301        candidates: 0,
302        path: path.to_owned(),
303        fallback_reason,
304    }
305}
306
307/// Native disk-store API shared by the concrete backends.
308pub trait LexicalStore {
309    /// Executes one query against the persistent native index.
310    fn search(&self, query: &SearchQuery) -> Result<SearchResult, StoreError>;
311    /// Returns exact canonical archive bytes retained in the index.
312    fn archive_bytes(&self) -> Result<Vec<u8>, StoreError>;
313    /// Flushes backend-managed data and closes are represented by dropping the store.
314    fn flush(&self) -> Result<(), StoreError>;
315    /// Returns logical bytes occupied by files below the store path after flush.
316    fn logical_bytes(&self) -> Result<u64, StoreError>;
317}
318
319pub(crate) fn require_fresh(path: &Path) -> Result<(), StoreError> {
320    if path.exists() {
321        return Err(StoreError::ExistingPath);
322    }
323    Ok(())
324}
325
326pub(crate) fn directory_bytes(path: &Path) -> Result<u64, StoreError> {
327    fn walk(path: &Path, total: &mut u64) -> Result<(), StoreError> {
328        for entry in std::fs::read_dir(path).map_err(StoreError::Io)? {
329            let entry = entry.map_err(StoreError::Io)?;
330            let metadata = entry.metadata().map_err(StoreError::Io)?;
331            if metadata.is_dir() {
332                walk(&entry.path(), total)?;
333            } else if metadata.is_file() {
334                *total = total.saturating_add(metadata.len());
335            }
336        }
337        Ok(())
338    }
339    let mut total = 0;
340    walk(path, &mut total)?;
341    Ok(total)
342}
343
344#[cfg(test)]
345mod identity_tests {
346    use super::*;
347    use vlajmaji_lexicon::{SourceFormat, SourceMetadata, TimePrecision};
348
349    #[test]
350    fn tuple_identity_is_length_prefixed_and_byte_exact() {
351        let mut lex = Lexicon::new();
352        let src = SourceMetadata::new(
353            "a\0b",
354            "x",
355            SourceFormat::LaLojbanJson,
356            "m",
357            "r",
358            None,
359            TimePrecision::Unknown,
360        );
361        lex.import_source(src, br#"[{"word":"c","definition":"d"}]"#, None)
362            .unwrap();
363        let left = record_id(lex.records()[0].identity());
364        let mut other = Lexicon::new();
365        let src = SourceMetadata::new(
366            "a",
367            "x",
368            SourceFormat::LaLojbanJson,
369            "m",
370            "r",
371            None,
372            TimePrecision::Unknown,
373        );
374        other
375            .import_source(src, br#"[{"word":"b\u0000c","definition":"d"}]"#, None)
376            .unwrap();
377        assert_ne!(left, record_id(other.records()[0].identity()));
378    }
379}
vlajmaji_store/lib.rs

vlajmaji_store/
lib.rs
