oxilite_core/
text.rs

1//! Optional full-text search over string literals with SQLite FTS5.
2//!
3//! When `StoreOptions::text_index` is set, `terms_fts` indexes the lexical form of every simple,
4//! language-tagged and directional string literal (triggers on `terms` keep it current), and
5//! the SPARQL function `oxl:textMatch(?literal, "fts query")` compiles to an FTS5 `MATCH`
6//! lookup. Without the index, the function is evaluated by the fallback evaluator with the
7//! same tokenization (case-insensitive words; `word*` prefixes; all terms required).
8//!
9// @lat: [[architecture#Text search]]
10
11use crate::encoding::PAYLOAD_BITS;
12use crate::sql::Statement;
13use oxrdf::{Literal, NamedNode, Term};
14
15/// `oxl:textMatch(?literal, "query")`: does the literal match the full-text query?
16pub const TEXT_MATCH: &str = "https://oxilite.dev/ns#textMatch";
17
18/// Ids of string literal kinds (simple / `xsd:string`, language-tagged, directional).
19fn is_text(x: &str) -> String {
20    format!("(({x}) >> {PAYLOAD_BITS}) IN (3, 4, 10)")
21}
22
23/// Schema statements of the text index (FTS5 over `terms`, kept current by triggers, and
24/// back-filled once when the index is enabled on an existing store).
25pub fn schema_statements() -> Vec<Statement> {
26    vec![
27        Statement::new(
28            "CREATE VIRTUAL TABLE IF NOT EXISTS terms_fts USING fts5(lex, content='terms', content_rowid='id', tokenize='unicode61 remove_diacritics 2')",
29        ),
30        Statement::new(format!(
31            "CREATE TRIGGER IF NOT EXISTS terms_fts_insert AFTER INSERT ON terms WHEN {} \
32             BEGIN INSERT INTO terms_fts(rowid, lex) VALUES (NEW.id, NEW.lex); END",
33            is_text("NEW.id")
34        )),
35        Statement::new(format!(
36            "CREATE TRIGGER IF NOT EXISTS terms_fts_delete AFTER DELETE ON terms WHEN {} \
37             BEGIN INSERT INTO terms_fts(terms_fts, rowid, lex) VALUES ('delete', OLD.id, OLD.lex); END",
38            is_text("OLD.id")
39        )),
40        Statement::new(format!(
41            "INSERT INTO terms_fts(rowid, lex) SELECT id, lex FROM terms WHERE {} \
42             AND NOT EXISTS (SELECT 1 FROM oxilite_meta WHERE key = 'text_index' AND value = '1')",
43            is_text("id")
44        )),
45        Statement::new("INSERT OR REPLACE INTO oxilite_meta(key, value) VALUES ('text_index', '1')"),
46    ]
47}
48
49fn tokens(text: &str) -> Vec<String> {
50    text.split(|c: char| !c.is_alphanumeric())
51        .filter(|t| !t.is_empty())
52        .map(str::to_lowercase)
53        .collect()
54}
55
56/// The fallback evaluation of `textMatch` (no index, or a query the compiler cannot express).
57pub fn text_match(args: &[Term]) -> Option<Term> {
58    let [Term::Literal(value), Term::Literal(query)] = args else {
59        return None;
60    };
61    let words = tokens(value.value());
62    let matched = query
63        .value()
64        .split_whitespace()
65        .map(|t| t.trim_matches('"'))
66        .filter(|t| !t.is_empty() && *t != "AND")
67        .all(|t| match t.strip_suffix('*') {
68            Some(prefix) => {
69                let p = prefix.to_lowercase();
70                words.iter().any(|w| w.starts_with(&p))
71            }
72            None => tokens(t).iter().all(|q| words.contains(q)),
73        });
74    Some(Literal::from(matched).into())
75}
76
77/// The function's name.
78pub fn text_match_name() -> NamedNode {
79    NamedNode::new_unchecked(TEXT_MATCH)
80}
oxilite_core/text.rs

oxilite_core/
text.rs
