llm_kernel/graph/
cjk.rs

1//! CJK-aware graph search (Rust-side segmentation, **no schema change**).
2//!
3//! The bundled FTS5 table uses the `trigram` tokenizer, which matches poorly
4//! for short CJK queries (a 1–2 character Japanese/Korean query produces no
5//! usable trigram). This module adds a parallel CJK search path that:
6//!
7//! 1. Segments the query in Rust — each CJK character becomes its own token,
8//!    Latin/digit runs stay intact ([`segment_cjk`]).
9//! 2. Matches tokens as substrings (`LIKE '%tok%'`) against `title`, `body`,
10//!    and `tags`, requiring every token to be present (AND semantics), ranked by
11//!    importance.
12//!
13//! Because this is plain `LIKE` over the existing columns, it touches **no
14//! SQLite DDL** — `init_graph_schema` is identical with or without `graph-cjk`,
15//! so a database created without the feature is safe to use with it and
16//! vice-versa. The cost is a scan per query (no FTS index), acceptable for the
17//! small-to-medium graphs this foundation targets.
18
19use rusqlite::Connection;
20
21use crate::error::{KernelError, Result};
22
23use super::types::{GraphNode, NODE_COLUMNS, escape_like, row_to_node};
24
25/// Returns `true` for CJK ideographs, kana, Hangul, and fullwidth forms.
26fn is_cjk_char(ch: char) -> bool {
27    matches!(
28        ch,
29        '\u{3040}'..='\u{30FF}'   // Hiragana, Katakana, Katakana phonetic extensions
30            | '\u{3400}'..='\u{4DBF}' // CJK Unified Ideographs Extension A
31            | '\u{4E00}'..='\u{9FFF}' // CJK Unified Ideographs
32            | '\u{F900}'..='\u{FAFF}' // CJK Compatibility Ideographs
33            | '\u{AC00}'..='\u{D7AF}' // Hangul Syllables
34            | '\u{FF00}'..='\u{FFEF}' // Fullwidth ASCII / Halfwidth Fullwidth forms
35    )
36}
37
38/// Insert a space after every CJK character so whitespace tokenization treats
39/// each CJK character as a separate token. Latin/digit/punctuation runs and
40/// existing whitespace are left untouched.
41///
42/// Examples: `"知識グラフ"` → `"知 識 グ ラ フ "`,
43/// `"rust ownership"` → `"rust ownership"`, `"Rustで非同期"` → `"Rust で 非 同 期 "`.
44pub fn segment_cjk(text: &str) -> String {
45    let mut out = String::with_capacity(text.len() + 8);
46    for ch in text.chars() {
47        out.push(ch);
48        if is_cjk_char(ch) {
49            out.push(' ');
50        }
51    }
52    out
53}
54
55/// Search nodes for a CJK (or mixed) query via **contiguous substring** matching.
56///
57/// The query is split on whitespace into terms; every term must appear as a
58/// contiguous, case-insensitive substring of `title`, `body`, or `tags`
59/// (AND semantics). A single-token CJK query like `"グラフ"` therefore matches
60/// only nodes containing that exact character run — not any node with the three
61/// characters scattered anywhere — which keeps precision high. Callers that
62/// want character-level recall for an unsegmented multi-word CJK query may
63/// pre-process it with [`segment_cjk`]. Results are ranked by importance DESC
64/// and capped at `limit`.
65pub fn search_nodes_cjk(conn: &Connection, query: &str, limit: usize) -> Result<Vec<GraphNode>> {
66    let terms: Vec<String> = query
67        .split_whitespace()
68        .map(|s| s.to_lowercase())
69        .filter(|s| !s.is_empty())
70        .collect();
71    if terms.is_empty() {
72        return Ok(Vec::new());
73    }
74
75    // Each term contributes a (title LIKE ? OR body LIKE ? OR tags LIKE ?) group;
76    // groups are AND-ed so every token must be present.
77    let term_cond = "(lower(title) LIKE ? ESCAPE '\\' OR lower(body) LIKE ? ESCAPE '\\' OR lower(tags) LIKE ? ESCAPE '\\')";
78    let where_clause = terms
79        .iter()
80        .map(|_| term_cond)
81        .collect::<Vec<_>>()
82        .join(" AND ");
83
84    let mut bind: Vec<Box<dyn rusqlite::ToSql>> = Vec::with_capacity(terms.len() * 3 + 1);
85    for t in &terms {
86        let pat = format!("%{}%", escape_like(t));
87        bind.push(Box::new(pat.clone()));
88        bind.push(Box::new(pat.clone()));
89        bind.push(Box::new(pat));
90    }
91    bind.push(Box::new(limit as i64));
92
93    let sql = format!(
94        "SELECT {NODE_COLUMNS} FROM nodes WHERE {where_clause} ORDER BY importance DESC LIMIT ?"
95    );
96    let refs: Vec<&dyn rusqlite::ToSql> = bind.iter().map(|b| b.as_ref()).collect();
97
98    let mut stmt = conn
99        .prepare(&sql)
100        .map_err(|e| KernelError::Store(e.to_string()))?;
101    let nodes: Vec<GraphNode> = stmt
102        .query_map(refs.as_slice(), row_to_node)
103        .map_err(|e| KernelError::Store(e.to_string()))?
104        .filter_map(|r| r.ok())
105        .collect();
106    Ok(nodes)
107}
108
109#[cfg(test)]
110mod tests {
111    use super::*;
112    use crate::graph::schema::init_graph_schema;
113    use crate::graph::store::upsert_node;
114
115    fn mem_db() -> Connection {
116        let conn = Connection::open_in_memory().unwrap();
117        init_graph_schema(&conn).unwrap();
118        conn
119    }
120
121    fn node(id: &str, title: &str, body: &str) -> GraphNode {
122        GraphNode {
123            id: id.to_string(),
124            node_type: "concept".to_string(),
125            title: title.to_string(),
126            body: body.to_string(),
127            tags: vec![],
128            projects: vec![],
129            agents: vec![],
130            created: "2026-01-01T00:00:00Z".to_string(),
131            updated: "2026-01-01T00:00:00Z".to_string(),
132            importance: 0.7,
133            access_count: 0,
134            accessed_at: String::new(),
135        }
136    }
137
138    #[test]
139    fn segment_separates_cjk_chars() {
140        assert_eq!(segment_cjk("知識"), "知 識 ");
141        // Latin runs are untouched.
142        assert_eq!(segment_cjk("rust async"), "rust async");
143        // Mixed: each CJK character is followed by a space; the Latin run is intact.
144        let seg = segment_cjk("Rustでトークン");
145        assert!(seg.starts_with("Rustで "), "got: {seg:?}");
146        assert!(seg.contains("ト "), "got: {seg:?}");
147    }
148
149    /// AC1: a CJK-titled node is found by a CJK query via the CJK search path.
150    #[test]
151    fn cjk_search_finds_cjk_node() {
152        let conn = mem_db();
153        upsert_node(
154            &conn,
155            &node("k1", "知識グラフの構築", "ナレッジベースをグラフ化する"),
156        )
157        .unwrap();
158        upsert_node(&conn, &node("k2", "Python GIL", "global interpreter lock")).unwrap();
159
160        // A short CJK query that trigram would not match reliably.
161        let hits = search_nodes_cjk(&conn, "グラフ", 10).unwrap();
162        assert_eq!(hits.len(), 1);
163        assert_eq!(hits[0].id, "k1");
164    }
165
166    /// AC1: multi-token CJK query requires every token.
167    #[test]
168    fn cjk_search_and_semantics() {
169        let conn = mem_db();
170        upsert_node(&conn, &node("k1", "知識グラフ", "説明本文")).unwrap();
171        // "グラフ 存在" — k1 has グラフ but not 存在 → no match.
172        let hits = search_nodes_cjk(&conn, "グラフ 存在", 10).unwrap();
173        assert!(hits.is_empty());
174        // "知識 グラフ" — both present → match.
175        let hits = search_nodes_cjk(&conn, "知識 グラフ", 10).unwrap();
176        assert_eq!(hits.len(), 1);
177    }
178
179    #[test]
180    fn empty_query_returns_nothing() {
181        let conn = mem_db();
182        upsert_node(&conn, &node("k1", "知識", "body")).unwrap();
183        assert!(search_nodes_cjk(&conn, "   ", 10).unwrap().is_empty());
184    }
185}
llm_kernel/graph/cjk.rs

llm_kernel/graph/
cjk.rs
