Skip to main content

clm_rust_core/
db.rs

1use std::env;
2use std::fs;
3use std::io::{Read, Write};
4use std::net::{TcpStream, Shutdown};
5use std::path::{Path, PathBuf};
6use std::time::Duration;
7use rusqlite::{params, Connection};
8use std::cell::RefCell;
9use std::collections::HashMap;
10use crate::error::{ClmError, StorageErrorCode, Result};
11
12thread_local! {
13    pub static TEST_SANDBOX_ACTIVE: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
14    pub static TEST_SANDBOX_DB_ID: std::cell::RefCell<Option<String>> = const { std::cell::RefCell::new(None) };
15    pub static TEST_SANDBOX_DEPTH: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
16    static IN_MEMORY_CONNS: RefCell<HashMap<String, Connection>> = RefCell::new(HashMap::new());
17}
18
19pub fn clear_in_memory_connections() {
20    IN_MEMORY_CONNS.with(|conns| conns.borrow_mut().clear());
21}
22
23/// Canonical MCard schema (v3.0.3) embedded at compile-time as single source of truth (SSOT).
24pub const CANONICAL_SCHEMA: &str = include_str!("../types/schemas/mcard_schema.sql");
25
26#[cfg(any(feature = "native", feature = "db"))]
27use std::sync::{Arc, Mutex, OnceLock};
28#[cfg(any(feature = "native", feature = "db"))]
29use r2d2::Pool;
30#[cfg(any(feature = "native", feature = "db"))]
31use r2d2_sqlite::SqliteConnectionManager;
32
33#[cfg(any(feature = "native", feature = "db"))]
34pub mod migrations;
35
36#[cfg(any(feature = "native", feature = "db"))]
37pub mod compaction;
38
39/// High-Concurrency Dual-Pool SQLite Manager.
40///
41/// Provides a dedicated single writer connection alongside an `r2d2` multi-reader
42/// connection pool configured with WAL mode, normal synchronous mode, and busy timeouts.
43#[cfg(any(feature = "native", feature = "db"))]
44#[derive(Clone)]
45pub struct DbPillarPool {
46    writer: Arc<Mutex<Connection>>,
47    reader_pool: Pool<SqliteConnectionManager>,
48    db_path: PathBuf,
49}
50
51#[cfg(any(feature = "native", feature = "db"))]
52impl DbPillarPool {
53    /// Initializes a dual-pool manager for the given pillar database path.
54    pub fn new(pillar: &str, clm_root: &str) -> Result<Self> {
55        let normalized = get_pillar_filename(pillar);
56        let db_path = get_db_path(&normalized, clm_root);
57        let db_path_str = db_path.to_string_lossy();
58
59        if !db_path_str.starts_with("file:")
60            && let Some(parent) = db_path.parent() {
61                let _ = fs::create_dir_all(parent);
62            }
63
64        #[cfg(unix)]
65        if !db_path_str.starts_with("file:") {
66            use std::os::unix::fs::PermissionsExt;
67            if let Ok(meta) = fs::metadata(&db_path) {
68                let current_mode = meta.permissions().mode();
69                if current_mode & 0o200 == 0 {
70                    let mut perms = meta.permissions();
71                    perms.set_mode(current_mode | 0o664);
72                    let _ = fs::set_permissions(&db_path, perms);
73                }
74            }
75            for ext in &["-shm", "-wal"] {
76                let aux_path = format!("{}{}", db_path.display(), ext);
77                if let Ok(meta) = fs::metadata(&aux_path) {
78                    let current_mode = meta.permissions().mode();
79                    if current_mode & 0o200 == 0 {
80                        let mut perms = meta.permissions();
81                        perms.set_mode(current_mode | 0o664);
82                        let _ = fs::set_permissions(&aux_path, perms);
83                    }
84                }
85            }
86        }
87
88        // 1. Initialize Writer Connection with WAL pragma & run migrations
89        let mut writer_conn = Connection::open_with_flags(
90            &db_path,
91            rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE
92                | rusqlite::OpenFlags::SQLITE_OPEN_CREATE
93                | rusqlite::OpenFlags::SQLITE_OPEN_URI,
94        )?;
95        writer_conn.busy_timeout(std::time::Duration::from_millis(5000))?;
96        let _ = writer_conn.pragma_update(None, "journal_mode", "WAL");
97        let _ = writer_conn.pragma_update(None, "synchronous", "NORMAL");
98        let _ = writer_conn.pragma_update(None, "foreign_keys", "ON");
99
100        // Apply versioned schema migrations on the writer connection
101        let _ = migrations::apply_migrations(&mut writer_conn, &normalized);
102
103        // 2. Initialize Reader Pool
104        let manager = SqliteConnectionManager::file(&db_path)
105            .with_flags(
106                rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE
107                    | rusqlite::OpenFlags::SQLITE_OPEN_CREATE
108                    | rusqlite::OpenFlags::SQLITE_OPEN_URI,
109            )
110            .with_init(|conn| {
111                let _ = conn.busy_timeout(std::time::Duration::from_millis(5000));
112                let _ = conn.pragma_update(None, "journal_mode", "WAL");
113                let _ = conn.pragma_update(None, "synchronous", "NORMAL");
114                let _ = conn.pragma_update(None, "query_only", true);
115                Ok(())
116            });
117
118        let reader_pool = Pool::builder()
119            .max_size(16)
120            .min_idle(Some(2))
121            .connection_timeout(std::time::Duration::from_millis(5000))
122            .build(manager)
123            .map_err(|e| {
124                ClmError::storage(
125                    StorageErrorCode::DatabaseUnavailable,
126                    format!("Failed to build SQLite reader connection pool for {}: {}", normalized, e),
127                )
128            })?;
129
130        Ok(Self {
131            writer: Arc::new(Mutex::new(writer_conn)),
132            reader_pool,
133            db_path,
134        })
135    }
136
137    /// Returns an `Arc<Mutex<Connection>>` handle to the dedicated writer connection.
138    pub fn writer(&self) -> Arc<Mutex<Connection>> {
139        self.writer.clone()
140    }
141
142    /// Acquires a connection from the multi-reader pool without locking the writer.
143    pub fn reader(&self) -> Result<r2d2::PooledConnection<SqliteConnectionManager>> {
144        self.reader_pool.get().map_err(|e| {
145            ClmError::storage(
146                StorageErrorCode::DatabaseUnavailable,
147                format!("Failed to acquire connection from reader pool: {}", e),
148            )
149        })
150    }
151
152    /// Returns the underlying database path or URI.
153    pub fn db_path(&self) -> &Path {
154        &self.db_path
155    }
156}
157
158#[cfg(any(feature = "native", feature = "db"))]
159static SHARED_POOLS: OnceLock<dashmap::DashMap<String, Arc<DbPillarPool>>> = OnceLock::new();
160
161#[cfg(any(feature = "native", feature = "db"))]
162static SHARED_CONNECTIONS: OnceLock<dashmap::DashMap<String, Arc<Mutex<Connection>>>> = OnceLock::new();
163
164#[cfg(any(feature = "native", feature = "db"))]
165pub fn get_pillar_pool(pillar: &str, clm_root: &str) -> Result<Arc<DbPillarPool>> {
166    let normalized = get_pillar_filename(pillar);
167    let db_path = get_db_path(&normalized, clm_root);
168    let key = format!("{}:{}", normalized, db_path.to_string_lossy());
169    let map = SHARED_POOLS.get_or_init(dashmap::DashMap::new);
170    if let Some(entry) = map.get(&key) {
171        return Ok(entry.clone());
172    }
173    let pool = Arc::new(DbPillarPool::new(&normalized, clm_root)?);
174    map.insert(key, pool.clone());
175    Ok(pool)
176}
177
178#[cfg(any(feature = "native", feature = "db"))]
179pub fn get_writer_connection(pillar: &str, clm_root: &str) -> Result<Arc<Mutex<Connection>>> {
180    let pool = get_pillar_pool(pillar, clm_root)?;
181    Ok(pool.writer())
182}
183
184#[cfg(any(feature = "native", feature = "db"))]
185pub fn get_reader_connection(pillar: &str, clm_root: &str) -> Result<r2d2::PooledConnection<SqliteConnectionManager>> {
186    let pool = get_pillar_pool(pillar, clm_root)?;
187    pool.reader()
188}
189
190#[cfg(any(feature = "native", feature = "db"))]
191pub fn get_shared_connection(pillar: &str, clm_root: &str) -> std::result::Result<Arc<Mutex<Connection>>, String> {
192    get_writer_connection(pillar, clm_root).map_err(|e| e.to_string())
193}
194
195#[cfg(any(feature = "native", feature = "db"))]
196pub fn clear_shared_connections() {
197    if let Some(map) = SHARED_CONNECTIONS.get() {
198        map.clear();
199    }
200    if let Some(pools) = SHARED_POOLS.get() {
201        pools.clear();
202    }
203}
204
205pub fn set_sandbox_depth(depth: usize) {
206    TEST_SANDBOX_DEPTH.with(|d| d.set(depth));
207}
208
209pub fn thread_local_depth_get() -> usize {
210    TEST_SANDBOX_DEPTH.with(|d| d.get())
211}
212
213pub fn thread_local_depth_increment() -> usize {
214    TEST_SANDBOX_DEPTH.with(|d| {
215        let c = d.get() + 1;
216        d.set(c);
217        c
218    })
219}
220
221pub fn thread_local_depth_decrement() {
222    TEST_SANDBOX_DEPTH.with(|d| {
223        let c = d.get();
224        if c > 0 {
225            d.set(c - 1);
226        }
227    })
228}
229
230pub struct DatabaseSandboxGuard {
231    prev_active: bool,
232    prev_db_id: Option<String>,
233}
234
235impl DatabaseSandboxGuard {
236    pub fn activate(db_id: &str) -> Self {
237        let prev_active = TEST_SANDBOX_ACTIVE.with(|active| active.get());
238        let prev_db_id = TEST_SANDBOX_DB_ID.with(|id| id.borrow().clone());
239        
240        TEST_SANDBOX_ACTIVE.with(|active| active.set(true));
241        TEST_SANDBOX_DB_ID.with(|id| {
242            *id.borrow_mut() = Some(db_id.to_string());
243        });
244        
245        DatabaseSandboxGuard {
246            prev_active,
247            prev_db_id,
248        }
249    }
250}
251
252impl Drop for DatabaseSandboxGuard {
253    fn drop(&mut self) {
254        TEST_SANDBOX_ACTIVE.with(|active| active.set(self.prev_active));
255        TEST_SANDBOX_DB_ID.with(|id| {
256            *id.borrow_mut() = self.prev_db_id.clone();
257        });
258        clear_in_memory_connections();
259    }
260}
261
262
263pub fn get_db_path(pillar: &str, clm_root: &str) -> PathBuf {
264    if env::var("CLM_MEM_TRIDB").unwrap_or_default() == "1" {
265        return PathBuf::from(format!("file:memdb_{}?mode=memory&cache=shared", get_pillar_filename(pillar)));
266    }
267    let is_sandbox = TEST_SANDBOX_ACTIVE.with(|active| active.get());
268    if is_sandbox {
269        let db_id = TEST_SANDBOX_DB_ID.with(|id| {
270            id.borrow().clone().unwrap_or_else(|| {
271                format!("{:?}", std::thread::current().id())
272            })
273        });
274        let depth = TEST_SANDBOX_DEPTH.with(|d| d.get());
275        let depth_suffix = if depth > 0 { format!("_depth_{}", depth) } else { "".to_string() };
276        PathBuf::from(format!("file:memdb_{}_{}{}?mode=memory&cache=shared", get_pillar_filename(pillar), db_id, depth_suffix))
277    } else {
278        let zero_fs = crate::utils::is_zero_fs();
279        if zero_fs {
280            let snap_dir = env::var("CLM_MEMBROKER_SNAP_DIR")
281                .unwrap_or_else(|_| "/tmp".to_string());
282            let snap_path = if cfg!(windows) && (snap_dir == "/tmp" || snap_dir == "\\tmp") {
283                std::env::temp_dir()
284            } else {
285                PathBuf::from(&snap_dir)
286            };
287            snap_path.join(format!("{}.db", get_pillar_filename(pillar)))
288        } else {
289            let fname = get_pillar_filename(pillar);
290            let db_dir = Path::new(clm_root).join("db");
291            let target_path = db_dir.join(format!("{}.db", fname));
292            // Backward-compatibility: if mcard_state requested but content_memory.db exists, fall back
293            if fname == "mcard_state" && !target_path.exists() && db_dir.join("content_memory.db").exists() {
294                db_dir.join("content_memory.db")
295            } else if fname == "sovereign_identity" && !target_path.exists() && db_dir.join("agent_identities.db").exists() {
296                db_dir.join("agent_identities.db")
297            } else {
298                target_path
299            }
300        }
301    }
302}
303
304fn get_pillar_filename(pillar: &str) -> String {
305    match pillar {
306        "mcard_state" | "mcard_state.db" => "mcard_state".to_string(),
307        "sovereign_identity" | "sovereign_identity.db" => "sovereign_identity".to_string(),
308        "agent_identity" | "agent_identities" => "agent_identities".to_string(),
309        "content_memory" | "content" | "memory" => "content_memory".to_string(),
310        "execution_log" | "execution" | "log" => "execution_log".to_string(),
311        "evolution" | "evol" => {
312            let now = chrono::Local::now();
313            let iso_week = chrono::Datelike::iso_week(&now);
314            format!("pkc_evol_{:04}_w{:02}", iso_week.year(), iso_week.week())
315        }
316        _ => "content_memory".to_string(),
317    }
318}
319
320/// Check if the membroker backend is active (CLM_DB_BACKEND=membroker and CLM_ZERO_FS=true).
321pub fn is_membroker_backend() -> bool {
322    env::var("CLM_DB_BACKEND").unwrap_or_default() == "membroker"
323        && crate::utils::is_zero_fs()
324}
325
326/// Send a SQL statement to the Python membroker via TCP (framed CLMQ protocol).
327/// Returns the response body on success.
328pub fn membroker_send(pillar: &str, sql: &str) -> Result<String> {
329    let port: u16 = env::var("CLM_MEMBROKER_PORT")
330        .unwrap_or_else(|_| "43740".to_string())
331        .parse()
332        .unwrap_or(43740);
333
334    let addr = format!("127.0.0.1:{}", port);
335    let timeout = Duration::from_secs(5);
336    let socket_addr: std::net::SocketAddr = addr
337        .parse()
338        .map_err(|e| ClmError::Evaluation(format!("Invalid membroker address: {}", e)))?;
339
340    let mut stream = TcpStream::connect_timeout(&socket_addr, timeout)
341        .map_err(|e| ClmError::Evaluation(format!("membroker connect failed: {}", e)))?;
342    stream.set_read_timeout(Some(timeout))?;
343    stream.set_write_timeout(Some(timeout))?;
344
345    let body = sql.as_bytes();
346    let header = format!("CLMQ\t{}\t{}\t\n", pillar, body.len());
347    stream.write_all(header.as_bytes())?;
348    stream.write_all(body)?;
349    stream.shutdown(Shutdown::Write)?;
350
351    let mut response = Vec::new();
352    stream.read_to_end(&mut response)?;
353    let resp_str = String::from_utf8_lossy(&response);
354
355    if resp_str.starts_with("CLMR\tOK\t") {
356        if let Some(pos) = resp_str.find('\n') {
357            Ok(resp_str[pos + 1..].to_string())
358        } else {
359            Ok(String::new())
360        }
361    } else {
362        Err(ClmError::Evaluation(format!("membroker error: {}", resp_str)))
363    }
364}
365
366/// Sync a single INSERT/REPLACE statement to the membroker if the backend is active.
367/// Silently skips if membroker is not available or not in use.
368pub fn membroker_sync(pillar: &str, sql: &str) {
369    if !is_membroker_backend() {
370        return;
371    }
372    let _ = membroker_send(pillar, sql);
373}
374
375
376#[path = "db_connection.rs"]
377mod db_connection;
378#[allow(unused_imports)]
379pub use db_connection::*;
380#[path = "db_handles.rs"]
381mod db_handles;
382#[allow(unused_imports)]
383pub use db_handles::*;
384#[cfg(test)]
385#[path = "db_tests.rs"]
386mod tests;
387
388
389
390