Skip to main content

clm_rust_core/layer2/
db.rs

1use crate::layer0::vocabulary::{HANDLE_FILE_MEMDB};
2use std::fs;
3use std::io::{Read, Write};
4use std::net::{TcpStream, Shutdown};
5use std::path::{Path, PathBuf};
6use std::time::Duration;
7use rusqlite::{params, Connection};
8use std::cell::RefCell;
9use std::collections::HashMap;
10use crate::error::{ClmError, StorageErrorCode, Result};
11
12thread_local! {
13    pub static TEST_SANDBOX_ACTIVE: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
14    pub static TEST_SANDBOX_DB_ID: std::cell::RefCell<Option<String>> = const { std::cell::RefCell::new(None) };
15    pub static TEST_SANDBOX_DEPTH: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
16    static IN_MEMORY_CONNS: RefCell<HashMap<String, Connection>> = RefCell::new(HashMap::new());
17}
18
19pub fn clear_in_memory_connections() {
20    IN_MEMORY_CONNS.with(|conns| conns.borrow_mut().clear());
21}
22
23/// Canonical MCard schema (v3.0.3) embedded at compile-time as single source of truth (SSOT).
24pub const CANONICAL_SCHEMA: &str = include_str!("../../types/schemas/mcard_schema.sql");
25
26#[cfg(any(feature = "native", feature = "db"))]
27use std::sync::{Arc, Mutex, OnceLock};
28#[cfg(any(feature = "native", feature = "db"))]
29use r2d2::Pool;
30#[cfg(any(feature = "native", feature = "db"))]
31use r2d2_sqlite::SqliteConnectionManager;
32
33#[cfg(any(feature = "native", feature = "db"))]
34#[path = "db/migrations.rs"]
35pub mod migrations;
36
37#[cfg(any(feature = "native", feature = "db"))]
38#[path = "db/compaction.rs"]
39pub mod compaction;
40
41/// High-Concurrency Dual-Pool SQLite Manager.
42///
43/// Provides a dedicated single writer connection alongside an `r2d2` multi-reader
44/// connection pool configured with WAL mode, normal synchronous mode, and busy timeouts.
45#[cfg(any(feature = "native", feature = "db"))]
46#[derive(Clone)]
47pub struct DbPillarPool {
48    writer: Arc<Mutex<Connection>>,
49    reader_pool: Pool<SqliteConnectionManager>,
50    db_path: PathBuf,
51}
52
53#[cfg(any(feature = "native", feature = "db"))]
54impl DbPillarPool {
55    /// Initializes a dual-pool manager for the given pillar database path.
56    pub fn new(pillar: &str, clm_root: &str) -> Result<Self> {
57        let normalized = get_pillar_filename(pillar);
58        let db_path = get_db_path(&normalized, clm_root);
59        let db_path_str = db_path.to_string_lossy();
60
61        if !db_path_str.starts_with(crate::layer0::vocabulary::URI_SCHEME_FILE)
62            && let Some(parent) = db_path.parent() {
63                let _ = fs::create_dir_all(parent);
64            }
65
66        #[cfg(unix)]
67        if !db_path_str.starts_with(crate::layer0::vocabulary::URI_SCHEME_FILE) {
68            use std::os::unix::fs::PermissionsExt;
69            if let Ok(meta) = fs::metadata(&db_path) {
70                let current_mode = meta.permissions().mode();
71                if current_mode & 0o200 == 0 {
72                    let mut perms = meta.permissions();
73                    perms.set_mode(current_mode | 0o664);
74                    let _ = fs::set_permissions(&db_path, perms);
75                }
76            }
77            for ext in &["-shm", "-wal"] {
78                let aux_path = format!("{}{}", db_path.display(), ext);
79                if let Ok(meta) = fs::metadata(&aux_path) {
80                    let current_mode = meta.permissions().mode();
81                    if current_mode & 0o200 == 0 {
82                        let mut perms = meta.permissions();
83                        perms.set_mode(current_mode | 0o664);
84                        let _ = fs::set_permissions(&aux_path, perms);
85                    }
86                }
87            }
88        }
89
90        // 1. Initialize Writer Connection with WAL pragma & run migrations
91        let mut writer_conn = Connection::open_with_flags(
92            &db_path,
93            rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE
94                | rusqlite::OpenFlags::SQLITE_OPEN_CREATE
95                | rusqlite::OpenFlags::SQLITE_OPEN_URI,
96        )?;
97        writer_conn.busy_timeout(std::time::Duration::from_millis(u64::from(crate::layer0::runtime_contract::SQLITE_BUSY_TIMEOUT_MS)))?;
98        let _ = writer_conn.pragma_update(None, "journal_mode", "WAL");
99        let _ = writer_conn.pragma_update(None, "synchronous", "NORMAL");
100        let _ = writer_conn.pragma_update(None, "foreign_keys", "ON");
101
102        // Apply versioned schema migrations on the writer connection
103        let _ = migrations::apply_migrations(&mut writer_conn, &normalized);
104
105        // 2. Initialize Reader Pool
106        let manager = SqliteConnectionManager::file(&db_path)
107            .with_flags(
108                rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE
109                    | rusqlite::OpenFlags::SQLITE_OPEN_CREATE
110                    | rusqlite::OpenFlags::SQLITE_OPEN_URI,
111            )
112            .with_init(|conn| {
113                let _ = conn.busy_timeout(std::time::Duration::from_millis(u64::from(crate::layer0::runtime_contract::SQLITE_BUSY_TIMEOUT_MS)));
114                let _ = conn.pragma_update(None, "journal_mode", "WAL");
115                let _ = conn.pragma_update(None, "synchronous", "NORMAL");
116                let _ = conn.pragma_update(None, "query_only", true);
117                Ok(())
118            });
119
120        let reader_pool = Pool::builder()
121            .max_size(16)
122            .min_idle(Some(2))
123            .connection_timeout(std::time::Duration::from_millis(u64::from(crate::layer0::runtime_contract::SQLITE_BUSY_TIMEOUT_MS)))
124            .build(manager)
125            .map_err(|e| {
126                ClmError::storage(
127                    StorageErrorCode::DatabaseUnavailable,
128                    format!("Failed to build SQLite reader connection pool for {}: {}", normalized, e),
129                )
130            })?;
131
132        Ok(Self {
133            writer: Arc::new(Mutex::new(writer_conn)),
134            reader_pool,
135            db_path,
136        })
137    }
138
139    /// Returns an `Arc<Mutex<Connection>>` handle to the dedicated writer connection.
140    pub fn writer(&self) -> Arc<Mutex<Connection>> {
141        self.writer.clone()
142    }
143
144    /// Acquires a connection from the multi-reader pool without locking the writer.
145    pub fn reader(&self) -> Result<r2d2::PooledConnection<SqliteConnectionManager>> {
146        self.reader_pool.get().map_err(|e| {
147            ClmError::storage(
148                StorageErrorCode::DatabaseUnavailable,
149                format!("Failed to acquire connection from reader pool: {}", e),
150            )
151        })
152    }
153
154    /// Returns the underlying database path or URI.
155    pub fn db_path(&self) -> &Path {
156        &self.db_path
157    }
158}
159
160#[cfg(any(feature = "native", feature = "db"))]
161static SHARED_POOLS: OnceLock<dashmap::DashMap<String, Arc<DbPillarPool>>> = OnceLock::new();
162
163#[cfg(any(feature = "native", feature = "db"))]
164static SHARED_CONNECTIONS: OnceLock<dashmap::DashMap<String, Arc<Mutex<Connection>>>> = OnceLock::new();
165
166#[cfg(any(feature = "native", feature = "db"))]
167pub fn get_pillar_pool(pillar: &str, clm_root: &str) -> Result<Arc<DbPillarPool>> {
168    let normalized = get_pillar_filename(pillar);
169    let db_path = get_db_path(&normalized, clm_root);
170    let key = format!("{}:{}", normalized, db_path.to_string_lossy());
171    let map = SHARED_POOLS.get_or_init(dashmap::DashMap::new);
172    if let Some(entry) = map.get(&key) {
173        return Ok(entry.clone());
174    }
175    let pool = Arc::new(DbPillarPool::new(&normalized, clm_root)?);
176    map.insert(key, pool.clone());
177    Ok(pool)
178}
179
180#[cfg(any(feature = "native", feature = "db"))]
181pub fn get_writer_connection(pillar: &str, clm_root: &str) -> Result<Arc<Mutex<Connection>>> {
182    let pool = get_pillar_pool(pillar, clm_root)?;
183    Ok(pool.writer())
184}
185
186#[cfg(any(feature = "native", feature = "db"))]
187pub fn get_reader_connection(pillar: &str, clm_root: &str) -> Result<r2d2::PooledConnection<SqliteConnectionManager>> {
188    let pool = get_pillar_pool(pillar, clm_root)?;
189    pool.reader()
190}
191
192#[cfg(any(feature = "native", feature = "db"))]
193pub fn get_shared_connection(pillar: &str, clm_root: &str) -> std::result::Result<Arc<Mutex<Connection>>, String> {
194    get_writer_connection(pillar, clm_root).map_err(|e| e.to_string())
195}
196
197#[cfg(any(feature = "native", feature = "db"))]
198pub fn clear_shared_connections() {
199    if let Some(map) = SHARED_CONNECTIONS.get() {
200        map.clear();
201    }
202    if let Some(pools) = SHARED_POOLS.get() {
203        pools.clear();
204    }
205}
206
207pub fn set_sandbox_depth(depth: usize) {
208    TEST_SANDBOX_DEPTH.with(|d| d.set(depth));
209}
210
211pub fn thread_local_depth_get() -> usize {
212    TEST_SANDBOX_DEPTH.with(|d| d.get())
213}
214
215pub fn thread_local_depth_increment() -> usize {
216    TEST_SANDBOX_DEPTH.with(|d| {
217        let c = d.get() + 1;
218        d.set(c);
219        c
220    })
221}
222
223pub fn thread_local_depth_decrement() {
224    TEST_SANDBOX_DEPTH.with(|d| {
225        let c = d.get();
226        if c > 0 {
227            d.set(c - 1);
228        }
229    })
230}
231
232pub struct DatabaseSandboxGuard {
233    prev_active: bool,
234    prev_db_id: Option<String>,
235}
236
237impl DatabaseSandboxGuard {
238    pub fn activate(db_id: &str) -> Self {
239        let prev_active = TEST_SANDBOX_ACTIVE.with(|active| active.get());
240        let prev_db_id = TEST_SANDBOX_DB_ID.with(|id| id.borrow().clone());
241        
242        TEST_SANDBOX_ACTIVE.with(|active| active.set(true));
243        TEST_SANDBOX_DB_ID.with(|id| {
244            *id.borrow_mut() = Some(db_id.to_string());
245        });
246        
247        DatabaseSandboxGuard {
248            prev_active,
249            prev_db_id,
250        }
251    }
252}
253
254impl Drop for DatabaseSandboxGuard {
255    fn drop(&mut self) {
256        TEST_SANDBOX_ACTIVE.with(|active| active.set(self.prev_active));
257        TEST_SANDBOX_DB_ID.with(|id| {
258            *id.borrow_mut() = self.prev_db_id.clone();
259        });
260        clear_in_memory_connections();
261    }
262}
263
264
265pub fn get_db_path(pillar: &str, clm_root: &str) -> PathBuf {
266    if crate::layer0::environment::var(crate::layer0::vocabulary_env::ENV_CLM_MEM_TRIDB).unwrap_or_default() == "1" {
267        return PathBuf::from(format!("{HANDLE_FILE_MEMDB}{}?mode=memory&cache=shared", get_pillar_filename(pillar)));
268    }
269    let is_sandbox = TEST_SANDBOX_ACTIVE.with(|active| active.get());
270    if is_sandbox {
271        let db_id = TEST_SANDBOX_DB_ID.with(|id| {
272            id.borrow().clone().unwrap_or_else(|| {
273                format!("{:?}", std::thread::current().id())
274            })
275        });
276        let depth = TEST_SANDBOX_DEPTH.with(|d| d.get());
277        let depth_suffix = if depth > 0 { format!("_depth_{}", depth) } else { "".to_string() };
278        PathBuf::from(format!("{HANDLE_FILE_MEMDB}{}_{}{}?mode=memory&cache=shared", get_pillar_filename(pillar), db_id, depth_suffix))
279    } else {
280        let zero_fs = crate::utils::is_zero_fs();
281        if zero_fs {
282            let snap_dir = crate::layer0::environment::var(crate::layer0::vocabulary_env::ENV_CLM_MEMBROKER_SNAP_DIR)
283                .unwrap_or_else(|_| "/tmp".to_string());
284            let snap_path = if cfg!(windows) && (snap_dir == "/tmp" || snap_dir == "\\tmp") {
285                std::env::temp_dir()
286            } else {
287                PathBuf::from(&snap_dir)
288            };
289            snap_path.join(format!("{}.db", get_pillar_filename(pillar)))
290        } else {
291            let fname = get_pillar_filename(pillar);
292            let db_dir = Path::new(clm_root).join("db");
293            let target_path = db_dir.join(format!("{}.db", fname));
294            // Backward-compatibility: if mcard_state requested but content_memory.db exists, fall back
295            if fname == "mcard_state" && !target_path.exists() && db_dir.join("content_memory.db").exists() {
296                db_dir.join("content_memory.db")
297            } else if fname == "sovereign_identity" && !target_path.exists() && db_dir.join("agent_identities.db").exists() {
298                db_dir.join("agent_identities.db")
299            } else {
300                target_path
301            }
302        }
303    }
304}
305
306fn get_pillar_filename(pillar: &str) -> String {
307    match pillar {
308        "mcard_state" | "mcard_state.db" => "mcard_state".to_string(),
309        "sovereign_identity" | "sovereign_identity.db" => "sovereign_identity".to_string(),
310        "agent_identity" | "agent_identities" => "agent_identities".to_string(),
311        crate::layer0::vocabulary_env::PILLAR_CONTENT_MEMORY | "content" | "memory" => crate::layer0::vocabulary_env::PILLAR_CONTENT_MEMORY.to_string(),
312        crate::layer0::vocabulary_env::PILLAR_EXECUTION_LOG | "execution" | "log" => crate::layer0::vocabulary_env::PILLAR_EXECUTION_LOG.to_string(),
313        "evolution" | "evol" => {
314            let now = chrono::Local::now();
315            let iso_week = chrono::Datelike::iso_week(&now);
316            format!("pkc_evol_{:04}_w{:02}", iso_week.year(), iso_week.week())
317        }
318        _ => crate::layer0::vocabulary_env::PILLAR_CONTENT_MEMORY.to_string(),
319    }
320}
321
322/// Check if the membroker backend is active (CLM_DB_BACKEND=membroker and CLM_ZERO_FS=true).
323pub fn is_membroker_backend() -> bool {
324    crate::layer0::environment::var(crate::layer0::vocabulary_env::ENV_CLM_DB_BACKEND).unwrap_or_default() == "membroker"
325        && crate::utils::is_zero_fs()
326}
327
328/// Send a SQL statement to the Python membroker via TCP (framed CLMQ protocol).
329/// Returns the response body on success.
330pub fn membroker_send(pillar: &str, sql: &str) -> Result<String> {
331    let port: u16 = crate::layer0::environment::var(crate::layer0::vocabulary_env::ENV_CLM_MEMBROKER_PORT)
332        .unwrap_or_else(|_| "43740".to_string())
333        .parse()
334        .unwrap_or(43740);
335
336    let addr = format!("127.0.0.1:{}", port);
337    let timeout = Duration::from_secs(5);
338    let socket_addr: std::net::SocketAddr = addr
339        .parse()
340        .map_err(|e| ClmError::Evaluation(format!("Invalid membroker address: {}", e)))?;
341
342    let mut stream = TcpStream::connect_timeout(&socket_addr, timeout)
343        .map_err(|e| ClmError::Evaluation(format!("membroker connect failed: {}", e)))?;
344    stream.set_read_timeout(Some(timeout))?;
345    stream.set_write_timeout(Some(timeout))?;
346
347    let body = sql.as_bytes();
348    let header = format!("CLMQ\t{}\t{}\t\n", pillar, body.len());
349    stream.write_all(header.as_bytes())?;
350    stream.write_all(body)?;
351    stream.shutdown(Shutdown::Write)?;
352
353    let mut response = Vec::new();
354    stream.read_to_end(&mut response)?;
355    let resp_str = String::from_utf8_lossy(&response);
356
357    if resp_str.starts_with("CLMR\tOK\t") {
358        if let Some(pos) = resp_str.find('\n') {
359            Ok(resp_str[pos + 1..].to_string())
360        } else {
361            Ok(String::new())
362        }
363    } else {
364        Err(ClmError::Evaluation(format!("membroker error: {}", resp_str)))
365    }
366}
367
368/// Sync a single INSERT/REPLACE statement to the membroker if the backend is active.
369/// Silently skips if membroker is not available or not in use.
370pub fn membroker_sync(pillar: &str, sql: &str) {
371    if !is_membroker_backend() {
372        return;
373    }
374    let _ = membroker_send(pillar, sql);
375}
376
377
378#[path = "db_connection.rs"]
379mod db_connection;
380#[allow(unused_imports)]
381pub use db_connection::*;
382#[path = "db_handles.rs"]
383mod db_handles;
384#[allow(unused_imports)]
385pub use db_handles::*;
386#[cfg(test)]
387#[path = "db_tests.rs"]
388mod tests;
389
390
391
392