use std::env;
use std::fs;
use std::io::{Read, Write};
use std::net::{TcpStream, Shutdown};
use std::path::{Path, PathBuf};
use std::time::Duration;
use rusqlite::{params, Connection};
use std::cell::RefCell;
use std::collections::HashMap;
use crate::error::{ClmError, StorageErrorCode, Result};
thread_local! {
pub static TEST_SANDBOX_ACTIVE: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
pub static TEST_SANDBOX_DB_ID: std::cell::RefCell<Option<String>> = const { std::cell::RefCell::new(None) };
pub static TEST_SANDBOX_DEPTH: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
static IN_MEMORY_CONNS: RefCell<HashMap<String, Connection>> = RefCell::new(HashMap::new());
}
pub fn clear_in_memory_connections() {
IN_MEMORY_CONNS.with(|conns| conns.borrow_mut().clear());
}
pub const CANONICAL_SCHEMA: &str = include_str!("../types/schemas/mcard_schema.sql");
#[cfg(any(feature = "native", feature = "db"))]
use std::sync::{Arc, Mutex, OnceLock};
#[cfg(any(feature = "native", feature = "db"))]
use r2d2::Pool;
#[cfg(any(feature = "native", feature = "db"))]
use r2d2_sqlite::SqliteConnectionManager;
#[cfg(any(feature = "native", feature = "db"))]
pub mod migrations;
#[cfg(any(feature = "native", feature = "db"))]
pub mod compaction;
#[cfg(any(feature = "native", feature = "db"))]
#[derive(Clone)]
pub struct DbPillarPool {
writer: Arc<Mutex<Connection>>,
reader_pool: Pool<SqliteConnectionManager>,
db_path: PathBuf,
}
#[cfg(any(feature = "native", feature = "db"))]
impl DbPillarPool {
pub fn new(pillar: &str, clm_root: &str) -> Result<Self> {
let normalized = get_pillar_filename(pillar);
let db_path = get_db_path(&normalized, clm_root);
let db_path_str = db_path.to_string_lossy();
if !db_path_str.starts_with("file:")
&& let Some(parent) = db_path.parent() {
let _ = fs::create_dir_all(parent);
}
let mut writer_conn = Connection::open_with_flags(
&db_path,
rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE
| rusqlite::OpenFlags::SQLITE_OPEN_CREATE
| rusqlite::OpenFlags::SQLITE_OPEN_URI,
)?;
writer_conn.busy_timeout(std::time::Duration::from_millis(5000))?;
let _ = writer_conn.pragma_update(None, "journal_mode", "WAL");
let _ = writer_conn.pragma_update(None, "synchronous", "NORMAL");
let _ = writer_conn.pragma_update(None, "foreign_keys", "ON");
let _ = migrations::apply_migrations(&mut writer_conn, &normalized);
let manager = SqliteConnectionManager::file(&db_path)
.with_flags(
rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE
| rusqlite::OpenFlags::SQLITE_OPEN_CREATE
| rusqlite::OpenFlags::SQLITE_OPEN_URI,
)
.with_init(|conn| {
let _ = conn.busy_timeout(std::time::Duration::from_millis(5000));
let _ = conn.pragma_update(None, "journal_mode", "WAL");
let _ = conn.pragma_update(None, "synchronous", "NORMAL");
let _ = conn.pragma_update(None, "query_only", true);
Ok(())
});
let reader_pool = Pool::builder()
.max_size(16)
.min_idle(Some(2))
.connection_timeout(std::time::Duration::from_millis(5000))
.build(manager)
.map_err(|e| {
ClmError::storage(
StorageErrorCode::DatabaseUnavailable,
format!("Failed to build SQLite reader connection pool for {}: {}", normalized, e),
)
})?;
Ok(Self {
writer: Arc::new(Mutex::new(writer_conn)),
reader_pool,
db_path,
})
}
pub fn writer(&self) -> Arc<Mutex<Connection>> {
self.writer.clone()
}
pub fn reader(&self) -> Result<r2d2::PooledConnection<SqliteConnectionManager>> {
self.reader_pool.get().map_err(|e| {
ClmError::storage(
StorageErrorCode::DatabaseUnavailable,
format!("Failed to acquire connection from reader pool: {}", e),
)
})
}
pub fn db_path(&self) -> &Path {
&self.db_path
}
}
#[cfg(any(feature = "native", feature = "db"))]
static SHARED_POOLS: OnceLock<dashmap::DashMap<String, Arc<DbPillarPool>>> = OnceLock::new();
#[cfg(any(feature = "native", feature = "db"))]
static SHARED_CONNECTIONS: OnceLock<dashmap::DashMap<String, Arc<Mutex<Connection>>>> = OnceLock::new();
#[cfg(any(feature = "native", feature = "db"))]
pub fn get_pillar_pool(pillar: &str, clm_root: &str) -> Result<Arc<DbPillarPool>> {
let normalized = get_pillar_filename(pillar);
let db_path = get_db_path(&normalized, clm_root);
let key = format!("{}:{}", normalized, db_path.to_string_lossy());
let map = SHARED_POOLS.get_or_init(dashmap::DashMap::new);
if let Some(entry) = map.get(&key) {
return Ok(entry.clone());
}
let pool = Arc::new(DbPillarPool::new(&normalized, clm_root)?);
map.insert(key, pool.clone());
Ok(pool)
}
#[cfg(any(feature = "native", feature = "db"))]
pub fn get_writer_connection(pillar: &str, clm_root: &str) -> Result<Arc<Mutex<Connection>>> {
let pool = get_pillar_pool(pillar, clm_root)?;
Ok(pool.writer())
}
#[cfg(any(feature = "native", feature = "db"))]
pub fn get_reader_connection(pillar: &str, clm_root: &str) -> Result<r2d2::PooledConnection<SqliteConnectionManager>> {
let pool = get_pillar_pool(pillar, clm_root)?;
pool.reader()
}
#[cfg(any(feature = "native", feature = "db"))]
pub fn get_shared_connection(pillar: &str, clm_root: &str) -> std::result::Result<Arc<Mutex<Connection>>, String> {
get_writer_connection(pillar, clm_root).map_err(|e| e.to_string())
}
#[cfg(any(feature = "native", feature = "db"))]
pub fn clear_shared_connections() {
if let Some(map) = SHARED_CONNECTIONS.get() {
map.clear();
}
if let Some(pools) = SHARED_POOLS.get() {
pools.clear();
}
}
pub fn set_sandbox_depth(depth: usize) {
TEST_SANDBOX_DEPTH.with(|d| d.set(depth));
}
pub fn thread_local_depth_get() -> usize {
TEST_SANDBOX_DEPTH.with(|d| d.get())
}
pub fn thread_local_depth_increment() -> usize {
TEST_SANDBOX_DEPTH.with(|d| {
let c = d.get() + 1;
d.set(c);
c
})
}
pub fn thread_local_depth_decrement() {
TEST_SANDBOX_DEPTH.with(|d| {
let c = d.get();
if c > 0 {
d.set(c - 1);
}
})
}
pub struct DatabaseSandboxGuard {
prev_active: bool,
prev_db_id: Option<String>,
}
impl DatabaseSandboxGuard {
pub fn activate(db_id: &str) -> Self {
let prev_active = TEST_SANDBOX_ACTIVE.with(|active| active.get());
let prev_db_id = TEST_SANDBOX_DB_ID.with(|id| id.borrow().clone());
TEST_SANDBOX_ACTIVE.with(|active| active.set(true));
TEST_SANDBOX_DB_ID.with(|id| {
*id.borrow_mut() = Some(db_id.to_string());
});
DatabaseSandboxGuard {
prev_active,
prev_db_id,
}
}
}
impl Drop for DatabaseSandboxGuard {
fn drop(&mut self) {
TEST_SANDBOX_ACTIVE.with(|active| active.set(self.prev_active));
TEST_SANDBOX_DB_ID.with(|id| {
*id.borrow_mut() = self.prev_db_id.clone();
});
clear_in_memory_connections();
}
}
pub fn get_db_path(pillar: &str, clm_root: &str) -> PathBuf {
if env::var("CLM_MEM_TRIDB").unwrap_or_default() == "1" {
return PathBuf::from(format!("file:memdb_{}?mode=memory&cache=shared", get_pillar_filename(pillar)));
}
let is_sandbox = TEST_SANDBOX_ACTIVE.with(|active| active.get());
if is_sandbox {
let db_id = TEST_SANDBOX_DB_ID.with(|id| {
id.borrow().clone().unwrap_or_else(|| {
format!("{:?}", std::thread::current().id())
})
});
let depth = TEST_SANDBOX_DEPTH.with(|d| d.get());
let depth_suffix = if depth > 0 { format!("_depth_{}", depth) } else { "".to_string() };
PathBuf::from(format!("file:memdb_{}_{}{}?mode=memory&cache=shared", get_pillar_filename(pillar), db_id, depth_suffix))
} else {
let zero_fs = crate::utils::is_zero_fs();
if zero_fs {
let snap_dir = env::var("CLM_MEMBROKER_SNAP_DIR")
.unwrap_or_else(|_| "/tmp".to_string());
Path::new(&snap_dir).join(format!("{}.db", get_pillar_filename(pillar)))
} else {
let fname = get_pillar_filename(pillar);
let db_dir = Path::new(clm_root).join("db");
let target_path = db_dir.join(format!("{}.db", fname));
if fname == "mcard_state" && !target_path.exists() && db_dir.join("content_memory.db").exists() {
db_dir.join("content_memory.db")
} else if fname == "sovereign_identity" && !target_path.exists() && db_dir.join("agent_identities.db").exists() {
db_dir.join("agent_identities.db")
} else {
target_path
}
}
}
}
fn get_pillar_filename(pillar: &str) -> String {
match pillar {
"mcard_state" | "mcard_state.db" => "mcard_state".to_string(),
"sovereign_identity" | "sovereign_identity.db" => "sovereign_identity".to_string(),
"agent_identity" | "agent_identities" => "agent_identities".to_string(),
"content_memory" | "content" | "memory" => "content_memory".to_string(),
"execution_log" | "execution" | "log" => "execution_log".to_string(),
"evolution" | "evol" => {
let now = chrono::Local::now();
let iso_week = chrono::Datelike::iso_week(&now);
format!("pkc_evol_{:04}_w{:02}", iso_week.year(), iso_week.week())
}
_ => "content_memory".to_string(),
}
}
pub fn is_membroker_backend() -> bool {
env::var("CLM_DB_BACKEND").unwrap_or_default() == "membroker"
&& crate::utils::is_zero_fs()
}
pub fn membroker_send(pillar: &str, sql: &str) -> Result<String> {
let port: u16 = env::var("CLM_MEMBROKER_PORT")
.unwrap_or_else(|_| "43740".to_string())
.parse()
.unwrap_or(43740);
let addr = format!("127.0.0.1:{}", port);
let timeout = Duration::from_secs(5);
let socket_addr: std::net::SocketAddr = addr
.parse()
.map_err(|e| ClmError::Evaluation(format!("Invalid membroker address: {}", e)))?;
let mut stream = TcpStream::connect_timeout(&socket_addr, timeout)
.map_err(|e| ClmError::Evaluation(format!("membroker connect failed: {}", e)))?;
stream.set_read_timeout(Some(timeout))?;
stream.set_write_timeout(Some(timeout))?;
let body = sql.as_bytes();
let header = format!("CLMQ\t{}\t{}\t\n", pillar, body.len());
stream.write_all(header.as_bytes())?;
stream.write_all(body)?;
stream.shutdown(Shutdown::Write)?;
let mut response = Vec::new();
stream.read_to_end(&mut response)?;
let resp_str = String::from_utf8_lossy(&response);
if resp_str.starts_with("CLMR\tOK\t") {
if let Some(pos) = resp_str.find('\n') {
Ok(resp_str[pos + 1..].to_string())
} else {
Ok(String::new())
}
} else {
Err(ClmError::Evaluation(format!("membroker error: {}", resp_str)))
}
}
pub fn membroker_sync(pillar: &str, sql: &str) {
if !is_membroker_backend() {
return;
}
let _ = membroker_send(pillar, sql);
}
pub fn get_connection(pillar: &str, clm_root: &str) -> Result<Connection> {
let db_path = get_db_path(pillar, clm_root);
let db_path_str = db_path.to_string_lossy();
if !db_path_str.starts_with("file:") {
if let Some(parent) = db_path.parent() {
let _ = fs::create_dir_all(parent);
}
} else {
let path_str = db_path_str.to_string();
let already_registered = IN_MEMORY_CONNS.with(|conns| conns.borrow().contains_key(&path_str));
if !already_registered
&& let Ok(keep_alive_conn) = Connection::open_with_flags(
&db_path,
rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE
| rusqlite::OpenFlags::SQLITE_OPEN_CREATE
| rusqlite::OpenFlags::SQLITE_OPEN_URI,
) {
let _ = keep_alive_conn.busy_timeout(std::time::Duration::from_millis(10000));
let _ = keep_alive_conn.pragma_update(None, "journal_mode", "WAL");
let _ = keep_alive_conn.pragma_update(None, "synchronous", "NORMAL");
let table_exists: bool = keep_alive_conn
.query_row(
"SELECT count(*) FROM sqlite_master WHERE type='table' AND name='card';",
[],
|row| row.get(0),
)
.unwrap_or(0) > 0;
if !table_exists {
let project_path = Path::new(clm_root);
let mut schema_file = project_path.join("types").join("schemas").join("mcard_schema.sql");
if !schema_file.exists() {
schema_file = project_path.join("schema").join("mcard_schema.sql");
}
if !schema_file.exists() {
schema_file = project_path.join("..").join("schema").join("mcard_schema.sql");
}
if schema_file.exists()
&& let Ok(schema_sql) = fs::read_to_string(&schema_file) {
let _ = keep_alive_conn.execute_batch(&schema_sql);
}
let _ = keep_alive_conn.execute("ALTER TABLE card ADD COLUMN g_time TEXT;", []);
}
IN_MEMORY_CONNS.with(|conns| {
conns.borrow_mut().insert(path_str, keep_alive_conn);
});
}
}
let conn = Connection::open_with_flags(
&db_path,
rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE
| rusqlite::OpenFlags::SQLITE_OPEN_CREATE
| rusqlite::OpenFlags::SQLITE_OPEN_URI,
)?;
conn.busy_timeout(std::time::Duration::from_millis(10000))?;
let _ = conn.pragma_update(None, "journal_mode", "WAL");
let _ = conn.pragma_update(None, "synchronous", "NORMAL");
let table_exists: bool = conn
.query_row(
"SELECT count(*) FROM sqlite_master WHERE type='table' AND name='card';",
[],
|row| row.get(0),
)
.unwrap_or(0) > 0;
if !table_exists {
let _ = conn.execute_batch(CANONICAL_SCHEMA);
}
let _ = conn.execute("ALTER TABLE card ADD COLUMN g_time TEXT;", []);
Ok(conn)
}
pub fn init_tridb(clm_root: &str) -> Result<()> {
let project_path = Path::new(clm_root);
let mut schema_file = project_path.join("types").join("schemas").join("mcard_schema.sql");
if !schema_file.exists() {
schema_file = project_path.join("..").join("types").join("schemas").join("mcard_schema.sql");
}
let schema_sql = if schema_file.exists() {
fs::read_to_string(&schema_file).unwrap_or_else(|_| CANONICAL_SCHEMA.to_string())
} else {
CANONICAL_SCHEMA.to_string()
};
let mut version = "unknown".to_string();
for line in schema_sql.lines() {
if line.contains("Version:") {
if let Some(pos) = line.find("Version:") {
let suffix = &line[pos + 8..].trim();
let parts: Vec<&str> = suffix.split_whitespace().collect();
if !parts.is_empty() {
version = parts[0].trim_matches(|c: char| !c.is_numeric() && c != '.').to_string();
}
}
break;
}
}
eprintln!("[INIT_TRIDB] Canonical Schema: {}", schema_file.display());
eprintln!("[INIT_TRIDB] Schema version: {}", version);
eprintln!();
let pillars = vec!["agent_identities", "content_memory", "execution_log"];
for pillar in pillars {
let conn = get_connection(pillar, clm_root)
.map_err(|e| ClmError::Evaluation(format!("Failed to open DB for {}: {}", pillar, e)))?;
let _ = conn.pragma_update(None, "journal_mode", "WAL");
conn.execute_batch(&schema_sql)
.map_err(|e| ClmError::Evaluation(format!("Failed to apply schema for {}: {}", pillar, e)))?;
let mut stmt = conn
.prepare("SELECT name FROM sqlite_master WHERE type='table';")?;
let table_rows = stmt
.query_map([], |row| row.get::<_, String>(0))?;
let mut has_card = false;
let mut has_registry = false;
let mut has_history = false;
for name in table_rows.flatten() {
match name.as_str() {
"card" => has_card = true,
"handle_registry" => has_registry = true,
"handle_history" => has_history = true,
_ => {}
}
}
if !has_card || !has_registry || !has_history {
return Err(ClmError::Evaluation(format!("DB {} is missing required tables", pillar)));
}
let schema_handle = format!("db:schema:{}:{}", pillar, version);
let schema_hash = crate::fs_utils::compute_sha256_hex(schema_sql.as_bytes());
let _ = handle_register_batch(pillar, clm_root, &schema_handle, &crate::hash::prefixed(&schema_hash), &schema_sql);
}
let mut lattice = crate::type_lattice::TypeLattice::new();
lattice.load_registry(clm_root);
eprintln!("[INIT_TRIDB] Type lattice loaded: {} types registered", lattice.registry.len());
Ok(())
}
pub fn sql_query_parameterized(
pillar: &str,
clm_root: &str,
query: &str,
params_vec: Vec<String>,
as_json: bool,
) -> Result<String> {
let conn = get_connection(pillar, clm_root)?;
let _ = conn.execute("SELECT 1 FROM card LIMIT 1;", []);
let query_trimmed = query.trim();
let is_batch = query_trimmed.contains("BEGIN;")
|| query_trimmed.contains("begin;")
|| query_trimmed.contains("COMMIT;")
|| query_trimmed.contains("commit;")
|| (query_trimmed.contains(';') && !query_trimmed.ends_with(';'))
|| query_trimmed.matches(';').count() > 1;
if is_batch {
conn.execute_batch(query)?;
if as_json {
return Ok("[]".to_string());
} else {
return Ok(String::new());
}
}
let mut stmt = conn.prepare(query)?;
let col_count = stmt.column_count();
let col_names: Vec<String> = stmt
.column_names()
.into_iter()
.map(|s| s.to_string())
.collect();
let params_ref: Vec<&dyn rusqlite::ToSql> = params_vec
.iter()
.map(|s| s as &dyn rusqlite::ToSql)
.collect();
let mut rows = stmt.query(params_ref.as_slice())?;
if as_json {
let mut results = Vec::new();
while let Some(row) = rows.next()? {
let mut row_map = serde_json::Map::new();
for i in 0..col_count {
let name = &col_names[i];
let value_ref = row.get_ref(i)?;
let json_val = match value_ref {
rusqlite::types::ValueRef::Null => serde_json::Value::Null,
rusqlite::types::ValueRef::Integer(n) => serde_json::Value::Number(n.into()),
rusqlite::types::ValueRef::Real(f) => {
if let Some(num) = serde_json::Number::from_f64(f) {
serde_json::Value::Number(num)
} else {
serde_json::Value::Null
}
}
rusqlite::types::ValueRef::Text(t) => {
let text = String::from_utf8_lossy(t).into_owned();
serde_json::Value::String(text)
}
rusqlite::types::ValueRef::Blob(b) => {
let text = String::from_utf8_lossy(b).into_owned();
serde_json::Value::String(text)
}
};
row_map.insert(name.clone(), json_val);
}
results.push(serde_json::Value::Object(row_map));
}
Ok(serde_json::Value::Array(results).to_string())
} else {
let mut results = Vec::new();
while let Some(row) = rows.next()? {
let mut row_parts = Vec::new();
for i in 0..col_count {
let val_ref = row.get_ref(i)?;
let s = match val_ref {
rusqlite::types::ValueRef::Null => "".to_string(),
rusqlite::types::ValueRef::Integer(n) => n.to_string(),
rusqlite::types::ValueRef::Real(f) => f.to_string(),
rusqlite::types::ValueRef::Text(t) => String::from_utf8_lossy(t).into_owned(),
rusqlite::types::ValueRef::Blob(b) => String::from_utf8_lossy(b).into_owned(),
};
row_parts.push(s);
}
results.push(row_parts.join("|"));
}
Ok(results.join("\n"))
}
}
pub fn get_card_cached(conn: &Connection, hash: &str) -> Result<Option<String>> {
let mut stmt = conn.prepare_cached("SELECT content FROM card WHERE hash = ?1")?;
let mut rows = stmt.query(params![hash])?;
if let Some(row) = rows.next()? {
let val: rusqlite::types::Value = row.get(0)?;
let content = match val {
rusqlite::types::Value::Text(s) => s,
rusqlite::types::Value::Blob(b) => {
String::from_utf8(b).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
}
_ => "".to_string(),
};
Ok(Some(content))
} else {
Ok(None)
}
}
pub fn card_exists_cached(conn: &Connection, hash: &str) -> Result<bool> {
let mut stmt = conn.prepare_cached("SELECT count(*) FROM card WHERE hash = ?1")?;
let count: i64 = stmt.query_row(params![hash], |row| row.get(0))?;
Ok(count > 0)
}
pub fn resolve_handle_hash_cached(conn: &Connection, handle: &str) -> Result<Option<String>> {
let mut stmt = conn.prepare_cached("SELECT current_hash FROM handle_registry WHERE handle = ?1")?;
let res: Option<String> = stmt.query_row(params![handle], |row| row.get(0)).ok();
Ok(res)
}
pub fn resolve_handle_content_cached(conn: &Connection, handle: &str) -> Result<Option<String>> {
if let Some(hash) = resolve_handle_hash_cached(conn, handle)? {
get_card_cached(conn, &hash)
} else {
Ok(None)
}
}
pub fn handle_register_batch(
pillar: &str,
clm_root: &str,
handle: &str,
hash_val: &str,
content: &str,
) -> Result<()> {
let now = crate::utils::now_iso8601();
#[cfg(any(feature = "native", feature = "db"))]
let created_at_opt = {
if let Ok(writer_arc) = get_writer_connection(pillar, clm_root) {
let mut conn = writer_arc
.lock()
.map_err(|e| ClmError::storage(StorageErrorCode::TransactionFailed, e.to_string()))?;
let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
let mut created_at = now.clone();
{
let mut stmt_card = tx.prepare_cached(
"INSERT OR IGNORE INTO card (hash, content, g_time) VALUES (?1, ?2, ?3)",
)?;
stmt_card.execute(params![hash_val, content, now])?;
let mut stmt_hist = tx.prepare_cached(
"INSERT INTO handle_history (handle, previous_hash, changed_at) \
SELECT hr.handle, hr.current_hash, ?1 FROM handle_registry hr \
JOIN card c ON c.hash = hr.current_hash \
WHERE hr.handle = ?2 AND hr.current_hash != ?3",
)?;
stmt_hist.execute(params![now, handle, hash_val])?;
let existing_created_at: Option<String> = {
let mut stmt_chk = tx.prepare_cached(
"SELECT created_at FROM handle_registry WHERE handle = ?1",
)?;
stmt_chk.query_row(params![handle], |row| row.get(0)).ok()
};
if let Some(ca) = existing_created_at {
created_at = ca;
}
let mut stmt_reg = tx.prepare_cached(
"INSERT OR REPLACE INTO handle_registry (handle, current_hash, created_at, updated_at) \
VALUES (?1, ?2, ?3, ?4)",
)?;
stmt_reg.execute(params![handle, hash_val, created_at, now])?;
}
tx.commit()?;
Some(created_at)
} else {
None
}
};
#[cfg(not(any(feature = "native", feature = "db")))]
let created_at_opt: Option<String> = None;
let created_at = if let Some(ca) = created_at_opt {
ca
} else {
let mut conn = get_connection(pillar, clm_root)?;
let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
let mut created_at = now.clone();
{
let mut stmt_card = tx.prepare_cached(
"INSERT OR IGNORE INTO card (hash, content, g_time) VALUES (?1, ?2, ?3)",
)?;
stmt_card.execute(params![hash_val, content, now])?;
let mut stmt_hist = tx.prepare_cached(
"INSERT INTO handle_history (handle, previous_hash, changed_at) \
SELECT hr.handle, hr.current_hash, ?1 FROM handle_registry hr \
JOIN card c ON c.hash = hr.current_hash \
WHERE hr.handle = ?2 AND hr.current_hash != ?3",
)?;
stmt_hist.execute(params![now, handle, hash_val])?;
let existing_created_at: Option<String> = {
let mut stmt_chk = tx.prepare_cached(
"SELECT created_at FROM handle_registry WHERE handle = ?1",
)?;
stmt_chk.query_row(params![handle], |row| row.get(0)).ok()
};
if let Some(ca) = existing_created_at {
created_at = ca;
}
let mut stmt_reg = tx.prepare_cached(
"INSERT OR REPLACE INTO handle_registry (handle, current_hash, created_at, updated_at) \
VALUES (?1, ?2, ?3, ?4)",
)?;
stmt_reg.execute(params![handle, hash_val, created_at, now])?;
}
tx.commit()?;
created_at
};
if is_membroker_backend() {
let esc_hash = hash_val.replace('\'', "''");
let esc_content = content.replace('\'', "''");
let esc_handle = handle.replace('\'', "''");
let esc_now = now.replace('\'', "''");
let esc_created = created_at.replace('\'', "''");
membroker_sync(pillar, &format!(
"INSERT OR IGNORE INTO card (hash, content, g_time) VALUES ('{}', '{}', '{}')",
esc_hash, esc_content, esc_now
));
membroker_sync(pillar, &format!(
"INSERT OR REPLACE INTO handle_registry (handle, current_hash, created_at, updated_at) VALUES ('{}', '{}', '{}', '{}')",
esc_handle, esc_hash, esc_created, esc_now
));
}
Ok(())
}
pub fn handle_delete_batch(pillar: &str, clm_root: &str, handle: &str) -> Result<()> {
let now = crate::utils::now_iso8601();
#[cfg(any(feature = "native", feature = "db"))]
{
if let Ok(writer_arc) = get_writer_connection(pillar, clm_root) {
let mut conn = writer_arc
.lock()
.map_err(|e| ClmError::storage(StorageErrorCode::TransactionFailed, e.to_string()))?;
let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
{
let mut stmt_hist = tx.prepare_cached(
"INSERT INTO handle_history (handle, previous_hash, changed_at) \
SELECT hr.handle, hr.current_hash, ?1 FROM handle_registry hr \
JOIN card c ON c.hash = hr.current_hash \
WHERE hr.handle = ?2",
)?;
stmt_hist.execute(params![now, handle])?;
let mut stmt_del_hist = tx.prepare_cached("DELETE FROM handle_history WHERE handle = ?1")?;
stmt_del_hist.execute(params![handle])?;
let mut stmt_del = tx.prepare_cached("DELETE FROM handle_registry WHERE handle = ?1")?;
stmt_del.execute(params![handle])?;
}
tx.commit()?;
return Ok(());
}
}
let mut conn = get_connection(pillar, clm_root)?;
let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
{
let mut stmt_hist = tx.prepare_cached(
"INSERT INTO handle_history (handle, previous_hash, changed_at) \
SELECT hr.handle, hr.current_hash, ?1 FROM handle_registry hr \
JOIN card c ON c.hash = hr.current_hash \
WHERE hr.handle = ?2",
)?;
stmt_hist.execute(params![now, handle])?;
let mut stmt_del_hist = tx.prepare_cached("DELETE FROM handle_history WHERE handle = ?1")?;
stmt_del_hist.execute(params![handle])?;
let mut stmt_del = tx.prepare_cached("DELETE FROM handle_registry WHERE handle = ?1")?;
stmt_del.execute(params![handle])?;
}
tx.commit()?;
Ok(())
}
pub fn resolve_handle_hash(pillar: &str, clm_root: &str, handle: &str) -> Result<Option<String>> {
#[cfg(any(feature = "native", feature = "db"))]
{
if let Ok(reader) = get_reader_connection(pillar, clm_root) {
return resolve_handle_hash_cached(&reader, handle);
}
}
let conn = get_connection(pillar, clm_root)?;
resolve_handle_hash_cached(&conn, handle)
}
pub fn resolve_handle_content(
pillar: &str,
clm_root: &str,
handle: &str,
) -> Result<Option<String>> {
#[cfg(any(feature = "native", feature = "db"))]
{
if let Ok(reader) = get_reader_connection(pillar, clm_root) {
return resolve_handle_content_cached(&reader, handle);
}
}
let conn = get_connection(pillar, clm_root)?;
resolve_handle_content_cached(&conn, handle)
}
pub fn card_exists(pillar: &str, clm_root: &str, hash: &str) -> Result<bool> {
#[cfg(any(feature = "native", feature = "db"))]
{
if let Ok(reader) = get_reader_connection(pillar, clm_root) {
return card_exists_cached(&reader, hash);
}
}
let conn = get_connection(pillar, clm_root)?;
card_exists_cached(&conn, hash)
}
pub fn setter_card_batch(
pillar: &str,
clm_root: &str,
hash: &str,
content: &str,
g_time: &str,
) -> Result<()> {
#[cfg(any(feature = "native", feature = "db"))]
{
if let Ok(writer_arc) = get_writer_connection(pillar, clm_root) {
let conn = writer_arc
.lock()
.map_err(|e| ClmError::storage(StorageErrorCode::TransactionFailed, e.to_string()))?;
let mut stmt = conn.prepare_cached(
"INSERT OR IGNORE INTO card (hash, content, g_time) VALUES (?1, ?2, ?3)",
)?;
stmt.execute(params![hash, content, g_time])?;
return Ok(());
}
}
let conn = get_connection(pillar, clm_root)?;
let mut stmt = conn.prepare_cached(
"INSERT OR IGNORE INTO card (hash, content, g_time) VALUES (?1, ?2, ?3)",
)?;
stmt.execute(params![hash, content, g_time])?;
Ok(())
}
pub fn emit_exec_mcard(
operation: &str,
content_hash: &str,
extra_yaml: &str,
a_hash: &str,
c_hash: &str,
clm_root: &str,
) -> Result<String> {
let is_sandbox = TEST_SANDBOX_ACTIVE.with(|active| active.get());
let zero_fs = is_sandbox || env::var("CLM_ZERO_FS").unwrap_or_default() == "true";
let data_root = Path::new(clm_root).join("data");
let mcard_dir = if zero_fs {
let tmp_dir = env::var("TMPDIR").unwrap_or_else(|_| "/tmp".to_string());
Path::new(&tmp_dir).join("data").join("execution_logs")
} else {
data_root.join("execution_logs")
};
if !zero_fs {
let _ = fs::create_dir_all(&mcard_dir);
}
let now = chrono::Local::now();
let g_time = now.format("%Y-%m-%dT%H:%M:%S%z").to_string();
let unix_ts = now.timestamp();
let short_hash = if content_hash.len() >= 8 {
&content_hash[0..8]
} else {
content_hash
};
let mcard_file = mcard_dir.join(format!("exec_{}_{}.yaml", unix_ts, short_hash));
let mut mcard_content = format!(
"mcard:\n g_time: \"{}\"\n content_hash: \"{}\"\n operation: \"{}\"\n agent_did: \"{}\"\n",
g_time, content_hash, operation, env::var("ACTIVE_AGENT_DID").unwrap_or_default()
);
if !extra_yaml.is_empty() {
for line in extra_yaml.lines() {
mcard_content.push_str(&format!(" {}\n", line));
}
}
if !a_hash.is_empty() || !c_hash.is_empty() {
mcard_content.push_str(" source:\n");
if !a_hash.is_empty() {
mcard_content.push_str(&format!(" a_hash: \"{}\"\n", a_hash));
}
if !c_hash.is_empty() {
mcard_content.push_str(&format!(" c_hash: \"{}\"\n", c_hash));
}
mcard_content.push_str(&format!(" operation: \"{}\"\n", operation));
if let Ok(runtime_facts) = env::var("CLM_RUNTIME_FACTS")
&& !runtime_facts.is_empty() {
mcard_content.push_str(" runtime_facts:\n");
for line in runtime_facts.lines() {
mcard_content.push_str(&format!(" {}\n", line));
}
}
} else {
if let Ok(runtime_facts) = env::var("CLM_RUNTIME_FACTS")
&& !runtime_facts.is_empty() {
mcard_content.push_str(" runtime_facts:\n");
for line in runtime_facts.lines() {
mcard_content.push_str(&format!(" {}\n", line));
}
}
}
if !zero_fs {
fs::write(&mcard_file, &mcard_content)?;
}
#[cfg(any(feature = "native", feature = "db"))]
let write_done = {
if let Ok(writer_arc) = get_writer_connection("execution_log", clm_root) {
if let Ok(conn) = writer_arc.lock() {
let mut stmt = conn.prepare_cached(
"INSERT OR REPLACE INTO card (hash, content, g_time) VALUES (?1, ?2, ?3)",
)?;
stmt.execute(params![content_hash, &mcard_content, &g_time])?;
true
} else {
false
}
} else {
false
}
};
#[cfg(not(any(feature = "native", feature = "db")))]
let write_done = false;
if !write_done {
let conn = get_connection("execution_log", clm_root)?;
let mut stmt = conn.prepare_cached(
"INSERT OR REPLACE INTO card (hash, content, g_time) VALUES (?1, ?2, ?3)",
)?;
stmt.execute(params![content_hash, &mcard_content, &g_time])?;
}
let esc_mcard = mcard_content.replace('\'', "''");
let esc_gtime = g_time.replace('\'', "''");
membroker_sync("execution_log", &format!(
"INSERT OR REPLACE INTO card (hash, content, g_time) VALUES ('{}', '{}', '{}')",
content_hash, esc_mcard, esc_gtime
));
Ok(mcard_file.to_string_lossy().to_string())
}
pub fn get_card_content(pillar: &str, clm_root: &str, hash: &str) -> Result<Option<String>> {
#[cfg(any(feature = "native", feature = "db"))]
{
if let Ok(reader) = get_reader_connection(pillar, clm_root) {
return get_card_cached(&reader, hash);
}
}
let conn = get_connection(pillar, clm_root)?;
get_card_cached(&conn, hash)
}
pub fn backup_db(pillar: &str, clm_root: &str) -> Result<Connection> {
let src = get_connection(pillar, clm_root)?;
let mut dst = Connection::open_in_memory()?;
{
let backup = rusqlite::backup::Backup::new(&src, &mut dst)?;
backup.run_to_completion(10_000, std::time::Duration::ZERO, None)?;
}
Ok(dst)
}
pub fn restore_db(pillar: &str, clm_root: &str, backup_conn: &Connection) -> Result<()> {
let mut dst = get_connection(pillar, clm_root)?;
{
let backup = rusqlite::backup::Backup::new(backup_conn, &mut dst)?;
backup.run_to_completion(10_000, std::time::Duration::ZERO, None)?;
}
Ok(())
}
#[cfg(any(feature = "native", all(feature = "db", feature = "async")))]
pub async fn query_db_async<F, R>(f: F) -> Result<R>
where
F: FnOnce() -> Result<R> + Send + 'static,
R: Send + 'static,
{
tokio::task::spawn_blocking(f)
.await
.map_err(|join_err| ClmError::Storage(crate::error::StorageErrorCode::QueryFailed, format!("Tokio worker join error: {}", join_err)))?
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_cached_statement_queries() {
let mut conn = Connection::open_in_memory().unwrap();
migrations::apply_migrations(&mut conn, "content_memory").unwrap();
let mut stmt = conn
.prepare_cached("INSERT OR IGNORE INTO card (hash, content, g_time) VALUES (?1, ?2, ?3)")
.unwrap();
stmt.execute(rusqlite::params!["test_hash_1", "test_content_1", 1000])
.unwrap();
let content = get_card_cached(&conn, "test_hash_1").unwrap();
assert_eq!(content, Some("test_content_1".to_string()));
let exists = card_exists_cached(&conn, "test_hash_1").unwrap();
assert!(exists);
let not_exists = card_exists_cached(&conn, "nonexistent").unwrap();
assert!(!not_exists);
let now = crate::utils::now_iso8601();
let mut stmt_h = conn
.prepare_cached("INSERT OR IGNORE INTO handle_registry (handle, current_hash, created_at, updated_at) VALUES (?1, ?2, ?3, ?4)")
.unwrap();
stmt_h.execute(params!["test_handle_1", "test_hash_1", &now, &now]).unwrap();
let resolved_hash = resolve_handle_hash_cached(&conn, "test_handle_1").unwrap();
assert_eq!(resolved_hash, Some("test_hash_1".to_string()));
let resolved_content = resolve_handle_content_cached(&conn, "test_handle_1").unwrap();
assert_eq!(resolved_content, Some("test_content_1".to_string()));
}
#[cfg(any(feature = "native", feature = "db"))]
#[test]
fn test_concurrent_readers_in_pool() {
let temp_dir = tempfile::tempdir().unwrap();
let clm_root = temp_dir.path().to_str().unwrap();
let pool = DbPillarPool::new("content_memory", clm_root).unwrap();
{
let writer_arc = pool.writer();
let mut conn = writer_arc.lock().unwrap();
let tx = conn.transaction().unwrap();
for i in 0..50 {
let hash = format!("hash_{:04}", i);
let content = format!("content_{:04}", i);
let mut stmt = tx
.prepare_cached("INSERT INTO card (hash, content, g_time) VALUES (?1, ?2, ?3)")
.unwrap();
stmt.execute(rusqlite::params![hash, content, i as i64])
.unwrap();
}
tx.commit().unwrap();
}
let mut handles = Vec::new();
for thread_idx in 0..16 {
let pool_clone = pool.clone();
let h = std::thread::spawn(move || {
let reader = pool_clone.reader().expect("Acquiring reader connection should succeed");
for j in 0..50 {
let card_idx = (thread_idx * 7 + j) % 50;
let target_hash = format!("hash_{:04}", card_idx);
let expected_content = format!("content_{:04}", card_idx);
let content = get_card_cached(&reader, &target_hash)
.expect("Read query should not fail");
assert_eq!(content, Some(expected_content));
}
});
handles.push(h);
}
for h in handles {
h.join().expect("Reader thread should complete without panicking");
}
}
#[tokio::test]
async fn test_query_db_async_isolation() {
let temp_dir = tempfile::tempdir().unwrap();
let clm_root = temp_dir.path().to_str().unwrap();
let pool = DbPillarPool::new("content_memory", clm_root).unwrap();
let pool_writer = pool.clone();
query_db_async(move || {
let writer_arc = pool_writer.writer();
let mut conn = writer_arc.lock().unwrap();
let tx = conn.transaction()?;
{
let mut stmt = tx.prepare_cached("INSERT OR REPLACE INTO card (hash, content, g_time) VALUES (?1, ?2, ?3)")?;
stmt.execute(rusqlite::params!["async_hash", "async_content", 12345])?;
}
tx.commit()?;
Ok(())
})
.await
.unwrap();
let pool_reader = pool.clone();
let res = query_db_async(move || {
let reader = pool_reader.reader()?;
let content = get_card_cached(&reader, "async_hash")?;
Ok(content)
})
.await
.unwrap();
assert_eq!(res, Some("async_content".to_string()));
}
}