use crate::dtob_ffi;
use crate::{decode_dtob_bytes, decode_dtob_file};
use crate::serve;
use std::fs;
use std::path::{Path, PathBuf};
pub enum HashLookup {
Found { xvuid: String, index: usize },
Missing,
}
fn nul_terminated_key(key: &str) -> String {
let mut out = key.to_owned();
out.push('\0');
out
}
pub fn map_file_path(maps_path: &Path, map_name: &str) -> PathBuf {
maps_path.join(format!("{}.dtob", map_name))
}
pub struct MapsCache {
cid_to_xvuid: Option<*mut dtob_ffi::DtobValue>,
ccid_to_xvuid: Option<*mut dtob_ffi::DtobValue>,
xvuid_to_snapshot_offset: Option<*mut dtob_ffi::DtobValue>,
xvuid_to_latest: Option<*mut dtob_ffi::DtobValue>,
xvuid_to_meta: Option<*mut dtob_ffi::DtobValue>,
}
impl MapsCache {
pub fn load(maps_path: &Path) -> Self {
let load = |name: &str| decode_dtob_file(&map_file_path(maps_path, name));
Self {
cid_to_xvuid: load("cid_to_xvuid"),
ccid_to_xvuid: load("ccid_to_xvuid"),
xvuid_to_snapshot_offset: load("xvuid_to_snapshot_offset"),
xvuid_to_latest: load("xvuid_to_latest"),
xvuid_to_meta: load("xvuid_to_meta"),
}
}
pub fn lookup_hash(&self, map_name: &str, hash: &str) -> HashLookup {
let root = match map_name {
"cid_to_xvuid" => self.cid_to_xvuid,
"ccid_to_xvuid" => self.ccid_to_xvuid,
_ => return HashLookup::Missing,
};
let Some(root) = root else { return HashLookup::Missing; };
let key = nul_terminated_key(hash);
unsafe {
let hits = dtob_ffi::ffi_kvset_get(root, key.as_ptr() as *const i8);
if hits.is_null() { return HashLookup::Missing; }
let count = dtob_ffi::ffi_dtob_array_len(hits);
if count == 0 { return HashLookup::Missing; }
let prefer_max = map_name == "cid_to_xvuid";
let mut best_xvuid: Option<String> = None;
let mut best_index = 0usize;
for i in 0..count {
let tuple = dtob_ffi::ffi_dtob_array_get(hits, i);
if tuple.is_null() { continue; }
let mut dummy = 0;
let xr_raw = dtob_ffi::ffi_dtob_get_raw(dtob_ffi::ffi_dtob_array_get(tuple, 0), &mut dummy);
let idx = dtob_ffi::ffi_dtob_get_u64(dtob_ffi::ffi_dtob_array_get(tuple, 1)) as usize;
if xr_raw.is_null() || dummy != 32 { continue; }
let xvuid = hex::encode(std::slice::from_raw_parts(xr_raw, 32));
match &best_xvuid {
None => { best_xvuid = Some(xvuid); best_index = idx; }
Some(_) => {
let better = if prefer_max { idx > best_index } else { idx < best_index };
if better { best_xvuid = Some(xvuid); best_index = idx; }
}
}
}
match best_xvuid {
Some(xvuid) => HashLookup::Found { xvuid, index: best_index },
None => HashLookup::Missing,
}
}
}
pub fn lookup_snapshot_offset(&self, xvuid: &str, snapshot_index: usize) -> Option<u64> {
let root = self.xvuid_to_snapshot_offset?;
let composite = nul_terminated_key(&format!("{}:{}", xvuid, snapshot_index));
unsafe {
let v = dtob_ffi::ffi_kvset_get(root, composite.as_ptr() as *const i8);
if v.is_null() { None } else { Some(dtob_ffi::ffi_dtob_get_u64(v)) }
}
}
pub fn lookup_latest(&self, xvuid: &str) -> Option<(usize, u64)> {
let root = self.xvuid_to_latest?;
let key = nul_terminated_key(xvuid);
unsafe {
let tuple = dtob_ffi::ffi_kvset_get(root, key.as_ptr() as *const i8);
if tuple.is_null() { return None; }
let latest_index = dtob_ffi::ffi_dtob_get_u64(dtob_ffi::ffi_dtob_array_get(tuple, 0)) as usize;
let snapshot_offset = dtob_ffi::ffi_dtob_get_u64(dtob_ffi::ffi_dtob_array_get(tuple, 1));
Some((latest_index, snapshot_offset))
}
}
pub fn lookup_title(&self, xvuid: &str) -> Option<String> {
self.lookup_meta_str(xvuid, 0)
}
pub fn lookup_meta_hex(&self, xvuid: &str, field_index: usize) -> Option<String> {
let root = self.xvuid_to_meta?;
let key = nul_terminated_key(xvuid);
unsafe {
let v = dtob_ffi::ffi_kvset_get(root, key.as_ptr() as *const i8);
if v.is_null() { return None; }
let v_field = dtob_ffi::ffi_dtob_array_get(v, field_index);
if v_field.is_null() { return None; }
let mut len = 0;
let raw = dtob_ffi::ffi_dtob_get_raw(v_field, &mut len);
if raw.is_null() || len == 0 { return None; }
Some(hex::encode(std::slice::from_raw_parts(raw, len)))
}
}
fn lookup_meta_str(&self, xvuid: &str, field_index: usize) -> Option<String> {
let root = self.xvuid_to_meta?;
let key = nul_terminated_key(xvuid);
unsafe {
let v = dtob_ffi::ffi_kvset_get(root, key.as_ptr() as *const i8);
if v.is_null() { return None; }
let v_field = dtob_ffi::ffi_dtob_array_get(v, field_index);
if v_field.is_null() { return None; }
let mut len = 0;
let raw = dtob_ffi::ffi_dtob_get_raw(v_field, &mut len);
if raw.is_null() || len == 0 { return None; }
Some(String::from_utf8_lossy(std::slice::from_raw_parts(raw, len)).to_string())
}
}
pub fn meta_root(&self) -> Option<*mut dtob_ffi::DtobValue> {
self.xvuid_to_meta
}
}
impl Drop for MapsCache {
fn drop(&mut self) {
unsafe {
for root in [self.cid_to_xvuid, self.ccid_to_xvuid, self.xvuid_to_snapshot_offset, self.xvuid_to_latest, self.xvuid_to_meta] {
if let Some(r) = root { dtob_ffi::ffi_dtob_free(r); }
}
}
}
}
pub fn rebuild_memcache_maps(repo_path: &Path, maps_path: &Path) {
let cid_to_xvuid = unsafe { dtob_ffi::ffi_kvset_new() };
let ccid_to_xvuid = unsafe { dtob_ffi::ffi_kvset_new() };
let xvuid_to_snapshot_offset = unsafe { dtob_ffi::ffi_kvset_new() };
let xvuid_to_latest = unsafe { dtob_ffi::ffi_kvset_new() };
let xvuid_to_meta = unsafe { dtob_ffi::ffi_kvset_new() };
let mut count = 0;
if let Ok(entries) = fs::read_dir(repo_path) {
for entry in entries.flatten() {
let path = entry.path();
if path.extension().and_then(|s| s.to_str()) != Some("dif") {
continue;
}
let file_name = path.file_stem().and_then(|s| s.to_str()).unwrap_or("");
let Ok(xvuid_bytes) = hex::decode(file_name) else {
continue;
};
if xvuid_bytes.len() != 32 {
continue;
}
let xvuid_hex = hex::encode(&xvuid_bytes);
let snapshot_offsets = serve::snapshot_offsets(&path).unwrap_or_default();
for (snapshot_index, byte_offset) in &snapshot_offsets {
let key = nul_terminated_key(&format!("{}:{}", xvuid_hex, snapshot_index));
unsafe {
dtob_ffi::ffi_kvset_put(
xvuid_to_snapshot_offset,
key.as_ptr() as *const i8,
dtob_ffi::ffi_uint_new(*byte_offset),
);
}
}
if let Ok(data) = fs::read(&path) {
unsafe {
let Some(root) = decode_dtob_bytes(&data) else {
continue;
};
let n = dtob_ffi::ffi_dtob_array_len(root);
let mut date_created = 0u64;
if n > 0 {
for i in 0..n {
let v_entry = dtob_ffi::ffi_dtob_array_get(root, i);
if v_entry.is_null() {
continue;
}
if i == 0 {
let v_meta_first = dtob_ffi::ffi_dtob_array_get(v_entry, 3);
if !v_meta_first.is_null() {
let v_ts = dtob_ffi::ffi_kvset_get(v_meta_first as *mut _, b"timestamp\0".as_ptr() as *const i8);
if !v_ts.is_null() {
date_created = dtob_ffi::ffi_dtob_get_u64(v_ts);
}
}
}
let v_ccid = dtob_ffi::ffi_dtob_array_get(v_entry, 0);
let v_cid = dtob_ffi::ffi_dtob_array_get(v_entry, 1);
let mut ccid_len = 0;
let mut cid_len = 0;
let ccid_ptr = dtob_ffi::ffi_dtob_get_raw(v_ccid, &mut ccid_len);
let cid_ptr = dtob_ffi::ffi_dtob_get_raw(v_cid, &mut cid_len);
if ccid_ptr.is_null() || ccid_len != 32 || cid_ptr.is_null() || cid_len != 32 {
continue;
}
let ccid_key = nul_terminated_key(&hex::encode(std::slice::from_raw_parts(ccid_ptr, 32)));
let entry_arr = dtob_ffi::ffi_array_new();
dtob_ffi::ffi_array_push(entry_arr, dtob_ffi::ffi_raw_new(xvuid_bytes.as_ptr(), 32));
dtob_ffi::ffi_array_push(entry_arr, dtob_ffi::ffi_uint_new(i as u64));
let existing_ccid = dtob_ffi::ffi_kvset_get(ccid_to_xvuid, ccid_key.as_ptr() as *const i8);
if !existing_ccid.is_null() {
dtob_ffi::ffi_array_push(existing_ccid, entry_arr);
} else {
let wrap_arr = dtob_ffi::ffi_array_new();
dtob_ffi::ffi_array_push(wrap_arr, entry_arr);
dtob_ffi::ffi_kvset_put(ccid_to_xvuid, ccid_key.as_ptr() as *const i8, wrap_arr);
}
let cid_key = nul_terminated_key(&hex::encode(std::slice::from_raw_parts(cid_ptr, 32)));
let entry_arr2 = dtob_ffi::ffi_array_new();
dtob_ffi::ffi_array_push(entry_arr2, dtob_ffi::ffi_raw_new(xvuid_bytes.as_ptr(), 32));
dtob_ffi::ffi_array_push(entry_arr2, dtob_ffi::ffi_uint_new(i as u64));
let existing_cid = dtob_ffi::ffi_kvset_get(cid_to_xvuid, cid_key.as_ptr() as *const i8);
if !existing_cid.is_null() {
dtob_ffi::ffi_array_push(existing_cid, entry_arr2);
} else {
let wrap_arr2 = dtob_ffi::ffi_array_new();
dtob_ffi::ffi_array_push(wrap_arr2, entry_arr2);
dtob_ffi::ffi_kvset_put(cid_to_xvuid, cid_key.as_ptr() as *const i8, wrap_arr2);
}
if i == n - 1 {
let snapshot_start = ((n as usize - 1) / crate::SNAPSHOT_INTERVAL) * crate::SNAPSHOT_INTERVAL;
let snapshot_offset = snapshot_offsets
.iter()
.find(|(idx, _)| *idx == snapshot_start)
.map(|(_, byte_offset)| *byte_offset)
.unwrap_or(0);
let latest_tuple = dtob_ffi::ffi_array_new();
dtob_ffi::ffi_array_push(latest_tuple, dtob_ffi::ffi_uint_new((n - 1) as u64));
dtob_ffi::ffi_array_push(latest_tuple, dtob_ffi::ffi_uint_new(snapshot_offset));
dtob_ffi::ffi_kvset_put(
xvuid_to_latest,
nul_terminated_key(&xvuid_hex).as_ptr() as *const i8,
latest_tuple,
);
let v_meta = dtob_ffi::ffi_dtob_array_get(v_entry, 3);
if !v_meta.is_null() {
let mut date_edited = date_created;
let v_ts = dtob_ffi::ffi_kvset_get(v_meta as *mut _, b"timestamp\0".as_ptr() as *const i8);
if !v_ts.is_null() {
date_edited = dtob_ffi::ffi_dtob_get_u64(v_ts);
}
let v_title = dtob_ffi::ffi_kvset_get(v_meta as *mut _, b"title\0".as_ptr() as *const i8);
let meta_arr = dtob_ffi::ffi_array_new();
if !v_title.is_null() {
let mut title_len = 0;
let title_ptr = dtob_ffi::ffi_dtob_get_raw(v_title, &mut title_len);
if !title_ptr.is_null() && title_len > 0 {
dtob_ffi::ffi_array_push(meta_arr, dtob_ffi::ffi_raw_new(title_ptr, title_len));
} else {
let s = b"Untitled\0";
dtob_ffi::ffi_array_push(meta_arr, dtob_ffi::ffi_raw_new(s.as_ptr(), s.len() - 1));
}
} else {
let s = b"Untitled\0";
dtob_ffi::ffi_array_push(meta_arr, dtob_ffi::ffi_raw_new(s.as_ptr(), s.len() - 1));
}
dtob_ffi::ffi_array_push(meta_arr, dtob_ffi::ffi_raw_new(cid_ptr, cid_len));
dtob_ffi::ffi_array_push(meta_arr, dtob_ffi::ffi_raw_new(ccid_ptr, ccid_len));
dtob_ffi::ffi_array_push(meta_arr, dtob_ffi::ffi_uint_new(date_created));
dtob_ffi::ffi_array_push(meta_arr, dtob_ffi::ffi_uint_new(date_edited));
dtob_ffi::ffi_kvset_put(
xvuid_to_meta,
nul_terminated_key(&xvuid_hex).as_ptr() as *const i8,
meta_arr,
);
}
}
}
}
dtob_ffi::ffi_dtob_free(root);
count += 1;
}
}
}
}
let _ = fs::create_dir_all(maps_path);
unsafe {
let dump_map = |m_ptr: *mut dtob_ffi::DtobValue, name: &str| {
let mut out_len = 0;
let enc = dtob_ffi::ffi_encode_dif(m_ptr, &mut out_len);
if !enc.is_null() && out_len > 0 {
let s = std::slice::from_raw_parts(enc, out_len);
let _ = fs::write(map_file_path(maps_path, name), s);
libc::free(enc as *mut libc::c_void);
}
dtob_ffi::ffi_dtob_free(m_ptr);
let pid_path = format!("/tmp/dtob-{}.pid", name);
if let Ok(pid_str) = fs::read_to_string(&pid_path) {
if let Ok(pid) = pid_str.trim().parse::<i32>() {
libc::kill(pid, libc::SIGHUP);
println!("Notified memcache daemon for {} (PID {})", name, pid);
}
}
};
dump_map(cid_to_xvuid, "cid_to_xvuid");
dump_map(ccid_to_xvuid, "ccid_to_xvuid");
dump_map(xvuid_to_snapshot_offset, "xvuid_to_snapshot_offset");
dump_map(xvuid_to_latest, "xvuid_to_latest");
dump_map(xvuid_to_meta, "xvuid_to_meta");
}
println!("Rebuilt memcache maps for {} xvuids. Saved correctly to {:?}", count, maps_path);
}