use std::collections::HashMap;
use std::io::Write;
use std::path::Path;
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
pub const HIT_LOG_REL_PATH: &str = ".memd/data/retrieval_exposures.jsonl";
pub const HIT_SUMMARY_REL_PATH: &str = ".memd/data/retrieval_exposures.summary.json";
const LEGACY_HIT_LOG_NAME: &str = "hit_counts.jsonl";
const LEGACY_HIT_SUMMARY_NAME: &str = "hit_counts.summary.json";
const EXPOSURE_LOG_NAME: &str = "retrieval_exposures.jsonl";
const EXPOSURE_SUMMARY_NAME: &str = "retrieval_exposures.summary.json";
const MAX_EXPOSURE_LOG_BYTES: u64 = 64 * 1024 * 1024;
pub const DEFAULT_SUMMARY_TTL_MS: i64 = 3_600_000;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HitRecord {
pub ts_ms: i64,
pub chunk_id: String,
pub tenant_id: String,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub project_id: Option<String>,
pub query_mode: String,
pub rank: usize,
pub score: f64,
#[serde(alias = "selected")]
pub rendered: bool,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct HitStats {
#[serde(alias = "hit_count")]
pub exposure_count: u32,
#[serde(alias = "selected_count")]
pub rendered_count: u32,
pub last_ts_ms: i64,
}
fn now_ms() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0)
}
const MAX_RECORD_BYTES: usize = 4096;
pub fn record_hits(records: &[HitRecord]) {
if records.is_empty() {
return;
}
let Ok(cwd) = std::env::current_dir() else {
return;
};
record_hits_in(&cwd, records);
}
pub fn record_hits_in(project_dir: &Path, records: &[HitRecord]) {
if records.is_empty() || !project_dir.join(".memd").is_dir() {
return;
}
migrate_legacy_exposure_files(&project_dir.join(".memd/data"));
write_hit_lines(&project_dir.join(HIT_LOG_REL_PATH), records);
}
pub fn record_hits_to_data_dir(data_dir: &Path, records: &[HitRecord]) {
if records.is_empty() {
return;
}
migrate_legacy_exposure_files(data_dir);
write_hit_lines(&data_dir.join(EXPOSURE_LOG_NAME), records);
}
fn write_hit_lines(path: &Path, records: &[HitRecord]) {
let Some(parent) = path.parent() else {
return;
};
if std::fs::create_dir_all(parent).is_err() {
return;
}
if std::fs::metadata(path)
.map(|metadata| metadata.len() >= MAX_EXPOSURE_LOG_BYTES)
.unwrap_or(false)
{
let rotated = parent.join(format!("retrieval_exposures.{}.jsonl", now_ms()));
let _ = std::fs::rename(path, rotated);
}
let Ok(mut file) = std::fs::OpenOptions::new()
.append(true)
.create(true)
.open(path)
else {
return;
};
for record in records {
let Ok(mut line) = serde_json::to_string(record) else {
continue;
};
line.push('\n');
if line.len() > MAX_RECORD_BYTES {
continue;
}
let _ = file.write_all(line.as_bytes());
}
}
#[derive(Debug, Serialize, Deserialize)]
struct SummaryCache {
generated_ms: i64,
window_days: u32,
stats: HashMap<String, HitStats>,
}
pub fn aggregate_hits(window_days: u32) -> HashMap<String, HitStats> {
let Ok(cwd) = std::env::current_dir() else {
return HashMap::new();
};
aggregate_hits_in(&cwd, window_days, DEFAULT_SUMMARY_TTL_MS)
}
pub fn aggregate_hits_in(
project_dir: &Path,
window_days: u32,
ttl_ms: i64,
) -> HashMap<String, HitStats> {
migrate_legacy_exposure_files(&project_dir.join(".memd/data"));
aggregate_from(
&project_dir.join(HIT_LOG_REL_PATH),
&project_dir.join(HIT_SUMMARY_REL_PATH),
window_days,
ttl_ms,
)
}
pub fn aggregate_hits_at_data_dir(
data_dir: &Path,
window_days: u32,
ttl_ms: i64,
) -> HashMap<String, HitStats> {
migrate_legacy_exposure_files(data_dir);
aggregate_from(
&data_dir.join(EXPOSURE_LOG_NAME),
&data_dir.join(EXPOSURE_SUMMARY_NAME),
window_days,
ttl_ms,
)
}
pub fn serve_counts_since(
data_dir: &Path,
since_ms: i64,
tenant_id: Option<&str>,
project_id: Option<&str>,
) -> HashMap<String, HitStats> {
migrate_legacy_exposure_files(data_dir);
scan_hit_log_since(
&data_dir.join(EXPOSURE_LOG_NAME),
since_ms,
tenant_id,
project_id,
)
}
fn migrate_legacy_exposure_files(data_dir: &Path) {
let legacy_log = data_dir.join(LEGACY_HIT_LOG_NAME);
let exposure_log = data_dir.join(EXPOSURE_LOG_NAME);
if legacy_log.exists() && !exposure_log.exists() {
let _ = std::fs::rename(legacy_log, exposure_log);
}
let legacy_summary = data_dir.join(LEGACY_HIT_SUMMARY_NAME);
let exposure_summary = data_dir.join(EXPOSURE_SUMMARY_NAME);
if legacy_summary.exists() && !exposure_summary.exists() {
let _ = std::fs::rename(legacy_summary, exposure_summary);
}
}
fn aggregate_from(
log_path: &Path,
summary_path: &Path,
window_days: u32,
ttl_ms: i64,
) -> HashMap<String, HitStats> {
let now = now_ms();
if let Some(cached) = read_fresh_cache(summary_path, window_days, ttl_ms, now) {
return cached;
}
let stats = scan_hit_log(log_path, window_days, now);
write_cache(summary_path, window_days, now, &stats);
stats
}
fn read_fresh_cache(
summary_path: &Path,
window_days: u32,
ttl_ms: i64,
now: i64,
) -> Option<HashMap<String, HitStats>> {
let text = std::fs::read_to_string(summary_path).ok()?;
let cache: SummaryCache = serde_json::from_str(&text).ok()?;
if cache.window_days != window_days {
return None;
}
if now.saturating_sub(cache.generated_ms) >= ttl_ms {
return None;
}
Some(cache.stats)
}
fn scan_hit_log(log_path: &Path, window_days: u32, now: i64) -> HashMap<String, HitStats> {
let cutoff = now.saturating_sub((window_days as i64).saturating_mul(86_400_000));
scan_hit_log_since(log_path, cutoff, None, None)
}
fn scan_hit_log_since(
log_path: &Path,
cutoff: i64,
tenant_id: Option<&str>,
project_id: Option<&str>,
) -> HashMap<String, HitStats> {
let mut stats: HashMap<String, HitStats> = HashMap::new();
let mut paths = vec![log_path.to_path_buf()];
if log_path.file_name().and_then(|name| name.to_str()) == Some(EXPOSURE_LOG_NAME) {
if let Some(parent) = log_path.parent() {
if let Ok(entries) = std::fs::read_dir(parent) {
paths.extend(entries.flatten().filter_map(|entry| {
let name = entry.file_name();
let name = name.to_str()?;
(name.starts_with("retrieval_exposures.") && name.ends_with(".jsonl"))
.then(|| entry.path())
}));
}
}
}
paths.sort();
paths.dedup();
for path in paths {
let Ok(text) = std::fs::read_to_string(path) else {
continue;
};
for line in text.lines() {
let line = line.trim();
if line.is_empty() {
continue;
}
let Ok(record) = serde_json::from_str::<HitRecord>(line) else {
continue;
};
if record.ts_ms < cutoff {
continue;
}
if tenant_id.is_some_and(|want| record.tenant_id != want) {
continue;
}
if project_id.is_some_and(|want| record.project_id.as_deref() != Some(want)) {
continue;
}
let entry = stats.entry(record.chunk_id).or_default();
entry.exposure_count = entry.exposure_count.saturating_add(1);
if record.rendered {
entry.rendered_count = entry.rendered_count.saturating_add(1);
}
entry.last_ts_ms = entry.last_ts_ms.max(record.ts_ms);
}
}
stats
}
fn write_cache(summary_path: &Path, window_days: u32, now: i64, stats: &HashMap<String, HitStats>) {
let Some(parent) = summary_path.parent() else {
return;
};
if std::fs::create_dir_all(parent).is_err() {
return;
}
let cache = SummaryCache {
generated_ms: now,
window_days,
stats: stats.clone(),
};
if let Ok(text) = serde_json::to_string(&cache) {
let _ = std::fs::write(summary_path, text);
}
}
pub fn query_mode_label(mode: &str) -> String {
mode.trim().to_ascii_lowercase().replace('_', "-")
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
fn record(chunk_id: &str, ts_ms: i64, rendered: bool) -> HitRecord {
HitRecord {
ts_ms,
chunk_id: chunk_id.to_string(),
tenant_id: "t".to_string(),
project_id: Some("p".to_string()),
query_mode: "find-failures".to_string(),
rank: 0,
score: 0.5,
rendered,
}
}
#[test]
fn record_hits_skips_without_memd_dir() {
let dir = tempdir().unwrap();
record_hits_in(dir.path(), &[record("c1", now_ms(), true)]);
assert!(!dir.path().join(HIT_LOG_REL_PATH).exists());
}
#[test]
fn record_hits_appends_one_line_per_record() {
let dir = tempdir().unwrap();
std::fs::create_dir_all(dir.path().join(".memd")).unwrap();
let now = now_ms();
record_hits_in(
dir.path(),
&[record("c1", now, true), record("c2", now, false)],
);
record_hits_in(dir.path(), &[record("c1", now, true)]);
let text = std::fs::read_to_string(dir.path().join(HIT_LOG_REL_PATH)).unwrap();
assert_eq!(text.lines().count(), 3);
}
#[test]
fn aggregate_groups_by_chunk_within_window() {
let dir = tempdir().unwrap();
std::fs::create_dir_all(dir.path().join(".memd")).unwrap();
let now = now_ms();
record_hits_in(
dir.path(),
&[
record("c1", now, true),
record("c1", now, false),
record("c2", now, true),
record("c3", now - 30 * 86_400_000, true),
],
);
let stats = aggregate_hits_in(dir.path(), 7, DEFAULT_SUMMARY_TTL_MS);
assert_eq!(stats["c1"].exposure_count, 2);
assert_eq!(stats["c1"].rendered_count, 1);
assert_eq!(stats["c2"].exposure_count, 1);
assert!(!stats.contains_key("c3"), "old record excluded by window");
}
#[test]
fn aggregate_respects_cache_ttl() {
let dir = tempdir().unwrap();
std::fs::create_dir_all(dir.path().join(".memd")).unwrap();
let now = now_ms();
record_hits_in(dir.path(), &[record("c1", now, true)]);
let first = aggregate_hits_in(dir.path(), 7, DEFAULT_SUMMARY_TTL_MS);
assert_eq!(first["c1"].exposure_count, 1);
record_hits_in(dir.path(), &[record("c1", now, true)]);
let cached = aggregate_hits_in(dir.path(), 7, DEFAULT_SUMMARY_TTL_MS);
assert_eq!(cached["c1"].exposure_count, 1, "fresh cache reused");
let rescanned = aggregate_hits_in(dir.path(), 7, 0);
assert_eq!(rescanned["c1"].exposure_count, 2, "expired cache rescanned");
}
#[test]
fn oversized_records_are_dropped_to_preserve_atomic_append() {
let dir = tempdir().unwrap();
std::fs::create_dir_all(dir.path().join(".memd")).unwrap();
let now = now_ms();
let mut huge = record("ok", now, true);
huge.chunk_id = "x".repeat(MAX_RECORD_BYTES + 100);
record_hits_in(dir.path(), &[huge, record("ok2", now, true)]);
let text = std::fs::read_to_string(dir.path().join(HIT_LOG_REL_PATH)).unwrap();
assert_eq!(text.lines().count(), 1, "oversized line must be dropped");
for line in text.lines() {
assert!(
line.len() < MAX_RECORD_BYTES,
"every persisted line must stay under the atomic-append boundary"
);
}
}
#[test]
fn aggregate_missing_log_is_empty() {
let dir = tempdir().unwrap();
std::fs::create_dir_all(dir.path().join(".memd")).unwrap();
assert!(aggregate_hits_in(dir.path(), 7, DEFAULT_SUMMARY_TTL_MS).is_empty());
}
fn record_in(chunk_id: &str, ts_ms: i64, rendered: bool, project: &str) -> HitRecord {
HitRecord {
ts_ms,
chunk_id: chunk_id.to_string(),
tenant_id: "t".to_string(),
project_id: Some(project.to_string()),
query_mode: "find-failures".to_string(),
rank: 0,
score: 0.5,
rendered,
}
}
#[test]
fn record_to_data_dir_writes_log_directly_under_data_dir() {
let dir = tempdir().unwrap();
let data_dir = dir.path().join("memd_data");
std::fs::create_dir_all(&data_dir).unwrap();
let now = now_ms();
record_hits_to_data_dir(&data_dir, &[record("c1", now, true)]);
let log = data_dir.join(EXPOSURE_LOG_NAME);
assert!(log.is_file(), "log must live directly under data_dir");
assert_eq!(
std::fs::read_to_string(&log).unwrap().lines().count(),
1,
"one line per record"
);
}
#[test]
fn serve_counts_since_filters_by_tenant_project_and_window() {
let dir = tempdir().unwrap();
let data_dir = dir.path().join("memd_data");
std::fs::create_dir_all(&data_dir).unwrap();
let now = now_ms();
let mut other_tenant = record_in("z", now, true, "p1");
other_tenant.tenant_id = "other".to_string();
record_hits_to_data_dir(
&data_dir,
&[
record_in("a", now, true, "p1"),
record_in("a", now, false, "p1"),
record_in("b", now, true, "p2"),
other_tenant,
record_in("c", now - 30 * 86_400_000, true, "p1"),
],
);
let all = serve_counts_since(&data_dir, now - 86_400_000, None, None);
assert_eq!(all["a"].exposure_count, 2);
assert_eq!(all["a"].rendered_count, 1);
assert_eq!(all["b"].exposure_count, 1);
assert!(
all.contains_key("z"),
"other-tenant chunk present when unfiltered"
);
assert!(!all.contains_key("c"), "old record excluded by cutoff");
let scoped = serve_counts_since(&data_dir, now - 86_400_000, Some("t"), None);
assert!(scoped.contains_key("a") && scoped.contains_key("b"));
assert!(
!scoped.contains_key("z"),
"other-tenant chunk filtered out by tenant scope"
);
let p1 = serve_counts_since(&data_dir, now - 86_400_000, Some("t"), Some("p1"));
assert!(p1.contains_key("a"));
assert!(!p1.contains_key("b"), "p2 chunk filtered out");
}
#[test]
fn aggregate_at_data_dir_reads_central_log() {
let dir = tempdir().unwrap();
let data_dir = dir.path().join("memd_data");
std::fs::create_dir_all(&data_dir).unwrap();
let now = now_ms();
record_hits_to_data_dir(
&data_dir,
&[record("c1", now, true), record("c1", now, false)],
);
let stats = aggregate_hits_at_data_dir(&data_dir, 7, 0);
assert_eq!(stats["c1"].exposure_count, 2);
assert_eq!(stats["c1"].rendered_count, 1);
}
}