use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::ffi::OsString;
use std::fs::{self, OpenOptions};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use super::args::ProjectScopeConfig;
use super::paths::absolutize_project_dir;
use super::report::memory_health_lines;
use crate::error::{MemdError, Result};
use crate::hit_stats::{
aggregate_hits_at_data_dir, aggregate_hits_in, HitStats, DEFAULT_SUMMARY_TTL_MS,
};
use crate::store::{is_unsupported_store_capability, OutcomePrior, Store, TenantManager};
use crate::types::{ChunkId, TenantId};
mod action;
mod collect;
mod evaluate;
mod rank;
mod render;
mod state;
#[cfg(test)]
mod tests;
pub(super) use action::explicit_agent_action;
use collect::collect_project_state;
use evaluate::evaluate_agent_usefulness;
pub(super) use evaluate::run_memory_md_eval;
pub(super) use rank::{build_repo_index, repo_doc_covering, RepoDoc};
use rank::{
dedupe_memory_md_union, filter_startup_takeaways, recompute_union_priorities,
reconcile_candidate_explanations, scan_takeaway_candidates, sort_takeaways,
suppress_finishes_covered_by_libraries, suppress_repo_covered,
suppress_unrelated_machine_takeaways, ScopedSuppressionReasons,
};
use render::render_memory_md;
const USER_PRESERVE_PRIORITY_THRESHOLD: u8 = 8;
const HIT_WINDOW_DAYS: u32 = 30;
const STALE_CHUNK_AGE_MS: i64 = 30 * 86_400_000;
const READABLE_SCAN_PAGE_SIZE: usize = 1_000;
const READABLE_SCAN_MAX_METADATA_ROWS: usize = 10_000;
const TAKEAWAY_CATEGORIES: &[(&str, u8)] = &[
("Decisions", 0),
("Validated Fixes", 1),
("Known Failures", 2),
("Commands/Paths", 3),
("Open Follow-ups", 4),
("Evidence", 5),
("Other Takeaways", 6),
];
#[derive(Debug)]
pub(super) struct MemoryMdOptions {
pub(super) tenant_id: Option<String>,
pub(super) project_id: Option<String>,
pub(super) project_dir: PathBuf,
pub(super) output: PathBuf,
pub(super) project_limit: usize,
pub(super) global_limit: usize,
pub(super) candidate_k: usize,
pub(super) explain_output: Option<PathBuf>,
}
#[derive(Debug)]
pub(super) struct MemoryMdEvalOptions {
pub(super) tenant_id: Option<String>,
pub(super) project_id: Option<String>,
pub(super) project_dir: PathBuf,
pub(super) output: PathBuf,
pub(super) project_limit: usize,
pub(super) candidate_k: usize,
pub(super) top_n: usize,
pub(super) min_useful_ratio: f64,
pub(super) max_generated_wrappers: usize,
pub(super) agent_usefulness: bool,
pub(super) gold_file: Option<PathBuf>,
}
pub(super) async fn refresh_memory_md_with_health<S: Store>(
store: &S,
tenant_manager: Option<&TenantManager>,
options: MemoryMdOptions,
) -> Result<Value> {
let project_dir = absolutize_project_dir(&options.project_dir)?;
let scope = read_project_scope(&project_dir)?;
let generated_unix_ms = now_ms();
let tenant_id = options
.tenant_id
.or_else(|| scope.as_ref().map(|scope| scope.tenant_id.clone()))
.ok_or_else(|| {
MemdError::ValidationError(
"memory-md requires --tenant-id or .memd/project_scope.json".to_string(),
)
})?;
let project_id = options
.project_id
.or_else(|| scope.as_ref().and_then(|scope| scope.project_id.clone()));
let tenant = TenantId::new(&tenant_id)?;
let candidate_k = options.candidate_k;
let project_limit = options.project_limit.min(10);
let global_limit = options.global_limit.min(10);
let health_lines = match memory_health_lines(
store,
tenant_manager,
tenant.as_str(),
project_id.as_deref(),
)
.await
{
Ok(lines) => lines,
Err(error) => {
tracing::debug!(?error, "memory health header skipped");
Vec::new()
}
};
let hit_stats = match tenant_manager {
Some(tm) => {
aggregate_hits_at_data_dir(tm.data_dir(), HIT_WINDOW_DAYS, DEFAULT_SUMMARY_TTL_MS)
}
None => aggregate_hits_in(&project_dir, HIT_WINDOW_DAYS, DEFAULT_SUMMARY_TTL_MS),
};
let project_state = collect_project_state(
store,
&tenant,
tenant.as_str(),
project_id.as_deref(),
&project_dir,
scope.as_ref(),
generated_unix_ms,
)
.await;
let (mut project_takeaways, mut global_takeaways, scan_explanations) =
scan_takeaway_candidates(store, &tenant, project_id.as_deref()).await?;
let (mut project_explanations, mut global_explanations): (Vec<_>, Vec<_>) = scan_explanations
.into_iter()
.partition(|explanation| explanation.section == "project");
let ranking_now_ms = now_ms() as i64;
let candidate_chunk_ids = project_takeaways
.iter()
.chain(global_takeaways.iter())
.filter_map(|takeaway| ChunkId::parse(&takeaway.chunk_id).ok())
.collect::<Vec<_>>();
let outcome_priors: HashMap<String, OutcomePrior> = match store
.outcome_priors(
&tenant,
project_id.as_deref(),
&candidate_chunk_ids,
ranking_now_ms,
)
.await
{
Ok(priors) => priors
.into_iter()
.map(|prior| (prior.chunk_id.to_string(), prior))
.collect(),
Err(error) if is_unsupported_store_capability(&error) => HashMap::new(),
Err(error) => return Err(error),
};
let union_breakdowns = recompute_union_priorities(
&mut project_takeaways,
&mut global_takeaways,
&hit_stats,
&outcome_priors,
ranking_now_ms,
);
let mut union_suppressed = ScopedSuppressionReasons::new();
for (section, takeaways) in [
("project", &mut project_takeaways),
("machine_wide", &mut global_takeaways),
] {
for chunk_id in suppress_finishes_covered_by_libraries(takeaways) {
union_suppressed.insert(
(section.to_string(), chunk_id),
"covered_by_library".to_string(),
);
}
for (chunk_id, reason) in filter_startup_takeaways(takeaways) {
union_suppressed.insert((section.to_string(), chunk_id), reason);
}
}
union_suppressed.extend(suppress_unrelated_machine_takeaways(&mut global_takeaways));
union_suppressed.extend(dedupe_memory_md_union(
&mut project_takeaways,
&mut global_takeaways,
));
let output_path = if options.output.is_absolute() {
options.output
} else {
project_dir.join(options.output)
};
let repo_index = build_repo_index(&project_dir, &output_path);
for (section, takeaways) in [
("project", &mut project_takeaways),
("machine_wide", &mut global_takeaways),
] {
for (chunk_id, reason) in suppress_repo_covered(takeaways, &repo_index) {
union_suppressed.insert((section.to_string(), chunk_id), reason);
}
}
sort_takeaways(&mut project_takeaways);
sort_takeaways(&mut global_takeaways);
project_takeaways.truncate(project_limit);
global_takeaways.truncate(global_limit);
reconcile_candidate_explanations(
&mut project_explanations,
"project",
&project_takeaways,
&union_suppressed,
&union_breakdowns,
);
reconcile_candidate_explanations(
&mut global_explanations,
"machine_wide",
&global_takeaways,
&union_suppressed,
&union_breakdowns,
);
let agent_usefulness =
evaluate_agent_usefulness(&project_state, &project_takeaways, &global_takeaways);
let rendered = render_memory_md(
&project_state,
&health_lines,
&project_takeaways,
&global_takeaways,
);
atomic_replace(&output_path, rendered.as_bytes())?;
let explain_output = if let Some(path) = options.explain_output {
let path = if path.is_absolute() {
path
} else {
project_dir.join(path)
};
let report = json!({
"tenant_id": tenant.to_string(),
"project_id": project_id.clone(),
"generated_unix_ms": generated_unix_ms,
"candidate_k": candidate_k,
"limits": {
"project": project_limit,
"machine_wide": global_limit,
},
"project_state": project_state.clone(),
"agent_usefulness": agent_usefulness.clone(),
"project": project_explanations,
"machine_wide": global_explanations,
});
let rendered_report = serde_json::to_string_pretty(&report)?;
atomic_replace(&path, rendered_report.as_bytes())?;
Some(path)
} else {
None
};
Ok(json!({
"tenant_id": tenant.to_string(),
"project_id": project_id,
"generated_unix_ms": generated_unix_ms,
"output": output_path,
"explain_output": explain_output,
"project_takeaways": project_takeaways.len(),
"global_takeaways": global_takeaways.len(),
"candidate_k": candidate_k,
"project_state": project_state,
"agent_usefulness": agent_usefulness
}))
}
pub(super) fn read_project_scope(project_dir: &Path) -> Result<Option<ProjectScopeConfig>> {
let path = project_dir.join(".memd/project_scope.json");
let text = match std::fs::read_to_string(&path) {
Ok(text) => text,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
match std::fs::symlink_metadata(&path) {
Err(metadata_error) if metadata_error.kind() == std::io::ErrorKind::NotFound => {
return Ok(None)
}
Ok(_) => {
return Err(MemdError::ValidationError(format!(
"unable to read {}: {error}",
path.display()
)))
}
Err(metadata_error) => {
return Err(MemdError::ValidationError(format!(
"unable to inspect {} after read failure ({error}): {metadata_error}",
path.display()
)))
}
}
}
Err(error) => {
return Err(MemdError::ValidationError(format!(
"unable to read {}: {error}",
path.display()
)))
}
};
let scope = serde_json::from_str(&text).map_err(|e| {
MemdError::ValidationError(format!("failed to parse {}: {e}", path.display()))
})?;
Ok(Some(scope))
}
fn atomic_replace(path: &Path, contents: &[u8]) -> Result<()> {
use std::io::Write;
let mut destination = path.to_path_buf();
let mut symlink_hops = 0;
loop {
match std::fs::symlink_metadata(&destination) {
Ok(metadata) if metadata.file_type().is_symlink() => {
if symlink_hops == 40 {
return Err(MemdError::ValidationError(format!(
"unable to resolve output symlink {}: too many symlink levels",
path.display()
)));
}
symlink_hops += 1;
let target = std::fs::read_link(&destination)?;
destination = if target.is_absolute() {
target
} else {
destination
.parent()
.unwrap_or_else(|| Path::new("."))
.join(target)
};
}
Ok(_) => break,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => break,
Err(error) => return Err(error.into()),
}
}
let parent = destination.parent().unwrap_or_else(|| Path::new("."));
let file_name = destination.file_name().ok_or_else(|| {
MemdError::ValidationError(format!(
"output path has no file name: {}",
destination.display()
))
})?;
let existing_permissions = match OpenOptions::new().write(true).open(&destination) {
Ok(file) => Some(file.metadata()?.permissions()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
Err(error) => return Err(error.into()),
};
let mut temp_name = OsString::from(".");
temp_name.push(file_name);
temp_name.push(format!(".{}.tmp", uuid::Uuid::now_v7()));
let temp_path = parent.join(temp_name);
let mut open_options = OpenOptions::new();
open_options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
open_options.mode(0o600);
}
let mut temp_file = open_options.open(&temp_path)?;
let publish_result = (|| -> std::io::Result<()> {
if let Some(permissions) = existing_permissions {
std::fs::set_permissions(&temp_path, permissions)?;
}
temp_file.write_all(contents)?;
drop(temp_file);
std::fs::rename(&temp_path, &destination)
})();
if let Err(error) = publish_result {
let _ = std::fs::remove_file(&temp_path);
return Err(error.into());
}
Ok(())
}
fn now_ms() -> u128 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_millis())
.unwrap_or(0)
}