use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fs;
use std::io::{self, Read};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::thread;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use super::args::{CliQueryMode, ProjectScopeConfig};
use super::paths::absolutize_project_dir;
use super::report::memory_health_lines;
use super::search::cli_search_payload;
use crate::error::{MemdError, Result};
use crate::hit_stats::{
aggregate_hits_at_data_dir, aggregate_hits_in, HitStats, DEFAULT_SUMMARY_TTL_MS,
};
use crate::store::{Store, TenantManager};
use crate::types::TenantId;
const PROJECT_QUERIES: &[(CliQueryMode, &str, &str)] = &[
(
CliQueryMode::FindHighlights,
"project_highlights",
"project takeaways best practices key decisions recurring issues important files paths how to solve",
),
(
CliQueryMode::FindDecisions,
"project_decisions",
"project architecture configuration deployment key decisions tradeoffs",
),
(
CliQueryMode::FindFailures,
"project_failures",
"project recurring failures bugs timeouts blockers fixes how to solve",
),
(
CliQueryMode::FindHighlights,
"project_library",
"consolidated highlight library ranked lessons future-agent uplift",
),
];
const GLOBAL_QUERIES: &[(CliQueryMode, &str, &str)] = &[
(
CliQueryMode::FindHighlights,
"global_highlights",
"machine wide reusable takeaways best practices recurring issues important paths how to solve",
),
(
CliQueryMode::FindDecisions,
"global_decisions",
"cross project general decisions best practices configuration deployment",
),
(
CliQueryMode::FindFailures,
"global_failures",
"cross project recurring failures timeouts blockers fixes how to solve",
),
(
CliQueryMode::FindHighlights,
"global_library",
"consolidated highlight library ranked lessons future-agent uplift",
),
];
const USER_PRESERVE_PRIORITY_THRESHOLD: u8 = 8;
const HIT_WINDOW_DAYS: u32 = 30;
const STALE_CHUNK_AGE_MS: i64 = 30 * 86_400_000;
const PROJECT_STATE_FILE_CAP_BYTES: u64 = 256 * 1024;
const HANDOFF_FILE_CAP_BYTES: u64 = 128 * 1024;
const GIT_STATUS_TIMEOUT: Duration = Duration::from_millis(1_500);
const READABLE_SCAN_PAGE_SIZE: usize = 1_000;
const READABLE_SCAN_MAX_METADATA_ROWS: usize = 10_000;
const TAKEAWAY_CATEGORIES: &[(&str, u8)] = &[
("Decisions", 0),
("Validated Fixes", 1),
("Known Failures", 2),
("Commands/Paths", 3),
("Open Follow-ups", 4),
("Evidence", 5),
("Other Takeaways", 6),
];
#[derive(Debug)]
pub(super) struct MemoryMdOptions {
pub(super) tenant_id: Option<String>,
pub(super) project_id: Option<String>,
pub(super) project_dir: PathBuf,
pub(super) output: PathBuf,
pub(super) project_limit: usize,
pub(super) global_limit: usize,
pub(super) candidate_k: usize,
pub(super) explain_output: Option<PathBuf>,
}
#[derive(Debug)]
pub(super) struct MemoryMdEvalOptions {
pub(super) tenant_id: Option<String>,
pub(super) project_id: Option<String>,
pub(super) project_dir: PathBuf,
pub(super) output: PathBuf,
pub(super) project_limit: usize,
pub(super) candidate_k: usize,
pub(super) top_n: usize,
pub(super) min_useful_ratio: f64,
pub(super) max_generated_wrappers: usize,
pub(super) agent_usefulness: bool,
pub(super) gold_file: Option<PathBuf>,
}
#[derive(Debug, Clone, Serialize, PartialEq)]
struct ProjectState {
generated_unix_ms: u128,
tenant_id: String,
project_id: Option<String>,
configured_project_dir: Option<String>,
resolved_project_dir: String,
scope_warnings: Vec<String>,
git: GitState,
latest_task: Option<StateSignal>,
latest_handoff: Option<StateSignal>,
latest_vcs: Option<StateSignal>,
next_actions: Vec<NextAction>,
memory: MemoryState,
collection_warnings: Vec<String>,
}
#[derive(Debug, Clone, Serialize, PartialEq)]
struct GitState {
available: bool,
not_git_repo: bool,
branch: Option<String>,
clean: Option<bool>,
changed_entries: usize,
summary: String,
warning: Option<String>,
}
#[derive(Debug, Clone, Serialize, PartialEq)]
struct StateSignal {
source_path: String,
line: Option<usize>,
heading: Option<String>,
text: String,
}
#[derive(Debug, Clone, Serialize, PartialEq)]
struct NextAction {
source_path: String,
line: usize,
heading: Option<String>,
text: String,
}
#[derive(Debug, Clone, Default, Serialize, PartialEq)]
struct MemoryState {
metadata_active_chunks: Option<usize>,
readable_active_chunks: Option<usize>,
unreadable_active_chunks: Option<usize>,
scan_warning: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
struct AgentUsefulnessMetrics {
latest_project_state_present: bool,
scope_present: bool,
git_state_present: bool,
git_state_present_or_not_git_repo: bool,
latest_work_present: bool,
next_action_present: bool,
no_open_tasks_detected: bool,
scope_health_present: bool,
memory_degraded_warning_present: bool,
fragment_count: usize,
duplicate_cluster_count: usize,
boilerplate_action_count: usize,
unrelated_machine_items: usize,
source_backed_next_actions: bool,
answerability_passed: bool,
}
#[derive(Debug, Clone)]
struct Takeaway {
chunk_id: String,
tenant_id: String,
project_id: Option<String>,
text: String,
score: f32,
priority: f32,
chunk_type: String,
timestamp_created: i64,
tags: Vec<String>,
sources: BTreeSet<String>,
occurrences: usize,
}
#[derive(Debug, Clone)]
struct RankedTakeawayCollection {
takeaways: Vec<Takeaway>,
explanations: Vec<MemoryMdCandidateExplanation>,
}
#[derive(Debug, Clone, Serialize, PartialEq)]
struct PriorityBreakdown {
explicit: f32,
kind_weight: f32,
type_weight: f32,
recurrence: f32,
multi_query: f32,
search_score: f32,
library_bonus: f32,
utility: f32,
staleness_penalty: f32,
total: f32,
}
#[derive(Debug, Clone, Serialize, PartialEq)]
struct MemoryMdCandidateExplanation {
section: String,
source: String,
query: String,
mode: String,
raw_rank: usize,
chunk_id: String,
tenant_id: String,
project_id: Option<String>,
chunk_type: String,
timestamp_created: i64,
search_score: f32,
priority_score: Option<f32>,
priority_breakdown: Option<PriorityBreakdown>,
display_status: String,
filter_reason: Option<String>,
display_rank: Option<usize>,
generated_digest: bool,
quality_flags: Vec<String>,
topic_key: Option<String>,
tags: Vec<String>,
matched_sources: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct TakeawayCategory {
heading: &'static str,
reason: &'static str,
order: u8,
}
#[derive(Debug, Clone, PartialEq)]
struct MemoryMdQualityReport {
displayed_count: usize,
useful_count: usize,
generated_wrapper_count: usize,
missing_reason_count: usize,
missing_action_count: usize,
useful_ratio: f64,
}
pub(super) async fn refresh_memory_md<S: Store>(
store: &S,
options: MemoryMdOptions,
) -> Result<Value> {
refresh_memory_md_with_health(store, None, options).await
}
pub(super) async fn refresh_memory_md_with_health<S: Store>(
store: &S,
tenant_manager: Option<&TenantManager>,
options: MemoryMdOptions,
) -> Result<Value> {
let project_dir = absolutize_project_dir(&options.project_dir)?;
let scope = read_project_scope(&project_dir)?;
let generated_unix_ms = now_ms();
let tenant_id = options
.tenant_id
.or_else(|| scope.as_ref().map(|scope| scope.tenant_id.clone()))
.ok_or_else(|| {
MemdError::ValidationError(
"memory-md requires --tenant-id or .memd/project_scope.json".to_string(),
)
})?;
let project_id = options
.project_id
.or_else(|| scope.as_ref().and_then(|scope| scope.project_id.clone()));
let tenant = TenantId::new(&tenant_id)?;
let candidate_k = options.candidate_k.clamp(1, 200);
let project_limit = options.project_limit.min(10);
let global_limit = options.global_limit.min(10);
let health_lines = match memory_health_lines(
store,
tenant_manager,
tenant.as_str(),
project_id.as_deref(),
)
.await
{
Ok(lines) => lines,
Err(error) => {
tracing::debug!(?error, "memory health header skipped");
Vec::new()
}
};
let hit_stats = match tenant_manager {
Some(tm) => {
aggregate_hits_at_data_dir(tm.data_dir(), HIT_WINDOW_DAYS, DEFAULT_SUMMARY_TTL_MS)
}
None => aggregate_hits_in(&project_dir, HIT_WINDOW_DAYS, DEFAULT_SUMMARY_TTL_MS),
};
let project_state = collect_project_state(
store,
&tenant,
tenant.as_str(),
project_id.as_deref(),
&project_dir,
scope.as_ref(),
generated_unix_ms,
)
.await;
let project_collection = if project_limit == 0 {
RankedTakeawayCollection {
takeaways: Vec::new(),
explanations: Vec::new(),
}
} else {
collect_ranked_takeaways_with_explanations(
store,
tenant.as_str(),
project_id.as_deref(),
PROJECT_QUERIES,
candidate_k,
project_limit,
&hit_stats,
"project",
)
.await?
};
let global_collection = if global_limit == 0 {
RankedTakeawayCollection {
takeaways: Vec::new(),
explanations: Vec::new(),
}
} else {
collect_ranked_takeaways_with_explanations(
store,
tenant.as_str(),
None,
GLOBAL_QUERIES,
candidate_k,
global_limit,
&hit_stats,
"machine_wide",
)
.await?
};
let RankedTakeawayCollection {
takeaways: project_takeaways,
explanations: project_explanations,
} = project_collection;
let RankedTakeawayCollection {
takeaways: global_takeaways,
explanations: global_explanations,
} = global_collection;
let agent_usefulness =
evaluate_agent_usefulness(&project_state, &project_takeaways, &global_takeaways);
let output_path = if options.output.is_absolute() {
options.output
} else {
project_dir.join(options.output)
};
let rendered = render_memory_md(
&project_state,
&health_lines,
&project_takeaways,
&global_takeaways,
);
std::fs::write(&output_path, rendered)?;
let explain_output = if let Some(path) = options.explain_output {
let path = if path.is_absolute() {
path
} else {
project_dir.join(path)
};
let report = json!({
"tenant_id": tenant.to_string(),
"project_id": project_id.clone(),
"generated_unix_ms": generated_unix_ms,
"candidate_k": candidate_k,
"limits": {
"project": project_limit,
"machine_wide": global_limit,
},
"project_state": project_state.clone(),
"agent_usefulness": agent_usefulness.clone(),
"project": project_explanations,
"machine_wide": global_explanations,
});
std::fs::write(&path, serde_json::to_string_pretty(&report)?)?;
Some(path)
} else {
None
};
Ok(json!({
"tenant_id": tenant.to_string(),
"project_id": project_id,
"generated_unix_ms": generated_unix_ms,
"output": output_path,
"explain_output": explain_output,
"project_takeaways": project_takeaways.len(),
"global_takeaways": global_takeaways.len(),
"candidate_k": candidate_k,
"project_state": project_state,
"agent_usefulness": agent_usefulness
}))
}
pub(super) async fn run_memory_md_eval<S: Store>(
store: &S,
tenant_manager: Option<&TenantManager>,
options: MemoryMdEvalOptions,
) -> Result<Value> {
if options.gold_file.is_some() {
return run_memory_md_gold_eval(store, tenant_manager, &options).await;
}
let project_dir = absolutize_project_dir(&options.project_dir)?;
let output = options.output.clone();
let rendered_path = if output.is_absolute() {
output.clone()
} else {
project_dir.join(&output)
};
let refresh = refresh_memory_md_with_health(
store,
tenant_manager,
MemoryMdOptions {
tenant_id: options.tenant_id,
project_id: options.project_id,
project_dir,
output,
project_limit: options.project_limit,
global_limit: if options.agent_usefulness { 2 } else { 0 },
candidate_k: options.candidate_k,
explain_output: None,
},
)
.await?;
let content = std::fs::read_to_string(&rendered_path)?;
let top_n = options.top_n.clamp(1, 10);
let report = evaluate_memory_md_quality(&content, top_n);
let min_useful_ratio = options.min_useful_ratio.clamp(0.0, 1.0);
let mut failures = Vec::new();
if report.displayed_count == 0 {
failures.push("no project takeaways were displayed".to_string());
}
if report.useful_ratio + f64::EPSILON < min_useful_ratio {
failures.push(format!(
"useful_ratio {:.3} below threshold {:.3}",
report.useful_ratio, min_useful_ratio
));
}
if report.generated_wrapper_count > options.max_generated_wrappers {
failures.push(format!(
"generated_wrapper_count {} exceeds threshold {}",
report.generated_wrapper_count, options.max_generated_wrappers
));
}
if report.missing_reason_count > 0 {
failures.push(format!(
"{} displayed items are missing reason metadata",
report.missing_reason_count
));
}
if report.missing_action_count > 0 {
failures.push(format!(
"{} displayed items are missing concrete agent action guidance",
report.missing_action_count
));
}
let agent_usefulness = if options.agent_usefulness {
let metrics: AgentUsefulnessMetrics = serde_json::from_value(
refresh
.get("agent_usefulness")
.cloned()
.unwrap_or_else(|| json!({})),
)
.map_err(|error| {
MemdError::ValidationError(format!("invalid agent_usefulness metrics: {error}"))
})?;
failures.extend(agent_usefulness_failures(&metrics));
Some(metrics)
} else {
None
};
let payload = json!({
"passed": failures.is_empty(),
"output": rendered_path,
"agent_usefulness": agent_usefulness,
"top_n": top_n,
"displayed_count": report.displayed_count,
"useful_count": report.useful_count,
"useful_ratio": report.useful_ratio,
"generated_wrapper_count": report.generated_wrapper_count,
"missing_reason_count": report.missing_reason_count,
"missing_action_count": report.missing_action_count,
"thresholds": {
"min_useful_ratio": min_useful_ratio,
"max_generated_wrappers": options.max_generated_wrappers,
},
"refresh": refresh,
"failures": failures,
});
if !failures.is_empty() {
return Err(MemdError::ValidationError(format!(
"memory-md quality thresholds failed: {}",
serde_json::to_string(&payload)?
)));
}
Ok(payload)
}
#[derive(Debug, Deserialize)]
struct MemoryMdGoldFile {
projects: Vec<MemoryMdGoldProject>,
}
#[derive(Debug, Deserialize)]
struct MemoryMdGoldProject {
name: Option<String>,
project_dir: PathBuf,
must_contain: Option<Vec<String>>,
must_not_contain: Option<Vec<String>>,
expected_git: Option<bool>,
max_fragments: Option<usize>,
max_unrelated_machine_items: Option<usize>,
}
async fn run_memory_md_gold_eval<S: Store>(
store: &S,
tenant_manager: Option<&TenantManager>,
options: &MemoryMdEvalOptions,
) -> Result<Value> {
let gold_path = options
.gold_file
.as_ref()
.expect("caller checked gold_file");
let gold_text = fs::read_to_string(gold_path)?;
let gold: MemoryMdGoldFile = serde_json::from_str(&gold_text).map_err(|error| {
MemdError::ValidationError(format!("failed to parse {}: {error}", gold_path.display()))
})?;
let mut project_results = Vec::new();
let mut failures = Vec::new();
for (idx, project) in gold.projects.iter().enumerate() {
let name = project
.name
.clone()
.unwrap_or_else(|| project.project_dir.display().to_string());
let output = std::env::temp_dir().join(format!("memd-memory-md-gold-{idx}.md"));
let refresh = refresh_memory_md_with_health(
store,
tenant_manager,
MemoryMdOptions {
tenant_id: options.tenant_id.clone(),
project_id: options.project_id.clone(),
project_dir: project.project_dir.clone(),
output: output.clone(),
project_limit: options.project_limit,
global_limit: 2,
candidate_k: options.candidate_k,
explain_output: None,
},
)
.await?;
let content = fs::read_to_string(&output)?;
let metrics: AgentUsefulnessMetrics = serde_json::from_value(
refresh
.get("agent_usefulness")
.cloned()
.unwrap_or_else(|| json!({})),
)
.map_err(|error| {
MemdError::ValidationError(format!("invalid agent_usefulness metrics: {error}"))
})?;
let mut project_failures = agent_usefulness_failures(&metrics);
if let Some(expected_git) = project.expected_git {
if let Some(failure) = expected_git_failure(&metrics, expected_git) {
project_failures.push(failure);
}
}
if let Some(max_fragments) = project.max_fragments {
if metrics.fragment_count > max_fragments {
project_failures.push(format!(
"fragment_count {} exceeds {}",
metrics.fragment_count, max_fragments
));
}
}
if let Some(max_unrelated) = project.max_unrelated_machine_items {
if metrics.unrelated_machine_items > max_unrelated {
project_failures.push(format!(
"unrelated_machine_items {} exceeds {}",
metrics.unrelated_machine_items, max_unrelated
));
}
}
for needle in project.must_contain.as_deref().unwrap_or(&[]) {
if !content.contains(needle) {
project_failures.push(format!("missing required text `{needle}`"));
}
}
for needle in project.must_not_contain.as_deref().unwrap_or(&[]) {
if content.contains(needle) {
project_failures.push(format!("forbidden text present `{needle}`"));
}
}
for failure in &project_failures {
failures.push(format!("{name}: {failure}"));
}
project_results.push(json!({
"name": name,
"project_dir": project.project_dir.display().to_string(),
"output": output,
"agent_usefulness": metrics,
"failures": project_failures,
}));
}
let payload = json!({
"passed": failures.is_empty(),
"gold_file": gold_path,
"projects": project_results,
"failures": failures,
});
if !failures.is_empty() {
return Err(MemdError::ValidationError(format!(
"memory-md gold-file thresholds failed: {}",
serde_json::to_string(&payload)?
)));
}
Ok(payload)
}
fn agent_usefulness_failures(metrics: &AgentUsefulnessMetrics) -> Vec<String> {
let mut failures = Vec::new();
if !metrics.latest_project_state_present {
failures.push("latest project state is missing".to_string());
}
if !metrics.scope_present {
failures.push("scope is missing".to_string());
}
if !metrics.git_state_present_or_not_git_repo {
failures.push("git state is missing for a git project".to_string());
}
if !metrics.latest_work_present {
failures.push("latest work signal is missing".to_string());
}
if !metrics.next_action_present && !metrics.no_open_tasks_detected {
failures.push("next actions are missing while open tasks exist".to_string());
}
if !metrics.scope_health_present {
failures.push("scope health is missing".to_string());
}
if metrics.fragment_count > 0 {
failures.push(format!(
"fragment_count {} exceeds threshold 0",
metrics.fragment_count
));
}
if metrics.boilerplate_action_count > 0 {
failures.push(format!(
"boilerplate_action_count {} exceeds threshold 0",
metrics.boilerplate_action_count
));
}
if metrics.unrelated_machine_items > 2 {
failures.push(format!(
"unrelated_machine_items {} exceeds threshold 2",
metrics.unrelated_machine_items
));
}
if !metrics.source_backed_next_actions {
failures.push("one or more next actions lack source path or line".to_string());
}
if !metrics.answerability_passed {
failures.push("answerability_passed=false".to_string());
}
failures
}
fn expected_git_failure(metrics: &AgentUsefulnessMetrics, expected_git: bool) -> Option<String> {
match (expected_git, metrics.git_state_present) {
(true, false) => Some("expected git state but git_state_present=false".to_string()),
(false, true) => Some("expected no git state but git_state_present=true".to_string()),
_ => None,
}
}
async fn collect_project_state<S: Store>(
store: &S,
tenant: &TenantId,
tenant_id: &str,
project_id: Option<&str>,
project_dir: &Path,
scope: Option<&ProjectScopeConfig>,
generated_unix_ms: u128,
) -> ProjectState {
let configured_project_dir = scope.map(|scope| scope.project_dir.clone());
let mut collection_warnings = Vec::new();
let mut scope_warnings = Vec::new();
if let Some(configured) = configured_project_dir.as_deref() {
if let Some(warning) = scope_path_drift_warning(configured, project_dir) {
scope_warnings.push(warning);
}
}
let git = collect_git_state(project_dir, "git");
let latest_vcs = git
.available
.then(|| collect_latest_git_commit(project_dir, "git"))
.flatten();
let task_scan = collect_task_state(project_dir);
collection_warnings.extend(task_scan.warnings);
let handoff_scan = collect_handoff_state(project_dir);
collection_warnings.extend(handoff_scan.warnings);
let memory = collect_memory_state(store, tenant, project_id).await;
ProjectState {
generated_unix_ms,
tenant_id: tenant_id.to_string(),
project_id: project_id.map(str::to_string),
configured_project_dir,
resolved_project_dir: canonical_or_lexical_path(project_dir).display().to_string(),
scope_warnings,
git,
latest_task: task_scan.latest_task,
latest_handoff: handoff_scan.latest_handoff,
latest_vcs,
next_actions: task_scan.next_actions,
memory,
collection_warnings,
}
}
#[derive(Debug, Default)]
struct TaskScan {
latest_task: Option<StateSignal>,
next_actions: Vec<NextAction>,
warnings: Vec<String>,
}
#[derive(Debug, Default)]
struct HandoffScan {
latest_handoff: Option<StateSignal>,
warnings: Vec<String>,
}
#[derive(Debug, Clone)]
struct Heading {
level: usize,
title: String,
line: usize,
}
fn collect_task_state(project_dir: &Path) -> TaskScan {
let path = project_dir.join("tasks/todo.md");
if !path.exists() {
return TaskScan::default();
}
let relative = relative_path(project_dir, &path);
let mut warnings = Vec::new();
let text = match read_text_capped(&path, PROJECT_STATE_FILE_CAP_BYTES) {
Ok((text, truncated)) => {
if truncated {
warnings.push(format!(
"{} was truncated to {} bytes while collecting project state",
relative, PROJECT_STATE_FILE_CAP_BYTES
));
}
text
}
Err(error) => {
return TaskScan {
warnings: vec![format!("could not read {relative}: {error}")],
..TaskScan::default()
};
}
};
let mut stack: Vec<Heading> = Vec::new();
let mut headings = Vec::new();
let mut next_actions = Vec::new();
for (idx, line) in text.lines().enumerate() {
let line_no = idx + 1;
if let Some(heading) = parse_heading(line, line_no) {
while stack
.last()
.map(|existing| existing.level >= heading.level)
.unwrap_or(false)
{
stack.pop();
}
stack.push(heading.clone());
headings.push(heading);
continue;
}
if let Some(action) = parse_next_action(line) {
let heading = stack.last().map(|heading| heading.title.clone());
next_actions.push(NextAction {
source_path: relative.clone(),
line: line_no,
heading,
text: action,
});
}
}
let latest_task = if let Some(first_action) = next_actions.first() {
Some(StateSignal {
source_path: first_action.source_path.clone(),
line: Some(first_action.line),
heading: first_action.heading.clone(),
text: format!("open action: {}", first_action.text),
})
} else if let Some(heading) = headings
.iter()
.rev()
.find(|heading| active_heading(&heading.title))
{
Some(StateSignal {
source_path: relative,
line: Some(heading.line),
heading: Some(heading.title.clone()),
text: "latest active section".to_string(),
})
} else {
headings
.iter()
.rev()
.find(|heading| completed_or_dated_heading(&heading.title))
.map(|heading| StateSignal {
source_path: relative,
line: Some(heading.line),
heading: Some(heading.title.clone()),
text: "latest completed or dated section".to_string(),
})
};
TaskScan {
latest_task,
next_actions,
warnings,
}
}
fn collect_handoff_state(project_dir: &Path) -> HandoffScan {
let dir = project_dir.join("docs/handoffs");
if !dir.exists() {
return HandoffScan::default();
}
let mut warnings = Vec::new();
let entries = match fs::read_dir(&dir) {
Ok(entries) => entries,
Err(error) => {
return HandoffScan {
warnings: vec![format!(
"could not read {}: {error}",
relative_path(project_dir, &dir)
)],
..HandoffScan::default()
};
}
};
let mut candidates = Vec::new();
for entry in entries {
let Ok(entry) = entry else {
warnings.push("could not read one docs/handoffs entry".to_string());
continue;
};
let path = entry.path();
if path
.components()
.any(|component| component.as_os_str() == "_archive")
{
continue;
}
if path.extension().and_then(|ext| ext.to_str()) != Some("md") {
continue;
}
let modified = entry
.metadata()
.and_then(|metadata| metadata.modified())
.unwrap_or(UNIX_EPOCH);
candidates.push((modified, path));
}
candidates.sort_by(|left, right| right.0.cmp(&left.0).then_with(|| left.1.cmp(&right.1)));
let Some((_, path)) = candidates.into_iter().next() else {
return HandoffScan {
warnings,
..HandoffScan::default()
};
};
let relative = relative_path(project_dir, &path);
let text = match read_text_capped(&path, HANDOFF_FILE_CAP_BYTES) {
Ok((text, truncated)) => {
if truncated {
warnings.push(format!(
"{} was truncated to {} bytes while collecting handoff state",
relative, HANDOFF_FILE_CAP_BYTES
));
}
text
}
Err(error) => {
warnings.push(format!("could not read {relative}: {error}"));
return HandoffScan {
warnings,
..HandoffScan::default()
};
}
};
let mut title = None;
let mut status_lines = Vec::new();
for (idx, line) in text.lines().enumerate() {
if title.is_none() {
title = parse_heading(line, idx + 1).map(|heading| (idx + 1, heading.title));
}
let trimmed = line.trim();
let lowered = trimmed.to_ascii_lowercase();
if lowered.starts_with("status:")
|| lowered.starts_with("- status:")
|| lowered.starts_with("next:")
|| lowered.starts_with("next step:")
|| lowered.starts_with("follow-up:")
{
status_lines.push(trimmed.trim_start_matches("- ").to_string());
}
if status_lines.len() >= 2 {
break;
}
}
let (line, title_text) = title.unwrap_or((1, relative.clone()));
let signal_text = if status_lines.is_empty() {
"latest handoff".to_string()
} else {
status_lines.join(" | ")
};
HandoffScan {
latest_handoff: Some(StateSignal {
source_path: relative,
line: Some(line),
heading: Some(title_text),
text: signal_text,
}),
warnings,
}
}
async fn collect_memory_state<S: Store>(
store: &S,
tenant: &TenantId,
project_id: Option<&str>,
) -> MemoryState {
let snapshot = match store.health_snapshot(tenant, project_id, 0).await {
Ok(Some(snapshot)) => snapshot,
Ok(None) => return MemoryState::default(),
Err(error) => {
return MemoryState {
scan_warning: Some(format!("memory health scan failed: {error}")),
..MemoryState::default()
};
}
};
let metadata_active = snapshot.counts.active_chunks;
let mut readable = 0usize;
let mut warning = None;
let mut offset = 0usize;
let scan_limit = metadata_active.min(READABLE_SCAN_MAX_METADATA_ROWS);
while offset < scan_limit {
let limit = READABLE_SCAN_PAGE_SIZE.min(scan_limit.saturating_sub(offset));
match store
.list_chunks_for_project(tenant, project_id, limit, offset)
.await
{
Ok(chunks) => readable = readable.saturating_add(chunks.len()),
Err(error) => {
warning = Some(format!(
"readable memory scan failed at offset {offset}: {error}"
));
break;
}
}
offset = offset.saturating_add(limit);
}
if metadata_active > scan_limit && warning.is_none() {
warning = Some(format!(
"readable memory scan partial: checked {scan_limit} of {metadata_active} active chunks; unreadable count may be understated"
));
}
let unreadable = scan_limit.saturating_sub(readable);
MemoryState {
metadata_active_chunks: Some(metadata_active),
readable_active_chunks: Some(readable),
unreadable_active_chunks: Some(unreadable),
scan_warning: warning,
}
}
fn collect_git_state(project_dir: &Path, git_binary: &str) -> GitState {
let mut command = Command::new(git_binary);
command
.arg("-C")
.arg(project_dir)
.args(["status", "--short", "--branch"]);
let output = match run_command_with_timeout(command, GIT_STATUS_TIMEOUT) {
Ok(output) => output,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
return GitState {
available: false,
not_git_repo: false,
branch: None,
clean: None,
changed_entries: 0,
summary: "git unavailable: executable not found".to_string(),
warning: Some("git unavailable: executable not found".to_string()),
};
}
Err(error) => {
return GitState {
available: false,
not_git_repo: false,
branch: None,
clean: None,
changed_entries: 0,
summary: format!("git unavailable: {error}"),
warning: Some(format!("git unavailable: {error}")),
};
}
};
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
let stdout = String::from_utf8_lossy(&output.stdout);
if output.timed_out {
return GitState {
available: false,
not_git_repo: false,
branch: None,
clean: None,
changed_entries: 0,
summary: "git unavailable: status timed out".to_string(),
warning: Some("git unavailable: status timed out".to_string()),
};
}
if !output.status_success {
let not_git_repo = stderr.contains("not a git repository");
let reason = if not_git_repo {
"not a git repository".to_string()
} else if stderr.is_empty() {
"git status failed".to_string()
} else {
stderr
};
return GitState {
available: false,
not_git_repo,
branch: None,
clean: None,
changed_entries: 0,
summary: format!("git unavailable: {reason}"),
warning: Some(format!("git unavailable: {reason}")),
};
}
let mut lines = stdout.lines();
let branch = lines
.next()
.and_then(|line| line.strip_prefix("## "))
.map(|line| line.split("...").next().unwrap_or(line).trim().to_string())
.filter(|branch| !branch.is_empty());
let changed_entries = lines.filter(|line| !line.trim().is_empty()).count();
let clean = changed_entries == 0;
let branch_label = branch.as_deref().unwrap_or("<unknown>");
let summary = if clean {
format!("branch `{branch_label}`; clean")
} else {
format!("branch `{branch_label}`; dirty ({changed_entries} changed entries)")
};
GitState {
available: true,
not_git_repo: false,
branch,
clean: Some(clean),
changed_entries,
summary,
warning: None,
}
}
fn collect_latest_git_commit(project_dir: &Path, git_binary: &str) -> Option<StateSignal> {
let mut command = Command::new(git_binary);
command.arg("-C").arg(project_dir).args([
"log",
"-1",
"--date=short",
"--pretty=format:%h %cd %s",
]);
let output = run_command_with_timeout(command, GIT_STATUS_TIMEOUT).ok()?;
if output.timed_out || !output.status_success {
return None;
}
let text = String::from_utf8_lossy(&output.stdout).trim().to_string();
if text.is_empty() {
return None;
}
Some(StateSignal {
source_path: ".git".to_string(),
line: None,
heading: Some("latest commit".to_string()),
text,
})
}
struct TimedCommandOutput {
status_success: bool,
timed_out: bool,
stdout: Vec<u8>,
stderr: Vec<u8>,
}
fn run_command_with_timeout(
mut command: Command,
timeout: Duration,
) -> std::io::Result<TimedCommandOutput> {
command.stdout(Stdio::piped()).stderr(Stdio::piped());
let mut child = command.spawn()?;
let stdout = child.stdout.take();
let stderr = child.stderr.take();
let stdout_reader = thread::spawn(move || read_child_pipe(stdout));
let stderr_reader = thread::spawn(move || read_child_pipe(stderr));
let started = Instant::now();
loop {
if let Some(status) = child.try_wait()? {
let stdout = join_child_pipe(stdout_reader)?;
let stderr = join_child_pipe(stderr_reader)?;
return Ok(TimedCommandOutput {
status_success: status.success(),
timed_out: false,
stdout,
stderr,
});
}
if started.elapsed() >= timeout {
let _ = child.kill();
let _ = child.wait();
let stdout = join_child_pipe(stdout_reader)?;
let stderr = join_child_pipe(stderr_reader)?;
return Ok(TimedCommandOutput {
status_success: false,
timed_out: true,
stdout,
stderr,
});
}
thread::sleep(Duration::from_millis(20));
}
}
fn read_child_pipe(mut pipe: Option<impl Read>) -> io::Result<Vec<u8>> {
let mut buffer = Vec::new();
if let Some(pipe) = pipe.as_mut() {
pipe.read_to_end(&mut buffer)?;
}
Ok(buffer)
}
fn join_child_pipe(handle: thread::JoinHandle<io::Result<Vec<u8>>>) -> io::Result<Vec<u8>> {
handle
.join()
.map_err(|_| io::Error::new(io::ErrorKind::Other, "command output reader panicked"))?
}
fn parse_heading(line: &str, line_no: usize) -> Option<Heading> {
let trimmed = line.trim_start();
let level = trimmed.chars().take_while(|ch| *ch == '#').count();
if level == 0 || !trimmed[level..].starts_with(' ') {
return None;
}
let title = trimmed[level..].trim().to_string();
(!title.is_empty()).then_some(Heading {
level,
title,
line: line_no,
})
}
fn parse_next_action(line: &str) -> Option<String> {
let trimmed = line.trim();
if trimmed.starts_with("- [x]")
|| trimmed.starts_with("- [X]")
|| trimmed.starts_with("* [x]")
|| trimmed.starts_with("* [X]")
{
return None;
}
let body = trimmed
.strip_prefix("- [ ]")
.or_else(|| trimmed.strip_prefix("* [ ]"))
.map(str::trim)
.or_else(|| strip_bullet(trimmed));
let candidate = body.unwrap_or(trimmed).trim();
let lowered = candidate.to_ascii_lowercase();
let explicit = lowered.starts_with("next step:")
|| lowered.starts_with("follow-up:")
|| lowered.starts_with("followup:")
|| lowered.starts_with("todo:")
|| lowered.starts_with("todo ")
|| lowered.starts_with("pending:")
|| lowered.starts_with("pending ");
if trimmed.starts_with("- [ ]") || trimmed.starts_with("* [ ]") || explicit {
Some(candidate.trim_end_matches('.').trim().to_string()).filter(|s| !s.is_empty())
} else {
None
}
}
fn strip_bullet(line: &str) -> Option<&str> {
line.strip_prefix("- ")
.or_else(|| line.strip_prefix("* "))
.map(str::trim)
}
fn active_heading(title: &str) -> bool {
let lowered = title.to_ascii_lowercase();
lowered.contains("in progress")
|| lowered.contains("todo")
|| lowered.contains("pending")
|| lowered.contains("open")
}
fn completed_or_dated_heading(title: &str) -> bool {
let lowered = title.to_ascii_lowercase();
lowered.contains("done")
|| lowered.contains("complete")
|| lowered.contains("completed")
|| lowered.contains("202")
}
fn read_text_capped(path: &Path, cap: u64) -> std::io::Result<(String, bool)> {
let mut file = fs::File::open(path)?;
let mut bytes = Vec::new();
file.by_ref()
.take(cap.saturating_add(1))
.read_to_end(&mut bytes)?;
let truncated = bytes.len() as u64 > cap;
if truncated {
bytes.truncate(cap as usize);
}
Ok((String::from_utf8_lossy(&bytes).to_string(), truncated))
}
fn scope_path_drift_warning(configured: &str, resolved_project_dir: &Path) -> Option<String> {
let configured_path = PathBuf::from(configured);
let configured_abs = if configured_path.is_absolute() {
configured_path
} else {
resolved_project_dir.join(configured_path)
};
let configured_norm = canonical_or_lexical_path(&configured_abs);
let resolved_norm = canonical_or_lexical_path(resolved_project_dir);
(configured_norm != resolved_norm).then(|| {
format!(
"scope mismatch: configured project_dir `{}` differs from resolved project_dir `{}`",
configured_norm.display(),
resolved_norm.display()
)
})
}
fn canonical_or_lexical_path(path: &Path) -> PathBuf {
fs::canonicalize(path).unwrap_or_else(|_| lexical_normalize_path(path))
}
fn lexical_normalize_path(path: &Path) -> PathBuf {
let mut normalized = PathBuf::new();
for component in path.components() {
match component {
std::path::Component::Prefix(prefix) => normalized.push(prefix.as_os_str()),
std::path::Component::RootDir => normalized.push(Path::new("/")),
std::path::Component::CurDir => {}
std::path::Component::ParentDir => {
normalized.pop();
}
std::path::Component::Normal(part) => normalized.push(part),
}
}
normalized
}
fn relative_path(project_dir: &Path, path: &Path) -> String {
path.strip_prefix(project_dir)
.unwrap_or(path)
.display()
.to_string()
}
async fn collect_ranked_takeaways_with_explanations<S: Store>(
store: &S,
tenant_id: &str,
project_id: Option<&str>,
queries: &[(CliQueryMode, &str, &str)],
candidate_k: usize,
limit: usize,
hit_stats: &HashMap<String, HitStats>,
section: &str,
) -> Result<RankedTakeawayCollection> {
let mut by_chunk: HashMap<String, Takeaway> = HashMap::new();
let mut explanations = Vec::new();
for (mode, source, query) in queries {
let payload = cli_search_payload(
store,
tenant_id.to_string(),
project_id.map(str::to_string),
(*query).to_string(),
candidate_k,
false,
false,
None,
*mode,
false,
false,
false,
)
.await?;
merge_payload_candidates(
&mut by_chunk,
&mut explanations,
&payload,
section,
source,
query,
*mode,
);
}
let tag_counts = recurring_tag_counts(by_chunk.values());
let now_ms = now_ms() as i64;
let mut breakdowns = HashMap::new();
let mut takeaways = by_chunk
.into_values()
.map(|mut takeaway| {
let breakdown = priority_breakdown(&takeaway, &tag_counts, hit_stats, now_ms);
takeaway.priority = breakdown.total;
breakdowns.insert(takeaway.chunk_id.clone(), breakdown);
takeaway
})
.collect::<Vec<_>>();
let scored_takeaways = takeaways.clone();
let mut suppressed_reasons = suppress_finishes_covered_by_libraries(&mut takeaways)
.into_iter()
.map(|id| (id, "covered_by_library".to_string()))
.collect::<HashMap<_, _>>();
suppressed_reasons.extend(filter_startup_takeaways(&mut takeaways, section));
takeaways.sort_by(|left, right| {
right
.priority
.partial_cmp(&left.priority)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| right.timestamp_created.cmp(&left.timestamp_created))
});
let displayed_ids = takeaways
.iter()
.take(limit)
.enumerate()
.map(|(idx, takeaway)| (takeaway.chunk_id.clone(), idx + 1))
.collect::<HashMap<_, _>>();
takeaways.truncate(limit);
finalize_candidate_explanations(
&mut explanations,
&scored_takeaways,
&suppressed_reasons,
&displayed_ids,
&breakdowns,
);
Ok(RankedTakeawayCollection {
takeaways,
explanations,
})
}
fn suppress_finishes_covered_by_libraries(takeaways: &mut Vec<Takeaway>) -> BTreeSet<String> {
let covered: BTreeSet<(Option<String>, String)> = takeaways
.iter()
.filter(|takeaway| is_library_digest(&takeaway.tags))
.flat_map(|takeaway| {
extract_covered_task_ids(&takeaway.text)
.into_iter()
.map(|id| (takeaway.project_id.clone(), id))
.collect::<Vec<_>>()
})
.collect();
if covered.is_empty() {
return BTreeSet::new();
}
let suppressed = takeaways
.iter()
.filter(|takeaway| is_suppressible_finish(takeaway, &covered))
.map(|takeaway| takeaway.chunk_id.clone())
.collect::<BTreeSet<_>>();
takeaways.retain(|takeaway| !suppressed.contains(&takeaway.chunk_id));
suppressed
}
fn filter_startup_takeaways(
takeaways: &mut Vec<Takeaway>,
section: &str,
) -> HashMap<String, String> {
let mut suppressed = HashMap::new();
for takeaway in takeaways.iter() {
let reason = if has_boilerplate_action(takeaway) {
Some("boilerplate_action")
} else if section == "machine_wide" && !is_machine_wide_startup_relevant(takeaway) {
Some("machine_wide_unrelated")
} else {
None
};
if let Some(reason) = reason {
suppressed.insert(takeaway.chunk_id.clone(), reason.to_string());
}
}
takeaways.retain(|takeaway| !suppressed.contains_key(&takeaway.chunk_id));
suppressed.extend(suppress_duplicate_topic_takeaways(takeaways));
suppressed
}
fn suppress_duplicate_topic_takeaways(takeaways: &mut Vec<Takeaway>) -> HashMap<String, String> {
let mut best_by_topic: BTreeMap<String, String> = BTreeMap::new();
for takeaway in takeaways.iter() {
let key = topic_key(takeaway);
match best_by_topic.get(&key) {
Some(existing_id) => {
let existing = takeaways
.iter()
.find(|candidate| candidate.chunk_id == *existing_id)
.expect("best id came from takeaways");
if takeaway_preferred(takeaway, existing) {
best_by_topic.insert(key, takeaway.chunk_id.clone());
}
}
None => {
best_by_topic.insert(key, takeaway.chunk_id.clone());
}
}
}
let mut suppressed = HashMap::new();
for takeaway in takeaways.iter() {
let key = topic_key(takeaway);
if best_by_topic.get(&key) != Some(&takeaway.chunk_id) {
suppressed.insert(takeaway.chunk_id.clone(), format!("duplicate_topic:{key}"));
}
}
takeaways.retain(|takeaway| !suppressed.contains_key(&takeaway.chunk_id));
suppressed
}
fn takeaway_preferred(candidate: &Takeaway, current: &Takeaway) -> bool {
candidate
.priority
.partial_cmp(¤t.priority)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| candidate.timestamp_created.cmp(¤t.timestamp_created))
.then_with(|| current.chunk_id.cmp(&candidate.chunk_id))
.is_gt()
}
fn topic_key(takeaway: &Takeaway) -> String {
let mut tag_key = takeaway
.tags
.iter()
.filter(|tag| {
tag.starts_with("topic:")
|| tag.starts_with("repo:")
|| tag.starts_with("task:id:")
|| tag.starts_with("ctx:subsystem:")
|| tag.starts_with("ctx:file:")
})
.cloned()
.collect::<Vec<_>>();
tag_key.sort();
tag_key.dedup();
if !tag_key.is_empty() {
return tag_key.into_iter().take(4).collect::<Vec<_>>().join("|");
}
normalized_topic_terms(&takeaway.text)
}
fn normalized_topic_terms(text: &str) -> String {
let first_line = text.lines().next().unwrap_or(text);
let mut words = Vec::new();
for raw in first_line.split(|ch: char| !ch.is_ascii_alphanumeric()) {
let word = raw.to_ascii_lowercase();
if word.len() < 4 || TOPIC_STOPWORDS.contains(&word.as_str()) {
continue;
}
words.push(word);
if words.len() >= 8 {
break;
}
}
if words.is_empty() {
summarize_text(text, 64).to_ascii_lowercase()
} else {
words.join("-")
}
}
const TOPIC_STOPWORDS: &[&str] = &[
"after",
"also",
"because",
"before",
"from",
"into",
"keep",
"lesson",
"lessons",
"memory",
"project",
"records",
"task",
"that",
"this",
"validation",
"with",
];
fn is_library_digest(tags: &[String]) -> bool {
let generated = tags.iter().any(|tag| tag == "task:status:generated");
let role = tags
.iter()
.any(|tag| tag == "task:role:highlight_library" || tag == "task:role:project_brief");
generated && role
}
fn is_suppressible_finish(
takeaway: &Takeaway,
covered: &BTreeSet<(Option<String>, String)>,
) -> bool {
let is_finish = takeaway
.tags
.iter()
.any(|tag| tag == "task:kind:task_finish");
if !is_finish {
return false;
}
if user_priority_at_least(&takeaway.tags, USER_PRESERVE_PRIORITY_THRESHOLD) {
return false;
}
takeaway.tags.iter().any(|tag| {
tag.strip_prefix("task:id:")
.map(|id| covered.contains(&(takeaway.project_id.clone(), id.to_string())))
.unwrap_or(false)
})
}
fn user_priority_at_least(tags: &[String], threshold: u8) -> bool {
tags.iter().any(|tag| {
let value = tag
.strip_prefix("priority:")
.or_else(|| tag.strip_prefix("importance:"));
match value.and_then(|v| v.parse::<f32>().ok()) {
Some(n) => n.is_finite() && n >= threshold as f32,
None => false,
}
})
}
fn extract_covered_task_ids(text: &str) -> Vec<String> {
let mut ids = Vec::new();
for line in text.lines() {
let trimmed = line.trim_start();
let Some(rest) = trimmed.strip_prefix("Covers tasks:") else {
continue;
};
for token in rest.split(',') {
let token = token.trim();
if let Some(id) = token.strip_prefix("task:id:") {
let id = id.trim().trim_end_matches('.').trim_end_matches(';');
if !id.is_empty() {
ids.push(id.to_string());
}
}
}
}
ids
}
fn merge_payload_candidates(
by_chunk: &mut HashMap<String, Takeaway>,
explanations: &mut Vec<MemoryMdCandidateExplanation>,
payload: &Value,
section: &str,
source: &str,
query: &str,
mode: CliQueryMode,
) {
let Some(results) = payload.get("results").and_then(Value::as_array) else {
return;
};
for (idx, result) in results.iter().enumerate() {
let Some(chunk_id) = result.get("chunk_id").and_then(Value::as_str) else {
continue;
};
let text = result
.get("text")
.and_then(Value::as_str)
.unwrap_or_default()
.trim();
let tags = result
.get("tags")
.and_then(Value::as_array)
.map(|items| {
items
.iter()
.filter_map(Value::as_str)
.map(str::to_string)
.collect::<Vec<_>>()
})
.unwrap_or_default();
let mut explanation =
candidate_explanation(section, source, query, mode, idx + 1, result, &tags, text);
if text.is_empty() {
explanation.display_status = "filtered".to_string();
explanation.filter_reason = Some("empty_text".to_string());
explanations.push(explanation);
continue;
}
if is_generated_digest_takeaway(&tags) {
explanation.display_status = "filtered".to_string();
explanation.filter_reason = Some("generated_digest_wrapper".to_string());
explanations.push(explanation);
continue;
}
if is_fragment_like_candidate(&tags, text) {
explanation.display_status = "filtered".to_string();
explanation.filter_reason = Some("fragment_like".to_string());
explanation.quality_flags.push("fragment_like".to_string());
explanations.push(explanation);
continue;
}
if tags.iter().any(|tag| tag.starts_with("kind:superseded")) {
explanation.display_status = "filtered".to_string();
explanation.filter_reason = Some("superseded_tag".to_string());
explanations.push(explanation);
continue;
}
let score = result.get("score").and_then(Value::as_f64).unwrap_or(0.0) as f32;
let entry = by_chunk
.entry(chunk_id.to_string())
.or_insert_with(|| Takeaway {
chunk_id: chunk_id.to_string(),
tenant_id: result
.get("tenant_id")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string(),
project_id: result
.get("project_id")
.and_then(Value::as_str)
.map(str::to_string),
text: text.to_string(),
score,
priority: 0.0,
chunk_type: result
.get("chunk_type")
.and_then(Value::as_str)
.unwrap_or("unknown")
.to_string(),
timestamp_created: result
.get("timestamp_created")
.and_then(Value::as_i64)
.unwrap_or_default(),
tags,
sources: BTreeSet::new(),
occurrences: 0,
});
entry.score = entry.score.max(score);
entry.occurrences = entry.occurrences.saturating_add(1);
entry.sources.insert(source.to_string());
explanations.push(explanation);
}
}
fn candidate_explanation(
section: &str,
source: &str,
query: &str,
mode: CliQueryMode,
raw_rank: usize,
result: &Value,
tags: &[String],
text: &str,
) -> MemoryMdCandidateExplanation {
let generated_digest = is_generated_digest_takeaway(tags)
|| text
.to_ascii_lowercase()
.contains("task digest status generated");
let mut quality_flags = Vec::new();
if is_fragment_like_candidate(tags, text) {
quality_flags.push("fragment_like".to_string());
}
if generated_digest
|| text
.to_ascii_lowercase()
.contains("task digest status generated")
{
quality_flags.push("generated_wrapper".to_string());
}
MemoryMdCandidateExplanation {
section: section.to_string(),
source: source.to_string(),
query: query.to_string(),
mode: query_mode_label(mode).to_string(),
raw_rank,
chunk_id: result
.get("chunk_id")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string(),
tenant_id: result
.get("tenant_id")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string(),
project_id: result
.get("project_id")
.and_then(Value::as_str)
.map(str::to_string),
chunk_type: result
.get("chunk_type")
.and_then(Value::as_str)
.unwrap_or("unknown")
.to_string(),
timestamp_created: result
.get("timestamp_created")
.and_then(Value::as_i64)
.unwrap_or_default(),
search_score: result.get("score").and_then(Value::as_f64).unwrap_or(0.0) as f32,
priority_score: None,
priority_breakdown: None,
display_status: "candidate".to_string(),
filter_reason: None,
display_rank: None,
generated_digest,
quality_flags,
topic_key: None,
tags: tags.to_vec(),
matched_sources: vec![source.to_string()],
}
}
fn finalize_candidate_explanations(
explanations: &mut [MemoryMdCandidateExplanation],
scored_takeaways: &[Takeaway],
suppressed_reasons: &HashMap<String, String>,
displayed_ids: &HashMap<String, usize>,
breakdowns: &HashMap<String, PriorityBreakdown>,
) {
let by_id = scored_takeaways
.iter()
.map(|takeaway| (takeaway.chunk_id.as_str(), takeaway))
.collect::<HashMap<_, _>>();
for explanation in explanations {
if explanation.display_status == "filtered" {
continue;
}
if let Some(takeaway) = by_id.get(explanation.chunk_id.as_str()) {
explanation.priority_score = Some(takeaway.priority);
explanation.priority_breakdown = breakdowns.get(&takeaway.chunk_id).cloned();
explanation.matched_sources = takeaway.sources.iter().cloned().collect();
explanation.topic_key = Some(topic_key(takeaway));
}
if let Some(reason) = suppressed_reasons.get(&explanation.chunk_id) {
explanation.display_status = "filtered".to_string();
explanation.filter_reason = Some(reason.clone());
explanation.quality_flags.push(reason.clone());
} else if let Some(rank) = displayed_ids.get(&explanation.chunk_id) {
explanation.display_status = "displayed".to_string();
explanation.display_rank = Some(*rank);
} else {
explanation.display_status = "filtered".to_string();
explanation.filter_reason = Some("below_display_limit".to_string());
}
}
}
fn query_mode_label(mode: CliQueryMode) -> &'static str {
match mode {
CliQueryMode::Generic => "generic",
CliQueryMode::BriefProject => "brief_project",
CliQueryMode::ResumeTask => "resume_task",
CliQueryMode::FindFailures => "find_failures",
CliQueryMode::FindDecisions => "find_decisions",
CliQueryMode::FindEvidence => "find_evidence",
CliQueryMode::FindHighlights => "find_highlights",
}
}
fn recurring_tag_counts<'a>(
takeaways: impl Iterator<Item = &'a Takeaway>,
) -> HashMap<String, usize> {
let mut counts = HashMap::new();
for takeaway in takeaways {
for tag in takeaway.tags.iter().filter(|tag| high_signal_tag(tag)) {
*counts.entry(tag.to_string()).or_insert(0) += 1;
}
}
counts
}
#[cfg(test)]
fn priority_score(
takeaway: &Takeaway,
tag_counts: &HashMap<String, usize>,
hit_stats: &HashMap<String, HitStats>,
now_ms: i64,
) -> f32 {
priority_breakdown(takeaway, tag_counts, hit_stats, now_ms).total
}
fn priority_breakdown(
takeaway: &Takeaway,
tag_counts: &HashMap<String, usize>,
hit_stats: &HashMap<String, HitStats>,
now_ms: i64,
) -> PriorityBreakdown {
let explicit = explicit_priority(&takeaway.tags).unwrap_or(0.0);
let kind_weight = takeaway
.tags
.iter()
.map(|tag| match tag.as_str() {
"kind:decision" => 12.0,
"kind:finish" => 10.0,
"kind:evidence" => 8.0,
"kind:run" => 5.0,
"kind:progress" => 3.0,
_ if tag.starts_with("ctx:file:") => 5.0,
_ if tag.starts_with("ctx:subsystem:") => 4.0,
_ => 0.0,
})
.sum::<f32>();
let type_weight = match takeaway.chunk_type.as_str() {
"decision" => 10.0,
"summary" => 6.0,
"research" => 5.0,
"trace" => 3.0,
"plan" => 2.0,
_ => 0.0,
};
let recurrence = takeaway
.tags
.iter()
.filter_map(|tag| tag_counts.get(tag))
.map(|count| count.saturating_sub(1).min(5) as f32)
.sum::<f32>();
let multi_query = takeaway.occurrences.saturating_sub(1).min(4) as f32 * 3.0;
let search_score = takeaway.score.clamp(0.0, 25.0) * 2.0;
let library_bonus = if takeaway.tags.iter().any(|tag| {
tag == "task:role:highlight_library"
|| tag == "task:role:project_brief"
|| tag.starts_with("kind:consolidated")
}) {
15.0
} else {
0.0
};
let utility = hit_stats
.get(&takeaway.chunk_id)
.map(|stats| (stats.selected_count.min(10) as f32) * 0.8)
.unwrap_or(0.0);
let staleness_penalty = if !hit_stats.contains_key(&takeaway.chunk_id)
&& takeaway.timestamp_created > 0
&& now_ms.saturating_sub(takeaway.timestamp_created) > STALE_CHUNK_AGE_MS
{
-2.0
} else {
0.0
};
let total = explicit
+ kind_weight
+ type_weight
+ recurrence
+ multi_query
+ search_score
+ library_bonus
+ utility
+ staleness_penalty;
PriorityBreakdown {
explicit,
kind_weight,
type_weight,
recurrence,
multi_query,
search_score,
library_bonus,
utility,
staleness_penalty,
total,
}
}
fn explicit_priority(tags: &[String]) -> Option<f32> {
tags.iter().find_map(|tag| {
let value = tag
.strip_prefix("priority:")
.or_else(|| tag.strip_prefix("importance:"))?;
let parsed = value.parse::<f32>().ok()?;
if !parsed.is_finite() {
return None;
}
let scaled = if parsed <= 10.0 {
parsed * 10.0
} else {
parsed.min(100.0)
};
Some(scaled.clamp(0.0, 100.0))
})
}
fn high_signal_tag(tag: &str) -> bool {
tag.starts_with("kind:")
|| tag.starts_with("ctx:")
|| tag.starts_with("priority:")
|| tag.starts_with("importance:")
}
fn is_generated_digest_takeaway(tags: &[String]) -> bool {
tags.iter().any(|tag| tag == "task:status:generated")
&& tags
.iter()
.any(|tag| tag.starts_with("task:role:") || tag.starts_with("task:digest:"))
}
fn is_fragment_like_takeaway(takeaway: &Takeaway) -> bool {
is_fragment_like_candidate(&takeaway.tags, &takeaway.text)
}
fn is_fragment_like_candidate(tags: &[String], text: &str) -> bool {
if tags.iter().any(|tag| {
tag.strip_prefix("chunk_index:")
.and_then(|value| value.parse::<usize>().ok())
.map(|idx| idx > 0)
.unwrap_or(false)
}) {
return true;
}
let trimmed = text.trim_start();
if trimmed.is_empty() {
return false;
}
if trimmed.starts_with("...")
|| trimmed
.chars()
.next()
.map(|ch| matches!(ch, ',' | ';' | ':' | ')' | ']' | '}'))
.unwrap_or(false)
{
return true;
}
let lowered = trimmed.to_ascii_lowercase();
const CONTINUATION_PREFIXES: &[&str] = &[
"and ",
"but ",
"which ",
"where ",
"then ",
"therefore ",
"from there ",
"as a result ",
];
CONTINUATION_PREFIXES
.iter()
.any(|prefix| lowered.starts_with(prefix))
}
fn has_boilerplate_action(takeaway: &Takeaway) -> bool {
explicit_agent_action(&takeaway.text).is_none()
&& takeaway_category(takeaway).reason == "ranked project takeaway"
}
fn is_machine_wide_startup_relevant(takeaway: &Takeaway) -> bool {
if takeaway.project_id.is_none() {
return true;
}
if user_priority_at_least(&takeaway.tags, USER_PRESERVE_PRIORITY_THRESHOLD)
&& takeaway.tags.iter().any(|tag| {
tag == "global"
|| tag == "always"
|| tag == "machine"
|| tag == "kind:consolidated"
|| tag.starts_with("global:")
|| tag.starts_with("machine:")
})
{
return true;
}
let lowered = takeaway.text.to_ascii_lowercase();
lowered.contains("machine-wide")
|| lowered.contains("all projects")
|| lowered.contains("global default")
|| lowered.contains("always ")
}
fn render_memory_md(
project_state: &ProjectState,
health_lines: &[String],
project_takeaways: &[Takeaway],
global_takeaways: &[Takeaway],
) -> String {
let mut out = String::new();
out.push_str("# memory.md\n\n");
out.push_str("Generated by `memd memory-md`.\n\n");
render_latest_project_state(&mut out, project_state);
if !health_lines.is_empty() {
out.push_str("## Memory health\n\n");
for line in health_lines {
out.push_str(&format!("- {line}\n"));
}
out.push('\n');
}
out.push_str("## Session-Start Use\n\n");
out.push_str("- Read this file before task-specific retrieval.\n");
out.push_str("- Refresh it at the start of substantive sessions with `memd memory-md`.\n");
out.push_str("- Then run task-specific `memd agent-context` or `memd search`.\n\n");
out.push_str("## Agent Guidance\n\n");
out.push_str("- Treat fact-library items as durable memory to verify against current files, logs, or tests before applying.\n");
out.push_str("- Use `Latest Project State` for the first resume pass; use `memd agent-context` for task-specific retrieval.\n\n");
out.push_str("## Scoring\n\n");
out.push_str("- Explicit `priority:N` or `importance:N` tags dominate when present.\n");
out.push_str("- Decisions, finishes, evidence, recurring tags, multi-query matches, and search score increase priority.\n");
out.push_str("- Repeated lessons should be recorded again with a higher `priority:N` tag when they keep mattering.\n\n");
render_section(&mut out, "Project Fact Library", project_takeaways);
if !global_takeaways.is_empty() {
render_section(&mut out, "Machine-Wide Fact Library", global_takeaways);
}
out
}
fn render_latest_project_state(out: &mut String, state: &ProjectState) {
out.push_str("## Latest Project State\n\n");
out.push_str("### Scope & Freshness\n\n");
out.push_str(&format!("- tenant_id: `{}`\n", state.tenant_id));
out.push_str(&format!(
"- project_id: `{}`\n",
state.project_id.as_deref().unwrap_or("<none>")
));
if let Some(configured) = state.configured_project_dir.as_deref() {
out.push_str(&format!("- configured_project_dir: `{configured}`\n"));
} else {
out.push_str("- configured_project_dir: `<none>`\n");
}
out.push_str(&format!(
"- resolved_project_dir: `{}`\n",
state.resolved_project_dir
));
out.push_str(&format!(
"- generated_unix_ms: `{}`\n\n",
state.generated_unix_ms
));
out.push_str("### Worktree\n\n");
out.push_str(&format!("- git: {}\n\n", state.git.summary));
out.push_str("### Latest Work\n\n");
if let Some(task) = &state.latest_task {
out.push_str(&format!("- task: {}\n", render_state_signal(task)));
} else {
out.push_str("- task: none detected in `tasks/todo.md`\n");
}
if let Some(handoff) = &state.latest_handoff {
out.push_str(&format!("- handoff: {}\n", render_state_signal(handoff)));
} else {
out.push_str("- handoff: none detected under `docs/handoffs/`\n");
}
if let Some(vcs) = &state.latest_vcs {
out.push_str(&format!("- vcs: {}\n", render_state_signal(vcs)));
}
out.push('\n');
out.push_str("### Next Actions\n\n");
if state.next_actions.is_empty() {
out.push_str("- No open next actions detected from `tasks/todo.md`.\n\n");
} else {
for (idx, action) in state.next_actions.iter().take(3).enumerate() {
out.push_str(&format!(
"{}. {} ([{}:{}])\n",
idx + 1,
inline_code_text(&action.text),
action.source_path,
action.line
));
}
out.push('\n');
}
out.push_str("### Memory Warnings\n\n");
let warnings = project_state_warnings(state);
if warnings.is_empty() {
out.push_str("- none detected\n\n");
} else {
for warning in warnings {
out.push_str(&format!("- {warning}\n"));
}
out.push('\n');
}
}
fn render_state_signal(signal: &StateSignal) -> String {
let source = match signal.line {
Some(line) => format!("{}:{}", signal.source_path, line),
None => signal.source_path.clone(),
};
match signal.heading.as_deref() {
Some(heading) if !heading.is_empty() => {
format!(
"{} - {} ([{}])",
heading,
inline_code_text(&signal.text),
source
)
}
_ => format!("{} ([{}])", inline_code_text(&signal.text), source),
}
}
fn project_state_warnings(state: &ProjectState) -> Vec<String> {
let mut warnings = Vec::new();
warnings.extend(state.scope_warnings.iter().cloned());
if let Some(warning) = &state.git.warning {
warnings.push(warning.clone());
}
if let Some(unreadable) = state.memory.unreadable_active_chunks {
if unreadable > 0 {
warnings.push(format!(
"memory degraded: {unreadable} active chunks could not be read from payload segments"
));
}
}
if let Some(warning) = &state.memory.scan_warning {
warnings.push(warning.clone());
}
warnings.extend(state.collection_warnings.iter().cloned());
warnings
}
fn evaluate_agent_usefulness(
state: &ProjectState,
project_takeaways: &[Takeaway],
global_takeaways: &[Takeaway],
) -> AgentUsefulnessMetrics {
let latest_project_state_present = true;
let scope_present = !state.tenant_id.is_empty() && !state.resolved_project_dir.is_empty();
let git_state_present = state.git.available;
let git_state_present_or_not_git_repo = state.git.available || state.git.not_git_repo;
let latest_work_present =
state.latest_task.is_some() || state.latest_handoff.is_some() || state.latest_vcs.is_some();
let next_action_present = !state.next_actions.is_empty();
let no_open_tasks_detected = state.next_actions.is_empty();
let scope_health_present = scope_health_checked(state);
let memory_degraded_warning_present = state
.memory
.unreadable_active_chunks
.map(|count| count > 0)
.unwrap_or(false);
let fragment_count = project_takeaways
.iter()
.chain(global_takeaways.iter())
.filter(|takeaway| is_fragment_like_takeaway(takeaway))
.count();
let duplicate_cluster_count = visible_duplicate_cluster_count(project_takeaways)
+ visible_duplicate_cluster_count(global_takeaways);
let boilerplate_action_count = project_takeaways
.iter()
.chain(global_takeaways.iter())
.filter(|takeaway| has_boilerplate_action(takeaway))
.count();
let unrelated_machine_items = global_takeaways
.iter()
.filter(|takeaway| !is_machine_wide_startup_relevant(takeaway))
.count();
let source_backed_next_actions = state
.next_actions
.iter()
.all(|action| !action.source_path.is_empty() && action.line > 0);
let answerability_passed = latest_project_state_present
&& scope_present
&& git_state_present_or_not_git_repo
&& latest_work_present
&& (next_action_present || no_open_tasks_detected)
&& fragment_count == 0
&& boilerplate_action_count == 0;
AgentUsefulnessMetrics {
latest_project_state_present,
scope_present,
git_state_present,
git_state_present_or_not_git_repo,
latest_work_present,
next_action_present,
no_open_tasks_detected,
scope_health_present,
memory_degraded_warning_present,
fragment_count,
duplicate_cluster_count,
boilerplate_action_count,
unrelated_machine_items,
source_backed_next_actions,
answerability_passed,
}
}
fn scope_health_checked(state: &ProjectState) -> bool {
if let Some(configured) = state.configured_project_dir.as_deref() {
scope_path_drift_warning(configured, Path::new(&state.resolved_project_dir)).is_none()
|| state
.scope_warnings
.iter()
.any(|warning| warning.starts_with("scope mismatch:"))
} else {
true
}
}
fn visible_duplicate_cluster_count(takeaways: &[Takeaway]) -> usize {
let mut counts = BTreeMap::new();
for takeaway in takeaways {
*counts.entry(topic_key(takeaway)).or_insert(0usize) += 1;
}
counts.values().filter(|count| **count > 1).count()
}
fn render_section(out: &mut String, title: &str, takeaways: &[Takeaway]) {
out.push_str(&format!("## {title}\n\n"));
if takeaways.is_empty() {
out.push_str("- No takeaways found yet.\n\n");
return;
}
let mut categorized = takeaways
.iter()
.map(|takeaway| (takeaway_category(takeaway), takeaway))
.collect::<Vec<_>>();
categorized.sort_by(|(left_category, left), (right_category, right)| {
left_category
.order
.cmp(&right_category.order)
.then_with(|| {
right
.priority
.partial_cmp(&left.priority)
.unwrap_or(std::cmp::Ordering::Equal)
})
.then_with(|| right.timestamp_created.cmp(&left.timestamp_created))
});
for (heading, _) in TAKEAWAY_CATEGORIES {
let group = categorized
.iter()
.filter(|(category, _)| category.heading == *heading)
.collect::<Vec<_>>();
if group.is_empty() {
continue;
}
out.push_str(&format!("### {heading}\n\n"));
for (idx, (category, takeaway)) in group.iter().enumerate() {
render_takeaway(out, idx + 1, takeaway, category.reason);
}
out.push('\n');
}
}
fn render_takeaway(out: &mut String, idx: usize, takeaway: &Takeaway, reason: &str) {
out.push_str(&format!(
"{}. {}\n",
idx,
summarize_text(&takeaway.text, 320)
));
out.push_str(&format!(
" - priority: `{:.1}`; chunk: `{}`; type: `{}`; tenant: `{}`",
takeaway.priority, takeaway.chunk_id, takeaway.chunk_type, takeaway.tenant_id
));
if let Some(project_id) = takeaway.project_id.as_deref() {
out.push_str(&format!("; project: `{project_id}`"));
}
if takeaway.timestamp_created > 0 {
out.push_str(&format!(
"; created_unix_ms: `{}`",
takeaway.timestamp_created
));
}
out.push('\n');
out.push_str(&format!(" - reason: `{reason}`\n"));
out.push_str(&format!(
" - agent action: `{}`\n",
inline_code_text(&agent_action_for_takeaway(takeaway, reason))
));
if !takeaway.tags.is_empty() {
out.push_str(&format!(
" - tags: `{}`\n",
takeaway
.tags
.iter()
.take(8)
.cloned()
.collect::<Vec<_>>()
.join(", ")
));
}
if !takeaway.sources.is_empty() {
out.push_str(&format!(
" - matched: `{}`\n",
takeaway
.sources
.iter()
.cloned()
.collect::<Vec<_>>()
.join(", ")
));
}
}
fn agent_action_for_takeaway(takeaway: &Takeaway, reason: &str) -> String {
if let Some(action) = explicit_agent_action(&takeaway.text) {
return summarize_text(&action, 240);
}
let evidence = summarize_text(&takeaway.text, 180);
match reason {
"decision or rationale" => {
format!("Apply this decision when the same scope appears: {evidence}")
}
"validated fix or result" => {
format!("Reuse this validated fix when the same failure appears: {evidence}")
}
"failure or root-cause evidence" => {
format!(
"Check for this known failure before retrying and avoid repeating it: {evidence}"
)
}
"command, path, or parameter evidence" => {
format!("Use or verify this command/path exactly when relevant: {evidence}")
}
"explicit follow-up" => {
format!(
"Treat this as pending work and resolve it before claiming completion: {evidence}"
)
}
"evidence or run result" => {
format!("Use this as evidence only after confirming it still matches current files or tests: {evidence}")
}
_ => format!("Translate this takeaway into a task-specific rule before acting: {evidence}"),
}
}
pub(super) fn explicit_agent_action(text: &str) -> Option<String> {
const MARKERS: &[(&str, &str)] = &[
("agent action:", ""),
("action:", ""),
("rule:", ""),
("do:", "Do "),
("use:", "Use "),
("avoid:", "Avoid "),
("prefer:", "Prefer "),
("check:", "Check "),
("verify:", "Verify "),
("next step:", "Do next: "),
("follow-up:", "Follow up: "),
("followup:", "Follow up: "),
];
for marker in explicit_action_markers(text, MARKERS) {
let body = explicit_action_body(text, marker.start, marker.marker);
if body.is_empty() {
continue;
}
let candidate = format!("{}{}", marker.prefix, body);
if is_concrete_agent_action_text(&candidate) {
return Some(candidate);
}
}
None
}
#[derive(Debug, Clone, Copy)]
struct ExplicitActionMarker<'a> {
start: usize,
marker: &'a str,
prefix: &'a str,
}
fn explicit_action_markers<'a>(
text: &str,
markers: &'a [(&'a str, &'a str)],
) -> Vec<ExplicitActionMarker<'a>> {
let lowered = text.to_ascii_lowercase();
let mut found = Vec::new();
for (marker, prefix) in markers {
let mut search_start = 0;
while let Some(relative_start) = lowered[search_start..].find(marker) {
let start = search_start + relative_start;
found.push(ExplicitActionMarker {
start,
marker,
prefix,
});
search_start = start + marker.len();
}
}
found.sort_by(|left, right| {
left.start
.cmp(&right.start)
.then_with(|| right.marker.len().cmp(&left.marker.len()))
});
found
}
fn explicit_action_body<'a>(text: &'a str, marker_start: usize, marker: &str) -> &'a str {
let body_start = marker_start + marker.len();
let lowered_tail = text[body_start..].to_ascii_lowercase();
let line_end = text[body_start..]
.find(|ch| matches!(ch, '\n' | '\r'))
.map(|offset| body_start + offset)
.unwrap_or(text.len());
let next_marker = [
"agent action:",
"action:",
"rule:",
"do:",
"use:",
"avoid:",
"prefer:",
"check:",
"verify:",
"next step:",
"follow-up:",
"followup:",
]
.iter()
.filter_map(|candidate| {
lowered_tail
.find(candidate)
.map(|offset| body_start + offset)
})
.min()
.unwrap_or(text.len());
let body_end = line_end.min(next_marker);
text[body_start..body_end]
.trim()
.trim_end_matches(|ch| matches!(ch, '.' | ';'))
.trim_end()
}
fn inline_code_text(text: &str) -> String {
text.replace('`', "'")
}
fn takeaway_category(takeaway: &Takeaway) -> TakeawayCategory {
let lowered = takeaway.text.to_ascii_lowercase();
let has_tag = |needle: &str| takeaway.tags.iter().any(|tag| tag == needle);
let has_source = |needle: &str| takeaway.sources.iter().any(|source| source == needle);
if takeaway.chunk_type == "decision"
|| has_tag("kind:decision")
|| lowered.contains("decision:")
|| lowered.contains("rationale:")
{
return category("Decisions", "decision or rationale");
}
if lowered.contains("fix:")
|| lowered.contains("validated fix")
|| lowered.contains("validated:")
|| lowered.contains("validation:")
|| lowered.contains("fixed by")
|| lowered.contains("solution:")
|| lowered.contains("passed")
|| lowered.contains("confirmed")
|| lowered.contains("reproduced")
|| lowered.contains("0 failures")
|| lowered.contains("no failures")
|| lowered.contains("resolved after")
|| lowered.contains("resolved by")
{
return category("Validated Fixes", "validated fix or result");
}
if has_tag("kind:failure")
|| lowered.contains("root cause")
|| lowered.contains("failed because")
|| lowered.contains("failure:")
|| lowered.contains("blocker")
{
return category("Known Failures", "failure or root-cause evidence");
}
if lowered.contains("command:")
|| lowered.contains("path:")
|| lowered.contains("parameter:")
|| lowered.contains("parameters:")
|| lowered.contains("/home/")
|| lowered.contains("crates/")
|| lowered.contains("tasks/")
|| lowered.contains(".rs")
|| lowered.contains(".md")
|| lowered.contains("http://")
|| lowered.contains("https://")
{
return category("Commands/Paths", "command, path, or parameter evidence");
}
if lowered.contains("next step:")
|| lowered.contains("follow-up:")
|| lowered.contains("followup:")
|| lowered.contains("followups:")
{
return category("Open Follow-ups", "explicit follow-up");
}
if has_tag("kind:evidence") || has_tag("kind:run") || has_source("project_highlights") {
return category("Evidence", "evidence or run result");
}
category("Other Takeaways", "ranked project takeaway")
}
fn category(heading: &'static str, reason: &'static str) -> TakeawayCategory {
let order = TAKEAWAY_CATEGORIES
.iter()
.find_map(|(candidate, order)| (*candidate == heading).then_some(*order))
.unwrap_or(u8::MAX);
TakeawayCategory {
heading,
reason,
order,
}
}
#[derive(Debug, Clone)]
struct DisplayedMemoryMdItem {
category: String,
text: String,
details: Vec<String>,
}
fn evaluate_memory_md_quality(content: &str, top_n: usize) -> MemoryMdQualityReport {
let items = parse_project_takeaways(content, top_n);
let displayed_count = items.len();
let useful_count = items
.iter()
.filter(|item| is_useful_display_item(item))
.count();
let generated_wrapper_count = items
.iter()
.filter(|item| is_generated_wrapper_display_item(item))
.count();
let missing_reason_count = items
.iter()
.filter(|item| !item.details.iter().any(|line| line.contains("reason: `")))
.count();
let missing_action_count = items
.iter()
.filter(|item| !has_concrete_agent_action(item))
.count();
let useful_ratio = if displayed_count == 0 {
0.0
} else {
useful_count as f64 / displayed_count as f64
};
MemoryMdQualityReport {
displayed_count,
useful_count,
generated_wrapper_count,
missing_reason_count,
missing_action_count,
useful_ratio,
}
}
fn parse_project_takeaways(content: &str, top_n: usize) -> Vec<DisplayedMemoryMdItem> {
let mut in_project = false;
let mut category = "Other Takeaways".to_string();
let mut items = Vec::new();
let mut current: Option<DisplayedMemoryMdItem> = None;
for line in content.lines() {
if line.starts_with("## ") {
if in_project {
if let Some(item) = current.take() {
items.push(item);
}
break;
}
let heading = line.trim();
in_project = heading == "## Project Takeaways" || heading == "## Project Fact Library";
continue;
}
if !in_project {
continue;
}
if let Some(heading) = line.strip_prefix("### ") {
if let Some(item) = current.take() {
items.push(item);
if items.len() >= top_n {
break;
}
}
category = heading.trim().to_string();
continue;
}
if let Some(text) = ordered_item_text(line) {
if let Some(item) = current.take() {
items.push(item);
if items.len() >= top_n {
break;
}
}
current = Some(DisplayedMemoryMdItem {
category: category.clone(),
text: text.to_string(),
details: Vec::new(),
});
continue;
}
if let Some(item) = current.as_mut() {
let trimmed = line.trim();
if !trimmed.is_empty() {
item.details.push(trimmed.to_string());
}
}
}
if items.len() < top_n {
if let Some(item) = current {
items.push(item);
}
}
items.truncate(top_n);
items
}
fn ordered_item_text(line: &str) -> Option<&str> {
let (number, rest) = line.split_once(". ")?;
if number.is_empty() || !number.chars().all(|ch| ch.is_ascii_digit()) {
return None;
}
Some(rest.trim())
}
fn is_useful_display_item(item: &DisplayedMemoryMdItem) -> bool {
if is_generated_wrapper_display_item(item) {
return false;
}
if !has_concrete_agent_action(item) {
return false;
}
matches!(
item.category.as_str(),
"Decisions"
| "Validated Fixes"
| "Known Failures"
| "Commands/Paths"
| "Open Follow-ups"
| "Evidence"
) || {
let lowered = item.text.to_ascii_lowercase();
lowered.contains("decision:")
|| lowered.contains("rationale:")
|| lowered.contains("validation:")
|| lowered.contains("validated")
|| lowered.contains("root cause")
|| lowered.contains("command:")
|| lowered.contains("path:")
|| lowered.contains("follow-up:")
|| lowered.contains("next step:")
}
}
fn has_concrete_agent_action(item: &DisplayedMemoryMdItem) -> bool {
item.details.iter().any(|line| {
let lowered = line.to_ascii_lowercase();
let Some(action) = lowered.strip_prefix("- agent action: `") else {
return false;
};
let action = action.trim_end_matches('`').trim();
is_concrete_agent_action_text(action)
&& !action.contains("translate this takeaway into a task-specific rule")
})
}
fn is_concrete_agent_action_text(action: &str) -> bool {
action.chars().count() >= 24 && contains_action_verb(action)
}
fn contains_action_verb(text: &str) -> bool {
text.split(|ch: char| !ch.is_ascii_alphabetic())
.any(|word| {
crate::write_admission::ACTION_VERBS.contains(&word.to_ascii_lowercase().as_str())
})
}
fn is_generated_wrapper_display_item(item: &DisplayedMemoryMdItem) -> bool {
let mut lowered = item.text.to_ascii_lowercase();
for line in &item.details {
lowered.push(' ');
lowered.push_str(&line.to_ascii_lowercase());
}
lowered.contains("task digest status generated")
|| lowered.contains("task:status:generated")
|| lowered.contains("task:role:highlight_library")
|| lowered.contains("task:digest:")
|| (lowered.contains("highlight library for") && lowered.contains("ranked lessons"))
}
fn summarize_text(text: &str, limit: usize) -> String {
let collapsed = text.split_whitespace().collect::<Vec<_>>().join(" ");
if collapsed.chars().count() <= limit {
return collapsed;
}
let mut out = collapsed
.chars()
.take(limit.saturating_sub(3))
.collect::<String>();
out.push_str("...");
out
}
fn read_project_scope(project_dir: &std::path::Path) -> Result<Option<ProjectScopeConfig>> {
let path = project_dir.join(".memd/project_scope.json");
if !path.exists() {
return Ok(None);
}
let text = std::fs::read_to_string(&path)?;
let scope = serde_json::from_str(&text).map_err(|e| {
MemdError::ValidationError(format!("failed to parse {}: {e}", path.display()))
})?;
Ok(Some(scope))
}
fn now_ms() -> u128 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_millis())
.unwrap_or(0)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::store::{
DuplicateHealth, HealthCounts, IndexCoverageHealth, PayloadHealth, StoreHealthSnapshot,
StoreStats,
};
use crate::types::{ChunkId, MemoryChunk};
#[test]
fn explicit_priority_scales_small_values_and_caps_large_values() {
assert_eq!(explicit_priority(&["priority:7".to_string()]), Some(70.0));
assert_eq!(
explicit_priority(&["importance:120".to_string()]),
Some(100.0)
);
}
#[test]
fn git_state_handles_clean_dirty_no_repo_and_missing_git() {
let no_repo = tempfile::tempdir().unwrap();
let no_repo_state = collect_git_state(no_repo.path(), "git");
if no_repo_state.warning.as_deref() == Some("git unavailable: executable not found") {
return;
}
assert!(no_repo_state.not_git_repo);
assert!(!no_repo_state.available);
let missing = collect_git_state(no_repo.path(), "definitely-not-a-git-binary");
assert!(!missing.available);
assert!(missing
.warning
.as_deref()
.unwrap_or_default()
.contains("executable not found"));
let repo = tempfile::tempdir().unwrap();
let init = Command::new("git")
.arg("-C")
.arg(repo.path())
.arg("init")
.output()
.unwrap();
assert!(init.status.success());
let clean = collect_git_state(repo.path(), "git");
assert!(clean.available);
assert_eq!(clean.clean, Some(true));
fs::write(repo.path().join("dirty.txt"), "dirty").unwrap();
let dirty = collect_git_state(repo.path(), "git");
assert!(dirty.available);
assert_eq!(dirty.clean, Some(false));
assert!(dirty.changed_entries > 0);
}
#[test]
fn task_state_uses_nested_heading_for_first_open_action() {
let dir = tempfile::tempdir().unwrap();
let tasks_dir = dir.path().join("tasks");
fs::create_dir_all(&tasks_dir).unwrap();
fs::write(
tasks_dir.join("todo.md"),
"# Completed\n\n- [x] old\n\n# Current\n\n## Deep Work\n\n- [ ] implement nested action\n",
)
.unwrap();
let scan = collect_task_state(dir.path());
let latest = scan.latest_task.expect("latest task");
assert_eq!(latest.heading.as_deref(), Some("Deep Work"));
assert_eq!(scan.next_actions.len(), 1);
assert_eq!(scan.next_actions[0].line, 9);
assert_eq!(scan.next_actions[0].source_path, "tasks/todo.md");
}
#[test]
fn task_state_ignores_checked_boxes_and_incidental_pending_text() {
let dir = tempfile::tempdir().unwrap();
let tasks_dir = dir.path().join("tasks");
fs::create_dir_all(&tasks_dir).unwrap();
fs::write(
tasks_dir.join("todo.md"),
"# Current\n\n- [x] resolved the pending migration\n- note: this pending migration is already closed\n- [ ] real open item\n",
)
.unwrap();
let scan = collect_task_state(dir.path());
assert_eq!(scan.next_actions.len(), 1);
assert_eq!(scan.next_actions[0].text, "real open item");
}
#[test]
fn handoff_state_renders_title_once_and_stable_spacing() {
let dir = tempfile::tempdir().unwrap();
let handoff_dir = dir.path().join("docs/handoffs");
fs::create_dir_all(&handoff_dir).unwrap();
fs::write(
handoff_dir.join("2026-06-28-release.md"),
"# Handoff 2026-06-28\n\nStatus: active\nNext step: ship release\n",
)
.unwrap();
let scan = collect_handoff_state(dir.path());
let handoff = scan.latest_handoff.expect("latest handoff");
assert_eq!(handoff.heading.as_deref(), Some("Handoff 2026-06-28"));
assert_eq!(handoff.text, "Status: active | Next step: ship release");
let rendered_signal = render_state_signal(&handoff);
assert!(!rendered_signal.contains("Handoff 2026-06-28 - Handoff 2026-06-28"));
let mut state = make_project_state();
state.latest_handoff = Some(handoff);
state.latest_vcs = None;
let mut rendered = String::new();
render_latest_project_state(&mut rendered, &state);
assert!(rendered.contains("- handoff: Handoff 2026-06-28 -"));
assert!(rendered.contains("Status: active"));
assert!(rendered.contains("Next step: ship release"));
assert!(!rendered.contains("\n\n\n### Next Actions"));
}
#[test]
fn scope_path_normalization_warns_only_on_real_drift() {
let dir = tempfile::tempdir().unwrap();
let configured_same = format!("{}/.", dir.path().display());
assert!(scope_path_drift_warning(&configured_same, dir.path()).is_none());
let drift = dir.path().join("other");
let warning =
scope_path_drift_warning(drift.to_str().unwrap(), dir.path()).expect("drift warning");
assert!(warning.contains("scope mismatch"));
}
#[test]
fn fragment_candidates_are_filtered_before_ranking() {
let payload = serde_json::json!({
"results": [
{
"chunk_id": "fragment",
"tenant_id": "t",
"project_id": "p",
"text": "and then continued from a prior chunk",
"score": 10.0,
"chunk_type": "summary",
"timestamp_created": 1,
"tags": ["chunk_index:1", "kind:finish"]
}
]
});
let mut by_chunk = HashMap::new();
let mut explanations = Vec::new();
merge_payload_candidates(
&mut by_chunk,
&mut explanations,
&payload,
"project",
"project_highlights",
"project takeaways",
CliQueryMode::FindHighlights,
);
assert!(by_chunk.is_empty());
let explanation = explanations.first().expect("explanation");
assert_eq!(explanation.filter_reason.as_deref(), Some("fragment_like"));
assert!(explanation
.quality_flags
.iter()
.any(|flag| flag == "fragment_like"));
}
#[test]
fn conditional_sentences_are_not_filtered_as_fragments() {
assert!(!is_fragment_like_candidate(
&[],
"When rerunning failed tasks, use --force to overwrite stale outputs."
));
assert!(!is_fragment_like_candidate(
&[],
"While debugging CI, inspect the failing job log before changing code."
));
assert!(!is_fragment_like_candidate(
&[],
"Because the release workflow publishes from main, push a release branch first."
));
assert!(is_fragment_like_candidate(
&[],
"and then continued from a prior chunk"
));
}
#[cfg(unix)]
#[test]
fn command_timeout_drains_large_output_while_waiting() {
let mut command = Command::new("sh");
command.arg("-c").arg(
"i=0; while [ \"$i\" -lt 20000 ]; do printf 'dirty-file-%05d xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\\n' \"$i\"; i=$((i + 1)); done",
);
let output = run_command_with_timeout(command, Duration::from_secs(5)).unwrap();
assert!(!output.timed_out);
assert!(output.status_success);
assert!(output.stdout.len() > 128 * 1024);
}
#[test]
fn expected_git_failure_checks_true_and_false_expectations() {
let mut metrics = make_agent_metrics();
metrics.git_state_present = false;
assert_eq!(
expected_git_failure(&metrics, true).as_deref(),
Some("expected git state but git_state_present=false")
);
assert!(expected_git_failure(&metrics, false).is_none());
metrics.git_state_present = true;
assert_eq!(
expected_git_failure(&metrics, false).as_deref(),
Some("expected no git state but git_state_present=true")
);
assert!(expected_git_failure(&metrics, true).is_none());
}
#[test]
fn startup_filters_collapse_duplicates_and_boilerplate_actions() {
let mut takeaways = vec![
make_takeaway(
"keep",
"Validation: MEMD_EMBED_DEVICE cpu override fixed GPU contention. Agent action: Use MEMD_EMBED_DEVICE=cpu when GPU contention blocks embedding.",
vec!["topic:embed-device", "priority:9"],
"summary",
),
make_takeaway(
"drop-duplicate",
"Validation: MEMD_EMBED_DEVICE cuda override fixed GPU contention. Agent action: Use MEMD_EMBED_DEVICE when GPU contention blocks embedding.",
vec!["topic:embed-device", "priority:8"],
"summary",
),
make_takeaway(
"drop-boilerplate",
"Routine note without a concrete action or category signal.",
vec![],
"summary",
),
];
for (idx, takeaway) in takeaways.iter_mut().enumerate() {
takeaway.priority = (90 - idx) as f32;
}
let suppressed = filter_startup_takeaways(&mut takeaways, "project");
let ids = takeaways
.iter()
.map(|takeaway| takeaway.chunk_id.as_str())
.collect::<Vec<_>>();
assert_eq!(ids, vec!["keep"]);
assert_eq!(
suppressed.get("drop-boilerplate").map(String::as_str),
Some("boilerplate_action")
);
assert!(suppressed
.get("drop-duplicate")
.map(|reason| reason.starts_with("duplicate_topic:"))
.unwrap_or(false));
}
#[tokio::test]
async fn memory_state_reports_project_scoped_unreadable_chunks() {
let store = FakeHealthStore::new(2, Vec::new());
let tenant = TenantId::new("t").unwrap();
let state = collect_memory_state(&store, &tenant, Some("p")).await;
assert_eq!(state.metadata_active_chunks, Some(2));
assert_eq!(state.readable_active_chunks, Some(0));
assert_eq!(state.unreadable_active_chunks, Some(2));
let project_state = ProjectState {
memory: state,
..make_project_state()
};
assert!(project_state_warnings(&project_state)
.iter()
.any(|warning| warning.contains("memory degraded: 2 active chunks")));
}
#[tokio::test]
async fn memory_state_caps_large_readable_scan_and_warns_partial() {
let store = FakeHealthStore::new(READABLE_SCAN_MAX_METADATA_ROWS + 5, Vec::new());
let tenant = TenantId::new("t").unwrap();
let state = collect_memory_state(&store, &tenant, Some("p")).await;
assert_eq!(
state.unreadable_active_chunks,
Some(READABLE_SCAN_MAX_METADATA_ROWS)
);
assert!(state
.scan_warning
.as_deref()
.unwrap_or_default()
.contains("readable memory scan partial"));
}
#[test]
fn render_memory_md_caps_long_takeaway_text() {
let takeaway = Takeaway {
chunk_id: "chunk-a".to_string(),
tenant_id: "tenant-a".to_string(),
project_id: Some("project-a".to_string()),
text: "word ".repeat(120),
score: 1.0,
priority: 42.0,
chunk_type: "summary".to_string(),
timestamp_created: 0,
tags: vec!["kind:finish".to_string()],
sources: BTreeSet::from(["project_highlights".to_string()]),
occurrences: 1,
};
let state = make_project_state();
let rendered = render_memory_md(&state, &[], &[takeaway], &[]);
assert!(rendered.contains("## Project Fact Library"));
assert!(rendered.contains("## Agent Guidance"));
assert!(rendered.contains("agent action: `Use this as evidence only after confirming"));
assert!(rendered.contains("chunk-a"));
assert!(rendered.contains("..."));
}
#[test]
fn render_memory_md_omits_empty_global_section() {
let takeaway = Takeaway {
chunk_id: "chunk-a".to_string(),
tenant_id: "tenant-a".to_string(),
project_id: Some("project-a".to_string()),
text: "validated project lesson".to_string(),
score: 1.0,
priority: 42.0,
chunk_type: "summary".to_string(),
timestamp_created: 0,
tags: vec!["kind:finish".to_string()],
sources: BTreeSet::from(["project_highlights".to_string()]),
occurrences: 1,
};
let state = make_project_state();
let rendered = render_memory_md(&state, &[], &[takeaway], &[]);
assert!(rendered.contains("## Project Fact Library"));
assert!(!rendered.contains("## Machine-Wide Fact Library"));
}
#[test]
fn render_memory_md_groups_takeaways_by_signal_category() {
let mut decision = make_takeaway(
"decision",
"Decision: keep project aliases explicit. Rationale: silent merging hides drift.",
vec!["kind:decision"],
"summary",
);
decision.priority = 80.0;
decision.timestamp_created = 42;
let mut fix = make_takeaway(
"fix",
"Validation: cargo test -p memd passed after the scoped startup change.",
vec!["kind:finish"],
"summary",
);
fix.priority = 70.0;
let mut command = make_takeaway(
"command",
"Command: memd memory-md --project-dir . --output memory.md",
vec!["kind:run"],
"trace",
);
command.priority = 60.0;
let state = make_project_state();
let rendered = render_memory_md(&state, &[], &[command, fix, decision], &[]);
assert!(rendered.contains("### Decisions"));
assert!(rendered.contains("### Validated Fixes"));
assert!(rendered.contains("### Commands/Paths"));
assert!(rendered.contains("reason: `decision or rationale`"));
assert!(rendered.contains("reason: `validated fix or result`"));
assert!(rendered.contains("reason: `command, path, or parameter evidence`"));
assert!(rendered.contains("agent action: `Apply this decision"));
assert!(rendered.contains("agent action: `Reuse this validated fix"));
assert!(rendered.contains("agent action: `Use or verify this command/path"));
assert!(rendered.contains("created_unix_ms: `42`"));
}
#[test]
fn memory_md_quality_report_scores_useful_items_and_wrappers() {
let content = r#"# memory.md
## Project Takeaways
### Decisions
1. Decision: keep project aliases explicit.
- reason: `decision or rationale`
- agent action: `Apply this decision when the same scope appears: keep project aliases explicit.`
### Other Takeaways
1. Task digest status generated. Summary: Highlight library for p contains 2 ranked lessons.
- reason: `ranked project takeaway`
- tags: `task:status:generated, task:role:highlight_library`
2. Routine status update with no reason.
## Machine-Wide Takeaways
1. Decision outside project section should not be counted.
- reason: `decision or rationale`
"#;
let report = evaluate_memory_md_quality(content, 10);
assert_eq!(report.displayed_count, 3);
assert_eq!(report.useful_count, 1);
assert_eq!(report.generated_wrapper_count, 1);
assert_eq!(report.missing_reason_count, 1);
assert_eq!(report.missing_action_count, 2);
assert!((report.useful_ratio - (1.0 / 3.0)).abs() < f64::EPSILON);
}
#[test]
fn explicit_agent_action_overrides_category_template() {
let takeaway = make_takeaway(
"action",
"Decision: keep cache keys scoped. Agent action: Verify tenant and project are present before reusing cached retrieval results.",
vec!["kind:decision"],
"summary",
);
let state = make_project_state();
let rendered = render_memory_md(&state, &[], &[takeaway], &[]);
assert!(rendered.contains(
"agent action: `Verify tenant and project are present before reusing cached retrieval results`"
));
}
#[test]
fn explicit_agent_action_skips_explanatory_marker_mentions() {
let takeaway = make_takeaway(
"action",
"Validation: memory quality gate passed. Docs mention Agent action: sentence. Agent action: Write every high-priority durable memory with a concrete action sentence that tells future agents what to verify or reuse.",
vec!["kind:finish"],
"summary",
);
let state = make_project_state();
let rendered = render_memory_md(&state, &[], &[takeaway], &[]);
assert!(rendered.contains(
"agent action: `Write every high-priority durable memory with a concrete action sentence that tells future agents what to verify or reuse`"
));
}
#[test]
fn explicit_agent_action_keeps_paths_and_versions() {
let takeaway = make_takeaway(
"action",
"Installed skill bundle. Agent action: Verify future agent sessions read the refreshed ~/.agents/skills/memd skill and use memd 0.61.0 before diagnosing memory-quality behavior.",
vec!["kind:finish"],
"summary",
);
let state = make_project_state();
let rendered = render_memory_md(&state, &[], &[takeaway], &[]);
assert!(rendered.contains(
"agent action: `Verify future agent sessions read the refreshed ~/.agents/skills/memd skill and use memd 0.61.0 before diagnosing memory-quality behavior`"
));
}
fn make_project_state() -> ProjectState {
ProjectState {
generated_unix_ms: 123,
tenant_id: "tenant-a".to_string(),
project_id: Some("project-a".to_string()),
configured_project_dir: Some("/tmp/project-a".to_string()),
resolved_project_dir: "/tmp/project-a".to_string(),
scope_warnings: Vec::new(),
git: GitState {
available: true,
not_git_repo: false,
branch: Some("main".to_string()),
clean: Some(true),
changed_entries: 0,
summary: "branch `main`; clean".to_string(),
warning: None,
},
latest_task: Some(StateSignal {
source_path: "tasks/todo.md".to_string(),
line: Some(1),
heading: Some("Current Work".to_string()),
text: "open action: run validation".to_string(),
}),
latest_handoff: None,
latest_vcs: None,
next_actions: vec![NextAction {
source_path: "tasks/todo.md".to_string(),
line: 2,
heading: Some("Current Work".to_string()),
text: "run validation".to_string(),
}],
memory: MemoryState::default(),
collection_warnings: Vec::new(),
}
}
fn make_agent_metrics() -> AgentUsefulnessMetrics {
AgentUsefulnessMetrics {
latest_project_state_present: true,
scope_present: true,
git_state_present: true,
git_state_present_or_not_git_repo: true,
latest_work_present: true,
next_action_present: true,
no_open_tasks_detected: false,
scope_health_present: true,
memory_degraded_warning_present: false,
fragment_count: 0,
duplicate_cluster_count: 0,
boilerplate_action_count: 0,
unrelated_machine_items: 0,
source_backed_next_actions: true,
answerability_passed: true,
}
}
struct FakeHealthStore {
active_chunks: usize,
readable_chunks: Vec<MemoryChunk>,
}
impl FakeHealthStore {
fn new(active_chunks: usize, readable_chunks: Vec<MemoryChunk>) -> Self {
Self {
active_chunks,
readable_chunks,
}
}
}
#[async_trait::async_trait]
impl Store for FakeHealthStore {
async fn add(&self, chunk: MemoryChunk) -> Result<ChunkId> {
Ok(chunk.chunk_id)
}
async fn add_batch(&self, chunks: Vec<MemoryChunk>) -> Result<Vec<ChunkId>> {
Ok(chunks.into_iter().map(|chunk| chunk.chunk_id).collect())
}
async fn get(
&self,
_tenant_id: &TenantId,
_chunk_id: &ChunkId,
) -> Result<Option<MemoryChunk>> {
Ok(None)
}
async fn search(
&self,
_tenant_id: &TenantId,
_query: &str,
_k: usize,
) -> Result<Vec<MemoryChunk>> {
Ok(Vec::new())
}
async fn list_chunks_for_project(
&self,
_tenant_id: &TenantId,
_project_id: Option<&str>,
limit: usize,
offset: usize,
) -> Result<Vec<MemoryChunk>> {
Ok(self
.readable_chunks
.iter()
.skip(offset)
.take(limit)
.cloned()
.collect())
}
async fn delete(&self, _tenant_id: &TenantId, _chunk_id: &ChunkId) -> Result<bool> {
Ok(false)
}
async fn stats(&self, _tenant_id: &TenantId) -> Result<StoreStats> {
Ok(StoreStats {
active_chunks: self.active_chunks,
..StoreStats::default()
})
}
async fn health_snapshot(
&self,
_tenant_id: &TenantId,
_project_id: Option<&str>,
_duplicate_limit: usize,
) -> Result<Option<StoreHealthSnapshot>> {
Ok(Some(StoreHealthSnapshot {
counts: HealthCounts {
active_chunks: self.active_chunks,
total_chunks: self.active_chunks,
..HealthCounts::default()
},
chunk_types_active: HashMap::new(),
chunk_types_all: HashMap::new(),
duplicates: DuplicateHealth::default(),
index_coverage: IndexCoverageHealth::default(),
payload: PayloadHealth::default(),
}))
}
}
fn make_takeaway(chunk_id: &str, text: &str, tags: Vec<&str>, chunk_type: &str) -> Takeaway {
Takeaway {
chunk_id: chunk_id.to_string(),
tenant_id: "t".to_string(),
project_id: Some("p".to_string()),
text: text.to_string(),
score: 0.0,
priority: 0.0,
chunk_type: chunk_type.to_string(),
timestamp_created: 0,
tags: tags.into_iter().map(str::to_string).collect(),
sources: BTreeSet::new(),
occurrences: 1,
}
}
#[test]
fn success_traces_are_not_filed_under_known_failures() {
let success = make_takeaway(
"success",
"Trace: ran alpha ETL end-to-end, 0 failures after retry patch was applied.",
vec!["kind:run"],
"trace",
);
let cat = takeaway_category(&success);
assert_ne!(cat.heading, "Known Failures", "got: {}", cat.heading);
assert_eq!(cat.heading, "Validated Fixes");
let mut via_query = make_takeaway(
"via-query",
"Benchmark: HNSW recall comparable to IVFFlat at this corpus size.",
vec![],
"summary",
);
via_query.sources.insert("project_failures".to_string());
assert_ne!(takeaway_category(&via_query).heading, "Known Failures");
}
#[test]
fn real_failures_still_classify_as_known_failures() {
let tagged = make_takeaway(
"tagged",
"Ingest run aborted on schema mismatch.",
vec!["kind:failure"],
"trace",
);
assert_eq!(takeaway_category(&tagged).heading, "Known Failures");
let root_cause = make_takeaway(
"root-cause",
"Root cause: NFS stall truncated the segment write; job failed because fsync never returned.",
vec![],
"summary",
);
assert_eq!(takeaway_category(&root_cause).heading, "Known Failures");
}
#[test]
fn extract_covered_task_ids_parses_summary_footer() {
let text = "Highlight library for foo contains 3 ranked lessons.\nCovers tasks: task:id:T1, task:id:T2, task:id:T3";
let ids = extract_covered_task_ids(text);
assert_eq!(
ids,
vec!["T1".to_string(), "T2".to_string(), "T3".to_string()]
);
}
#[test]
fn task_finish_suppressed_when_covered_by_highlight() {
let mut takeaways = vec![
make_takeaway(
"digest1",
"Highlight library contains lessons.\nCovers tasks: task:id:T1, task:id:T2",
vec!["task:role:highlight_library", "task:status:generated"],
"summary",
),
make_takeaway(
"raw1",
"Task T1 finished successfully.",
vec!["task:kind:task_finish", "task:id:T1"],
"summary",
),
make_takeaway(
"raw2",
"Task T3 finished — not covered.",
vec!["task:kind:task_finish", "task:id:T3"],
"summary",
),
];
suppress_finishes_covered_by_libraries(&mut takeaways);
let ids: Vec<&str> = takeaways.iter().map(|t| t.chunk_id.as_str()).collect();
assert!(ids.contains(&"digest1"));
assert!(!ids.contains(&"raw1"), "covered finish should be dropped");
assert!(ids.contains(&"raw2"), "uncovered finish should survive");
}
#[test]
fn user_explicit_priority_high_survives_suppression() {
let mut takeaways = vec![
make_takeaway(
"digest1",
"Covers tasks: task:id:T1",
vec!["task:role:highlight_library", "task:status:generated"],
"summary",
),
make_takeaway(
"raw1",
"Task T1 finished — but operator marked it high priority.",
vec!["task:kind:task_finish", "task:id:T1", "priority:9"],
"summary",
),
];
suppress_finishes_covered_by_libraries(&mut takeaways);
let ids: Vec<&str> = takeaways.iter().map(|t| t.chunk_id.as_str()).collect();
assert!(ids.contains(&"raw1"), "priority>=8 finish must survive");
}
#[test]
fn unverified_role_tag_does_not_suppress() {
let mut takeaways = vec![
make_takeaway(
"spoof",
"Covers tasks: task:id:T1",
vec!["task:role:highlight_library"],
"summary",
),
make_takeaway(
"raw1",
"Task T1 finished successfully.",
vec!["task:kind:task_finish", "task:id:T1"],
"summary",
),
];
suppress_finishes_covered_by_libraries(&mut takeaways);
let ids: Vec<&str> = takeaways.iter().map(|t| t.chunk_id.as_str()).collect();
assert!(ids.contains(&"raw1"), "spoofed digest must not suppress");
}
#[test]
fn cross_project_finish_is_not_suppressed() {
let mut digest = make_takeaway(
"digest_p",
"Covers tasks: task:id:T1",
vec!["task:role:highlight_library", "task:status:generated"],
"summary",
);
digest.project_id = Some("project-a".to_string());
let mut raw = make_takeaway(
"raw_other",
"Task T1 finished in a different project.",
vec!["task:kind:task_finish", "task:id:T1"],
"summary",
);
raw.project_id = Some("project-b".to_string());
let mut takeaways = vec![digest, raw];
suppress_finishes_covered_by_libraries(&mut takeaways);
let ids: Vec<&str> = takeaways.iter().map(|t| t.chunk_id.as_str()).collect();
assert!(
ids.contains(&"raw_other"),
"finish from another project must not be suppressed"
);
}
#[test]
fn decimal_priority_survives_suppression() {
let mut takeaways = vec![
make_takeaway(
"digest1",
"Covers tasks: task:id:T1",
vec!["task:role:highlight_library", "task:status:generated"],
"summary",
),
make_takeaway(
"raw1",
"Task T1 finished — operator set a decimal priority.",
vec!["task:kind:task_finish", "task:id:T1", "priority:8.5"],
"summary",
),
];
suppress_finishes_covered_by_libraries(&mut takeaways);
let ids: Vec<&str> = takeaways.iter().map(|t| t.chunk_id.as_str()).collect();
assert!(ids.contains(&"raw1"), "priority:8.5 finish must survive");
}
#[test]
fn library_bonus_outranks_raw_chunk() {
let counts = HashMap::new();
let hit_stats = HashMap::new();
let library = make_takeaway(
"digest",
"Highlight library text.",
vec!["task:role:highlight_library", "kind:finish"],
"summary",
);
let raw = make_takeaway("raw", "Plain finish.", vec!["kind:finish"], "summary");
let lib_score = priority_score(&library, &counts, &hit_stats, 0);
let raw_score = priority_score(&raw, &counts, &hit_stats, 0);
assert!(
lib_score > raw_score + 10.0,
"library bonus should dominate: lib={lib_score}, raw={raw_score}"
);
}
#[test]
fn priority_score_boosts_frequently_hit_chunks() {
let counts = HashMap::new();
let cold = make_takeaway("cold", "text", vec!["kind:finish"], "summary");
let hot = make_takeaway("hot", "text", vec!["kind:finish"], "summary");
let mut hits = HashMap::new();
hits.insert(
"hot".to_string(),
HitStats {
hit_count: 7,
selected_count: 7,
last_ts_ms: 1,
},
);
let cold_score = priority_score(&cold, &counts, &hits, 1);
let hot_score = priority_score(&hot, &counts, &hits, 1);
assert!(
hot_score > cold_score + 5.0,
"hot chunk should outrank cold: hot={hot_score}, cold={cold_score}"
);
}
#[test]
fn priority_score_demotes_unused_old_chunks() {
let counts = HashMap::new();
let hits: HashMap<String, HitStats> = HashMap::new();
let recent = {
let mut t = make_takeaway("recent", "text", vec!["kind:finish"], "summary");
t.timestamp_created = 1_000_000;
t
};
let stale = {
let mut t = make_takeaway("stale", "text", vec!["kind:finish"], "summary");
t.timestamp_created = 1_000_000; t
};
let now_recent = 1_000_000 + 3 * 3_600_000;
let now_stale = 1_000_000 + 365 * 86_400_000;
let recent_score = priority_score(&recent, &counts, &hits, now_recent);
let stale_score = priority_score(&stale, &counts, &hits, now_stale);
assert!(
recent_score > stale_score,
"stale chunk must be demoted: recent={recent_score}, stale={stale_score}"
);
}
#[test]
fn generated_digest_candidates_are_skipped_from_takeaways() {
let tags = vec![
"task:status:generated".to_string(),
"task:role:decision_library".to_string(),
];
assert!(is_generated_digest_takeaway(&tags));
assert!(!is_generated_digest_takeaway(&[
"kind:decision".to_string(),
"priority:9".to_string(),
]));
let payload = serde_json::json!({
"results": [
{
"chunk_id": "digest",
"tenant_id": "t",
"project_id": "p",
"text": "Task digest status generated. Summary: Highlight library for p contains 2 ranked lessons.",
"score": 25.0,
"chunk_type": "summary",
"timestamp_created": 10,
"tags": tags
},
{
"chunk_id": "raw",
"tenant_id": "t",
"project_id": "p",
"text": "Validated fix: use the stable project scope when restoring the gateway.",
"score": 4.0,
"chunk_type": "summary",
"timestamp_created": 11,
"tags": ["kind:finish"]
}
]
});
let mut by_chunk = HashMap::new();
let mut explanations = Vec::new();
merge_payload_candidates(
&mut by_chunk,
&mut explanations,
&payload,
"project",
"project_highlights",
"project takeaways",
CliQueryMode::FindHighlights,
);
assert!(!by_chunk.contains_key("digest"));
assert!(by_chunk.contains_key("raw"));
assert_eq!(
explanations
.iter()
.find(|item| item.chunk_id == "digest")
.and_then(|item| item.filter_reason.as_deref()),
Some("generated_digest_wrapper")
);
assert_eq!(
explanations
.iter()
.find(|item| item.chunk_id == "raw")
.map(|item| item.display_status.as_str()),
Some("candidate")
);
}
}