use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::fs;
use std::path::{Path, PathBuf};
use std::time::UNIX_EPOCH;
use serde_json::{json, Value};
use super::memory_md::{build_repo_index, repo_doc_covering, RepoDoc};
use super::paths::absolutize_project_dir;
use super::scope;
use crate::error::{MemdError, Result};
use crate::store::{OutcomeEvent, OutcomeKind, OutcomeVerifier, RetrievalEpisodeId, Store};
use crate::types::{ChunkId, TenantId};
const MIN_LITERAL_LEN: usize = 12;
const MAX_LITERALS_PER_CHUNK: usize = 10;
const MAX_EXAMPLES: usize = 5;
const MAX_EXAMPLE_EXCERPT_CHARS: usize = 160;
const STATE_FILE: &str = ".memd/data/outcome_scan_state.json";
const SESSION_DIR_MAX_DEPTH: usize = 8;
#[derive(Debug)]
pub(super) struct OutcomeScanOptions {
pub(super) project_dir: PathBuf,
pub(super) sessions_dir: Option<PathBuf>,
pub(super) since_days: u64,
pub(super) dry_run: bool,
}
#[derive(Debug)]
struct Serve {
line: usize,
episode_id: String,
chunk_ids: Vec<String>,
}
#[derive(Debug)]
struct Action {
line: usize,
text: String,
timestamp_ms: Option<i64>,
}
#[derive(Debug, Default)]
struct SessionSignals {
serves: Vec<Serve>,
actions: Vec<Action>,
}
#[derive(Debug)]
struct Hit {
chunk_id: String,
literal: String,
serve_line: usize,
action_line: usize,
action_timestamp_ms: i64,
action_excerpt: String,
}
pub(super) async fn run_outcome_scan<S: Store>(
store: &S,
options: OutcomeScanOptions,
) -> Result<Value> {
let project_dir = absolutize_project_dir(&options.project_dir)?;
let (tenant_id, _project_id) = scope::resolve_required(&project_dir, None, None)?;
let tenant = TenantId::new(&tenant_id)?;
let sessions_dir = match options.sessions_dir {
Some(dir) => dir,
None => dirs::home_dir()
.ok_or_else(|| {
MemdError::ValidationError(
"cannot resolve the home directory for the default sessions dir; pass --sessions-dir".to_string(),
)
})?
.join(".codex")
.join("sessions"),
};
let now_ms = std::time::SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_millis() as i64)
.unwrap_or(0);
let cutoff_ms = now_ms.saturating_sub((options.since_days as i64).saturating_mul(86_400_000));
let state_path = project_dir.join(STATE_FILE);
let mut state = load_scan_state(&state_path);
let mut session_files = Vec::new();
collect_jsonl_files(&sessions_dir, 0, &mut session_files);
session_files.sort();
let repo_index: Vec<RepoDoc> = build_repo_index(&project_dir, &project_dir.join("memory.md"));
let mut files_scanned = 0usize;
let mut files_skipped_unchanged = 0usize;
let mut files_skipped_age = 0usize;
let mut files_unreadable = 0usize;
let mut files_with_serves = 0usize;
let mut serves_found = 0usize;
let mut chunks_skipped_missing = 0usize;
let mut chunks_skipped_repo_covered = 0usize;
let mut candidate_hits = 0usize;
let mut events_written = 0usize;
let mut events_planned = 0usize;
let mut skipped_missing_episode = 0usize;
let mut skipped_already_recorded = 0usize;
let mut skipped_not_rendered = 0usize;
let mut skipped_outside_window = 0usize;
let mut record_errors = 0usize;
let mut served_chunk_ids = BTreeSet::new();
let mut examples = Vec::new();
let mut literal_cache: HashMap<String, Vec<String>> = HashMap::new();
for path in &session_files {
let Ok(metadata) = fs::metadata(path) else {
files_unreadable += 1;
continue;
};
let mtime_ms = metadata
.modified()
.ok()
.and_then(|time| time.duration_since(UNIX_EPOCH).ok())
.map(|duration| duration.as_millis() as i64)
.unwrap_or(0);
if mtime_ms < cutoff_ms {
files_skipped_age += 1;
continue;
}
let state_key = path.display().to_string();
if state.get(&state_key) == Some(&mtime_ms) {
files_skipped_unchanged += 1;
continue;
}
let Ok(text) = fs::read_to_string(path) else {
files_unreadable += 1;
continue;
};
files_scanned += 1;
if find_episode_ids(&text).is_empty() {
if !options.dry_run {
state.insert(state_key, mtime_ms);
}
continue;
}
let signals = parse_session(&text);
if signals.serves.is_empty() {
if !options.dry_run {
state.insert(state_key, mtime_ms);
}
continue;
}
files_with_serves += 1;
serves_found += signals.serves.len();
for serve in &signals.serves {
for chunk_id in &serve.chunk_ids {
served_chunk_ids.insert(chunk_id.clone());
if literal_cache.contains_key(chunk_id) {
continue;
}
let literals = match resolve_chunk_literals(
store,
&tenant,
chunk_id,
&project_dir,
&repo_index,
)
.await?
{
ChunkLiterals::Missing => {
chunks_skipped_missing += 1;
Vec::new()
}
ChunkLiterals::RepoCovered => {
chunks_skipped_repo_covered += 1;
Vec::new()
}
ChunkLiterals::Literals(literals) => literals,
};
literal_cache.insert(chunk_id.clone(), literals);
}
}
let hits = compute_hits(&signals, &literal_cache);
candidate_hits += hits.len();
if options.dry_run {
for (_, hit) in hits
.iter()
.take(MAX_EXAMPLES.saturating_sub(examples.len()))
{
examples.push(json!({
"session": path.file_name().map(|name| name.to_string_lossy().into_owned()),
"chunk_id": hit.chunk_id,
"literal": hit.literal,
"serve_line": hit.serve_line,
"action_line": hit.action_line,
"action_excerpt": hit.action_excerpt,
}));
}
}
let mut per_episode: BTreeMap<String, Vec<&Hit>> = BTreeMap::new();
for (episode_id, hit) in &hits {
per_episode.entry(episode_id.clone()).or_default().push(hit);
}
for (episode_id, episode_hits) in per_episode {
let Ok(episode_uuid) = RetrievalEpisodeId::parse(&episode_id) else {
continue;
};
let Some((episode, items)) =
store.get_retrieval_episode(&tenant, &episode_uuid).await?
else {
skipped_missing_episode += 1;
continue;
};
let session_name = path
.file_name()
.map(|name| name.to_string_lossy().into_owned())
.unwrap_or_else(|| path.display().to_string());
let evidence_prefix = format!("codex:{session_name}:");
if store
.list_outcomes_for_episode(&tenant, &episode_uuid)
.await?
.iter()
.any(|existing| {
existing
.evidence_reference
.as_deref()
.is_some_and(|reference| reference.starts_with(&evidence_prefix))
})
{
skipped_already_recorded += 1;
continue;
}
let rendered = items
.iter()
.filter(|item| item.rendered)
.map(|item| item.chunk_id.to_string())
.collect::<HashSet<_>>();
let credited_hits = episode_hits
.into_iter()
.filter(|hit| rendered.contains(&hit.chunk_id))
.collect::<Vec<_>>();
let used_chunk_ids = credited_hits
.iter()
.map(|hit| hit.chunk_id.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.filter_map(|chunk_id| ChunkId::parse(&chunk_id).ok())
.collect::<Vec<_>>();
if used_chunk_ids.is_empty() {
skipped_not_rendered += 1;
continue;
}
let evidence_action = credited_hits
.iter()
.min_by_key(|hit| hit.action_line)
.expect("per-episode hit group is non-empty");
if evidence_action.action_timestamp_ms < episode.created_at_ms
|| evidence_action.action_timestamp_ms > episode.expires_at_ms
{
skipped_outside_window += 1;
continue;
}
let event = OutcomeEvent::new(
episode_uuid,
OutcomeKind::ObservedUsed,
OutcomeVerifier::ExternalTool,
used_chunk_ids,
Vec::new(),
Some(format!(
"codex:{session_name}:{}",
evidence_action.action_line
)),
evidence_action.action_timestamp_ms,
);
if options.dry_run {
events_planned += 1;
} else {
match store.record_outcome(&tenant, event).await {
Ok(()) => events_written += 1,
Err(error) => {
record_errors += 1;
tracing::warn!(?error, episode_id, "outcome-scan record_outcome failed");
}
}
}
}
if !options.dry_run {
state.insert(state_key, mtime_ms);
}
}
if !options.dry_run {
save_scan_state(&state_path, &state)?;
}
Ok(json!({
"tenant_id": tenant.to_string(),
"sessions_dir": sessions_dir,
"dry_run": options.dry_run,
"files_found": session_files.len(),
"files_scanned": files_scanned,
"files_skipped_unchanged": files_skipped_unchanged,
"files_skipped_age": files_skipped_age,
"files_unreadable": files_unreadable,
"files_with_serves": files_with_serves,
"serves_found": serves_found,
"served_chunks": served_chunk_ids.len(),
"chunks_skipped_missing": chunks_skipped_missing,
"chunks_skipped_repo_covered": chunks_skipped_repo_covered,
"candidate_hits": candidate_hits,
"events_planned": events_planned,
"events_written": events_written,
"events_skipped": {
"missing_episode": skipped_missing_episode,
"already_recorded": skipped_already_recorded,
"not_rendered": skipped_not_rendered,
"outside_window": skipped_outside_window,
"record_error": record_errors,
},
"examples": examples,
}))
}
enum ChunkLiterals {
Missing,
RepoCovered,
Literals(Vec<String>),
}
async fn resolve_chunk_literals<S: Store>(
store: &S,
tenant: &TenantId,
chunk_id: &str,
project_dir: &Path,
repo_index: &[RepoDoc],
) -> Result<ChunkLiterals> {
let Ok(parsed) = ChunkId::parse(chunk_id) else {
return Ok(ChunkLiterals::Missing);
};
let Some(chunk) = store.get(tenant, &parsed).await? else {
return Ok(ChunkLiterals::Missing);
};
if repo_doc_covering(&chunk.text, repo_index).is_some() {
return Ok(ChunkLiterals::RepoCovered);
}
let literals = extract_literals(&chunk.text)
.into_iter()
.filter(|literal| !literal_is_repo_path(project_dir, literal))
.collect();
Ok(ChunkLiterals::Literals(literals))
}
fn collect_jsonl_files(dir: &Path, depth: usize, files: &mut Vec<PathBuf>) {
if depth > SESSION_DIR_MAX_DEPTH {
return;
}
let Ok(entries) = fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
collect_jsonl_files(&path, depth + 1, files);
} else if path.extension().is_some_and(|ext| ext == "jsonl") {
files.push(path);
}
}
}
fn load_scan_state(path: &Path) -> HashMap<String, i64> {
fs::read_to_string(path)
.ok()
.and_then(|text| serde_json::from_str(&text).ok())
.unwrap_or_default()
}
fn save_scan_state(path: &Path, state: &HashMap<String, i64>) -> Result<()> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)?;
}
fs::write(path, serde_json::to_string_pretty(&state)?)?;
Ok(())
}
fn parse_session(text: &str) -> SessionSignals {
let mut signals = SessionSignals::default();
for (index, line) in text.lines().enumerate() {
let line_number = index + 1;
let Ok(value) = serde_json::from_str::<Value>(line) else {
continue;
};
let Some(payload) = value.get("payload") else {
continue;
};
match payload.get("type").and_then(Value::as_str) {
Some("custom_tool_call") | Some("function_call") => {
let action_text = payload
.get("input")
.and_then(Value::as_str)
.or_else(|| payload.get("arguments").and_then(Value::as_str));
if let Some(action_text) = action_text {
signals.actions.push(Action {
line: line_number,
text: action_text.to_string(),
timestamp_ms: value.get("timestamp").and_then(Value::as_str).and_then(
|timestamp| crate::structural::parse_iso_datetime(timestamp).ok(),
),
});
}
}
Some("custom_tool_call_output") | Some("function_call_output") => {
let Some(output) = payload.get("output") else {
continue;
};
let mut rendered = String::new();
collect_strings(output, &mut rendered);
let episode_ids = find_episode_ids(&rendered);
if episode_ids.is_empty() {
continue;
}
let chunk_ids = find_uuids(&rendered)
.into_iter()
.filter(|uuid| !episode_ids.contains(uuid))
.collect::<Vec<_>>();
for episode_id in episode_ids {
signals.serves.push(Serve {
line: line_number,
episode_id,
chunk_ids: chunk_ids.clone(),
});
}
}
_ => {}
}
}
signals
}
fn collect_strings(value: &Value, out: &mut String) {
match value {
Value::String(text) => {
out.push_str(text);
out.push('\n');
}
Value::Array(items) => {
for item in items {
collect_strings(item, out);
}
}
Value::Object(map) => {
for item in map.values() {
collect_strings(item, out);
}
}
_ => {}
}
}
fn find_episode_ids(text: &str) -> BTreeSet<String> {
const KEY: &str = "retrieval_episode_id";
let mut ids = BTreeSet::new();
let bytes = text.as_bytes();
let mut search = 0;
while let Some(found) = text[search..].find(KEY) {
let mut pos = search + found + KEY.len();
while pos < bytes.len() && matches!(bytes[pos], b'"' | b'\\' | b':' | b' ' | b'=' | b'`') {
pos += 1;
}
if let Some(uuid) = uuid_at(bytes, pos) {
ids.insert(uuid);
}
search += found + KEY.len();
}
ids
}
fn find_uuids(text: &str) -> Vec<String> {
let bytes = text.as_bytes();
let mut found = Vec::new();
let mut seen = HashSet::new();
let mut index = 0;
while index + 36 <= bytes.len() {
if let Some(uuid) = uuid_at(bytes, index) {
if seen.insert(uuid.clone()) {
found.push(uuid);
}
index += 36;
} else {
index += 1;
}
}
found
}
fn uuid_at(bytes: &[u8], start: usize) -> Option<String> {
const GROUPS: [usize; 5] = [8, 4, 4, 4, 12];
let is_hex = |byte: u8| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte);
if start > 0 && is_hex(bytes[start - 1]) {
return None;
}
let mut pos = start;
for (group, length) in GROUPS.iter().enumerate() {
if group > 0 {
if bytes.get(pos) != Some(&b'-') {
return None;
}
pos += 1;
}
for _ in 0..*length {
match bytes.get(pos) {
Some(&byte) if is_hex(byte) => pos += 1,
_ => return None,
}
}
}
if bytes.get(pos).is_some_and(|byte| is_hex(*byte)) {
return None;
}
Some(String::from_utf8_lossy(&bytes[start..pos]).into_owned())
}
fn extract_literals(chunk_text: &str) -> Vec<String> {
let mut literals = Vec::new();
let mut seen = HashSet::new();
let push = |literal: &str, literals: &mut Vec<String>, seen: &mut HashSet<String>| {
if literals.len() < MAX_LITERALS_PER_CHUNK
&& literal_qualifies(literal)
&& seen.insert(literal.to_string())
{
literals.push(literal.to_string());
}
};
for (index, span) in chunk_text.split('`').enumerate() {
if index % 2 == 1 {
push(span.trim(), &mut literals, &mut seen);
}
}
for token in chunk_text.split_whitespace() {
let token = token
.trim_matches(|ch: char| {
matches!(
ch,
',' | ';' | ':' | '(' | ')' | '[' | ']' | '{' | '}' | '"' | '\'' | '`'
)
})
.trim_end_matches('.');
if token.contains('/') || token.starts_with("--") {
push(token, &mut literals, &mut seen);
}
}
literals
}
fn literal_qualifies(literal: &str) -> bool {
literal.len() >= MIN_LITERAL_LEN
&& literal
.chars()
.any(|ch| ch.is_ascii_digit() || matches!(ch, '/' | '_' | '.' | '-'))
}
fn literal_is_repo_path(project_dir: &Path, literal: &str) -> bool {
let candidate = Path::new(literal);
let resolved = if candidate.is_absolute() {
candidate.to_path_buf()
} else {
project_dir.join(candidate)
};
resolved.starts_with(project_dir) && resolved.exists()
}
fn compute_hits(
signals: &SessionSignals,
literals_by_chunk: &HashMap<String, Vec<String>>,
) -> Vec<(String, Hit)> {
let mut hits = Vec::new();
let mut credited = HashSet::new();
let mut serves = signals.serves.iter().collect::<Vec<_>>();
serves.sort_by_key(|serve| std::cmp::Reverse(serve.line));
for serve in serves {
for chunk_id in &serve.chunk_ids {
let Some(literals) = literals_by_chunk.get(chunk_id) else {
continue;
};
for literal in literals {
let seen_before = signals
.actions
.iter()
.any(|action| action.line < serve.line && action.text.contains(literal));
if seen_before {
continue;
}
let Some(action) = signals.actions.iter().find(|action| {
action.line > serve.line
&& action.timestamp_ms.is_some()
&& action.text.contains(literal)
}) else {
continue;
};
if !credited.insert((chunk_id.clone(), literal.clone())) {
continue;
}
hits.push((
serve.episode_id.clone(),
Hit {
chunk_id: chunk_id.clone(),
literal: literal.clone(),
serve_line: serve.line,
action_line: action.line,
action_timestamp_ms: action
.timestamp_ms
.expect("matched actions have native timestamps"),
action_excerpt: excerpt_around(&action.text, literal),
},
));
}
}
}
hits
}
fn excerpt_around(text: &str, literal: &str) -> String {
let center = text.find(literal).unwrap_or(0);
let mut start = center.saturating_sub(MAX_EXAMPLE_EXCERPT_CHARS / 2);
while start > 0 && !text.is_char_boundary(start) {
start -= 1;
}
let mut end = (center + literal.len() + MAX_EXAMPLE_EXCERPT_CHARS / 2).min(text.len());
while end < text.len() && !text.is_char_boundary(end) {
end += 1;
}
text[start..end].replace('\n', " ")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::store::{
stable_query_hash, MemoryStore, RankingPolicyMode, RetrievalEpisode, RetrievalEpisodeItem,
};
use crate::types::{ChunkType, MemoryChunk};
#[test]
fn literal_extraction_keeps_backticks_paths_and_flags() {
let text = "Fix lives in `crates/memd/src/cli/args.rs` — search with \
--dedupe-by-source; run scripts/run_bench.sh, then check \
docs/handoffs/2026-07-13_reliable.md. Short `x.rs` and \
--force and plain twelvecharword do not qualify.";
let literals = extract_literals(text);
assert!(literals.contains(&"crates/memd/src/cli/args.rs".to_string()));
assert!(literals.contains(&"--dedupe-by-source".to_string()));
assert!(literals.contains(&"scripts/run_bench.sh".to_string()));
assert!(literals.contains(&"docs/handoffs/2026-07-13_reliable.md".to_string()));
assert!(!literals.iter().any(|l| l == "x.rs"), "below min length");
assert!(!literals.iter().any(|l| l == "--force"), "below min length");
assert!(
!literals.iter().any(|l| l == "twelvecharword"),
"no /0-9_.- character and not a path/flag/backtick span"
);
let many = (0..20)
.map(|i| format!("dir{i}/file_number_{i}.rs"))
.collect::<Vec<_>>()
.join(" ");
assert_eq!(extract_literals(&many).len(), MAX_LITERALS_PER_CHUNK);
}
fn serve(line: usize, episode: &str, chunks: &[&str]) -> Serve {
Serve {
line,
episode_id: episode.to_string(),
chunk_ids: chunks.iter().map(|c| c.to_string()).collect(),
}
}
fn action(line: usize, text: &str) -> Action {
Action {
line,
text: text.to_string(),
timestamp_ms: Some(1_700_000_000_000 + line as i64),
}
}
#[test]
fn hit_requires_use_after_serve_and_absence_before() {
let signals = SessionSignals {
serves: vec![serve(10, "ep-1", &["chunk-a", "chunk-b"])],
actions: vec![
action(5, "cat docs/known_before_serve.md"),
action(
20,
"bash scripts/learned_from_memory.sh docs/known_before_serve.md",
),
],
};
let literals = HashMap::from([
(
"chunk-a".to_string(),
vec!["scripts/learned_from_memory.sh".to_string()],
),
(
"chunk-b".to_string(),
vec!["docs/known_before_serve.md".to_string()],
),
]);
let hits = compute_hits(&signals, &literals);
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].1.chunk_id, "chunk-a");
assert_eq!(hits[0].1.literal, "scripts/learned_from_memory.sh");
assert_eq!(hits[0].1.serve_line, 10);
assert_eq!(hits[0].1.action_line, 20);
}
#[test]
fn repeated_serves_credit_the_nearest_one_once() {
let signals = SessionSignals {
serves: vec![
serve(10, "ep-early", &["chunk-a"]),
serve(20, "ep-late", &["chunk-a"]),
],
actions: vec![action(30, "bash scripts/learned_from_memory.sh")],
};
let literals = HashMap::from([(
"chunk-a".to_string(),
vec!["scripts/learned_from_memory.sh".to_string()],
)]);
let hits = compute_hits(&signals, &literals);
assert_eq!(hits.len(), 1, "one use must not credit two episodes");
assert_eq!(hits[0].0, "ep-late", "credit the nearest preceding serve");
let signals = SessionSignals {
serves: vec![
serve(10, "ep-early", &["chunk-a"]),
serve(20, "ep-late", &["chunk-a"]),
],
actions: vec![action(15, "bash scripts/learned_from_memory.sh")],
};
let hits = compute_hits(&signals, &literals);
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].0, "ep-early");
}
#[test]
fn tool_outputs_never_count_as_actions() {
let episode = "019f0000-0000-7000-8000-000000000001";
let chunk = "019f0000-0000-7000-8000-000000000002";
let session = format!(
concat!(
"{{\"timestamp\":\"2026-09-15T12:00:00Z\",\"type\":\"response_item\",\"payload\":{{\"type\":\"custom_tool_call_output\",",
"\"output\":[{{\"type\":\"input_text\",\"text\":\"chunk {chunk} ... ",
"\\\"retrieval_episode_id\\\": \\\"{episode}\\\"\"}}]}}}}\n",
"{{\"timestamp\":\"2026-09-15T12:00:01Z\",\"type\":\"response_item\",\"payload\":{{\"type\":\"custom_tool_call_output\",",
"\"output\":\"rerendered scripts/from_memory_only.sh\"}}}}\n",
"{{\"timestamp\":\"2026-09-15T12:00:02.345Z\",\"type\":\"response_item\",\"payload\":{{\"type\":\"function_call\",",
"\"arguments\":\"{{\\\"cmd\\\":\\\"bash scripts/from_memory_only.sh\\\"}}\"}}}}\n",
),
chunk = chunk,
episode = episode,
);
let signals = parse_session(&session);
assert_eq!(signals.serves.len(), 1);
assert_eq!(signals.serves[0].episode_id, episode);
assert_eq!(signals.serves[0].chunk_ids, vec![chunk.to_string()]);
assert_eq!(
signals.actions.len(),
1,
"outputs must never become actions"
);
assert_eq!(signals.actions[0].line, 3);
assert_eq!(
signals.actions[0].timestamp_ms,
Some(crate::structural::parse_iso_datetime("2026-09-15T12:00:02.345Z").unwrap())
);
let literals = HashMap::from([(
chunk.to_string(),
vec!["scripts/from_memory_only.sh".to_string()],
)]);
let hits = compute_hits(&signals, &literals);
assert_eq!(hits.len(), 1, "the input on line 3 is genuine usage");
assert_eq!(hits[0].1.action_line, 3);
}
#[tokio::test]
async fn current_custom_tool_reuse_records_native_time_once_after_append() {
let project = tempfile::tempdir().unwrap();
let sessions = tempfile::tempdir().unwrap();
let tenant = TenantId::new("outcome_scan_current_log").unwrap();
let store = MemoryStore::new();
let literal = "scripts/from_memory_observation.sh";
let unrendered_literal = "scripts/unrendered_memory.sh";
let chunk_id = store
.add(MemoryChunk::new(
tenant.clone(),
format!("Run `{literal}` after setup."),
ChunkType::Doc,
))
.await
.unwrap();
let unrendered_chunk_id = store
.add(MemoryChunk::new(
tenant.clone(),
format!("Do not render `{unrendered_literal}`."),
ChunkType::Doc,
))
.await
.unwrap();
let action_timestamp = "2026-09-15T12:00:02.345Z";
let action_timestamp_ms = crate::structural::parse_iso_datetime(action_timestamp).unwrap();
let episode_id = RetrievalEpisodeId::new();
store
.record_retrieval_episode(
RetrievalEpisode {
episode_id: episode_id.clone(),
tenant_id: tenant.clone(),
project_id: None,
query_hash: stable_query_hash("current custom tool records"),
query_mode: "generic".to_string(),
requested_k: 2,
fetched_k: 2,
rendered_k: 1,
policy_version: crate::store::OUTCOME_POLICY_VERSION.to_string(),
policy_mode: RankingPolicyMode::Shadow,
task_id: None,
thread_id: None,
created_at_ms: action_timestamp_ms - 1_000,
expires_at_ms: action_timestamp_ms + 86_400_000,
},
vec![
RetrievalEpisodeItem {
episode_id: episode_id.clone(),
chunk_id: chunk_id.clone(),
origin_tenant_id: tenant.clone(),
origin_project_id: None,
original_rank: 0,
original_score: 1.0,
lane_scores_json: "{}".to_string(),
outcome_adjustment: 0.0,
served_rank: Some(0),
shadow_rank: Some(0),
rendered: true,
source_dedup_group: None,
},
RetrievalEpisodeItem {
episode_id: episode_id.clone(),
chunk_id: unrendered_chunk_id.clone(),
origin_tenant_id: tenant.clone(),
origin_project_id: None,
original_rank: 1,
original_score: 0.5,
lane_scores_json: "{}".to_string(),
outcome_adjustment: 0.0,
served_rank: None,
shadow_rank: Some(1),
rendered: false,
source_dedup_group: None,
},
],
)
.await
.unwrap();
std::fs::create_dir_all(project.path().join(".memd")).unwrap();
std::fs::write(
project.path().join(".memd/project_scope.json"),
serde_json::json!({
"tenant_id": tenant.as_str(),
"project_id": null,
"interface": "cli",
"cli_command": "memd",
"agent_context_output": ".memd/context.md",
"project_dir": "."
})
.to_string(),
)
.unwrap();
let search_output = serde_json::json!({
"retrieval_episode_id": episode_id.to_string(),
"results": [
{"chunk_id": chunk_id.to_string()},
{"chunk_id": unrendered_chunk_id.to_string()}
]
})
.to_string();
let session_path = sessions.path().join("rollout-2026-09-15-current.jsonl");
let records = [
serde_json::json!({
"timestamp": "2026-09-15T12:00:01Z",
"type": "response_item",
"ordinal": 1,
"payload": {
"type": "custom_tool_call_output",
"id": "output-1",
"call_id": "call-1",
"output": [
{"type": "text", "text": search_output},
{"type": "text", "text": ""}
],
"internal_chat_message_metadata_passthrough": {}
}
}),
serde_json::json!({
"timestamp": "2026-09-15T11:00:00Z",
"type": "response_item",
"ordinal": 2,
"payload": {
"type": "custom_tool_call",
"id": "input-1",
"status": "completed",
"call_id": "call-2",
"name": "functions.exec",
"input": format!(
"await tools.exec_command({{cmd: \\\"bash {unrendered_literal}\\\"}});"
),
"internal_chat_message_metadata_passthrough": {}
}
}),
serde_json::json!({
"timestamp": action_timestamp,
"type": "response_item",
"ordinal": 3,
"payload": {
"type": "custom_tool_call",
"id": "input-2",
"status": "completed",
"call_id": "call-3",
"name": "functions.exec",
"input": format!(
"await tools.exec_command({{cmd: \\\"bash {literal}\\\"}});"
),
"internal_chat_message_metadata_passthrough": {}
}
}),
];
std::fs::write(
&session_path,
records
.iter()
.map(Value::to_string)
.collect::<Vec<_>>()
.join("\n")
+ "\n",
)
.unwrap();
let options = || OutcomeScanOptions {
project_dir: project.path().to_path_buf(),
sessions_dir: Some(sessions.path().to_path_buf()),
since_days: 30,
dry_run: false,
};
let first = run_outcome_scan(&store, options()).await.unwrap();
assert_eq!(first["events_written"], 1);
let events = store
.list_outcomes_for_episode(&tenant, &episode_id)
.await
.unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].outcome, OutcomeKind::ObservedUsed);
assert!(!events[0].ranking_eligible);
assert_eq!(events[0].used_chunk_ids, vec![chunk_id]);
assert_eq!(events[0].timestamp_ms, action_timestamp_ms);
assert_eq!(
events[0].evidence_reference.as_deref(),
Some("codex:rollout-2026-09-15-current.jsonl:3")
);
std::thread::sleep(std::time::Duration::from_millis(2));
let mut file = std::fs::OpenOptions::new()
.append(true)
.open(&session_path)
.unwrap();
use std::io::Write as _;
writeln!(
file,
"{}",
serde_json::json!({
"timestamp": "2026-09-15T18:00:00Z",
"type": "response_item",
"ordinal": 4,
"payload": {
"type": "custom_tool_call",
"id": "input-3",
"status": "completed",
"call_id": "call-4",
"name": "functions.exec",
"input": format!(
"await tools.exec_command({{cmd: \\\"bash {literal}\\\"}});"
),
"internal_chat_message_metadata_passthrough": {}
}
})
)
.unwrap();
drop(file);
let second = run_outcome_scan(&store, options()).await.unwrap();
assert_eq!(second["events_written"], 0);
assert_eq!(second["events_skipped"]["already_recorded"], 1);
let events = store
.list_outcomes_for_episode(&tenant, &episode_id)
.await
.unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].timestamp_ms, action_timestamp_ms);
}
#[test]
fn null_episode_id_is_not_a_serve() {
let session = concat!(
"{\"type\":\"response_item\",\"payload\":{\"type\":\"custom_tool_call_output\",",
"\"output\":\"\\\"retrieval_episode_id\\\": null\"}}\n",
);
assert!(parse_session(session).serves.is_empty());
}
#[test]
fn repo_path_literals_are_dropped() {
let dir = tempfile::tempdir().unwrap();
let project_dir = dir.path();
std::fs::create_dir_all(project_dir.join("src")).unwrap();
std::fs::write(project_dir.join("src/real_file_on_disk.rs"), "x").unwrap();
assert!(literal_is_repo_path(
project_dir,
"src/real_file_on_disk.rs"
));
assert!(literal_is_repo_path(
project_dir,
project_dir
.join("src/real_file_on_disk.rs")
.to_str()
.unwrap(),
));
assert!(!literal_is_repo_path(project_dir, "src/not_on_disk.rs"));
assert!(!literal_is_repo_path(
Path::new("/nonexistent-project"),
project_dir
.join("src/real_file_on_disk.rs")
.to_str()
.unwrap(),
));
}
}