Skip to main content

meerkat_mobkit/memory/
distiller.rs

1//! Distiller — extraction from evidence
2//! (docs/design/agent-memory-architecture.md §8.4).
3//!
4//! Runs **off-turn**: a bounded one-shot structured LLM call over a window
5//! of session evidence, producing `remember`/`update` proposals that land
6//! through the authored-write seam with `MemoryAuthor::Distiller` — never
7//! above `AgentObserved`, quarantined when the evidence window is tainted
8//! (§10.1) or when the window closed at a `reset()` boundary (§8.4).
9//!
10//! ## Harness: detached now, fork later (deliberate, not a shortcut)
11//!
12//! §8.4 prefers a `Session::fork` harness so the extraction call shares the
13//! parent's prompt cache. The primitive exists in meerkat 0.7.9
14//! (`Session::fork` / `fork_at`, meerkat-core `session.rs:3644/3764`), but
15//! MobKit can only reach it through the mob layer's fork launch mode
16//! (`meerkat_mob::launch::MemberLaunchMode::Fork` + `ForkContext`,
17//! `launch.rs:23/52`; one-shot `MobRuntimeHandle::fork_helper`,
18//! `handle.rs:5946`) — which spawns a **live member carrying the parent's
19//! full tool surface**. §8.4's fork containment ("every call gated to
20//! read-only + propose/remember") needs an authorization/capability layer
21//! that does not exist yet, and a fork-helper run gives back free text, not
22//! this stage's validated structured ops. Shipping the fork path now would
23//! mean an uncontained extractor with live tools. So P2 ships the
24//! **detached bounded re-read** first-class — a one-shot client obtained
25//! through the same `AgentFactory::build_llm_client_for_identity` seam the
26//! Selector uses (§8.1), over a transcript slice read from the persistent
27//! session store — with cost bounded by the §8.1 guards.
28//!
29//! TODO(§8.4 fork harness): when a capability-gated tool-authorization
30//! layer lands, add a fork-based path via
31//! `SpawnMemberSpec.launch_mode = MemberLaunchMode::Fork { source_member_id,
32//! fork_context: ForkContext::FullHistory }` (the O(1) CoW
33//! `Session::fork`), keeping the parent's tool list byte-identical for
34//! prompt-cache sharing and moving containment to the authorization layer.
35//! The detached path stays as the cross-process / cache-expired fallback.
36//!
37//! ## Triggers
38//!
39//! (a) **Completed interactions** — the observe-only agent-event stream
40//! (same surface as the taint tracker), coalesced per session and
41//! throttled (≥ `min_interactions` runs completed AND ≥
42//! `MIN_SECONDS_BETWEEN_RUNS` since the last extraction). (b) **Session
43//! rotation** — the identity runtime's respawn/retire/delete paths call
44//! [`DistillerEngine::distill_now`] before rotation (bounded by
45//! [`PRE_ROTATION_TIMEOUT`]; rotation never hangs on distillation), and
46//! reset/resume-fallback spawn it detached off the critical path.
47//! (c) **Compaction** — `AgentEvent::CompactionCompleted` is observable on
48//! the agent-event stream; the discarded content survives only in
49//! meerkat's session semantic memory, so the post-compaction run reads the
50//! discard range host-side via `MemoryStore::enumerate_scoped` over that
51//! session's scope (ask 8: exact, provenance-ordered, paged — read-only
52//! [`HnswDiscardSource`]; opened lazily, drained, dropped — D3's re-index
53//! cost is paid once per harvest, never held).
54
55use std::collections::HashMap;
56use std::path::{Path, PathBuf};
57use std::sync::{Arc, Mutex};
58use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
59
60use async_trait::async_trait;
61use futures::StreamExt;
62use serde::Deserialize;
63
64use meerkat_client::{LlmClient, LlmDoneOutcome, LlmError, LlmEvent, LlmRequest};
65use meerkat_core::event::AgentEvent;
66use meerkat_core::{Message, Provider, UserMessage};
67
68use crate::identity_first::agent_memory::{
69    AgentMemoryError, AgentMemoryProvider, MEMORY_TOOL_NAME, compact_whitespace,
70    truncate_utf8_boundary,
71};
72use crate::memory::guards::{BackgroundBudget, BackgroundBudgetConfig};
73use crate::memory::records::{
74    EvidenceRef, ManifestTier, MemoryAuthor, MemoryKind, MemoryScope, NewMemoryRecord, RecordMeta,
75};
76use crate::memory::selector::FactorySelectorHandle;
77use crate::memory::taint::{MemberAgentEventSink, SessionTaintTracker};
78
79/// Embedded prompt bundle (crate-local copy of
80/// `memory-evals/prompts/distiller-v0.md`; a unit test enforces byte
81/// equality so the calibration artifact and the shipped default cannot
82/// drift — same pattern as the Selector).
83pub const EMBEDDED_PROMPT_V0: &str = include_str!("distiller_prompt_v0.md");
84
85const MANIFEST_PLACEHOLDER: &str = "{{existing_manifest}}";
86const TOMBSTONES_PLACEHOLDER: &str = "{{recent_tombstones}}";
87const TRANSCRIPT_PLACEHOLDER: &str = "{{transcript}}";
88
89/// Pre-rotation distillation budget: respawn/retire/delete wait at most
90/// this long before proceeding with rotation (§8.4 — rotation must never
91/// hang on distillation; a timed-out run is a loud skip).
92pub const PRE_ROTATION_TIMEOUT: Duration = Duration::from_secs(15);
93/// Per-identity coalescing floor between interaction-triggered runs.
94pub const MIN_SECONDS_BETWEEN_RUNS: u64 = 120;
95/// Tombstone window rendered into the prompt's "never re-create" list.
96const TOMBSTONE_LOOKBACK_MS: u64 = 7 * 24 * 60 * 60 * 1000;
97/// Row cap for one compaction-discard harvest query.
98const COMPACTION_HARVEST_LIMIT: usize = 64;
99/// Per-message and total byte bounds on the rendered transcript window.
100const MAX_TRANSCRIPT_MESSAGE_BYTES: usize = 4 * 1024;
101const MAX_TRANSCRIPT_TOTAL_BYTES: usize = 48 * 1024;
102/// Window-state entries are bounded; least-recently-active evict first.
103const MAX_TRACKED_WINDOWS: usize = 4096;
104/// Output budget for the structured op list.
105const DEFAULT_MAX_OUTPUT_TOKENS: u32 = 2048;
106
107// ---------------------------------------------------------------------------
108// Errors
109// ---------------------------------------------------------------------------
110
111#[derive(Debug)]
112pub enum DistillerError {
113    Profile(String),
114    Auth(String),
115    Client(String),
116    Parse(String),
117    Store(String),
118    Transcript(String),
119}
120
121impl std::fmt::Display for DistillerError {
122    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
123        match self {
124            Self::Profile(msg) => write!(f, "distiller profile error: {msg}"),
125            Self::Auth(msg) => write!(f, "distiller auth error: {msg}"),
126            Self::Client(msg) => write!(f, "distiller client error: {msg}"),
127            Self::Parse(msg) => write!(f, "distiller parse error: {msg}"),
128            Self::Store(msg) => write!(f, "distiller store error: {msg}"),
129            Self::Transcript(msg) => write!(f, "distiller transcript error: {msg}"),
130        }
131    }
132}
133
134impl std::error::Error for DistillerError {}
135
136// ---------------------------------------------------------------------------
137// Calibration profile (§11)
138// ---------------------------------------------------------------------------
139
140#[derive(Debug, Clone, Deserialize)]
141pub struct DistillerParams {
142    #[serde(default = "default_temperature")]
143    pub temperature: f32,
144    #[serde(default = "default_max_output_tokens")]
145    pub max_output_tokens: u32,
146    /// Manifest rows rendered into the prompt (newest-first when truncating).
147    #[serde(default = "default_max_manifest_records")]
148    pub max_manifest_records: usize,
149    #[serde(default = "default_max_tombstones")]
150    pub max_tombstones: usize,
151}
152
153fn default_temperature() -> f32 {
154    0.0
155}
156fn default_max_output_tokens() -> u32 {
157    DEFAULT_MAX_OUTPUT_TOKENS
158}
159fn default_max_manifest_records() -> usize {
160    200
161}
162fn default_max_tombstones() -> usize {
163    32
164}
165
166impl Default for DistillerParams {
167    fn default() -> Self {
168        Self {
169            temperature: default_temperature(),
170            max_output_tokens: default_max_output_tokens(),
171            max_manifest_records: default_max_manifest_records(),
172            max_tombstones: default_max_tombstones(),
173        }
174    }
175}
176
177/// A loaded distiller calibration profile (§11), prompt template resolved.
178#[derive(Debug, Clone)]
179pub struct DistillerProfile {
180    pub stage: String,
181    pub version: String,
182    pub model: String,
183    pub provider: Provider,
184    pub prompt_bundle: String,
185    pub prompt_template: String,
186    pub params: DistillerParams,
187}
188
189#[derive(Debug, Deserialize)]
190struct RawProfile {
191    stage: String,
192    version: String,
193    model: String,
194    #[serde(default)]
195    provider: Option<String>,
196    prompt_bundle: String,
197    #[serde(default)]
198    params: Option<DistillerParams>,
199}
200
201impl DistillerProfile {
202    /// The embedded default: `memory-evals/profiles/distiller-v0.toml` with
203    /// the prompt compiled in. The model tier is a calibration decision
204    /// (§11/§16); the config's `distiller.model` override adjusts it
205    /// per-deployment without a new profile file.
206    pub fn embedded_default() -> Self {
207        Self {
208            stage: "distiller".to_string(),
209            version: "0".to_string(),
210            model: "claude-haiku-4-5".to_string(),
211            provider: Provider::Anthropic,
212            prompt_bundle: "prompts/distiller-v0.md".to_string(),
213            prompt_template: EMBEDDED_PROMPT_V0.to_string(),
214            params: DistillerParams::default(),
215        }
216    }
217
218    /// Replace the profile's model (the config-block override). Fail-loud:
219    /// the model must resolve in the catalog unless a provider is already
220    /// pinned by the profile.
221    pub fn with_model_override(mut self, model: &str) -> Result<Self, DistillerError> {
222        let model = model.trim();
223        if model.is_empty() {
224            return Err(DistillerError::Profile(
225                "distiller model override must not be empty".to_string(),
226            ));
227        }
228        self.provider = meerkat_models::infer_provider(model).ok_or_else(|| {
229            DistillerError::Profile(format!(
230                "distiller model override '{model}' is not in the model catalog"
231            ))
232        })?;
233        self.model = model.to_string();
234        Ok(self)
235    }
236
237    /// Load an external calibration profile (fail-loud), same layout rules
238    /// as the Selector's loader.
239    pub fn load(path: &Path) -> Result<Self, DistillerError> {
240        let text = std::fs::read_to_string(path).map_err(|err| {
241            DistillerError::Profile(format!("cannot read profile '{}': {err}", path.display()))
242        })?;
243        let raw: RawProfile = toml::from_str(&text).map_err(|err| {
244            DistillerError::Profile(format!("invalid profile '{}': {err}", path.display()))
245        })?;
246        if raw.stage != "distiller" {
247            return Err(DistillerError::Profile(format!(
248                "profile '{}' is for stage '{}', not 'distiller'",
249                path.display(),
250                raw.stage
251            )));
252        }
253        if raw.model.trim().is_empty() || raw.model == "PLACEHOLDER" {
254            return Err(DistillerError::Profile(format!(
255                "profile '{}' does not name a model",
256                path.display()
257            )));
258        }
259        let provider = match raw.provider.as_deref() {
260            Some(name) => Provider::parse_strict(name).ok_or_else(|| {
261                DistillerError::Profile(format!(
262                    "profile '{}': unknown provider '{name}'",
263                    path.display()
264                ))
265            })?,
266            None => meerkat_models::infer_provider(&raw.model).ok_or_else(|| {
267                DistillerError::Profile(format!(
268                    "profile '{}': model '{}' is not in the catalog; set `provider` explicitly",
269                    path.display(),
270                    raw.model
271                ))
272            })?,
273        };
274        let base = path.parent().unwrap_or_else(|| Path::new("."));
275        let candidates = [
276            base.join(&raw.prompt_bundle),
277            base.parent()
278                .unwrap_or_else(|| Path::new("."))
279                .join(&raw.prompt_bundle),
280        ];
281        let bundle_path = candidates.iter().find(|p| p.is_file()).ok_or_else(|| {
282            DistillerError::Profile(format!(
283                "profile '{}': prompt_bundle '{}' does not resolve",
284                path.display(),
285                raw.prompt_bundle
286            ))
287        })?;
288        let prompt_template = std::fs::read_to_string(bundle_path).map_err(|err| {
289            DistillerError::Profile(format!(
290                "cannot read prompt bundle '{}': {err}",
291                bundle_path.display()
292            ))
293        })?;
294        let profile = Self {
295            stage: raw.stage,
296            version: raw.version,
297            model: raw.model,
298            provider,
299            prompt_bundle: raw.prompt_bundle,
300            prompt_template,
301            params: raw.params.unwrap_or_default(),
302        };
303        profile.validate()?;
304        Ok(profile)
305    }
306
307    fn validate(&self) -> Result<(), DistillerError> {
308        for placeholder in [
309            MANIFEST_PLACEHOLDER,
310            TOMBSTONES_PLACEHOLDER,
311            TRANSCRIPT_PLACEHOLDER,
312        ] {
313            if !self.prompt_template.contains(placeholder) {
314                return Err(DistillerError::Profile(format!(
315                    "prompt bundle '{}' is missing placeholder `{placeholder}`",
316                    self.prompt_bundle
317                )));
318            }
319        }
320        Ok(())
321    }
322}
323
324// ---------------------------------------------------------------------------
325// Config (`agent_memory.distiller { ... }`)
326// ---------------------------------------------------------------------------
327
328/// Distiller config block. `enabled` defaults **off** for this landing:
329/// flipping the default is a calibration-scorecard decision (§11), like the
330/// Selector's.
331#[derive(Debug, Clone, PartialEq, Eq)]
332pub struct DistillerConfig {
333    pub enabled: bool,
334    /// §8.1 hard per-window cap on distillation runs, per realm.
335    pub runs_per_hour: u32,
336    /// Interaction-trigger threshold: completed runs per session before an
337    /// extraction is considered.
338    pub min_interactions: u32,
339    /// Optional model override applied to the embedded default profile.
340    pub model: Option<String>,
341}
342
343impl Default for DistillerConfig {
344    fn default() -> Self {
345        Self {
346            enabled: false,
347            runs_per_hour: crate::memory::guards::DEFAULT_RUNS_PER_HOUR,
348            min_interactions: 3,
349            model: None,
350        }
351    }
352}
353
354// ---------------------------------------------------------------------------
355// Structured output: proposed ops
356// ---------------------------------------------------------------------------
357
358/// Epistemic attribution (§8.4 doctrine rule 5). Mechanically mirrored into
359/// an `epistemic:*` tag on the landed record, same convention as the
360/// Recorder tool.
361#[derive(Debug, Clone, Copy, PartialEq, Eq)]
362pub enum Epistemic {
363    OperatorSaid,
364    Observed,
365}
366
367impl Epistemic {
368    fn parse(value: &str) -> Option<Self> {
369        match value {
370            "operator_said" => Some(Self::OperatorSaid),
371            "observed" => Some(Self::Observed),
372            _ => None,
373        }
374    }
375}
376
377#[derive(Debug, Clone, PartialEq, Eq)]
378pub enum ProposedAction {
379    Remember,
380    Update { target_id: String },
381}
382
383/// One validated distiller proposal.
384#[derive(Debug, Clone, PartialEq)]
385pub struct ProposedOp {
386    pub action: ProposedAction,
387    pub kind: MemoryKind,
388    pub title: String,
389    pub description: String,
390    pub body: String,
391    pub tags: Vec<String>,
392    pub epistemic: Epistemic,
393    pub evidence_range: Option<(u64, u64)>,
394}
395
396#[derive(Debug, Deserialize)]
397struct RawOp {
398    action: String,
399    #[serde(default)]
400    target_id: Option<String>,
401    #[serde(default)]
402    kind: Option<String>,
403    title: String,
404    #[serde(default)]
405    description: String,
406    body: String,
407    #[serde(default)]
408    tags: Vec<String>,
409    epistemic: String,
410    #[serde(default)]
411    evidence_range: Option<(u64, u64)>,
412}
413
414/// Strict parse of the model's op list: exactly one JSON array (fenced or
415/// prefixed output tolerated by extracting the outermost `[..]`).
416pub fn parse_ops(reply: &str) -> Result<Vec<RawParsedOp>, String> {
417    let trimmed = reply.trim();
418    let raw: Vec<RawOp> = match serde_json::from_str(trimmed) {
419        Ok(raw) => raw,
420        Err(first_err) => {
421            let (Some(start), Some(end)) = (trimmed.find('['), trimmed.rfind(']')) else {
422                return Err(format!("no JSON array in reply: {first_err}"));
423            };
424            if start >= end {
425                return Err(format!("no JSON array in reply: {first_err}"));
426            }
427            serde_json::from_str(&trimmed[start..=end]).map_err(|err| err.to_string())?
428        }
429    };
430    Ok(raw.into_iter().map(RawParsedOp).collect())
431}
432
433/// A syntactically-parsed op awaiting semantic validation. Opaque on
434/// purpose: callers go through [`validate_op`].
435#[derive(Debug)]
436pub struct RawParsedOp(RawOp);
437
438/// Semantic validation of one parsed op against the manifest the model was
439/// shown. Invalid ops are per-op skips (warned by the caller), not run
440/// failures — one bad op must not discard the window's good ones.
441pub fn validate_op(op: RawParsedOp, manifest_ids: &[String]) -> Result<ProposedOp, String> {
442    let raw = op.0;
443    let action = match raw.action.as_str() {
444        "remember" => ProposedAction::Remember,
445        "update" => {
446            let target = raw
447                .target_id
448                .as_deref()
449                .map(str::trim)
450                .filter(|id| !id.is_empty())
451                .ok_or_else(|| "update op is missing target_id".to_string())?;
452            if !manifest_ids.iter().any(|id| id == target) {
453                return Err(format!(
454                    "update op targets '{target}', which is not in the manifest"
455                ));
456            }
457            ProposedAction::Update {
458                target_id: target.to_string(),
459            }
460        }
461        other => return Err(format!("unknown action '{other}'")),
462    };
463    let kind = match raw.kind.as_deref() {
464        None => MemoryKind::Fact,
465        Some(kind) => {
466            MemoryKind::parse(kind).ok_or_else(|| format!("unknown record kind '{kind}'"))?
467        }
468    };
469    let epistemic = Epistemic::parse(&raw.epistemic)
470        .ok_or_else(|| format!("unknown epistemic status '{}'", raw.epistemic))?;
471    let title = compact_whitespace(&raw.title);
472    if title.is_empty() {
473        return Err("op has an empty title".to_string());
474    }
475    let body = raw.body.trim().to_string();
476    if body.is_empty() {
477        return Err("op has an empty body".to_string());
478    }
479    if let Some((start, end)) = raw.evidence_range
480        && start > end
481    {
482        return Err(format!("evidence_range [{start}, {end}] is inverted"));
483    }
484    Ok(ProposedOp {
485        action,
486        kind,
487        title,
488        description: compact_whitespace(&raw.description),
489        body,
490        tags: raw.tags,
491        epistemic,
492        evidence_range: raw.evidence_range,
493    })
494}
495
496// ---------------------------------------------------------------------------
497// Evidence sources
498// ---------------------------------------------------------------------------
499
500/// One transcript message in the evidence window, with its **absolute**
501/// position in the persisted session (the `[N]` index the prompt shows and
502/// `evidence_range` cites).
503#[derive(Debug, Clone, PartialEq, Eq)]
504pub struct TranscriptMessage {
505    pub index: u64,
506    pub role: &'static str,
507    pub text: String,
508}
509
510/// The evidence window: messages `[start_index, end_index)` of the session
511/// transcript. `end_index` is the cursor the engine advances to.
512#[derive(Debug, Clone, PartialEq, Eq)]
513pub struct TranscriptSlice {
514    pub session_key: String,
515    pub start_index: u64,
516    pub end_index: u64,
517    pub messages: Vec<TranscriptMessage>,
518    /// Ask 4 refinement (0.7.12): the transcript head revision id at the
519    /// instant this slice was read, so distilled records can pin the exact
520    /// revision their evidence was extracted from (§7.1). `None` when the
521    /// source cannot report a revision.
522    pub head_revision: Option<String>,
523}
524
525/// How the engine reads persisted transcripts. The real implementation is
526/// [`SessionStoreTranscriptSource`]; tests supply scripted slices.
527#[async_trait]
528pub trait TranscriptSource: Send + Sync {
529    /// Messages from `from_index` to the current end of the persisted
530    /// transcript; `None` when the session does not exist in the store.
531    async fn read(
532        &self,
533        session_key: &str,
534        from_index: u64,
535    ) -> Result<Option<TranscriptSlice>, DistillerError>;
536}
537
538/// Reads the persistent session store (`SessionStore::load`, the same
539/// surface the console history uses). The transcript is durable and
540/// positionally indexed, and survives member teardown/retire/reset — which
541/// is what makes the detached reset-boundary distillation possible.
542pub struct SessionStoreTranscriptSource {
543    store: Arc<dyn meerkat::SessionStore>,
544}
545
546impl SessionStoreTranscriptSource {
547    pub fn new(store: Arc<dyn meerkat::SessionStore>) -> Self {
548        Self { store }
549    }
550}
551
552#[async_trait]
553impl TranscriptSource for SessionStoreTranscriptSource {
554    async fn read(
555        &self,
556        session_key: &str,
557        from_index: u64,
558    ) -> Result<Option<TranscriptSlice>, DistillerError> {
559        let session_id = meerkat_core::types::SessionId::parse(session_key).map_err(|err| {
560            DistillerError::Transcript(format!("invalid session key '{session_key}': {err}"))
561        })?;
562        let session = self
563            .store
564            .load(&session_id)
565            .await
566            .map_err(|err| DistillerError::Transcript(err.to_string()))?;
567        let Some(session) = session else {
568            return Ok(None);
569        };
570        // Ask 4 refinement: pin the head revision the extractor is about to
571        // read. `transcript_revision()` is fallible serialization; a failure
572        // degrades to None ("head at capture time") rather than blocking the
573        // distillation.
574        let head_revision = session.transcript_revision().ok();
575        let all = session.messages();
576        let end_index = all.len() as u64;
577        let start_index = from_index.min(end_index);
578        let messages = all[start_index as usize..]
579            .iter()
580            .enumerate()
581            .filter_map(|(offset, message)| {
582                let index = start_index + offset as u64;
583                project_message(message).map(|(role, text)| TranscriptMessage {
584                    index,
585                    role,
586                    text: truncate_utf8_boundary(
587                        &compact_whitespace(&text),
588                        MAX_TRANSCRIPT_MESSAGE_BYTES,
589                    ),
590                })
591            })
592            .collect();
593        Ok(Some(TranscriptSlice {
594            session_key: session_key.to_string(),
595            start_index,
596            end_index,
597            messages,
598            head_revision,
599        }))
600    }
601}
602
603/// Text projection of one transcript message. The system prompt is not
604/// evidence (it is configuration, and it would dwarf the window); tool
605/// results are evidence (operator-visible ground truth) but rendered
606/// tersely.
607fn project_message(message: &Message) -> Option<(&'static str, String)> {
608    match message {
609        Message::System(_) => None,
610        Message::SystemNotice(notice) => notice
611            .body
612            .as_deref()
613            .map(|body| ("system notice", body.to_string())),
614        Message::User(user) => Some(("user", user.text_content())),
615        Message::BlockAssistant(assistant) => Some((
616            "assistant",
617            assistant.text_blocks().collect::<Vec<_>>().join("\n"),
618        )),
619        Message::ToolResults { results, .. } => {
620            let text = results
621                .iter()
622                .map(|result| meerkat_core::types::text_content(&result.content))
623                .collect::<Vec<_>>()
624                .join("\n");
625            Some(("tool results", text))
626        }
627    }
628}
629
630/// One compaction-discard row harvested from meerkat's session semantic
631/// memory (§8.4 trigger (c)).
632#[derive(Debug, Clone, PartialEq)]
633pub struct DiscardEntry {
634    pub content: String,
635    /// Source message offsets in the **pre-compaction** transcript
636    /// (`MemorySource::Compaction.source_range`).
637    pub range: Option<(u64, u64)>,
638}
639
640/// Host-side read over a session's compaction discards.
641#[async_trait]
642pub trait CompactionDiscardSource: Send + Sync {
643    async fn read_discards(
644        &self,
645        session_key: &str,
646        limit: usize,
647    ) -> Result<Vec<DiscardEntry>, DistillerError>;
648
649    /// Ask 2 (landed in meerkat 0.7.12): reclaim a permanently-orphaned
650    /// session's semantic-memory rows via `MemoryStore::drop_scope`. Session
651    /// ids rotate under respawn/reset and vanish under delete, stranding rows
652    /// that nothing will ever search yet that every future agent build in the
653    /// realm re-embeds (defect D3, storage half). Callers MUST run this only
654    /// AFTER distillation has preserved what mattered, and ONLY for causes
655    /// that permanently abandon the id. Returns the number of rows dropped
656    /// (0 when unsupported — the default is a no-op for stores/test doubles
657    /// without a drop facility).
658    async fn drop_scope(&self, session_key: &str) -> Result<usize, DistillerError> {
659        let _ = session_key;
660        Ok(0)
661    }
662}
663
664/// Reads meerkat's own session semantic memory store at
665/// `<persistent_state>/memory` (the path `AgentFactory` opens it at,
666/// meerkat `factory.rs:5416`). **Read-only host-side access to meerkat's
667/// store** — sanctioned by §8.4; this is not a MobKit retrieval index and
668/// stays outside the §12 bright line, which governs the bundled store.
669///
670/// Cost note (D3): `HnswMemoryStore::open` rebuilds its in-RAM index from
671/// the SQLite rows on every open, so the store is opened lazily per
672/// harvest and dropped afterwards.
673///
674/// Ask 8 (landed in meerkat 0.7.12): the harvest now reads the discard range
675/// via `MemoryStore::enumerate_scoped` — exact, provenance-ordered (durable
676/// id), paged — instead of the old generous-limit `search` approximation.
677/// Enumeration is read-only and additive; it does not touch indexing/search
678/// semantics. The per-page `limit` is the total safety ceiling on rows read;
679/// if a scope exceeds it the harvest stops and logs (never silently), but at
680/// per-session compaction-row counts the full scope fits well within it.
681pub struct HnswDiscardSource {
682    dir: PathBuf,
683}
684
685/// Raw scope rows fetched per `enumerate_scoped` page.
686const HARVEST_PAGE_ROWS: usize = 256;
687
688impl HnswDiscardSource {
689    pub fn new(dir: impl Into<PathBuf>) -> Self {
690        Self { dir: dir.into() }
691    }
692}
693
694#[async_trait]
695impl CompactionDiscardSource for HnswDiscardSource {
696    async fn read_discards(
697        &self,
698        session_key: &str,
699        limit: usize,
700    ) -> Result<Vec<DiscardEntry>, DistillerError> {
701        use meerkat_core::memory::{MemoryEnumerationRequest, MemorySearchScope, MemoryStore};
702
703        if !self.dir.is_dir() {
704            // No session semantic memory in this deployment: nothing was
705            // preserved at compaction, so there is nothing to harvest.
706            return Ok(Vec::new());
707        }
708        let session_id = meerkat_core::types::SessionId::parse(session_key).map_err(|err| {
709            DistillerError::Transcript(format!("invalid session key '{session_key}': {err}"))
710        })?;
711        let dir = self.dir.clone();
712        let store = tokio::task::spawn_blocking(move || meerkat_memory::HnswMemoryStore::open(dir))
713            .await
714            .map_err(|err| DistillerError::Store(err.to_string()))?
715            .map_err(|err| DistillerError::Store(err.to_string()))?;
716        let scope = MemorySearchScope::for_session(session_id);
717
718        // Ask 8: page the scope exactly, in durable-id order, until it is
719        // exhausted or the safety ceiling is hit — no relevance ranking, no
720        // silent truncation.
721        let mut entries = Vec::new();
722        let mut offset = 0usize;
723        loop {
724            let remaining = limit.saturating_sub(entries.len());
725            if remaining == 0 {
726                break;
727            }
728            let page = store
729                .enumerate_scoped(
730                    &scope,
731                    MemoryEnumerationRequest {
732                        limit: HARVEST_PAGE_ROWS.min(remaining),
733                        offset,
734                        source_overlap: None,
735                        indexed_after: None,
736                    },
737                )
738                .await
739                .map_err(|err| DistillerError::Store(err.to_string()))?;
740            for record in page.records {
741                entries.push(DiscardEntry {
742                    range: record
743                        .metadata
744                        .source
745                        .source_range()
746                        .map(|range| (range.start(), range.end())),
747                    content: record.content,
748                });
749            }
750            match page.next_offset {
751                Some(next) if entries.len() < limit => offset = next,
752                Some(_) => {
753                    tracing::warn!(
754                        session_key,
755                        limit,
756                        "compaction discard harvest hit its row ceiling; \
757                         scope has more rows than were read"
758                    );
759                    break;
760                }
761                None => break,
762            }
763        }
764        Ok(entries)
765    }
766
767    async fn drop_scope(&self, session_key: &str) -> Result<usize, DistillerError> {
768        use meerkat_core::memory::{MemoryOwner, MemoryStore};
769
770        if !self.dir.is_dir() {
771            // No session semantic memory in this deployment: nothing to reap.
772            return Ok(0);
773        }
774        let session_id = meerkat_core::types::SessionId::parse(session_key).map_err(|err| {
775            DistillerError::Transcript(format!("invalid session key '{session_key}': {err}"))
776        })?;
777        let dir = self.dir.clone();
778        let store = tokio::task::spawn_blocking(move || meerkat_memory::HnswMemoryStore::open(dir))
779            .await
780            .map_err(|err| DistillerError::Store(err.to_string()))?
781            .map_err(|err| DistillerError::Store(err.to_string()))?;
782        let receipt = store
783            .drop_scope(&MemoryOwner::canonical_session(session_id))
784            .await
785            .map_err(|err| DistillerError::Store(err.to_string()))?;
786        Ok(receipt.dropped_entries)
787    }
788}
789
790// ---------------------------------------------------------------------------
791// Tombstones (prompt guard) — the mechanical backstop is the staged
792// validator's tombstone-recreation rejection; this list closes the
793// paraphrase gap (§8.4 "never re-create these").
794// ---------------------------------------------------------------------------
795
796#[derive(Debug, Clone, PartialEq, Eq)]
797pub struct TombstoneMeta {
798    pub title: String,
799    pub kind: MemoryKind,
800    pub tombstoned_at_ms: u64,
801}
802
803#[async_trait]
804pub trait TombstoneSource: Send + Sync {
805    async fn recent_tombstones(
806        &self,
807        scope: &MemoryScope,
808        since_ms: u64,
809        limit: usize,
810    ) -> Result<Vec<TombstoneMeta>, AgentMemoryError>;
811}
812
813// ---------------------------------------------------------------------------
814// Client acquisition (§8.1 — same factory seam as the Selector)
815// ---------------------------------------------------------------------------
816
817#[async_trait]
818pub trait DistillerClientHandle: Send + Sync {
819    async fn client(&self) -> Result<Arc<dyn LlmClient>, DistillerError>;
820    fn invalidate(&self);
821}
822
823/// Thin wrapper over the Selector's factory handle: one client-acquisition
824/// path for every judgment stage (§8.1 dogma rule 7).
825pub struct FactoryDistillerHandle {
826    inner: FactorySelectorHandle,
827}
828
829impl FactoryDistillerHandle {
830    pub fn new(
831        store_path: impl Into<PathBuf>,
832        config: meerkat::Config,
833        realm: impl Into<String>,
834        profile: &DistillerProfile,
835    ) -> Self {
836        Self {
837            inner: FactorySelectorHandle::for_model(
838                store_path,
839                config,
840                realm,
841                &profile.model,
842                profile.provider,
843            ),
844        }
845    }
846}
847
848#[async_trait]
849impl DistillerClientHandle for FactoryDistillerHandle {
850    async fn client(&self) -> Result<Arc<dyn LlmClient>, DistillerError> {
851        use crate::memory::selector::{SelectorError, SelectorHandle};
852        self.inner.client().await.map_err(|err| match err {
853            SelectorError::Auth(msg) => DistillerError::Auth(msg),
854            other => DistillerError::Client(other.to_string()),
855        })
856    }
857
858    fn invalidate(&self) {
859        use crate::memory::selector::SelectorHandle;
860        self.inner.invalidate();
861    }
862}
863
864// ---------------------------------------------------------------------------
865// Prompt rendering + the extraction call
866// ---------------------------------------------------------------------------
867
868pub fn render_prompt(
869    profile: &DistillerProfile,
870    manifest: &[RecordMeta],
871    tombstones: &[TombstoneMeta],
872    transcript_text: &str,
873) -> String {
874    let manifest_text = if manifest.is_empty() {
875        "(no records)".to_string()
876    } else {
877        manifest
878            .iter()
879            .take(profile.params.max_manifest_records)
880            .map(crate::memory::selector::render_manifest_row)
881            .collect::<Vec<_>>()
882            .join("\n")
883    };
884    let tombstones_text = if tombstones.is_empty() {
885        "(none)".to_string()
886    } else {
887        tombstones
888            .iter()
889            .take(profile.params.max_tombstones)
890            .map(|tombstone| {
891                format!(
892                    "- [{}] {}",
893                    tombstone.kind.as_str(),
894                    compact_whitespace(&tombstone.title)
895                )
896            })
897            .collect::<Vec<_>>()
898            .join("\n")
899    };
900    profile
901        .prompt_template
902        .replace(MANIFEST_PLACEHOLDER, &manifest_text)
903        .replace(TOMBSTONES_PLACEHOLDER, &tombstones_text)
904        .replace(TRANSCRIPT_PLACEHOLDER, transcript_text)
905}
906
907/// Render the evidence window with `[N]` indices, bounded by the total
908/// transcript byte budget (oldest messages drop first — the window's tail
909/// is the freshest evidence).
910pub fn render_transcript(slice: &TranscriptSlice) -> String {
911    let mut lines: Vec<String> = Vec::new();
912    let mut total = 0usize;
913    for message in slice.messages.iter().rev() {
914        let line = format!("[{}] {}: {}", message.index, message.role, message.text);
915        if total + line.len() + 1 > MAX_TRANSCRIPT_TOTAL_BYTES && !lines.is_empty() {
916            lines.push("(earlier messages omitted for budget)".to_string());
917            break;
918        }
919        total += line.len() + 1;
920        lines.push(line);
921    }
922    lines.reverse();
923    lines.join("\n")
924}
925
926fn render_discards(entries: &[DiscardEntry]) -> String {
927    let mut lines: Vec<String> = vec![
928        "(compaction-discarded content recovered from session semantic memory; \
929         [N-M] are pre-compaction message offsets)"
930            .to_string(),
931    ];
932    let mut total = 0usize;
933    for entry in entries {
934        let prefix = match entry.range {
935            Some((start, end)) => format!("[{start}-{end}]"),
936            None => "[?]".to_string(),
937        };
938        let line = format!(
939            "{prefix} {}",
940            truncate_utf8_boundary(
941                &compact_whitespace(&entry.content),
942                MAX_TRANSCRIPT_MESSAGE_BYTES
943            )
944        );
945        if total + line.len() + 1 > MAX_TRANSCRIPT_TOTAL_BYTES && lines.len() > 1 {
946            lines.push("(further discards omitted for budget)".to_string());
947            break;
948        }
949        total += line.len() + 1;
950        lines.push(line);
951    }
952    lines.join("\n")
953}
954
955async fn complete_text(
956    client: &dyn LlmClient,
957    profile: &DistillerProfile,
958    prompt: String,
959) -> Result<String, DistillerError> {
960    let request = LlmRequest::new(
961        &profile.model,
962        vec![Message::User(UserMessage::text(prompt))],
963    )
964    .with_max_tokens(profile.params.max_output_tokens)
965    .with_temperature(profile.params.temperature);
966    let mut stream = client.stream(&request);
967    let mut text = String::new();
968    while let Some(event) = stream.next().await {
969        match event.map_err(classify_llm_error)? {
970            LlmEvent::TextDelta { delta, .. } => text.push_str(&delta),
971            LlmEvent::Done { outcome } => match outcome {
972                LlmDoneOutcome::Success { .. } => break,
973                LlmDoneOutcome::Error { error } => return Err(classify_llm_error(error)),
974            },
975            _ => {}
976        }
977    }
978    Ok(text)
979}
980
981fn classify_llm_error(error: LlmError) -> DistillerError {
982    match error {
983        LlmError::AuthenticationFailed { .. } | LlmError::InvalidApiKey => {
984            DistillerError::Auth(error.to_string())
985        }
986        other => DistillerError::Client(other.to_string()),
987    }
988}
989
990/// One extraction call: render, call, strict-parse with exactly one repair
991/// round-trip (the Selector's containment shape).
992pub async fn extract(
993    profile: &DistillerProfile,
994    client: &dyn LlmClient,
995    manifest: &[RecordMeta],
996    tombstones: &[TombstoneMeta],
997    transcript_text: &str,
998) -> Result<Vec<RawParsedOp>, DistillerError> {
999    let prompt = render_prompt(profile, manifest, tombstones, transcript_text);
1000    let reply = complete_text(client, profile, prompt).await?;
1001    match parse_ops(&reply) {
1002        Ok(ops) => Ok(ops),
1003        Err(first_err) => {
1004            let repair_prompt = format!(
1005                "The following reply was supposed to be exactly one JSON array of memory ops \
1006                 (each {{\"action\": \"remember\" | \"update\", \"target_id\"?, \"kind\", \
1007                 \"title\", \"description\", \"body\", \"tags\", \"epistemic\", \
1008                 \"evidence_range\"?}}) but did not parse ({first_err}). Reply with ONLY the \
1009                 corrected JSON array, no other text.\n\n{reply}"
1010            );
1011            let repaired = complete_text(client, profile, repair_prompt).await?;
1012            parse_ops(&repaired).map_err(DistillerError::Parse)
1013        }
1014    }
1015}
1016
1017// ---------------------------------------------------------------------------
1018// The engine
1019// ---------------------------------------------------------------------------
1020
1021/// Why an extraction window is closing. `Reset` is the only cause whose
1022/// distillates land wholesale-quarantined (§8.4 — reset is the operator's
1023/// escape hatch; quarantine preserves the re-dream option where "off" would
1024/// destroy evidence once session GC lands upstream).
1025#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1026pub enum DistillCause {
1027    Interactions,
1028    Respawn,
1029    Retire,
1030    Delete,
1031    Reset,
1032    ResumeFallback,
1033    Compaction,
1034}
1035
1036impl DistillCause {
1037    pub fn as_str(&self) -> &'static str {
1038        match self {
1039            Self::Interactions => "interactions",
1040            Self::Respawn => "respawn",
1041            Self::Retire => "retire",
1042            Self::Delete => "delete",
1043            Self::Reset => "reset",
1044            Self::ResumeFallback => "resume_fallback",
1045            Self::Compaction => "compaction",
1046        }
1047    }
1048
1049    /// §8.4: reset-boundary distillates land `Quarantined` pending steward
1050    /// review; respawn/retire distill normally (recovery/continuity paths).
1051    pub fn quarantines_output(&self) -> bool {
1052        matches!(self, Self::Reset)
1053    }
1054}
1055
1056/// Per-(identity, session) window state: the cursor adaptation of CC's
1057/// interaction-id mutual-exclusion trick — transcript position instead of
1058/// interaction ids, because persisted meerkat transcripts are positionally
1059/// indexed and carry no interaction ids (meerkat-core `session_store.rs`).
1060#[derive(Default, Clone)]
1061struct WindowState {
1062    cursor: u64,
1063    completed_runs: u32,
1064    /// The agent's own `memory` tool wrote during this window: skip
1065    /// extraction for the window and advance the cursor (§8.4 mutual
1066    /// exclusion — applies to the interaction trigger only; rotation
1067    /// windows are final and distill regardless, with the pre-injected
1068    /// manifest as the duplication guard).
1069    recorder_wrote: bool,
1070    generation: u64,
1071    last_run_at: Option<Instant>,
1072    last_activity_at: Option<Instant>,
1073    in_flight: bool,
1074}
1075
1076/// Outcome of one `distill_now` call, for logs and tests.
1077#[derive(Debug, Clone, PartialEq, Eq)]
1078pub enum DistillOutcome {
1079    Skipped {
1080        reason: String,
1081    },
1082    Completed {
1083        run_id: String,
1084        written: usize,
1085        quarantined: usize,
1086    },
1087}
1088
1089/// Compaction-skip reasons that mean "there was nothing to harvest", not
1090/// "the harvest failed" — shared with the §8.6 ordering check so the
1091/// strings cannot drift apart.
1092pub(crate) const SKIP_NO_DISCARD_SOURCE: &str = "no compaction discard source wired";
1093pub(crate) const SKIP_NO_DISCARDS: &str = "no compaction discards to harvest";
1094
1095impl DistillOutcome {
1096    /// §8.6 ordering invariant input: whether a compaction-cause run left
1097    /// the boundary's evidence harvested (completed) or provably empty
1098    /// (nothing to harvest). Budget denials, read failures, and extraction
1099    /// failures are NOT satisfied — hygiene must not discard material the
1100    /// distiller never saw.
1101    pub fn compaction_harvest_satisfied(&self) -> bool {
1102        match self {
1103            Self::Completed { .. } => true,
1104            Self::Skipped { reason } => {
1105                reason == SKIP_NO_DISCARDS || reason == SKIP_NO_DISCARD_SOURCE
1106            }
1107        }
1108    }
1109}
1110
1111/// Post-compaction follow-up hook (§8.6 trigger sequencing): invoked with
1112/// `(identity, session_key, outcome)` after every compaction-cause
1113/// distillation attempt, on the same detached task — so a wired Hygienist
1114/// runs strictly AFTER the distiller's harvest for that boundary.
1115pub type CompactionFollowUp = Arc<dyn Fn(&str, &str, &DistillOutcome) + Send + Sync>;
1116
1117/// Compaction-observation hook: invoked synchronously with
1118/// `(identity, session_key)` the moment a `CompactionCompleted` event is
1119/// observed — before, and regardless of, the harvest attempt (skipped and
1120/// budget-denied harvests included). For harvest/hygiene sequencing
1121/// observers only. Deliberately NOT the recall budget-reset path: distiller
1122/// sinks register only when the distiller is enabled, so the coordinator's
1123/// `on_session_compacted` reset is driven by the gateway's always-on
1124/// member-event sink instead — do not couple it back here.
1125pub type CompactionObserved = Arc<dyn Fn(&str, &str) + Send + Sync>;
1126
1127pub struct DistillerEngine {
1128    profile: DistillerProfile,
1129    config: DistillerConfig,
1130    handle: Arc<dyn DistillerClientHandle>,
1131    provider: Arc<dyn AgentMemoryProvider>,
1132    tombstones: Arc<dyn TombstoneSource>,
1133    transcripts: Arc<dyn TranscriptSource>,
1134    compaction: Option<Arc<dyn CompactionDiscardSource>>,
1135    tracker: Option<SessionTaintTracker>,
1136    budget: BackgroundBudget,
1137    realm: String,
1138    /// §9.3 timeline sink (optional; tracing stays the fallback surface).
1139    events: Mutex<Option<Arc<dyn crate::memory::events::MemoryEventSink>>>,
1140    /// (identity, session_key) → window state.
1141    windows: Mutex<HashMap<(String, String), WindowState>>,
1142    /// Test-only override for the pre-rotation timeout.
1143    pre_rotation_timeout: Duration,
1144    run_counter: std::sync::atomic::AtomicU64,
1145    /// §8.6 trigger sequencing: the Hygienist's post-compaction pass runs
1146    /// through this hook, strictly after the harvest attempt.
1147    compaction_follow_up: Mutex<Option<CompactionFollowUp>>,
1148    /// Synchronous compaction-observation hook (see [`CompactionObserved`]).
1149    compaction_observed: Mutex<Option<CompactionObserved>>,
1150}
1151
1152impl DistillerEngine {
1153    #[allow(clippy::too_many_arguments)]
1154    pub fn new(
1155        profile: DistillerProfile,
1156        config: DistillerConfig,
1157        handle: Arc<dyn DistillerClientHandle>,
1158        provider: Arc<dyn AgentMemoryProvider>,
1159        tombstones: Arc<dyn TombstoneSource>,
1160        transcripts: Arc<dyn TranscriptSource>,
1161        compaction: Option<Arc<dyn CompactionDiscardSource>>,
1162        tracker: Option<SessionTaintTracker>,
1163        realm: impl Into<String>,
1164    ) -> Self {
1165        let budget = BackgroundBudget::new(BackgroundBudgetConfig {
1166            runs_per_window: config.runs_per_hour,
1167            window: Duration::from_hours(1),
1168            max_concurrent: crate::memory::guards::DEFAULT_MAX_CONCURRENT,
1169        });
1170        Self {
1171            profile,
1172            config,
1173            handle,
1174            provider,
1175            tombstones,
1176            transcripts,
1177            compaction,
1178            tracker,
1179            budget,
1180            realm: realm.into(),
1181            events: Mutex::new(None),
1182            windows: Mutex::new(HashMap::new()),
1183            pre_rotation_timeout: PRE_ROTATION_TIMEOUT,
1184            run_counter: std::sync::atomic::AtomicU64::new(0),
1185            compaction_follow_up: Mutex::new(None),
1186            compaction_observed: Mutex::new(None),
1187        }
1188    }
1189
1190    #[cfg(test)]
1191    fn with_pre_rotation_timeout(mut self, timeout: Duration) -> Self {
1192        self.pre_rotation_timeout = timeout;
1193        self
1194    }
1195
1196    /// Wire the §9.3 timeline sink (skipped pre-rotation distillations,
1197    /// budget denials). Also threads it into the engine's budget guard.
1198    pub fn set_event_sink(&self, sink: Arc<dyn crate::memory::events::MemoryEventSink>) {
1199        self.budget.set_event_sink(sink.clone());
1200        *self
1201            .events
1202            .lock()
1203            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(sink);
1204    }
1205
1206    pub fn pre_rotation_timeout(&self) -> Duration {
1207        self.pre_rotation_timeout
1208    }
1209
1210    /// Wire the §8.6 post-compaction follow-up (the Hygienist). Invoked
1211    /// after every compaction-cause distillation attempt with its outcome.
1212    pub fn set_compaction_follow_up(&self, hook: CompactionFollowUp) {
1213        *self
1214            .compaction_follow_up
1215            .lock()
1216            .unwrap_or_else(|err| err.into_inner()) = Some(hook);
1217    }
1218
1219    /// Wire the synchronous compaction-observation hook (see
1220    /// [`CompactionObserved`]).
1221    pub fn set_compaction_observed(&self, hook: CompactionObserved) {
1222        *self
1223            .compaction_observed
1224            .lock()
1225            .unwrap_or_else(|err| err.into_inner()) = Some(hook);
1226    }
1227
1228    /// Fire the compaction-observation hook. Called by the trigger sink at
1229    /// the earliest session-attributed observation of a compaction, before
1230    /// the harvest is even spawned.
1231    fn note_compaction_observed(&self, identity: &str, session_key: &str) {
1232        let hook = self
1233            .compaction_observed
1234            .lock()
1235            .unwrap_or_else(|err| err.into_inner())
1236            .clone();
1237        if let Some(hook) = hook {
1238            hook(identity, session_key);
1239        }
1240    }
1241
1242    /// Transcript index up to which interaction/rotation distillation has
1243    /// run for `(identity, session_key)` (§8.4 window cursor). 0 when the
1244    /// window was never distilled. Read-only: never creates window state.
1245    pub fn distilled_cursor(&self, identity: &str, session_key: &str) -> u64 {
1246        self.windows
1247            .lock()
1248            .unwrap_or_else(std::sync::PoisonError::into_inner)
1249            .get(&(identity.to_string(), session_key.to_string()))
1250            .map(|state| state.cursor)
1251            .unwrap_or(0)
1252    }
1253
1254    fn with_window<T>(
1255        &self,
1256        identity: &str,
1257        session_key: &str,
1258        f: impl FnOnce(&mut WindowState) -> T,
1259    ) -> T {
1260        let mut windows = self
1261            .windows
1262            .lock()
1263            .unwrap_or_else(std::sync::PoisonError::into_inner);
1264        if windows.len() >= MAX_TRACKED_WINDOWS
1265            && !windows.contains_key(&(identity.to_string(), session_key.to_string()))
1266            && let Some(oldest) = windows
1267                .iter()
1268                .min_by_key(|(_, state)| state.last_activity_at)
1269                .map(|(key, _)| key.clone())
1270        {
1271            windows.remove(&oldest);
1272        }
1273        let state = windows
1274            .entry((identity.to_string(), session_key.to_string()))
1275            .or_default();
1276        state.last_activity_at = Some(Instant::now());
1277        f(state)
1278    }
1279
1280    /// Session-context hint from the identity runtime (delivery and
1281    /// lifecycle paths): binds the continuity generation the session's
1282    /// `EvidenceRef`s carry. Sessions only ever observed (never delivered
1283    /// to through the runtime) keep generation 0 — documented coarseness,
1284    /// resolved when an upstream generation fact exists on the stream.
1285    pub fn note_session_generation(&self, identity: &str, session_key: &str, generation: u64) {
1286        self.with_window(identity, session_key, |state| {
1287            state.generation = generation;
1288        });
1289    }
1290
1291    fn note_recorder_write(&self, identity: &str, session_key: &str) {
1292        self.with_window(identity, session_key, |state| {
1293            state.recorder_wrote = true;
1294        });
1295    }
1296
1297    /// Interaction-trigger bookkeeping: returns true when thresholds say an
1298    /// extraction should run now (the caller spawns it).
1299    fn note_run_completed(&self, identity: &str, session_key: &str) -> bool {
1300        let min_interactions = self.config.min_interactions;
1301        self.with_window(identity, session_key, |state| {
1302            state.completed_runs += 1;
1303            if state.in_flight {
1304                return false;
1305            }
1306            if state.completed_runs < min_interactions {
1307                return false;
1308            }
1309            if let Some(last) = state.last_run_at
1310                && last.elapsed() < Duration::from_secs(MIN_SECONDS_BETWEEN_RUNS)
1311            {
1312                return false;
1313            }
1314            state.in_flight = true;
1315            true
1316        })
1317    }
1318
1319    fn identity_scope(&self, identity: &str) -> MemoryScope {
1320        MemoryScope::Identity {
1321            realm: self.realm.clone(),
1322            identity: identity.to_string(),
1323        }
1324    }
1325
1326    fn mint_run_id(&self) -> String {
1327        let seq = self
1328            .run_counter
1329            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1330        format!("distill-{}-{seq}", now_ms())
1331    }
1332
1333    /// One extraction run over the window `[cursor, end)` of
1334    /// `session_key`'s transcript (or the compaction-discard harvest for
1335    /// [`DistillCause::Compaction`]). Budget-gated, mutually exclusive with
1336    /// Recorder writes, evidence-taint-aware through the store's write gate.
1337    pub async fn distill_now(
1338        self: &Arc<Self>,
1339        identity: &str,
1340        session_key: &str,
1341        cause: DistillCause,
1342    ) -> DistillOutcome {
1343        let outcome = self.distill_inner(identity, session_key, cause).await;
1344        self.with_window(identity, session_key, |state| {
1345            state.in_flight = false;
1346        });
1347        if cause == DistillCause::Compaction {
1348            let follow_up = self
1349                .compaction_follow_up
1350                .lock()
1351                .unwrap_or_else(|err| err.into_inner())
1352                .clone();
1353            if let Some(follow_up) = follow_up {
1354                follow_up(identity, session_key, &outcome);
1355            }
1356        }
1357        match &outcome {
1358            DistillOutcome::Skipped { reason } => {
1359                tracing::debug!(
1360                    identity,
1361                    session_key,
1362                    cause = cause.as_str(),
1363                    reason,
1364                    "agent memory distiller: run skipped"
1365                );
1366            }
1367            DistillOutcome::Completed {
1368                run_id,
1369                written,
1370                quarantined,
1371            } => {
1372                tracing::info!(
1373                    identity,
1374                    session_key,
1375                    cause = cause.as_str(),
1376                    run_id,
1377                    written,
1378                    quarantined,
1379                    "agent memory distiller: run completed"
1380                );
1381            }
1382        }
1383        outcome
1384    }
1385
1386    async fn distill_inner(
1387        self: &Arc<Self>,
1388        identity: &str,
1389        session_key: &str,
1390        cause: DistillCause,
1391    ) -> DistillOutcome {
1392        let (cursor, recorder_wrote, generation) =
1393            self.with_window(identity, session_key, |state| {
1394                (state.cursor, state.recorder_wrote, state.generation)
1395            });
1396
1397        // Evidence read comes before the budget gate: an empty window must
1398        // not burn a budgeted run.
1399        let (evidence_text, evidence_range, window_end, head_revision) = match cause {
1400            DistillCause::Compaction => {
1401                let Some(compaction) = self.compaction.as_ref() else {
1402                    return DistillOutcome::Skipped {
1403                        reason: SKIP_NO_DISCARD_SOURCE.to_string(),
1404                    };
1405                };
1406                let entries = match compaction
1407                    .read_discards(session_key, COMPACTION_HARVEST_LIMIT)
1408                    .await
1409                {
1410                    Ok(entries) => entries,
1411                    Err(err) => {
1412                        tracing::warn!(
1413                            identity,
1414                            session_key,
1415                            error = %err,
1416                            "agent memory distiller: compaction harvest failed"
1417                        );
1418                        return DistillOutcome::Skipped {
1419                            reason: format!("compaction harvest failed: {err}"),
1420                        };
1421                    }
1422                };
1423                if entries.is_empty() {
1424                    return DistillOutcome::Skipped {
1425                        reason: SKIP_NO_DISCARDS.to_string(),
1426                    };
1427                }
1428                let range = discard_evidence_range(&entries);
1429                // Discards predate the current transcript head (they were
1430                // evicted at compaction), so there is no head revision to pin.
1431                (render_discards(&entries), range, None, None)
1432            }
1433            _ => {
1434                let slice = match self.transcripts.read(session_key, cursor).await {
1435                    Ok(Some(slice)) => slice,
1436                    Ok(None) => {
1437                        return DistillOutcome::Skipped {
1438                            reason: "session not found in the session store".to_string(),
1439                        };
1440                    }
1441                    Err(err) => {
1442                        tracing::warn!(
1443                            identity,
1444                            session_key,
1445                            error = %err,
1446                            "agent memory distiller: transcript read failed"
1447                        );
1448                        return DistillOutcome::Skipped {
1449                            reason: format!("transcript read failed: {err}"),
1450                        };
1451                    }
1452                };
1453                if slice.messages.is_empty() {
1454                    return DistillOutcome::Skipped {
1455                        reason: "empty evidence window".to_string(),
1456                    };
1457                }
1458                if cause == DistillCause::Interactions && recorder_wrote {
1459                    // CC's mutual-exclusion trick: the agent already curated
1460                    // this window itself; skip and advance the cursor.
1461                    self.with_window(identity, session_key, |state| {
1462                        state.cursor = slice.end_index;
1463                        state.recorder_wrote = false;
1464                        state.completed_runs = 0;
1465                    });
1466                    return DistillOutcome::Skipped {
1467                        reason: "recorder wrote in window (mutual exclusion)".to_string(),
1468                    };
1469                }
1470                let range = Some((slice.start_index, slice.end_index.saturating_sub(1)));
1471                let head_revision = slice.head_revision.clone();
1472                (
1473                    render_transcript(&slice),
1474                    range,
1475                    Some(slice.end_index),
1476                    head_revision,
1477                )
1478            }
1479        };
1480
1481        // §8.1 resource guard: consulted before every run, loud on deny.
1482        let _permit = match self.budget.try_acquire(&self.realm, "distiller") {
1483            Ok(permit) => permit,
1484            Err(denied) => {
1485                return DistillOutcome::Skipped {
1486                    reason: format!("budget denied: {denied}"),
1487                };
1488            }
1489        };
1490
1491        // §8.4 reset quarantine: mark the boundary in the tracker so the
1492        // store's write gate quarantines every record citing this session —
1493        // the write law stays at the store seam, not in this caller.
1494        if cause.quarantines_output()
1495            && let Some(tracker) = self.tracker.as_ref()
1496        {
1497            tracker.mark_reset_boundary(session_key);
1498        }
1499
1500        let scope = self.identity_scope(identity);
1501        let manifest = match self
1502            .provider
1503            .manifest(std::slice::from_ref(&scope), ManifestTier::Full)
1504            .await
1505        {
1506            Ok(manifest) => manifest,
1507            Err(err) => {
1508                tracing::warn!(identity, error = %err, "agent memory distiller: manifest read failed");
1509                return DistillOutcome::Skipped {
1510                    reason: format!("manifest read failed: {err}"),
1511                };
1512            }
1513        };
1514        let tombstones = match self
1515            .tombstones
1516            .recent_tombstones(
1517                &scope,
1518                now_ms().saturating_sub(TOMBSTONE_LOOKBACK_MS),
1519                self.profile.params.max_tombstones,
1520            )
1521            .await
1522        {
1523            Ok(tombstones) => tombstones,
1524            Err(err) => {
1525                tracing::warn!(identity, error = %err, "agent memory distiller: tombstone read failed");
1526                return DistillOutcome::Skipped {
1527                    reason: format!("tombstone read failed: {err}"),
1528                };
1529            }
1530        };
1531
1532        let client = match self.handle.client().await {
1533            Ok(client) => client,
1534            Err(err) => {
1535                tracing::warn!(identity, error = %err, "agent memory distiller: client acquisition failed");
1536                return DistillOutcome::Skipped {
1537                    reason: format!("client acquisition failed: {err}"),
1538                };
1539            }
1540        };
1541        let raw_ops = match extract(
1542            &self.profile,
1543            &*client,
1544            &manifest,
1545            &tombstones,
1546            &evidence_text,
1547        )
1548        .await
1549        {
1550            Ok(ops) => ops,
1551            Err(DistillerError::Auth(message)) => {
1552                // One re-resolve, mirroring the Selector's auth containment.
1553                tracing::warn!(error = %message, "distiller auth failure; re-resolving client");
1554                self.handle.invalidate();
1555                let retried = match self.handle.client().await {
1556                    Ok(client) => {
1557                        extract(
1558                            &self.profile,
1559                            &*client,
1560                            &manifest,
1561                            &tombstones,
1562                            &evidence_text,
1563                        )
1564                        .await
1565                    }
1566                    Err(err) => Err(err),
1567                };
1568                match retried {
1569                    Ok(ops) => ops,
1570                    Err(err) => {
1571                        tracing::warn!(identity, error = %err, "agent memory distiller: extraction failed");
1572                        return DistillOutcome::Skipped {
1573                            reason: format!("extraction failed: {err}"),
1574                        };
1575                    }
1576                }
1577            }
1578            Err(err) => {
1579                tracing::warn!(identity, error = %err, "agent memory distiller: extraction failed");
1580                return DistillOutcome::Skipped {
1581                    reason: format!("extraction failed: {err}"),
1582                };
1583            }
1584        };
1585
1586        let manifest_ids: Vec<String> = manifest.iter().map(|meta| meta.id.clone()).collect();
1587        let run_id = self.mint_run_id();
1588        let author = MemoryAuthor::Distiller {
1589            run_id: run_id.clone(),
1590        };
1591        let mut written = 0usize;
1592        let mut quarantined = 0usize;
1593        for raw in raw_ops {
1594            let op = match validate_op(raw, &manifest_ids) {
1595                Ok(op) => op,
1596                Err(reason) => {
1597                    tracing::warn!(run_id, reason, "agent memory distiller: op dropped");
1598                    continue;
1599                }
1600            };
1601            let record = self.build_record(
1602                &op,
1603                session_key,
1604                generation,
1605                evidence_range,
1606                head_revision.as_deref(),
1607            );
1608            let result = match &op.action {
1609                ProposedAction::Remember => {
1610                    self.provider
1611                        .remember_authored(&scope, record, author.clone())
1612                        .await
1613                }
1614                ProposedAction::Update { target_id } => {
1615                    self.provider
1616                        .supersede_authored(&scope, target_id, record, author.clone())
1617                        .await
1618                }
1619            };
1620            match result {
1621                Ok(receipt) => {
1622                    written += 1;
1623                    if matches!(
1624                        receipt.status,
1625                        crate::memory::records::RecordStatus::Quarantined { .. }
1626                    ) {
1627                        quarantined += 1;
1628                    }
1629                }
1630                Err(err) => {
1631                    // Includes the staged validator's tombstone-recreation
1632                    // reject — the mechanical backstop behind the prompt
1633                    // guard.
1634                    tracing::warn!(run_id, error = %err, "agent memory distiller: write rejected");
1635                }
1636            }
1637        }
1638
1639        self.with_window(identity, session_key, |state| {
1640            // Interaction-window bookkeeping is consumed only by runs that
1641            // actually covered the window [cursor, end). A compaction-cause
1642            // harvest reads the discard ledger (window_end=None) — clearing
1643            // `recorder_wrote` there would defeat the §8.4 mutual exclusion
1644            // for a window the agent already curated, and zeroing
1645            // `completed_runs` would silently defer the interaction trigger.
1646            if let Some(end) = window_end {
1647                state.cursor = end;
1648                state.recorder_wrote = false;
1649                state.completed_runs = 0;
1650            }
1651            state.last_run_at = Some(Instant::now());
1652        });
1653        DistillOutcome::Completed {
1654            run_id,
1655            written,
1656            quarantined,
1657        }
1658    }
1659
1660    fn build_record(
1661        &self,
1662        op: &ProposedOp,
1663        session_key: &str,
1664        generation: u64,
1665        window_range: Option<(u64, u64)>,
1666        revision: Option<&str>,
1667    ) -> NewMemoryRecord {
1668        let mut tags = op.tags.clone();
1669        if op.epistemic == Epistemic::OperatorSaid
1670            && !tags.iter().any(|tag| tag == "epistemic:operator_said")
1671        {
1672            // Same convention as the Recorder tool: attribution rides as a
1673            // tag so recall and the steward see the claim's nature.
1674            tags.push("epistemic:operator_said".to_string());
1675        }
1676        // Model-cited range wins when it stays inside the window; a range
1677        // outside the evidence the model was shown is a hallucinated
1678        // citation and falls back to the window bounds.
1679        let range = match (op.evidence_range, window_range) {
1680            (Some((start, end)), Some((window_start, window_end)))
1681                if start >= window_start && end <= window_end =>
1682            {
1683                Some((start, end))
1684            }
1685            (_, window) => window,
1686        };
1687        NewMemoryRecord {
1688            kind: op.kind,
1689            title: op.title.clone(),
1690            description: op.description.clone(),
1691            body: op.body.clone(),
1692            tags,
1693            evidence: vec![EvidenceRef {
1694                session_id: session_key.to_string(),
1695                generation,
1696                // Ask 4 refinement: pin the transcript head revision the
1697                // evidence was read at (None on the compaction path, whose
1698                // discards predate the current head).
1699                revision: revision.map(str::to_string),
1700                range,
1701            }],
1702            verification: None,
1703        }
1704    }
1705
1706    /// Pre-rotation hook body (respawn/retire/delete): bounded, best-effort
1707    /// — rotation proceeds on timeout with a loud skip.
1708    pub async fn distill_before_rotation(
1709        self: &Arc<Self>,
1710        identity: &str,
1711        session_key: &str,
1712        cause: DistillCause,
1713    ) {
1714        let timeout = self.pre_rotation_timeout;
1715        match tokio::time::timeout(timeout, self.distill_now(identity, session_key, cause)).await {
1716            Ok(_) => {}
1717            Err(_) => {
1718                tracing::warn!(
1719                    identity,
1720                    session_key,
1721                    cause = cause.as_str(),
1722                    timeout_ms = timeout.as_millis() as u64,
1723                    "agent memory distiller: pre-rotation distillation timed out; \
1724                     rotation proceeds without it"
1725                );
1726                if let Some(sink) = self
1727                    .events
1728                    .lock()
1729                    .unwrap_or_else(std::sync::PoisonError::into_inner)
1730                    .as_ref()
1731                {
1732                    sink.emit(
1733                        crate::memory::events::MemoryTimelineEvent::DistillationTimedOut {
1734                            identity: identity.to_string(),
1735                            session_key: session_key.to_string(),
1736                            cause: cause.as_str().to_string(),
1737                        },
1738                    );
1739                }
1740            }
1741        }
1742    }
1743
1744    /// Ask 2 GC: reclaim a permanently-orphaned session's semantic-memory
1745    /// rows after its knowledge has been distilled. Best-effort and bounded —
1746    /// a rotation must never fail on cleanup. Caller MUST have already run
1747    /// `distill_before_rotation` for this session and MUST only invoke this
1748    /// for causes that permanently abandon the session id (respawn/reset mint
1749    /// a fresh id; delete discards it) — never for resumable retires.
1750    pub async fn drop_orphaned_session_scope(
1751        self: &Arc<Self>,
1752        session_key: &str,
1753        cause: DistillCause,
1754    ) {
1755        let Some(compaction) = self.compaction.as_ref() else {
1756            return;
1757        };
1758        match compaction.drop_scope(session_key).await {
1759            Ok(0) => {}
1760            Ok(dropped) => tracing::debug!(
1761                session_key,
1762                cause = cause.as_str(),
1763                dropped,
1764                "agent memory GC: reclaimed orphaned session semantic-memory rows"
1765            ),
1766            Err(err) => tracing::warn!(
1767                session_key,
1768                cause = cause.as_str(),
1769                error = %err,
1770                "agent memory GC: dropping orphaned session scope failed; \
1771                 rows remain (re-embed tax persists), rotation proceeds"
1772            ),
1773        }
1774    }
1775
1776    /// Detached distillation (reset / resume-fallback / compaction): never
1777    /// on any critical path. The session store outlives the session, so the
1778    /// read stays valid after teardown.
1779    pub fn spawn_detached(
1780        self: &Arc<Self>,
1781        identity: &str,
1782        session_key: &str,
1783        cause: DistillCause,
1784    ) {
1785        let engine = self.clone();
1786        let identity = identity.to_string();
1787        let session_key = session_key.to_string();
1788        tokio::spawn(async move {
1789            engine.distill_now(&identity, &session_key, cause).await;
1790        });
1791    }
1792}
1793
1794fn discard_evidence_range(entries: &[DiscardEntry]) -> Option<(u64, u64)> {
1795    let mut bounds: Option<(u64, u64)> = None;
1796    for entry in entries {
1797        if let Some((start, end)) = entry.range {
1798            bounds = Some(match bounds {
1799                None => (start, end),
1800                Some((lo, hi)) => (lo.min(start), hi.max(end)),
1801            });
1802        }
1803    }
1804    bounds
1805}
1806
1807// ---------------------------------------------------------------------------
1808// Observe-stream trigger sink (rides the same member-event observer as the
1809// taint tracker)
1810// ---------------------------------------------------------------------------
1811
1812pub struct DistillerTriggers {
1813    engine: Arc<DistillerEngine>,
1814}
1815
1816impl DistillerTriggers {
1817    pub fn new(engine: Arc<DistillerEngine>) -> Self {
1818        Self { engine }
1819    }
1820}
1821
1822impl MemberAgentEventSink for DistillerTriggers {
1823    fn observe(&self, identity: &str, envelope: &meerkat_core::event::EventEnvelope<AgentEvent>) {
1824        match &envelope.payload {
1825            AgentEvent::ToolCallRequested { name, args, .. } if name == MEMORY_TOOL_NAME => {
1826                let is_write = args
1827                    .as_value()
1828                    .get("action")
1829                    .and_then(serde_json::Value::as_str)
1830                    .is_some_and(|action| {
1831                        matches!(action, "remember" | "update" | "forget" | "propose_to_mob")
1832                    });
1833                if is_write && let Some(session) = current_session_of(envelope) {
1834                    self.engine.note_recorder_write(identity, &session);
1835                }
1836            }
1837            AgentEvent::RunCompleted { session_id, .. } => {
1838                let session = session_id.to_string();
1839                if self.engine.note_run_completed(identity, &session) {
1840                    self.engine
1841                        .spawn_detached(identity, &session, DistillCause::Interactions);
1842                }
1843            }
1844            AgentEvent::CompactionCompleted { .. } => {
1845                if let Some(session) = current_session_of(envelope) {
1846                    // Sequencing observers hear about the boundary here —
1847                    // synchronously, before the detached harvest, and even
1848                    // when that harvest is later skipped or budget-denied.
1849                    self.engine.note_compaction_observed(identity, &session);
1850                    self.engine
1851                        .spawn_detached(identity, &session, DistillCause::Compaction);
1852                } else {
1853                    tracing::warn!(
1854                        identity,
1855                        "agent memory distiller: compaction event without session \
1856                         attribution; harvest skipped"
1857                    );
1858                }
1859            }
1860            _ => {}
1861        }
1862    }
1863}
1864
1865/// Session attribution for non-run-scoped events: the envelope's source
1866/// identity when it names a session.
1867fn current_session_of(envelope: &meerkat_core::event::EventEnvelope<AgentEvent>) -> Option<String> {
1868    match &envelope.source {
1869        meerkat_core::event::EventSourceIdentity::Session { session_id } => {
1870            Some(session_id.to_string())
1871        }
1872        _ => None,
1873    }
1874}
1875
1876fn now_ms() -> u64 {
1877    SystemTime::now()
1878        .duration_since(UNIX_EPOCH)
1879        .map(|duration| duration.as_millis() as u64)
1880        .unwrap_or(0)
1881}
1882
1883#[cfg(test)]
1884#[allow(clippy::expect_used, clippy::redundant_clone, clippy::unwrap_used)]
1885mod tests {
1886    use super::*;
1887    use crate::identity_first::agent_memory::{
1888        AgentMemoryRecallRequest, AgentMemoryRecord, AuthoredWriteReceipt,
1889    };
1890    use crate::memory::records::RecordStatus;
1891    use futures::stream;
1892    use std::sync::Mutex as StdMutex;
1893
1894    // Ask 2+8 (meerkat 0.7.12): the compaction-discard harvest reads its
1895    // session's semantic-memory scope EXACTLY via `enumerate_scoped`, and a
1896    // permanently-orphaned scope is reclaimed via `drop_scope`. This exercises
1897    // both against a real `HnswMemoryStore` seeded with compaction-discard
1898    // rows (proving the source_range mapping and the round-trip drop).
1899    #[tokio::test]
1900    async fn hnsw_discard_source_enumerates_then_drops_scope()
1901    -> Result<(), Box<dyn std::error::Error>> {
1902        use meerkat_core::memory::{
1903            MemoryIndexRequest, MemoryIndexScope, MemoryMetadata, MemorySource, MemoryStore,
1904            MessageRange,
1905        };
1906        use meerkat_core::types::{MemoryIndexableContent, SessionId};
1907
1908        let dir = tempfile::tempdir()?;
1909        let session = SessionId::new();
1910
1911        // Seed two discarded ranges into meerkat's session semantic memory.
1912        {
1913            let store = meerkat_memory::HnswMemoryStore::open(dir.path())?;
1914            let scope = MemoryIndexScope::for_session(session.clone());
1915            for (start, end, text) in [
1916                (0u64, 2u64, "operator prefers terse status updates"),
1917                (2u64, 5u64, "root cause was an expired upstream TLS cert"),
1918            ] {
1919                let request = MemoryIndexRequest::new(
1920                    scope.clone(),
1921                    MemoryIndexableContent::Indexable(text.to_string()),
1922                    MemoryMetadata {
1923                        session_id: session.clone(),
1924                        source: MemorySource::Compaction {
1925                            source_range: MessageRange::new(start, end)?,
1926                        },
1927                        indexed_at: meerkat_core::time_compat::SystemTime::now(),
1928                    },
1929                )?;
1930                store.index_scoped(request).await?;
1931            }
1932        }
1933
1934        let source = HnswDiscardSource::new(dir.path());
1935
1936        // Exact enumeration: both rows, with their source ranges intact.
1937        let discards = source
1938            .read_discards(&session.to_string(), COMPACTION_HARVEST_LIMIT)
1939            .await?;
1940        assert_eq!(discards.len(), 2, "enumeration must return every scope row");
1941        assert!(discards.iter().any(|d| d.range == Some((0, 2))));
1942        assert!(
1943            discards
1944                .iter()
1945                .any(|d| d.content.contains("expired upstream TLS cert"))
1946        );
1947
1948        // GC: dropping the scope reclaims exactly those rows; a re-read is empty.
1949        let dropped = source.drop_scope(&session.to_string()).await?;
1950        assert_eq!(dropped, 2, "drop_scope must reclaim every seeded row");
1951        let after = source
1952            .read_discards(&session.to_string(), COMPACTION_HARVEST_LIMIT)
1953            .await?;
1954        assert!(after.is_empty(), "dropped scope must enumerate empty");
1955        Ok(())
1956    }
1957
1958    // -- scripted LLM -------------------------------------------------------
1959
1960    struct ScriptedLlm {
1961        replies: StdMutex<Vec<String>>,
1962        prompts: StdMutex<Vec<String>>,
1963    }
1964
1965    impl ScriptedLlm {
1966        fn new(replies: Vec<&str>) -> Self {
1967            Self {
1968                replies: StdMutex::new(replies.into_iter().map(str::to_string).collect()),
1969                prompts: StdMutex::new(Vec::new()),
1970            }
1971        }
1972
1973        fn prompts(&self) -> Vec<String> {
1974            self.prompts
1975                .lock()
1976                .unwrap_or_else(std::sync::PoisonError::into_inner)
1977                .clone()
1978        }
1979    }
1980
1981    #[async_trait]
1982    impl LlmClient for ScriptedLlm {
1983        fn stream<'a>(&'a self, request: &'a LlmRequest) -> meerkat_client::types::LlmStream<'a> {
1984            let prompt = request
1985                .messages
1986                .iter()
1987                .map(|message| match message {
1988                    Message::User(user) => user.text_content(),
1989                    _ => String::new(),
1990                })
1991                .collect::<Vec<_>>()
1992                .join("\n");
1993            self.prompts
1994                .lock()
1995                .unwrap_or_else(std::sync::PoisonError::into_inner)
1996                .push(prompt);
1997            let reply = {
1998                let mut replies = self
1999                    .replies
2000                    .lock()
2001                    .unwrap_or_else(std::sync::PoisonError::into_inner);
2002                if replies.is_empty() {
2003                    String::new()
2004                } else {
2005                    replies.remove(0)
2006                }
2007            };
2008            Box::pin(stream::iter(vec![
2009                Ok(LlmEvent::TextDelta {
2010                    delta: reply,
2011                    meta: None,
2012                }),
2013                Ok(LlmEvent::Done {
2014                    outcome: LlmDoneOutcome::Success {
2015                        stop_reason: meerkat_core::StopReason::EndTurn,
2016                    },
2017                }),
2018            ]))
2019        }
2020
2021        fn provider(&self) -> Provider {
2022            Provider::Other
2023        }
2024
2025        async fn health_check(&self) -> Result<(), LlmError> {
2026            Ok(())
2027        }
2028    }
2029
2030    struct ScriptedHandle {
2031        client: Arc<ScriptedLlm>,
2032    }
2033
2034    #[async_trait]
2035    impl DistillerClientHandle for ScriptedHandle {
2036        async fn client(&self) -> Result<Arc<dyn LlmClient>, DistillerError> {
2037            Ok(self.client.clone())
2038        }
2039        fn invalidate(&self) {}
2040    }
2041
2042    /// Handle whose client acquisition never resolves — the pre-rotation
2043    /// timeout test's hang.
2044    struct HangingHandle;
2045
2046    #[async_trait]
2047    impl DistillerClientHandle for HangingHandle {
2048        async fn client(&self) -> Result<Arc<dyn LlmClient>, DistillerError> {
2049            futures::future::pending().await
2050        }
2051        fn invalidate(&self) {}
2052    }
2053
2054    // -- scripted provider / sources ---------------------------------------
2055
2056    #[derive(Default)]
2057    struct CapturingProvider {
2058        remembers: StdMutex<Vec<(MemoryScope, NewMemoryRecord, MemoryAuthor)>>,
2059        supersedes: StdMutex<Vec<(MemoryScope, String, NewMemoryRecord, MemoryAuthor)>>,
2060        manifest: StdMutex<Vec<RecordMeta>>,
2061        quarantine_all: bool,
2062    }
2063
2064    #[async_trait]
2065    impl AgentMemoryProvider for CapturingProvider {
2066        async fn recall(
2067            &self,
2068            _request: AgentMemoryRecallRequest,
2069        ) -> Result<Vec<AgentMemoryRecord>, AgentMemoryError> {
2070            Ok(Vec::new())
2071        }
2072
2073        async fn manifest(
2074            &self,
2075            _scopes: &[MemoryScope],
2076            _tier: ManifestTier,
2077        ) -> Result<Vec<RecordMeta>, AgentMemoryError> {
2078            Ok(self
2079                .manifest
2080                .lock()
2081                .unwrap_or_else(std::sync::PoisonError::into_inner)
2082                .clone())
2083        }
2084
2085        fn supports_manifest(&self) -> bool {
2086            true
2087        }
2088
2089        async fn remember_authored(
2090            &self,
2091            scope: &MemoryScope,
2092            record: NewMemoryRecord,
2093            author: MemoryAuthor,
2094        ) -> Result<AuthoredWriteReceipt, AgentMemoryError> {
2095            self.remembers
2096                .lock()
2097                .unwrap_or_else(std::sync::PoisonError::into_inner)
2098                .push((scope.clone(), record, author));
2099            Ok(AuthoredWriteReceipt {
2100                memory_id: "mem-new".to_string(),
2101                status: if self.quarantine_all {
2102                    RecordStatus::Quarantined {
2103                        reason: "test".to_string(),
2104                    }
2105                } else {
2106                    RecordStatus::Active
2107                },
2108            })
2109        }
2110
2111        async fn supersede_authored(
2112            &self,
2113            scope: &MemoryScope,
2114            prior: &str,
2115            record: NewMemoryRecord,
2116            author: MemoryAuthor,
2117        ) -> Result<AuthoredWriteReceipt, AgentMemoryError> {
2118            self.supersedes
2119                .lock()
2120                .unwrap_or_else(std::sync::PoisonError::into_inner)
2121                .push((scope.clone(), prior.to_string(), record, author));
2122            Ok(AuthoredWriteReceipt {
2123                memory_id: "mem-updated".to_string(),
2124                status: RecordStatus::Active,
2125            })
2126        }
2127
2128        fn supports_authored_writes(&self) -> bool {
2129            true
2130        }
2131    }
2132
2133    struct StaticTombstones(Vec<TombstoneMeta>);
2134
2135    #[async_trait]
2136    impl TombstoneSource for StaticTombstones {
2137        async fn recent_tombstones(
2138            &self,
2139            _scope: &MemoryScope,
2140            _since_ms: u64,
2141            _limit: usize,
2142        ) -> Result<Vec<TombstoneMeta>, AgentMemoryError> {
2143            Ok(self.0.clone())
2144        }
2145    }
2146
2147    struct StaticTranscript(Option<TranscriptSlice>);
2148
2149    #[async_trait]
2150    impl TranscriptSource for StaticTranscript {
2151        async fn read(
2152            &self,
2153            _session_key: &str,
2154            from_index: u64,
2155        ) -> Result<Option<TranscriptSlice>, DistillerError> {
2156            Ok(self.0.clone().map(|mut slice| {
2157                slice.messages.retain(|message| message.index >= from_index);
2158                slice.start_index = from_index.max(slice.start_index);
2159                slice
2160            }))
2161        }
2162    }
2163
2164    fn slice(messages: &[(&'static str, &str)]) -> TranscriptSlice {
2165        TranscriptSlice {
2166            session_key: "sess-1".to_string(),
2167            start_index: 0,
2168            end_index: messages.len() as u64,
2169            messages: messages
2170                .iter()
2171                .enumerate()
2172                .map(|(index, (role, text))| TranscriptMessage {
2173                    index: index as u64,
2174                    role,
2175                    text: text.to_string(),
2176                })
2177                .collect(),
2178            head_revision: Some("rev-head".to_string()),
2179        }
2180    }
2181
2182    fn engine_with(
2183        replies: Vec<&str>,
2184        provider: Arc<CapturingProvider>,
2185        transcript: Option<TranscriptSlice>,
2186        tombstones: Vec<TombstoneMeta>,
2187        tracker: Option<SessionTaintTracker>,
2188        config: DistillerConfig,
2189    ) -> (Arc<DistillerEngine>, Arc<ScriptedLlm>) {
2190        let client = Arc::new(ScriptedLlm::new(replies));
2191        let engine = Arc::new(DistillerEngine::new(
2192            DistillerProfile::embedded_default(),
2193            config,
2194            Arc::new(ScriptedHandle {
2195                client: client.clone(),
2196            }),
2197            provider,
2198            Arc::new(StaticTombstones(tombstones)),
2199            Arc::new(StaticTranscript(transcript)),
2200            None,
2201            tracker,
2202            "family",
2203        ));
2204        (engine, client)
2205    }
2206
2207    fn enabled_config() -> DistillerConfig {
2208        DistillerConfig {
2209            enabled: true,
2210            ..DistillerConfig::default()
2211        }
2212    }
2213
2214    const NOOP_REPLY: &str = "[]";
2215    const ONE_OP_REPLY: &str = r#"[{"action": "remember", "kind": "gotcha",
2216        "title": "Cargo goes through the wrapper",
2217        "description": "When running cargo commands in this repo",
2218        "body": "Operator said: \"always use ./scripts/repo-cargo, never raw cargo\".",
2219        "tags": [], "epistemic": "operator_said", "evidence_range": [1, 2]}]"#;
2220
2221    // -- prompt rendering ---------------------------------------------------
2222
2223    #[test]
2224    fn prompt_renders_manifest_tombstones_and_transcript() {
2225        let profile = DistillerProfile::embedded_default();
2226        let manifest = vec![RecordMeta {
2227            id: "mem-1".to_string(),
2228            kind: MemoryKind::Gotcha,
2229            title: "Wrapper cargo".to_string(),
2230            description: "When building".to_string(),
2231            age_days: 2,
2232            rank: Some(1),
2233        }];
2234        let tombstones = vec![TombstoneMeta {
2235            title: "Operator phone number".to_string(),
2236            kind: MemoryKind::Fact,
2237            tombstoned_at_ms: 5,
2238        }];
2239        let transcript = render_transcript(&slice(&[
2240            ("user", "please run the build"),
2241            ("assistant", "running it now"),
2242        ]));
2243        let prompt = render_prompt(&profile, &manifest, &tombstones, &transcript);
2244        assert!(prompt.contains("- mem-1 [gotcha, saved 2 days ago, rank 1] Wrapper cargo"));
2245        assert!(prompt.contains("- [fact] Operator phone number"));
2246        assert!(prompt.contains("[0] user: please run the build"));
2247        assert!(prompt.contains("[1] assistant: running it now"));
2248        assert!(!prompt.contains("{{existing_manifest}}"));
2249        assert!(!prompt.contains("{{recent_tombstones}}"));
2250        assert!(!prompt.contains("{{transcript}}"));
2251    }
2252
2253    #[test]
2254    fn prompt_renders_empty_sections_honestly() {
2255        let profile = DistillerProfile::embedded_default();
2256        let prompt = render_prompt(&profile, &[], &[], "(empty)");
2257        assert!(prompt.contains("(no records)"));
2258        assert!(prompt.contains("(none)"));
2259    }
2260
2261    // -- parse + repair -----------------------------------------------------
2262
2263    #[tokio::test]
2264    async fn extract_parses_ops_and_tolerates_fences() -> Result<(), Box<dyn std::error::Error>> {
2265        let fenced = format!("```json\n{ONE_OP_REPLY}\n```");
2266        let client = ScriptedLlm::new(vec![&fenced]);
2267        let profile = DistillerProfile::embedded_default();
2268        let ops = extract(&profile, &client, &[], &[], "[0] user: hi").await?;
2269        assert_eq!(ops.len(), 1);
2270        assert_eq!(client.prompts().len(), 1, "fenced JSON needs no repair");
2271        let op = validate_op(ops.into_iter().next().unwrap(), &[]).expect("valid op");
2272        assert_eq!(op.kind, MemoryKind::Gotcha);
2273        assert_eq!(op.epistemic, Epistemic::OperatorSaid);
2274        assert_eq!(op.evidence_range, Some((1, 2)));
2275        Ok(())
2276    }
2277
2278    #[tokio::test]
2279    async fn extract_repairs_malformed_output_once() -> Result<(), Box<dyn std::error::Error>> {
2280        let client = ScriptedLlm::new(vec!["Here are my thoughts, no JSON", NOOP_REPLY]);
2281        let profile = DistillerProfile::embedded_default();
2282        let ops = extract(&profile, &client, &[], &[], "[0] user: hi").await?;
2283        assert!(ops.is_empty());
2284        let prompts = client.prompts();
2285        assert_eq!(prompts.len(), 2, "exactly one repair round-trip");
2286        assert!(prompts[1].contains("ONLY the corrected JSON array"));
2287        Ok(())
2288    }
2289
2290    #[tokio::test]
2291    async fn extract_errors_after_failed_repair() {
2292        let client = ScriptedLlm::new(vec!["nope", "still nope"]);
2293        let profile = DistillerProfile::embedded_default();
2294        let result = extract(&profile, &client, &[], &[], "[0] user: hi").await;
2295        assert!(
2296            matches!(result, Err(DistillerError::Parse(_))),
2297            "{result:?}"
2298        );
2299    }
2300
2301    #[test]
2302    fn validate_op_enforces_action_kind_epistemic_and_targets() {
2303        let parse_one = |json: &str| -> Result<ProposedOp, String> {
2304            let ops = parse_ops(json).expect("parses");
2305            validate_op(
2306                ops.into_iter().next().expect("one op"),
2307                &["mem-1".to_string()],
2308            )
2309        };
2310        // Unknown epistemic status is a per-op reject.
2311        let err =
2312            parse_one(r#"[{"action":"remember","title":"t","body":"b","epistemic":"vibes"}]"#)
2313                .expect_err("unknown epistemic");
2314        assert!(err.contains("epistemic"), "{err}");
2315        // Updates must target a manifest record.
2316        let err = parse_one(
2317            r#"[{"action":"update","target_id":"mem-9","title":"t","body":"b","epistemic":"observed"}]"#,
2318        )
2319        .expect_err("unknown target");
2320        assert!(err.contains("not in the manifest"), "{err}");
2321        let ok = parse_one(
2322            r#"[{"action":"update","target_id":"mem-1","title":"t","body":"b","epistemic":"observed"}]"#,
2323        )
2324        .expect("valid update");
2325        assert_eq!(
2326            ok.action,
2327            ProposedAction::Update {
2328                target_id: "mem-1".to_string()
2329            }
2330        );
2331        // Inverted evidence ranges are hallucinated citations.
2332        let err = parse_one(
2333            r#"[{"action":"remember","title":"t","body":"b","epistemic":"observed","evidence_range":[9,2]}]"#,
2334        )
2335        .expect_err("inverted range");
2336        assert!(err.contains("inverted"), "{err}");
2337    }
2338
2339    // -- engine: writes, evidence, author -----------------------------------
2340
2341    #[tokio::test]
2342    async fn distill_writes_through_authored_seam_with_distiller_author_and_evidence() {
2343        let provider = Arc::new(CapturingProvider::default());
2344        let (engine, _client) = engine_with(
2345            vec![ONE_OP_REPLY],
2346            provider.clone(),
2347            Some(slice(&[
2348                ("user", "use the wrapper"),
2349                ("assistant", "noted"),
2350                ("user", "always ./scripts/repo-cargo, never raw cargo"),
2351            ])),
2352            Vec::new(),
2353            None,
2354            enabled_config(),
2355        );
2356        engine.note_session_generation("identity:a", "sess-1", 3);
2357        let outcome = engine
2358            .distill_now("identity:a", "sess-1", DistillCause::Retire)
2359            .await;
2360        assert!(
2361            matches!(outcome, DistillOutcome::Completed { written: 1, .. }),
2362            "{outcome:?}"
2363        );
2364        let remembers = provider.remembers.lock().unwrap();
2365        assert_eq!(remembers.len(), 1);
2366        let (scope, record, author) = &remembers[0];
2367        assert_eq!(
2368            *scope,
2369            MemoryScope::Identity {
2370                realm: "family".to_string(),
2371                identity: "identity:a".to_string()
2372            }
2373        );
2374        assert!(matches!(author, MemoryAuthor::Distiller { .. }));
2375        assert!(author.is_llm(), "Distiller must classify as an LLM author");
2376        assert_eq!(record.evidence.len(), 1);
2377        let evidence = &record.evidence[0];
2378        assert_eq!(evidence.session_id, "sess-1");
2379        assert_eq!(evidence.generation, 3);
2380        // Ask 4 refinement: transcript-path evidence pins the head revision
2381        // the slice was read at (the StaticTranscript reports "rev-head").
2382        assert_eq!(evidence.revision.as_deref(), Some("rev-head"));
2383        assert_eq!(
2384            evidence.range,
2385            Some((1, 2)),
2386            "model-cited range within window"
2387        );
2388        assert!(record.tags.contains(&"epistemic:operator_said".to_string()));
2389    }
2390
2391    #[tokio::test]
2392    async fn hallucinated_evidence_range_falls_back_to_window_bounds() {
2393        let reply = r#"[{"action": "remember", "kind": "fact", "title": "t",
2394            "description": "", "body": "b", "tags": [],
2395            "epistemic": "observed", "evidence_range": [90, 95]}]"#;
2396        let provider = Arc::new(CapturingProvider::default());
2397        let (engine, _client) = engine_with(
2398            vec![reply],
2399            provider.clone(),
2400            Some(slice(&[("user", "a"), ("assistant", "b")])),
2401            Vec::new(),
2402            None,
2403            enabled_config(),
2404        );
2405        engine
2406            .distill_now("identity:a", "sess-1", DistillCause::Retire)
2407            .await;
2408        let remembers = provider.remembers.lock().unwrap();
2409        assert_eq!(remembers[0].1.evidence[0].range, Some((0, 1)));
2410    }
2411
2412    #[tokio::test]
2413    async fn update_ops_supersede_the_target_record() {
2414        let reply = r#"[{"action": "update", "target_id": "mem-1", "kind": "gotcha",
2415            "title": "t", "description": "", "body": "b", "tags": [],
2416            "epistemic": "observed"}]"#;
2417        let provider = Arc::new(CapturingProvider::default());
2418        provider.manifest.lock().unwrap().push(RecordMeta {
2419            id: "mem-1".to_string(),
2420            kind: MemoryKind::Gotcha,
2421            title: "old".to_string(),
2422            description: String::new(),
2423            age_days: 1,
2424            rank: None,
2425        });
2426        let (engine, _client) = engine_with(
2427            vec![reply],
2428            provider.clone(),
2429            Some(slice(&[("user", "a")])),
2430            Vec::new(),
2431            None,
2432            enabled_config(),
2433        );
2434        engine
2435            .distill_now("identity:a", "sess-1", DistillCause::Retire)
2436            .await;
2437        let supersedes = provider.supersedes.lock().unwrap();
2438        assert_eq!(supersedes.len(), 1);
2439        assert_eq!(supersedes[0].1, "mem-1");
2440        assert!(provider.remembers.lock().unwrap().is_empty());
2441    }
2442
2443    // -- cursor mutual exclusion -------------------------------------------
2444
2445    #[tokio::test]
2446    async fn recorder_write_in_window_skips_extraction_and_advances_cursor() {
2447        let provider = Arc::new(CapturingProvider::default());
2448        let (engine, client) = engine_with(
2449            vec![ONE_OP_REPLY, ONE_OP_REPLY],
2450            provider.clone(),
2451            Some(slice(&[("user", "a"), ("assistant", "b")])),
2452            Vec::new(),
2453            None,
2454            enabled_config(),
2455        );
2456        engine.note_recorder_write("identity:a", "sess-1");
2457        let outcome = engine
2458            .distill_now("identity:a", "sess-1", DistillCause::Interactions)
2459            .await;
2460        assert!(
2461            matches!(&outcome, DistillOutcome::Skipped { reason } if reason.contains("mutual exclusion")),
2462            "{outcome:?}"
2463        );
2464        assert!(
2465            client.prompts().is_empty(),
2466            "no LLM call in a skipped window"
2467        );
2468        assert!(provider.remembers.lock().unwrap().is_empty());
2469
2470        // The cursor advanced past the curated window: the next run sees an
2471        // empty window, not the same messages again.
2472        let outcome = engine
2473            .distill_now("identity:a", "sess-1", DistillCause::Interactions)
2474            .await;
2475        assert!(
2476            matches!(&outcome, DistillOutcome::Skipped { reason } if reason.contains("empty")),
2477            "{outcome:?}"
2478        );
2479
2480        // Rotation causes are NOT excluded: retire distills the window even
2481        // after a recorder write (window is final; manifest guards dupes).
2482        let (engine, _client) = engine_with(
2483            vec![ONE_OP_REPLY],
2484            provider.clone(),
2485            Some(slice(&[("user", "a")])),
2486            Vec::new(),
2487            None,
2488            enabled_config(),
2489        );
2490        engine.note_recorder_write("identity:a", "sess-1");
2491        let outcome = engine
2492            .distill_now("identity:a", "sess-1", DistillCause::Retire)
2493            .await;
2494        assert!(
2495            matches!(outcome, DistillOutcome::Completed { .. }),
2496            "{outcome:?}"
2497        );
2498    }
2499
2500    // -- trigger throttling + budget guard -----------------------------------
2501
2502    #[test]
2503    fn interaction_trigger_respects_min_interactions_and_in_flight() {
2504        let provider = Arc::new(CapturingProvider::default());
2505        let (engine, _client) = engine_with(
2506            vec![],
2507            provider,
2508            None,
2509            Vec::new(),
2510            None,
2511            DistillerConfig {
2512                enabled: true,
2513                min_interactions: 3,
2514                ..DistillerConfig::default()
2515            },
2516        );
2517        assert!(!engine.note_run_completed("identity:a", "sess-1"));
2518        assert!(!engine.note_run_completed("identity:a", "sess-1"));
2519        assert!(
2520            engine.note_run_completed("identity:a", "sess-1"),
2521            "third run trips"
2522        );
2523        // in_flight set by the trip: further completions coalesce.
2524        assert!(!engine.note_run_completed("identity:a", "sess-1"));
2525    }
2526
2527    #[tokio::test]
2528    async fn budget_guard_skips_runs_beyond_the_window_cap() {
2529        let provider = Arc::new(CapturingProvider::default());
2530        let (engine, _client) = engine_with(
2531            vec![NOOP_REPLY, NOOP_REPLY],
2532            provider,
2533            Some(slice(&[("user", "a")])),
2534            Vec::new(),
2535            None,
2536            DistillerConfig {
2537                enabled: true,
2538                runs_per_hour: 1,
2539                ..DistillerConfig::default()
2540            },
2541        );
2542        let first = engine
2543            .distill_now("identity:a", "sess-1", DistillCause::Retire)
2544            .await;
2545        assert!(
2546            matches!(first, DistillOutcome::Completed { .. }),
2547            "{first:?}"
2548        );
2549        // Fresh window content so the second attempt reaches the guard.
2550        engine.with_window("identity:a", "sess-1", |state| state.cursor = 0);
2551        let second = engine
2552            .distill_now("identity:a", "sess-1", DistillCause::Retire)
2553            .await;
2554        assert!(
2555            matches!(&second, DistillOutcome::Skipped { reason } if reason.contains("budget denied")),
2556            "{second:?}"
2557        );
2558    }
2559
2560    // -- trigger sink over the observe stream --------------------------------
2561
2562    fn envelope(
2563        session: &meerkat_core::types::SessionId,
2564        payload: AgentEvent,
2565    ) -> meerkat_core::event::EventEnvelope<AgentEvent> {
2566        meerkat_core::event::EventEnvelope {
2567            event_id: Default::default(),
2568            source: meerkat_core::event::EventSourceIdentity::Session {
2569                session_id: session.clone(),
2570            },
2571            seq: 0,
2572            mob_id: None,
2573            timestamp_ms: 0,
2574            payload,
2575        }
2576    }
2577
2578    #[tokio::test]
2579    async fn trigger_sink_marks_memory_tool_writes_but_not_reads() {
2580        let provider = Arc::new(CapturingProvider::default());
2581        let session = meerkat_core::types::SessionId::new();
2582        let session_key = session.to_string();
2583        let make = |transcript: TranscriptSlice| {
2584            engine_with(
2585                vec![NOOP_REPLY],
2586                provider.clone(),
2587                Some(transcript),
2588                Vec::new(),
2589                None,
2590                enabled_config(),
2591            )
2592        };
2593        let tool_call = |action: &str| AgentEvent::ToolCallRequested {
2594            id: "t-1".to_string(),
2595            name: MEMORY_TOOL_NAME.to_string(),
2596            args: meerkat_core::event::ToolCallArguments::from_value(serde_json::json!({
2597                "action": action, "title": "t", "body": "b"
2598            }))
2599            .expect("object args"),
2600        };
2601
2602        // A memory WRITE trips the mutual-exclusion flag...
2603        let mut transcript = slice(&[("user", "a")]);
2604        transcript.session_key = session_key.clone();
2605        let (engine, _client) = make(transcript.clone());
2606        let sink = DistillerTriggers::new(engine.clone());
2607        sink.observe("identity:a", &envelope(&session, tool_call("remember")));
2608        let outcome = engine
2609            .distill_now("identity:a", &session_key, DistillCause::Interactions)
2610            .await;
2611        assert!(
2612            matches!(&outcome, DistillOutcome::Skipped { reason } if reason.contains("mutual exclusion")),
2613            "{outcome:?}"
2614        );
2615
2616        // ...a memory READ (recall) does not.
2617        let (engine, _client) = make(transcript);
2618        let sink = DistillerTriggers::new(engine.clone());
2619        sink.observe("identity:a", &envelope(&session, tool_call("recall")));
2620        let outcome = engine
2621            .distill_now("identity:a", &session_key, DistillCause::Interactions)
2622            .await;
2623        assert!(
2624            matches!(outcome, DistillOutcome::Completed { .. }),
2625            "{outcome:?}"
2626        );
2627    }
2628
2629    // -- reset quarantine -----------------------------------------------------
2630
2631    #[tokio::test]
2632    async fn reset_cause_marks_boundary_so_evidence_quarantines() {
2633        let tracker = SessionTaintTracker::new(Default::default());
2634        let provider = Arc::new(CapturingProvider::default());
2635        let (engine, _client) = engine_with(
2636            vec![ONE_OP_REPLY],
2637            provider,
2638            Some(slice(&[("user", "a"), ("assistant", "b"), ("user", "c")])),
2639            Vec::new(),
2640            Some(tracker.clone()),
2641            enabled_config(),
2642        );
2643        assert!(tracker.evidence_quarantine_reason("sess-1").is_none());
2644        engine
2645            .distill_now("identity:a", "sess-1", DistillCause::Reset)
2646            .await;
2647        let reason = tracker
2648            .evidence_quarantine_reason("sess-1")
2649            .expect("reset boundary marked before writes");
2650        assert!(reason.contains("reset"), "{reason}");
2651    }
2652
2653    // -- pre-rotation timeout --------------------------------------------------
2654
2655    #[tokio::test]
2656    async fn pre_rotation_distillation_never_blocks_rotation() {
2657        let provider = Arc::new(CapturingProvider::default());
2658        let engine = Arc::new(
2659            DistillerEngine::new(
2660                DistillerProfile::embedded_default(),
2661                enabled_config(),
2662                Arc::new(HangingHandle),
2663                provider,
2664                Arc::new(StaticTombstones(Vec::new())),
2665                Arc::new(StaticTranscript(Some(slice(&[("user", "a")])))),
2666                None,
2667                None,
2668                "family",
2669            )
2670            .with_pre_rotation_timeout(Duration::from_millis(50)),
2671        );
2672        let started = Instant::now();
2673        engine
2674            .distill_before_rotation("identity:a", "sess-1", DistillCause::Respawn)
2675            .await;
2676        assert!(
2677            started.elapsed() < Duration::from_secs(2),
2678            "pre-rotation hook must return at the timeout, not hang"
2679        );
2680    }
2681
2682    // -- profile ----------------------------------------------------------------
2683
2684    #[test]
2685    fn embedded_prompt_matches_calibration_bundle() -> Result<(), Box<dyn std::error::Error>> {
2686        let bundle =
2687            Path::new(env!("CARGO_MANIFEST_DIR")).join("../memory-evals/prompts/distiller-v0.md");
2688        if !bundle.is_file() {
2689            return Ok(());
2690        }
2691        let text = std::fs::read_to_string(bundle)?;
2692        assert_eq!(
2693            text, EMBEDDED_PROMPT_V0,
2694            "memory-evals/prompts/distiller-v0.md and \
2695             src/memory/distiller_prompt_v0.md have drifted"
2696        );
2697        Ok(())
2698    }
2699
2700    #[test]
2701    fn embedded_default_profile_validates_and_names_a_catalog_model() {
2702        let profile = DistillerProfile::embedded_default();
2703        profile.validate().expect("embedded profile must validate");
2704        assert_eq!(
2705            meerkat_models::infer_provider(&profile.model),
2706            Some(profile.provider),
2707            "embedded default model must resolve in the catalog"
2708        );
2709    }
2710
2711    #[test]
2712    fn external_profile_loads_from_evals_layout() -> Result<(), Box<dyn std::error::Error>> {
2713        let path = Path::new(env!("CARGO_MANIFEST_DIR"))
2714            .join("../memory-evals/profiles/distiller-v0.toml");
2715        if !path.is_file() {
2716            return Ok(());
2717        }
2718        let profile = DistillerProfile::load(&path)?;
2719        assert_eq!(profile.stage, "distiller");
2720        assert_eq!(profile.prompt_template, EMBEDDED_PROMPT_V0);
2721        Ok(())
2722    }
2723
2724    #[test]
2725    fn model_override_is_fail_loud() {
2726        let profile = DistillerProfile::embedded_default();
2727        assert!(profile.clone().with_model_override("not-a-model").is_err());
2728        assert!(profile.clone().with_model_override("  ").is_err());
2729        let overridden = profile
2730            .with_model_override("claude-haiku-4-5")
2731            .expect("catalog model accepted");
2732        assert_eq!(overridden.model, "claude-haiku-4-5");
2733    }
2734
2735    // -- §8.6 seams: harvest classification, follow-up hook, cursor ----------
2736
2737    #[test]
2738    fn compaction_harvest_satisfaction_classifies_skip_reasons() {
2739        assert!(
2740            DistillOutcome::Completed {
2741                run_id: "r".to_string(),
2742                written: 0,
2743                quarantined: 0,
2744            }
2745            .compaction_harvest_satisfied()
2746        );
2747        assert!(
2748            DistillOutcome::Skipped {
2749                reason: SKIP_NO_DISCARDS.to_string(),
2750            }
2751            .compaction_harvest_satisfied()
2752        );
2753        assert!(
2754            DistillOutcome::Skipped {
2755                reason: SKIP_NO_DISCARD_SOURCE.to_string(),
2756            }
2757            .compaction_harvest_satisfied()
2758        );
2759        for reason in [
2760            "budget denied: window budget exhausted (2/2 runs)",
2761            "compaction harvest failed: io",
2762            "extraction failed: parse",
2763        ] {
2764            assert!(
2765                !DistillOutcome::Skipped {
2766                    reason: reason.to_string(),
2767                }
2768                .compaction_harvest_satisfied(),
2769                "{reason}"
2770            );
2771        }
2772    }
2773
2774    #[tokio::test]
2775    async fn compaction_follow_up_fires_with_outcome_and_cursor_is_readable() {
2776        let provider = Arc::new(CapturingProvider::default());
2777        // No compaction source wired ⇒ the run skips with the benign
2778        // "nothing to harvest" reason, and the follow-up still fires.
2779        let (engine, _client) = engine_with(
2780            vec![NOOP_REPLY],
2781            provider,
2782            Some(slice(&[("user", "hello")])),
2783            Vec::new(),
2784            None,
2785            enabled_config(),
2786        );
2787        let seen: Arc<StdMutex<Vec<(String, String, bool)>>> = Arc::new(StdMutex::new(Vec::new()));
2788        let sink = seen.clone();
2789        engine.set_compaction_follow_up(Arc::new(move |identity, session, outcome| {
2790            sink.lock()
2791                .unwrap_or_else(std::sync::PoisonError::into_inner)
2792                .push((
2793                    identity.to_string(),
2794                    session.to_string(),
2795                    outcome.compaction_harvest_satisfied(),
2796                ));
2797        }));
2798        engine
2799            .distill_now("identity:a", "sess-1", DistillCause::Compaction)
2800            .await;
2801        // Non-compaction causes never fire the hook.
2802        engine
2803            .distill_now("identity:a", "sess-1", DistillCause::Interactions)
2804            .await;
2805        let seen = seen
2806            .lock()
2807            .unwrap_or_else(std::sync::PoisonError::into_inner)
2808            .clone();
2809        assert_eq!(
2810            seen,
2811            vec![("identity:a".to_string(), "sess-1".to_string(), true)]
2812        );
2813        // The §8.4 window cursor is readable without creating state.
2814        assert_eq!(engine.distilled_cursor("identity:never", "sess-x"), 0);
2815        engine.with_window("identity:a", "sess-1", |state| state.cursor = 7);
2816        assert_eq!(engine.distilled_cursor("identity:a", "sess-1"), 7);
2817    }
2818
2819    struct StaticDiscards(Vec<DiscardEntry>);
2820
2821    #[async_trait]
2822    impl CompactionDiscardSource for StaticDiscards {
2823        async fn read_discards(
2824            &self,
2825            _session_key: &str,
2826            _limit: usize,
2827        ) -> Result<Vec<DiscardEntry>, DistillerError> {
2828            Ok(self.0.clone())
2829        }
2830    }
2831
2832    #[tokio::test]
2833    async fn compaction_harvest_preserves_interaction_window_state() {
2834        let provider = Arc::new(CapturingProvider::default());
2835        let client = Arc::new(ScriptedLlm::new(vec![ONE_OP_REPLY]));
2836        let engine = Arc::new(DistillerEngine::new(
2837            DistillerProfile::embedded_default(),
2838            enabled_config(),
2839            Arc::new(ScriptedHandle { client }),
2840            provider,
2841            Arc::new(StaticTombstones(Vec::new())),
2842            Arc::new(StaticTranscript(Some(slice(&[
2843                ("user", "use the wrapper"),
2844                ("assistant", "noted"),
2845            ])))),
2846            Some(Arc::new(StaticDiscards(vec![DiscardEntry {
2847                content: "discarded: wrapper reminder".to_string(),
2848                range: Some((0, 5)),
2849            }]))),
2850            None,
2851            "family",
2852        ));
2853        // The agent's Recorder wrote in the interaction window, and one run
2854        // completed toward the interaction trigger.
2855        engine.note_recorder_write("identity:a", "sess-1");
2856        assert!(!engine.note_run_completed("identity:a", "sess-1"));
2857
2858        // A compaction-cause harvest COMPLETES over the discard ledger…
2859        let outcome = engine
2860            .distill_now("identity:a", "sess-1", DistillCause::Compaction)
2861            .await;
2862        assert!(
2863            matches!(outcome, DistillOutcome::Completed { written: 1, .. }),
2864            "{outcome:?}"
2865        );
2866
2867        // …without consuming the interaction window it never covered: the
2868        // cursor stays put, the §8.4 mutual-exclusion flag survives, and the
2869        // interaction counter is not silently deferred.
2870        let (cursor, recorder_wrote, completed_runs) =
2871            engine.with_window("identity:a", "sess-1", |state| {
2872                (state.cursor, state.recorder_wrote, state.completed_runs)
2873            });
2874        assert_eq!(cursor, 0, "compaction never advances the transcript cursor");
2875        assert!(
2876            recorder_wrote,
2877            "compaction-cause harvests must not clear the recorder \
2878             mutual-exclusion flag for a window they did not distill"
2879        );
2880        assert_eq!(
2881            completed_runs, 1,
2882            "compaction-cause harvests must not zero the interaction counter"
2883        );
2884
2885        // The next interactions-cause run therefore still skips the window
2886        // the agent already curated itself.
2887        let outcome = engine
2888            .distill_now("identity:a", "sess-1", DistillCause::Interactions)
2889            .await;
2890        assert!(
2891            matches!(&outcome, DistillOutcome::Skipped { reason } if reason.contains("mutual exclusion")),
2892            "{outcome:?}"
2893        );
2894    }
2895
2896    #[tokio::test]
2897    async fn compaction_observed_hook_fires_at_observation_even_without_discards() {
2898        let provider = Arc::new(CapturingProvider::default());
2899        // No discard source wired: the harvest itself will skip — the
2900        // observation hook must fire regardless (sequencing observers must
2901        // not depend on the harvest being viable).
2902        let (engine, _client) = engine_with(
2903            vec![NOOP_REPLY],
2904            provider,
2905            Some(slice(&[("user", "hello")])),
2906            Vec::new(),
2907            None,
2908            enabled_config(),
2909        );
2910        let seen: Arc<StdMutex<Vec<(String, String)>>> = Arc::new(StdMutex::new(Vec::new()));
2911        let observed = seen.clone();
2912        engine.set_compaction_observed(Arc::new(move |identity, session| {
2913            observed
2914                .lock()
2915                .unwrap_or_else(std::sync::PoisonError::into_inner)
2916                .push((identity.to_string(), session.to_string()));
2917        }));
2918
2919        let session = meerkat_core::types::SessionId::new();
2920        let sink = DistillerTriggers::new(engine.clone());
2921        sink.observe(
2922            "identity:a",
2923            &envelope(
2924                &session,
2925                AgentEvent::CompactionCompleted {
2926                    summary_tokens: 10,
2927                    messages_before: 20,
2928                    messages_after: 2,
2929                },
2930            ),
2931        );
2932        // Fired synchronously at observation time — no detached harvest to
2933        // await.
2934        assert_eq!(
2935            seen.lock()
2936                .unwrap_or_else(std::sync::PoisonError::into_inner)
2937                .clone(),
2938            vec![("identity:a".to_string(), session.to_string())]
2939        );
2940
2941        // Non-compaction events never fire it.
2942        sink.observe(
2943            "identity:a",
2944            &envelope(
2945                &session,
2946                AgentEvent::RunCompleted {
2947                    session_id: session.clone(),
2948                    result: "done".to_string(),
2949                    structured_output: None,
2950                    extraction_required: false,
2951                    usage: Default::default(),
2952                    terminal_cause_kind: None,
2953                },
2954            ),
2955        );
2956        assert_eq!(
2957            seen.lock()
2958                .unwrap_or_else(std::sync::PoisonError::into_inner)
2959                .len(),
2960            1
2961        );
2962    }
2963}